By using AWS re:Post, you agree to the AWS re:Post Terms of Use
AWS re:Postre:Post

Key element doesnt match scheme

0

I am running into a problem with batch_write() and pushing it into my dynamodb table. Below is the code that I am running along with what the batch output looks like:

Scheme:

def createItemOrderTable(self, tableName: str) -> int:
        try:
            response = self.dynamodb.create_table(
                TableName=tableName+"_ORDERS",
                KeySchema=[
                    {
                        'AttributeName': 'issued', # It is called issued in the esi pull
                        'KeyType': 'HASH'  # Hash Key
                    },
                    {
                        'AttributeName': 'order_id', 
                        'KeyType': 'RANGE'  # Sort Key
                    }
                ],
                AttributeDefinitions=[
                    {
                        'AttributeName': 'issued',
                        'AttributeType': 'S'  # string data type
                    },
                    {
                        'AttributeName': 'order_id',
                        'AttributeType': 'N'  # number data type
                    }
                ],
                ProvisionedThroughput={
                    'ReadCapacityUnits': 50,
                    'WriteCapacityUnits': 50
                }
            )
            print(f'[createNewTable]: New table {tableName} created')
            return 200
        except ClientError  as err:
            if err.response['ResponseMetadata']['HTTPStatusCode'] == 400:
                print(f"[createNewTable]: {err.response['message']} Table already created")
            else:
                print(f"[createNewTable]: Error msg {err.response['message']}")
            return err.response['ResponseMetadata']['HTTPStatusCode']

Function to push data:

 def pushItemOrdersToDynamo(self,tableName: str)->None:
        response = self.newTable.createItemOrderTable(tableName)
        print(response)
        if response == 200:
            print(f'Create new table {tableName}')
            table = self.dynamodb.Table(tableName)
            table.wait_until_exists()
            print("Done waiting")
            itemjson = itemPrices.getAllItemOrderHistory(item[tableName].value,region.THE_FORGE.value)
            jsondata = json.loads(itemjson, parse_float=Decimal)
            with table.batch_writer() as batch:
                for myDict in jsondata:
                    print(myDict)
                    response = batch.put_item(Item = myDict)
        else:
            print("Just update the table")
            table = self.dynamodb.Table(tableName)
            itemjson = itemPrices.getAllItemOrderHistory(item[tableName].value,region.THE_FORGE.value)
            jsondata = json.loads(itemjson, parse_float=Decimal)
            with table.batch_writer() as batch:
                for myDict in jsondata:
                    print(myDict)
                    response = batch.put_item(Item = myDict)
        print(f'Finished pushing the {tableName} table')```

How im calling it:

start = time.perf_counter()
    dynamodb = dynamodbPushData.pushData()
    for i in list(ItemIdEnum.item):
        # print(i.name)
        dynamodb.pushItemOrdersToDynamo(str(i.name))
        # dynamodb.pushPriceHistoryToDynamo(str(i.name))
    end = time.perf_counter()
    print(end-start)

How the json looks from the batch point of view:

{'duration': 7, 'is_buy_order': False, 'issued': '2023-08-24T13:40:47Z', 'location_id': 60003148, 'min_volume': 1, 'order_id': 6578976651, 'price': Decimal('30.6'), 'range': 'region', 'system_id': 30000134, 'type_id': 18, 'volume_remain': 6309, 'volume_total': 6309}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-07-06T18:46:42Z', 'location_id': 60006430, 'min_volume': 1, 'order_id': 6547446777, 'price': Decimal('40.0'), 'range': 'region', 'system_id': 30000135, 'type_id': 18, 'volume_remain': 28885, 'volume_total': 28885}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-20T01:04:35Z', 'location_id': 60003112, 'min_volume': 1, 'order_id': 6576115006, 'price': Decimal('31.0'), 'range': 'region', 'system_id': 30000136, 'type_id': 18, 'volume_remain': 14000, 'volume_total': 14000}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-16T02:07:27Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6572827930, 'price': Decimal('44.98'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 194942, 'volume_total': 1000000}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-21T19:48:15Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6577349828, 'price': Decimal('43.99'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 572172, 'volume_total': 586172}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-21T11:56:30Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6577079055, 'price': Decimal('44.97'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 33024, 'volume_total': 47024}
{'duration': 3, 'is_buy_order': False, 'issued': '2023-08-24T05:02:42Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6578789814, 'price': Decimal('43.93'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 13714, 'volume_total': 100000}
{'duration': 7, 'is_buy_order': False, 'issued': '2023-08-25T09:46:29Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6579505847, 'price': Decimal('43.87'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 100000, 'volume_total': 100000}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-24T10:27:03Z', 'location_id': 60003754, 'min_volume': 1, 'order_id': 6578897995, 'price': Decimal('35.0'), 'range': 'region', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 250, 'volume_total': 250}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-07-25T19:00:24Z', 'location_id': 60003049, 'min_volume': 1, 'order_id': 6559866725, 'price': Decimal('31.0'), 'range': 'region', 'system_id': 30000161, 'type_id': 18, 'volume_remain': 14427, 'volume_total': 14427}
{'duration': 30, 'is_buy_order': False, 'issued': '2023-08-24T18:24:41Z', 'location_id': 60000520, 'min_volume': 1, 'order_id': 6579137550, 'price': Decimal('30.0'), 'range': 'region', 'system_id': 30000184, 'type_id': 18, 'volume_remain': 36, 'volume_total': 36}
{'duration': 1, 'is_buy_order': False, 'issued': '2023-08-25T12:44:00Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6578760373, 'price': Decimal('43.86'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 819180, 'volume_total': 948989}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-25T14:53:14Z', 'location_id': 60006430, 'min_volume': 1, 'order_id': 6579642015, 'price': Decimal('40.0'), 'range': 'region', 'system_id': 30000135, 'type_id': 18, 'volume_remain': 30127, 'volume_total': 30127}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-25T16:44:48Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6577405145, 'price': Decimal('43.85'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 1602463, 'volume_total': 2013393}
{'duration': 90, 'is_buy_order': False, 'issued': '2023-08-25T18:32:11Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6579089343, 'price': Decimal('43.84'), 'range': 'region', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 201480, 'volume_total': 365561}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-08-21T01:21:39Z', 'location_id': 60004228, 'min_volume': 1, 'order_id': 6511337107, 'price': Decimal('25.05'), 'range': 'solarsystem', 'system_id': 30000129, 'type_id': 18, 'volume_remain': 5948883, 'volume_total': 6000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-08-18T03:54:04Z', 'location_id': 60003337, 'min_volume': 100000, 'order_id': 6512149734, 'price': Decimal('17.31'), 'range': '10', 'system_id': 30000139, 'type_id': 18, 'volume_remain': 3820420, 'volume_total': 20000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-05-28T09:06:50Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6507180858, 'price': Decimal('26.16'), 'range': '1', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 4273197, 'volume_total': 5000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-06-22T21:13:00Z', 'location_id': 60003760, 'min_volume': 1, 'order_id': 6508764420, 'price': Decimal('26.45'), 'range': 'station', 'system_id': 30000142, 'type_id': 18, 'volume_remain': 64145, 'volume_total': 500000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-07-27T15:21:58Z', 'location_id': 1029209158478, 'min_volume': 1, 'order_id': 6560935552, 'price': Decimal('30.21'), 'range': '1', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 18434816, 'volume_total': 20000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-08-21T06:57:26Z', 'location_id': 1029036975486, 'min_volume': 1, 'order_id': 6570893701, 'price': Decimal('26.46'), 'range': '1', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 1937267, 'volume_total': 10000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-08-20T08:06:08Z', 'location_id': 1029209158478, 'min_volume': 1, 'order_id': 6576287871, 'price': Decimal('30.23'), 'range': '1', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 149407, 'volume_total': 1000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-07-08T15:09:05Z', 'location_id': 1029209158478, 'min_volume': 1, 'order_id': 6548000885, 'price': Decimal('30.18'), 'range': '1', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 9285600, 'volume_total': 10000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-08-24T13:33:33Z', 'location_id': 1029036975486, 'min_volume': 1, 'order_id': 6572427336, 'price': Decimal('25.17'), 'range': '3', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 185732, 'volume_total': 1000000}
{'duration': 90, 'is_buy_order': True, 'issued': '2023-08-10T21:31:07Z', 'location_id': 1029209158478, 'min_volume': 1, 'order_id': 6540201085, 'price': Decimal('17.3'), 'range': 'region', 'system_id': 30000144, 'type_id': 18, 'volume_remain': 3176652, 'volume_total': 3999999}

Here is the exact error im getting from running this:

raise error_class(parsed_response, operation_name) botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the BatchWriteItem operation: The provided key element does not match the schema

Topics
Database
Tags
Amazon DynamoDB
Language
English
Rob
asked a year ago405 views
1 Answer
0
Accepted Answer

The problem was my lookup for the table was incorrect

Rob
answered a year ago

You are not logged in. Log in to post an answer.

A good answer clearly answers the question and provides constructive feedback and encourages professional growth in the question asker.

Guidelines for Answering Questions

Relevant content