AWS DynamoDBのページング †TODO:
以下、Paginatorsを使用してページネーションを行う方法を記載する テスト用テーブル 及び データの作成 †create_test_data_for_paginator.py import boto3 import copy TABLE_NAME = 'TestPaginator' dynamodb = boto3.resource('dynamodb') def delete_table(): """テーブル削除 """ try: table = dynamodb.Table(TABLE_NAME) response = table.delete() except Exception as e: print(e) pass def create_table(): """テーブル作成 """ try: dynamodb.create_table( TableName=TABLE_NAME, KeySchema=[ {'AttributeName': 'pkey', 'KeyType': 'HASH'}, {'AttributeName': 'skey', 'KeyType': 'RANGE'} ], AttributeDefinitions=[ {'AttributeName': 'pkey', 'AttributeType': 'S'}, {'AttributeName': 'skey', 'AttributeType': 'S'} ], ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 } ) except Exception as e: print(e) pass def create_test_data(): """テストデータ作成 """ init_arguments = {'RequestItems': {TABLE_NAME: []}} arguments = copy.deepcopy(init_arguments) requests = arguments['RequestItems'][TABLE_NAME] for i in range(100): group = int(i/10) item = { 'pkey': 'pkey-{0:03d}'.format(group), 'skey': 'skey-{0:03d}'.format(i), 'str_value': 'str-{0:03d}'.format(i), 'num_value': i } requests.append({'PutRequest': {'Item': item}}) if len(requests) == 25: response = dynamodb.batch_write_item(**arguments) arguments = copy.deepcopy(init_arguments) requests = arguments['RequestItems'][TABLE_NAME] if len(requests) > 0: response = dynamodb.batch_write_item(**arguments) if __name__ == '__main__': # テーブル削除 #delete_table() # テーブル作成 create_table() # テストデータ作成 create_test_data() ページネーションのテスト †test_paginator.py import boto3 TABLE_NAME = 'TestPaginator' PAGE_SIZE=10 MAX_ITEMS=10 def scan_paginator(): next_token = None client = boto3.client('dynamodb') paginator = client.get_paginator('scan') #next_token = {'pkey': {'S': 'pkey-008'}, 'skey': {'S': 'skey-001'}} i = 0 response = None while response is None or next_token is not None: arguments = { 'TableName': TABLE_NAME, 'Select': 'ALL_ATTRIBUTES', # 'ALL_ATTRIBUTES'|'ALL_PROJECTED_ATTRIBUTES'|'SPECIFIC_ATTRIBUTES'|'COUNT' 'ConsistentRead': True, # True時はScanの開始前に登録完了したすべての書き込みがスキャン応答に含まれることを保証する 'PaginationConfig': { 'MaxItems': MAX_ITEMS, 'PageSize': PAGE_SIZE, }, 'ReturnConsumedCapacity': 'INDEXES', #'ScanIndexForward': True } if next_token is not None: #arguments['PaginationConfig']['StartingToken'] = next_token arguments['ExclusiveStartKey'] = next_token print(f'### next_token: {next_token}') else: print(f'### next_token: None') response = paginator.paginate(**arguments) next_token = None for rec in response: for item in rec["Items"]: i += 1 print(f'{i}: {item}') #print(f'{i}: {rec["Items"]}') #print('### ConsumedCapacity: {} ###'.format(rec['ConsumedCapacity'])) if 'LastEvaluatedKey' in rec: next_token = rec['LastEvaluatedKey'] #if response["NextToken"]: # next_token = response["NextToken"] # TODO: NextToken返ってこない.. PaginationConfigの設定がおかしい? if __name__ == '__main__': scan_paginator() 実行結果 †python3 test_paginator.py ### next_token: None 1: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-070'}, 'num_value': {'N': '70'}, 'skey': {'S': 'skey-070'}} 2: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-071'}, 'num_value': {'N': '71'}, 'skey': {'S': 'skey-071'}} 3: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-072'}, 'num_value': {'N': '72'}, 'skey': {'S': 'skey-072'}} 4: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-073'}, 'num_value': {'N': '73'}, 'skey': {'S': 'skey-073'}} 5: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-074'}, 'num_value': {'N': '74'}, 'skey': {'S': 'skey-074'}} 6: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-075'}, 'num_value': {'N': '75'}, 'skey': {'S': 'skey-075'}} 7: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-076'}, 'num_value': {'N': '76'}, 'skey': {'S': 'skey-076'}} 8: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-077'}, 'num_value': {'N': '77'}, 'skey': {'S': 'skey-077'}} 9: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-078'}, 'num_value': {'N': '78'}, 'skey': {'S': 'skey-078'}} 10: {'pkey': {'S': 'pkey-007'}, 'str_value': {'S': 'str-079'}, 'num_value': {'N': '79'}, 'skey': {'S': 'skey-079'}} ### next_token: {'pkey': {'S': 'pkey-007'}, 'skey': {'S': 'skey-079'}} 11: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-030'}, 'num_value': {'N': '30'}, 'skey': {'S': 'skey-030'}} 12: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-031'}, 'num_value': {'N': '31'}, 'skey': {'S': 'skey-031'}} 13: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-032'}, 'num_value': {'N': '32'}, 'skey': {'S': 'skey-032'}} 14: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-033'}, 'num_value': {'N': '33'}, 'skey': {'S': 'skey-033'}} 15: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-034'}, 'num_value': {'N': '34'}, 'skey': {'S': 'skey-034'}} 16: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-035'}, 'num_value': {'N': '35'}, 'skey': {'S': 'skey-035'}} 17: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-036'}, 'num_value': {'N': '36'}, 'skey': {'S': 'skey-036'}} 18: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-037'}, 'num_value': {'N': '37'}, 'skey': {'S': 'skey-037'}} 19: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-038'}, 'num_value': {'N': '38'}, 'skey': {'S': 'skey-038'}} 20: {'pkey': {'S': 'pkey-003'}, 'str_value': {'S': 'str-039'}, 'num_value': {'N': '39'}, 'skey': {'S': 'skey-039'}} ### next_token: {'pkey': {'S': 'pkey-003'}, 'skey': {'S': 'skey-039'}} 21: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-050'}, 'num_value': {'N': '50'}, 'skey': {'S': 'skey-050'}} 22: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-051'}, 'num_value': {'N': '51'}, 'skey': {'S': 'skey-051'}} 23: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-052'}, 'num_value': {'N': '52'}, 'skey': {'S': 'skey-052'}} 24: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-053'}, 'num_value': {'N': '53'}, 'skey': {'S': 'skey-053'}} 25: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-054'}, 'num_value': {'N': '54'}, 'skey': {'S': 'skey-054'}} 26: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-055'}, 'num_value': {'N': '55'}, 'skey': {'S': 'skey-055'}} 27: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-056'}, 'num_value': {'N': '56'}, 'skey': {'S': 'skey-056'}} 28: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-057'}, 'num_value': {'N': '57'}, 'skey': {'S': 'skey-057'}} 29: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-058'}, 'num_value': {'N': '58'}, 'skey': {'S': 'skey-058'}} 30: {'pkey': {'S': 'pkey-005'}, 'str_value': {'S': 'str-059'}, 'num_value': {'N': '59'}, 'skey': {'S': 'skey-059'}} ### next_token: {'pkey': {'S': 'pkey-005'}, 'skey': {'S': 'skey-059'}} 31: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-040'}, 'num_value': {'N': '40'}, 'skey': {'S': 'skey-040'}} 32: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-041'}, 'num_value': {'N': '41'}, 'skey': {'S': 'skey-041'}} 33: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-042'}, 'num_value': {'N': '42'}, 'skey': {'S': 'skey-042'}} 34: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-043'}, 'num_value': {'N': '43'}, 'skey': {'S': 'skey-043'}} 35: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-044'}, 'num_value': {'N': '44'}, 'skey': {'S': 'skey-044'}} 36: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-045'}, 'num_value': {'N': '45'}, 'skey': {'S': 'skey-045'}} 37: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-046'}, 'num_value': {'N': '46'}, 'skey': {'S': 'skey-046'}} 38: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-047'}, 'num_value': {'N': '47'}, 'skey': {'S': 'skey-047'}} 39: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-048'}, 'num_value': {'N': '48'}, 'skey': {'S': 'skey-048'}} 40: {'pkey': {'S': 'pkey-004'}, 'str_value': {'S': 'str-049'}, 'num_value': {'N': '49'}, 'skey': {'S': 'skey-049'}} ### next_token: {'pkey': {'S': 'pkey-004'}, 'skey': {'S': 'skey-049'}} 41: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-010'}, 'num_value': {'N': '10'}, 'skey': {'S': 'skey-010'}} 42: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-011'}, 'num_value': {'N': '11'}, 'skey': {'S': 'skey-011'}} 43: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-012'}, 'num_value': {'N': '12'}, 'skey': {'S': 'skey-012'}} 44: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-013'}, 'num_value': {'N': '13'}, 'skey': {'S': 'skey-013'}} 45: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-014'}, 'num_value': {'N': '14'}, 'skey': {'S': 'skey-014'}} 46: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-015'}, 'num_value': {'N': '15'}, 'skey': {'S': 'skey-015'}} 47: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-016'}, 'num_value': {'N': '16'}, 'skey': {'S': 'skey-016'}} 48: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-017'}, 'num_value': {'N': '17'}, 'skey': {'S': 'skey-017'}} 49: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-018'}, 'num_value': {'N': '18'}, 'skey': {'S': 'skey-018'}} 50: {'pkey': {'S': 'pkey-001'}, 'str_value': {'S': 'str-019'}, 'num_value': {'N': '19'}, 'skey': {'S': 'skey-019'}} ### next_token: {'pkey': {'S': 'pkey-001'}, 'skey': {'S': 'skey-019'}} 51: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-000'}, 'num_value': {'N': '0'}, 'skey': {'S': 'skey-000'}} 52: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-001'}, 'num_value': {'N': '1'}, 'skey': {'S': 'skey-001'}} 53: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-002'}, 'num_value': {'N': '2'}, 'skey': {'S': 'skey-002'}} 54: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-003'}, 'num_value': {'N': '3'}, 'skey': {'S': 'skey-003'}} 55: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-004'}, 'num_value': {'N': '4'}, 'skey': {'S': 'skey-004'}} 56: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-005'}, 'num_value': {'N': '5'}, 'skey': {'S': 'skey-005'}} 57: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-006'}, 'num_value': {'N': '6'}, 'skey': {'S': 'skey-006'}} 58: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-007'}, 'num_value': {'N': '7'}, 'skey': {'S': 'skey-007'}} 59: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-008'}, 'num_value': {'N': '8'}, 'skey': {'S': 'skey-008'}} 60: {'pkey': {'S': 'pkey-000'}, 'str_value': {'S': 'str-009'}, 'num_value': {'N': '9'}, 'skey': {'S': 'skey-009'}} ### next_token: {'pkey': {'S': 'pkey-000'}, 'skey': {'S': 'skey-009'}} 61: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-020'}, 'num_value': {'N': '20'}, 'skey': {'S': 'skey-020'}} 62: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-021'}, 'num_value': {'N': '21'}, 'skey': {'S': 'skey-021'}} 63: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-022'}, 'num_value': {'N': '22'}, 'skey': {'S': 'skey-022'}} 64: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-023'}, 'num_value': {'N': '23'}, 'skey': {'S': 'skey-023'}} 65: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-024'}, 'num_value': {'N': '24'}, 'skey': {'S': 'skey-024'}} 66: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-025'}, 'num_value': {'N': '25'}, 'skey': {'S': 'skey-025'}} 67: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-026'}, 'num_value': {'N': '26'}, 'skey': {'S': 'skey-026'}} 68: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-027'}, 'num_value': {'N': '27'}, 'skey': {'S': 'skey-027'}} 69: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-028'}, 'num_value': {'N': '28'}, 'skey': {'S': 'skey-028'}} 70: {'pkey': {'S': 'pkey-002'}, 'str_value': {'S': 'str-029'}, 'num_value': {'N': '29'}, 'skey': {'S': 'skey-029'}} ### next_token: {'pkey': {'S': 'pkey-002'}, 'skey': {'S': 'skey-029'}} 71: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-060'}, 'num_value': {'N': '60'}, 'skey': {'S': 'skey-060'}} 72: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-061'}, 'num_value': {'N': '61'}, 'skey': {'S': 'skey-061'}} 73: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-062'}, 'num_value': {'N': '62'}, 'skey': {'S': 'skey-062'}} 74: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-063'}, 'num_value': {'N': '63'}, 'skey': {'S': 'skey-063'}} 75: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-064'}, 'num_value': {'N': '64'}, 'skey': {'S': 'skey-064'}} 76: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-065'}, 'num_value': {'N': '65'}, 'skey': {'S': 'skey-065'}} 77: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-066'}, 'num_value': {'N': '66'}, 'skey': {'S': 'skey-066'}} 78: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-067'}, 'num_value': {'N': '67'}, 'skey': {'S': 'skey-067'}} 79: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-068'}, 'num_value': {'N': '68'}, 'skey': {'S': 'skey-068'}} 80: {'pkey': {'S': 'pkey-006'}, 'str_value': {'S': 'str-069'}, 'num_value': {'N': '69'}, 'skey': {'S': 'skey-069'}} ### next_token: {'pkey': {'S': 'pkey-006'}, 'skey': {'S': 'skey-069'}} 81: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-090'}, 'num_value': {'N': '90'}, 'skey': {'S': 'skey-090'}} 82: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-091'}, 'num_value': {'N': '91'}, 'skey': {'S': 'skey-091'}} 83: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-092'}, 'num_value': {'N': '92'}, 'skey': {'S': 'skey-092'}} 84: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-093'}, 'num_value': {'N': '93'}, 'skey': {'S': 'skey-093'}} 85: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-094'}, 'num_value': {'N': '94'}, 'skey': {'S': 'skey-094'}} 86: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-095'}, 'num_value': {'N': '95'}, 'skey': {'S': 'skey-095'}} 87: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-096'}, 'num_value': {'N': '96'}, 'skey': {'S': 'skey-096'}} 88: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-097'}, 'num_value': {'N': '97'}, 'skey': {'S': 'skey-097'}} 89: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-098'}, 'num_value': {'N': '98'}, 'skey': {'S': 'skey-098'}} 90: {'pkey': {'S': 'pkey-009'}, 'str_value': {'S': 'str-099'}, 'num_value': {'N': '99'}, 'skey': {'S': 'skey-099'}} ### next_token: {'pkey': {'S': 'pkey-009'}, 'skey': {'S': 'skey-099'}} 91: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-080'}, 'num_value': {'N': '80'}, 'skey': {'S': 'skey-080'}} 92: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-081'}, 'num_value': {'N': '81'}, 'skey': {'S': 'skey-081'}} 93: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-082'}, 'num_value': {'N': '82'}, 'skey': {'S': 'skey-082'}} 94: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-083'}, 'num_value': {'N': '83'}, 'skey': {'S': 'skey-083'}} 95: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-084'}, 'num_value': {'N': '84'}, 'skey': {'S': 'skey-084'}} 96: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-085'}, 'num_value': {'N': '85'}, 'skey': {'S': 'skey-085'}} 97: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-086'}, 'num_value': {'N': '86'}, 'skey': {'S': 'skey-086'}} 98: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-087'}, 'num_value': {'N': '87'}, 'skey': {'S': 'skey-087'}} 99: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-088'}, 'num_value': {'N': '88'}, 'skey': {'S': 'skey-088'}} 100: {'pkey': {'S': 'pkey-008'}, 'str_value': {'S': 'str-089'}, 'num_value': {'N': '89'}, 'skey': {'S': 'skey-089'}} ### next_token: {'pkey': {'S': 'pkey-008'}, 'skey': {'S': 'skey-089'}} 注意点など †・別セッションで取得した ExclusiveStartKey を指定しても動作する。 |