From ccfb775519b1be6b391ed040e318a690cf09d2e5 Mon Sep 17 00:00:00 2001 From: DavidMrts Date: Wed, 1 May 2024 14:07:12 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Dynamodb:=20fix=20list?= =?UTF-8?q?=20more=20than=20100=20tables=20(#31935)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm --- .../connectors/source-dynamodb/metadata.yaml | 2 +- .../source/dynamodb/DynamodbOperations.java | 23 ++++++++++++++++--- docs/integrations/sources/dynamodb.md | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml index d7092572ef15..1a6cf19cd0be 100644 --- a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 50401137-8871-4c5a-abb7-1f5fda35545a - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-dynamodb documentationUrl: https://docs.airbyte.com/integrations/sources/dynamodb githubIssueLabel: source-dynamodb diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java index 27b1fbb3fa8d..bbc2f5dfd38e 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java @@ -22,6 +22,8 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest; +import software.amazon.awssdk.services.dynamodb.model.ListTablesRequest; +import software.amazon.awssdk.services.dynamodb.model.ListTablesResponse; import software.amazon.awssdk.services.dynamodb.model.ScanRequest; public class DynamodbOperations extends AbstractDatabase implements Closeable { @@ -56,9 +58,24 @@ private void initMappers() { } public List listTables() { - return dynamoDbClient.listTables() - // filter on table status? - .tableNames(); + List tableNames = new ArrayList<>(); + ListTablesRequest listTablesRequest = ListTablesRequest.builder().build(); + boolean completed = false; + + while (!completed) { + ListTablesResponse listTablesResponse = dynamoDbClient.listTables(listTablesRequest); + tableNames.addAll(listTablesResponse.tableNames()); + + if (listTablesResponse.lastEvaluatedTableName() == null) { + completed = true; + } else { + listTablesRequest = listTablesRequest.toBuilder() + .exclusiveStartTableName(listTablesResponse.lastEvaluatedTableName()) + .build(); + } + } + + return tableNames; } public List primaryKey(String tableName) { diff --git a/docs/integrations/sources/dynamodb.md b/docs/integrations/sources/dynamodb.md index a329fd2ab49c..a39ca606400e 100644 --- a/docs/integrations/sources/dynamodb.md +++ b/docs/integrations/sources/dynamodb.md @@ -75,6 +75,7 @@ the underlying role executing the container workload in AWS. | Version | Date | Pull Request | Subject | |:--------| :--------- | :-------------------------------------------------------- |:-----------------------------------------------------------------------| +| 0.3.1 | 2024-05-01 | [31935](https://github.com/airbytehq/airbyte/pull/31935) | Fix list more than 100 tables | | 0.3.0 | 2024-04-24 | [37530](https://github.com/airbytehq/airbyte/pull/37530) | Allow role based access | | 0.2.3 | 2024-02-13 | [35232](https://github.com/airbytehq/airbyte/pull/35232) | Adopt CDK 0.20.4 | | 0.2.2 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version |