feat(source-s3): bump cdk to have preserve sub directories option in …

…copy raw files (file-transfer) (#51474)
airbytehq · Jan 16, 2025 · c494847 · c494847
1 parent d4c848c
commit c494847
Show file tree

Hide file tree

Showing 6 changed files with 166 additions and 260 deletions.
diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json
@@ -405,6 +405,12 @@
                 "const": "use_file_transfer",
                 "enum": ["use_file_transfer"],
                 "type": "string"
+              },
+              "preserve_directory_structure": {
+                "default": true,
+                "description": "If enabled, sends subdirectory folder structure along with source file names to the destination. Otherwise, files will be synced by their names only. This option is ignored when file-based replication is not enabled.",
+                "title": "Preserve Sub-Directories in File Paths",
+                "type": "boolean"
               }
             },
             "description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.",

diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json
@@ -405,6 +405,12 @@
                 "const": "use_file_transfer",
                 "enum": ["use_file_transfer"],
                 "type": "string"
+              },
+              "preserve_directory_structure": {
+                "default": true,
+                "description": "If enabled, sends subdirectory folder structure along with source file names to the destination. Otherwise, files will be synced by their names only. This option is ignored when file-based replication is not enabled.",
+                "title": "Preserve Sub-Directories in File Paths",
+                "type": "boolean"
               }
             },
             "description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.",

diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml
@@ -10,7 +10,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
-  dockerImageTag: 4.11.4
+  dockerImageTag: 4.12.0-rc.1
   dockerRepository: airbyte/source-s3
   documentationUrl: https://docs.airbyte.com/integrations/sources/s3
   githubIssueLabel: source-s3
@@ -30,7 +30,7 @@ data:
   releaseStage: generally_available
   releases:
     rolloutConfiguration:
-      enableProgressiveRollout: false
+      enableProgressiveRollout: true
     breakingChanges:
       4.0.0:
         message: UX improvement, multi-stream support and deprecation of some parsing features
@@ -47,6 +47,8 @@ data:
     - suite: unitTests
     - suite: liveTests
       testConnections:
+        - name: s3_100_mb_dev_null
+          id: 3dbae341-863a-43c0-bf00-d6a7473c7364
         - name: s3_v4_jsonl_config_dev_null
           id: 01f508b1-5690-47af-940c-a777d471762f
         - name: s3_v4_csv_skip_rows_no_header_config_dev_null