diff --git a/airbyte-integrations/connectors/source-s3/.coveragerc b/airbyte-integrations/connectors/source-s3/.coveragerc index 0c7476c81073..4c1de9ec0853 100644 --- a/airbyte-integrations/connectors/source-s3/.coveragerc +++ b/airbyte-integrations/connectors/source-s3/.coveragerc @@ -3,4 +3,5 @@ omit = source_s3/exceptions.py source_s3/stream.py source_s3/utils.py - source_s3/source_files_abstract/source.py \ No newline at end of file + source_s3/source_files_abstract/source.py + source_s3/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/main.py b/airbyte-integrations/connectors/source-s3/main.py index c3b6b0bc32ed..cb0007d5581b 100644 --- a/airbyte-integrations/connectors/source-s3/main.py +++ b/airbyte-integrations/connectors/source-s3/main.py @@ -3,40 +3,7 @@ # -import sys -import traceback -from datetime import datetime -from typing import List - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type -from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader - - -def get_source(args: List[str]): - catalog_path = AirbyteEntrypoint.extract_catalog(args) - try: - return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor) - except Exception: - print( - AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.ERROR, - emitted_at=int(datetime.now().timestamp() * 1000), - error=AirbyteErrorTraceMessage( - message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", - stack_trace=traceback.format_exc(), - ), - ), - ).json() - ) - return None - +from source_s3.run import run if __name__ == "__main__": - _args = sys.argv[1:] - source = get_source(_args) - - if source: - launch(source, _args) + run() diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index f3bed3cd1ff4..8f13506605fd 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.3.0 + dockerImageTag: 4.3.1 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index 3171ff163f0e..e2d0000f2949 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -33,4 +33,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-s3=source_s3.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/run.py b/airbyte-integrations/connectors/source-s3/source_s3/run.py new file mode 100644 index 000000000000..67379f1ec4ab --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/run.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader + + +def get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + try: + return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = get_source(_args) + + if source: + launch(source, _args) diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 533d25b29285..36e5eb5fef31 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -256,6 +256,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.3.1 | 2024-01-04 | [33937](https://github.com/airbytehq/airbyte/pull/33937) | Prepare for airbyte-lib | | 4.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | | 4.2.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 4.2.3 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | @@ -334,4 +335,4 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | 0.1.3 | 2021-08-04 | [5197](https://github.com/airbytehq/airbyte/pull/5197) | Fixed bug where sync could hang indefinitely on schema inference | | 0.1.2 | 2021-08-02 | [5135](https://github.com/airbytehq/airbyte/pull/5135) | Fixed bug in spec so it displays in UI correctly | | 0.1.1 | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990/commits/ff5f70662c5f84eabc03526cddfcc9d73c58c0f4) | Fixed documentation url in source definition | -| 0.1.0 | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990) | Created S3 source connector | \ No newline at end of file +| 0.1.0 | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990) | Created S3 source connector |