From 99079f92511bbaa452e3d81686d8e8e2115f7e8b Mon Sep 17 00:00:00 2001 From: askarpets Date: Wed, 21 Feb 2024 14:32:31 +0200 Subject: [PATCH] Source Amazon Ads: migrate source to YamlDeclarativeSource with custom check_connection --- .../declarative_source_adapter.py | 37 ++++ .../source_amazon_ads/manifest.yaml | 169 ++++++++++++++++++ .../source_amazon_ads/run.py | 3 +- .../source_amazon_ads/spec.yaml | 165 ----------------- 4 files changed, 208 insertions(+), 166 deletions(-) create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py new file mode 100644 index 000000000000..ede81f77e8cb --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from logging import Logger +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.models import AirbyteConnectionStatus +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog + + +class DeclarativeSourceAdapter(YamlDeclarativeSource): + def __init__(self, source: AbstractSource) -> None: + super().__init__(path_to_yaml="manifest.yaml") + self._source = source + + def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + return self._source.check(logger, config) + + def discover(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteCatalog: + return self._source.discover(logger, config) + + def read( + self, + logger: Logger, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, + ) -> Iterator[AirbyteMessage]: + return self._source.read(logger, config, catalog, state) + + # def _validate_source(self) -> None: + # # TODO ??? + # return diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml new file mode 100644 index 000000000000..b7e8c86d7e3d --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml @@ -0,0 +1,169 @@ +version: 0.60.1 +type: source_amazon_ads.SourceAmazonAds +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/amazon-ads + connection_specification: + title: Amazon Ads Spec + type: object + properties: + auth_type: + title: Auth Type + const: oauth2.0 + order: 0 + type: string + client_id: + title: Client ID + description: + The client ID of your Amazon Ads developer application. See the + docs + for more information. + order: 1 + type: string + airbyte_secret: true + client_secret: + title: Client Secret + description: + The client secret of your Amazon Ads developer application. See + the docs + for more information. + airbyte_secret: true + order: 2 + type: string + refresh_token: + title: Refresh Token + description: + Amazon Ads refresh token. See the docs + for more information on how to obtain this token. + airbyte_secret: true + order: 3 + type: string + region: + title: Region + description: + Region to pull data from (EU/NA/FE). See docs + for more details. + enum: + - NA + - EU + - FE + type: string + default: NA + order: 4 + start_date: + title: Start Date + description: + The Start date for collecting reports, should not be more than + 60 days in the past. In YYYY-MM-DD format + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + format: date + examples: + - "2022-10-10" + - "2022-10-22" + order: 5 + type: string + profiles: + title: Profile IDs + description: 'Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.' + order: 6 + type: array + items: + type: integer + marketplace_ids: + title: Marketplace IDs + description: "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID." + order: 7 + type: array + items: + type: string + state_filter: + title: State Filter + description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely. + items: + type: string + enum: + - enabled + - paused + - archived + type: array + uniqueItems: true + order: 8 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 9 + report_record_types: + title: Report Record Types + description: + Optional configuration which accepts an array of string of record types. + Leave blank for default behaviour to pull all report types. + Use this config option only if you want to pull specific report type(s). + See docs + for more details + items: + type: string + enum: + - adGroups + - asins + - asins_keywords + - asins_targets + - campaigns + - keywords + - productAds + - targets + type: array + uniqueItems: true + order: 10 + required: + - client_id + - client_secret + - refresh_token + additionalProperties: true + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - auth_type + predicate_value: oauth2.0 + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + region: + type: string + path_in_connector_config: + - region + complete_oauth_output_specification: + type: object + additionalProperties: true + properties: + refresh_token: + type: string + path_in_connector_config: + - refresh_token + complete_oauth_server_input_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + path_in_connector_config: + - client_id + client_secret: + type: string + path_in_connector_config: + - client_secret +streams: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py index a8012240de66..0436d379599e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py @@ -8,9 +8,10 @@ from airbyte_cdk.entrypoint import launch from source_amazon_ads import SourceAmazonAds from source_amazon_ads.config_migrations import MigrateStartDate +from source_amazon_ads.declarative_source_adapter import DeclarativeSourceAdapter def run(): - source = SourceAmazonAds() + source = DeclarativeSourceAdapter(source=SourceAmazonAds()) MigrateStartDate.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml deleted file mode 100644 index 0e703cb4ca3a..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ /dev/null @@ -1,165 +0,0 @@ ---- -documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads -connectionSpecification: - title: Amazon Ads Spec - type: object - properties: - auth_type: - title: Auth Type - const: oauth2.0 - order: 0 - type: string - client_id: - title: Client ID - description: - The client ID of your Amazon Ads developer application. See the - docs - for more information. - order: 1 - type: string - airbyte_secret: true - client_secret: - title: Client Secret - description: - The client secret of your Amazon Ads developer application. See - the docs - for more information. - airbyte_secret: true - order: 2 - type: string - refresh_token: - title: Refresh Token - description: - Amazon Ads refresh token. See the docs - for more information on how to obtain this token. - airbyte_secret: true - order: 3 - type: string - region: - title: Region - description: - Region to pull data from (EU/NA/FE). See docs - for more details. - enum: - - NA - - EU - - FE - type: string - default: NA - order: 4 - start_date: - title: Start Date - description: - The Start date for collecting reports, should not be more than - 60 days in the past. In YYYY-MM-DD format - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - format: date - examples: - - "2022-10-10" - - "2022-10-22" - order: 5 - type: string - profiles: - title: Profile IDs - description: 'Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.' - order: 6 - type: array - items: - type: integer - marketplace_ids: - title: Marketplace IDs - description: "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID." - order: 7 - type: array - items: - type: string - state_filter: - title: State Filter - description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely. - items: - type: string - enum: - - enabled - - paused - - archived - type: array - uniqueItems: true - order: 8 - look_back_window: - title: "Look Back Window" - description: "The amount of days to go back in time to get the updated data from Amazon Ads" - examples: - - 3 - - 10 - type: "integer" - default: 3 - order: 9 - report_record_types: - title: Report Record Types - description: - Optional configuration which accepts an array of string of record types. - Leave blank for default behaviour to pull all report types. - Use this config option only if you want to pull specific report type(s). - See docs - for more details - items: - type: string - enum: - - adGroups - - asins - - asins_keywords - - asins_targets - - campaigns - - keywords - - productAds - - targets - type: array - uniqueItems: true - order: 10 - required: - - client_id - - client_secret - - refresh_token - additionalProperties: true -advanced_auth: - auth_flow_type: oauth2.0 - predicate_key: - - auth_type - predicate_value: oauth2.0 - oauth_config_specification: - oauth_user_input_from_connector_config_specification: - type: object - additionalProperties: false - properties: - region: - type: string - path_in_connector_config: - - region - complete_oauth_output_specification: - type: object - additionalProperties: true - properties: - refresh_token: - type: string - path_in_connector_config: - - refresh_token - complete_oauth_server_input_specification: - type: object - additionalProperties: true - properties: - client_id: - type: string - client_secret: - type: string - complete_oauth_server_output_specification: - type: object - additionalProperties: true - properties: - client_id: - type: string - path_in_connector_config: - - client_id - client_secret: - type: string - path_in_connector_config: - - client_secret