🐛 Source Amazon Ads: Fix unexpected column for SponsoredProductCampai…

…gns and SponsoredBrandsKeywords (#30679)

Co-authored-by: lazebnyi <[email protected]>

airbyte-integrations/connectors/source-amazon-ads/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=3.2.0
+LABEL io.airbyte.version=3.3.0
 LABEL io.airbyte.name=airbyte/source-amazon-ads

airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml

@@ -61,5 +61,7 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: integration_tests/spec.json
+        backward_compatibility_tests_config:
+          disable_for_version: "3.2.0"
 connector_image: airbyte/source-amazon-ads:dev
 test_strictness_level: high

airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json

@@ -13,6 +13,7 @@
       "client_id": {
         "title": "Client ID",
         "description": "The client ID of your Amazon Ads developer application. See the <a href=\"https://advertising.amazon.com/API/docs/en-us/get-started/generate-api-tokens#retrieve-your-client-id-and-client-secret\">docs</a> for more information.",
+        "airbyte_secret": true,
         "order": 1,
         "type": "string"
       },
@@ -43,7 +44,9 @@
         "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format",
         "examples": ["2022-10-10", "2022-10-22"],
         "order": 5,
-        "type": "string"
+        "type": "string",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        "format": "date"
       },
       "profiles": {
         "title": "Profile IDs",

airbyte-integrations/connectors/source-amazon-ads/main.py

@@ -7,7 +7,9 @@
 
 from airbyte_cdk.entrypoint import launch
 from source_amazon_ads import SourceAmazonAds
+from source_amazon_ads.config_migrations import MigrateStartDate
 
 if __name__ == "__main__":
     source = SourceAmazonAds()
+    MigrateStartDate.migrate(sys.argv[1:], source)
     launch(source, sys.argv[1:])

airbyte-integrations/connectors/source-amazon-ads/metadata.yaml

@@ -8,7 +8,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
-  dockerImageTag: 3.2.0
+  dockerImageTag: 3.3.0
   dockerRepository: airbyte/source-amazon-ads
   githubIssueLabel: source-amazon-ads
   icon: amazonads.svg

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py

@@ -0,0 +1,104 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import logging
+from typing import Any, List, Mapping
+
+from airbyte_cdk.config_observation import create_connector_config_control_message
+from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.sources import Source
+from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
+
+logger = logging.getLogger("airbyte_logger")
+
+
+class MigrateStartDate:
+    """
+    This class stands for migrating the config at runtime,
+    while providing the backward compatibility when falling back to the previous source version.
+
+    Delete start_date field if it set to None or an empty string ("").
+    """
+
+    message_repository: MessageRepository = InMemoryMessageRepository()
+    # Key used to identify start date in the configuration.
+    key: str = "start_date"
+
+    @classmethod
+    def should_migrate(cls, config: Mapping[str, Any]) -> bool:
+        """
+        Determines if a configuration requires migration.
+
+        Args:
+        - config (Mapping[str, Any]): The configuration data to check.
+
+        Returns:
+        - True: If the configuration requires migration.
+        - False: Otherwise.
+        """
+        return not config.get(cls.key, "skip_if_start_date_in_config")
+
+    @classmethod
+    def delete_from_config(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]:
+        """
+        Removes the specified key from the configuration.
+
+        Args:
+        - config (Mapping[str, Any]): The configuration from which the key should be removed.
+        - source (Source, optional): The data source. Defaults to None.
+
+        Returns:
+        - Mapping[str, Any]: The configuration after removing the key.
+        """
+        config.pop(cls.key, None)  # Safely remove the key if it exists.
+        return config
+
+    @classmethod
+    def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        """
+        Modifies the configuration and then saves it back to the source.
+
+        Args:
+        - config_path (str): The path where the configuration is stored.
+        - source (Source): The data source.
+        - config (Mapping[str, Any]): The current configuration.
+
+        Returns:
+        - Mapping[str, Any]: The updated configuration.
+        """
+        migrated_config = cls.delete_from_config(config, source)
+        source.write_config(migrated_config, config_path)
+        return migrated_config
+
+    @classmethod
+    def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None:
+        """
+        Emits the control messages related to configuration migration.
+
+        Args:
+        - migrated_config (Mapping[str, Any]): The migrated configuration.
+        """
+        cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
+        for message in cls.message_repository._message_queue:
+            print(message.json(exclude_unset=True))
+
+    @classmethod
+    def migrate(cls, args: List[str], source: Source) -> None:
+        """
+        Orchestrates the configuration migration process.
+
+        It first checks if the `--config` argument is provided, and if so,
+        determines whether migration is needed, and then performs the migration
+        if required.
+
+        Args:
+        - args (List[str]): List of command-line arguments.
+        - source (Source): The data source.
+        """
+        config_path = AirbyteEntrypoint(source).extract_config(args)
+        if config_path:
+            config = source.read_config(config_path)
+            if cls.should_migrate(config):
+                cls.emit_control_message(cls.modify_and_save(config_path, source, config))

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py

@@ -52,3 +52,8 @@ class BrandsAdGroup(CatalogModel):
     adGroupId: Decimal
     name: str
     bid: int
+    keywordId: Decimal
+    keywordText: str
+    nativeLanguageKeyword: str
+    matchType: str
+    state: str

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py

@@ -32,6 +32,7 @@ class ProductCampaign(CatalogModel):
     endDate: str = None
     premiumBidAdjustment: bool
     bidding: Bidding
+    networks: str
 
 
 class ProductAdGroups(CatalogModel):

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml

@@ -17,6 +17,7 @@ connectionSpecification:
         for more information.
       order: 1
       type: string
+      airbyte_secret: true
     client_secret:
       title: Client Secret
       description:
@@ -51,6 +52,8 @@ connectionSpecification:
       description:
         The Start date for collecting reports, should not be more than
         60 days in the past. In YYYY-MM-DD format
+      pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+      format: date
       examples:
         - "2022-10-10"
         - "2022-10-22"

airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py

@@ -11,6 +11,7 @@
 from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
+from source_amazon_ads.config_migrations import MigrateStartDate
 
 
 def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]:
@@ -41,6 +42,7 @@ def read_full_refresh(stream_instance: Stream):
 def command_check(source: Source, config):
     logger = mock.MagicMock()
     connector_config, _ = split_config(config)
+    connector_config = MigrateStartDate.modify_and_save("unit_tests/config.json", source, connector_config)
     if source.check_config_against_spec:
         source_spec: ConnectorSpecification = source.spec(logger)
         check_config_against_spec_or_exit(connector_config, source_spec)

docs/integrations/sources/amazon-ads.md

@@ -109,6 +109,7 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 3.3.0   | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for `SponsoredProductCampaigns` and `SponsoredBrandsKeywords`                             |
 | 3.2.0   | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue                                                              |
 | 3.1.2   | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs                                                                                  |
 | 3.1.1   | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups                                                           |