🚨 🚨 🐍 Source Google Ads: fix custom queries (#30705)

airbytehq · Sep 29, 2023 · 955e20b · 955e20b
1 parent 3822b09
commit 955e20b
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 51 deletions.
diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile
@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.11.1
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-google-ads
diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml
@@ -15,6 +15,8 @@ acceptance_tests:
   discovery:
     tests:
       - config_path: "secrets/config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.11.1" # schemas for custom queries are fixed in 1.0.0
   basic_read:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml
@@ -6,7 +6,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
-  dockerImageTag: 0.11.1
+  dockerImageTag: 1.0.0
   dockerRepository: airbyte/source-google-ads
   githubIssueLabel: source-google-ads
   icon: google-adwords.svg
@@ -36,6 +36,11 @@ data:
       - display_keyword_performance_report
       - shopping_performance_report
       - display_topics_performance_report
+  releases:
+    breakingChanges:
+      1.0.0:
+        message: This release introduces fix to custom query schemas creation. User should refresh their schemas and data before update.
+        upgradeDeadline: "2023-10-31"
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads
   tags:
     - language:python

diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py
@@ -52,6 +52,8 @@ def get_json_schema(self) -> Dict[str, Any]:
             "STRING": "string",
             "BOOLEAN": "boolean",
             "DATE": "string",
+            "MESSAGE": "string",
+            "ENUM": "string",
         }
         fields = list(self.config["query"].fields)
         if self.cursor_field:
@@ -62,25 +64,16 @@ def get_json_schema(self) -> Dict[str, Any]:
             node = google_schema.get(field)
             # Data type return in enum format: "GoogleAdsFieldDataType.<data_type>"
             google_data_type = node.data_type.name
+            field_value = {"type": [google_datatype_mapping.get(google_data_type, "string"), "null"]}
+
+            if google_data_type == "DATE":
+                field_value["format"] = "date"
+
             if google_data_type == "ENUM":
                 field_value = {"type": "string", "enum": list(node.enum_values)}
-                if node.is_repeated:
-                    field_value = {"type": ["null", "array"], "items": field_value}
-            elif google_data_type == "MESSAGE":
-                # Represents protobuf message and could be anything, set custom
-                # attribute "protobuf_message" to convert it to a string (or
-                # array of strings) later.
-                # https://developers.google.com/google-ads/api/reference/rpc/v11/GoogleAdsFieldDataTypeEnum.GoogleAdsFieldDataType?hl=en#message
-                if node.is_repeated:
-                    output_type = ["array", "null"]
-                else:
-                    output_type = ["string", "null"]
-                field_value = {"type": output_type, "protobuf_message": True}
-            else:
-                output_type = [google_datatype_mapping.get(google_data_type, "string"), "null"]
-                field_value = {"type": output_type}
-                if google_data_type == "DATE":
-                    field_value["format"] = "date"
+
+            if node.is_repeated:
+                field_value = {"type": ["null", "array"], "items": field_value}
 
             local_json_schema["properties"][field] = field_value
 

diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py
@@ -178,17 +178,8 @@ def get_field_value(field_value: GoogleAdsRow, field: str, schema_type: Mapping[
         # For example:
         # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en).
         # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en).
-        #
-        if not (isinstance(field_value, (list, int, float, str, bool, dict)) or field_value is None):
+        if not isinstance(field_value, (list, int, float, str, bool, dict)) and field_value is not None:
             field_value = str(field_value)
-        # In case of custom query field has MESSAGE type it represents protobuf
-        # message and could be anything, convert it to a string or array of
-        # string if it has "repeated" flag on metadata
-        if schema_type.get("protobuf_message"):
-            if "array" in schema_type.get("type"):
-                field_value = [str(field) for field in field_value]
-            else:
-                field_value = str(field_value)
 
         return field_value
 

diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py
@@ -27,31 +27,29 @@ def __init__(self, **entries):
 
 
 def test_get_json_schema():
-    query_object = MagicMock(
-        return_value={
-            "a": Obj(data_type=Obj(name="ENUM"), is_repeated=False, enum_values=["a", "aa"]),
-            "b": Obj(data_type=Obj(name="ENUM"), is_repeated=True, enum_values=["b", "bb"]),
-            "c": Obj(data_type=Obj(name="MESSAGE"), is_repeated=False),
-            "d": Obj(data_type=Obj(name="MESSAGE"), is_repeated=True),
-            "e": Obj(data_type=Obj(name="STRING")),
-            "f": Obj(data_type=Obj(name="DATE")),
-        }
-    )
-    instance = CustomQueryMixin(config={"query": Obj(fields=["a", "b", "c", "d", "e", "f"])})
+    query_object = MagicMock(return_value={
+        'a': Obj(data_type=Obj(name='ENUM'), is_repeated=False, enum_values=['a', 'aa']),
+        'b': Obj(data_type=Obj(name='ENUM'), is_repeated=True,  enum_values=['b', 'bb']),
+        'c': Obj(data_type=Obj(name='MESSAGE'), is_repeated=False),
+        'd': Obj(data_type=Obj(name='MESSAGE'), is_repeated=True),
+        'e': Obj(data_type=Obj(name='STRING'), is_repeated=False),
+        'f': Obj(data_type=Obj(name='DATE'), is_repeated=False),
+    })
+    instance = CustomQueryMixin(config={'query': Obj(fields=['a', 'b', 'c', 'd', 'e', 'f'])})
     instance.cursor_field = None
     instance.google_ads_client = Obj(get_fields_metadata=query_object)
     schema = instance.get_json_schema()
 
     assert schema == {
-        "$schema": "http://json-schema.org/draft-07/schema#",
-        "additionalProperties": True,
-        "type": "object",
-        "properties": {
-            "a": {"type": "string", "enum": ["a", "aa"]},
-            "b": {"type": ["null", "array"], "items": {"type": "string", "enum": ["b", "bb"]}},
-            "c": {"type": ["string", "null"], "protobuf_message": True},
-            "d": {"type": ["array", "null"], "protobuf_message": True},
-            "e": {"type": ["string", "null"]},
-            "f": {"type": ["string", "null"], "format": "date"},
-        },
+        '$schema': 'http://json-schema.org/draft-07/schema#',
+        'additionalProperties': True,
+        'type': 'object',
+        'properties': {
+            'a': {'type': 'string', 'enum': ['a', 'aa']},
+            'b': {'type': ['null', 'array'], 'items': {'type': 'string', 'enum': ['b', 'bb']}},
+            'c': {'type': ['string', 'null']},
+            'd': {'type': ['null', 'array'], 'items': {'type': ['string', 'null']}},
+            'e': {'type': ['string', 'null']},
+            'f': {'type': ['string', 'null'], 'format': 'date'},
+        }
     }
diff --git a/docs/integrations/sources/google-ads-migrations.md b/docs/integrations/sources/google-ads-migrations.md
@@ -0,0 +1,5 @@
+# Google Ads Migration Guide
+
+## Upgrading to 1.0.0
+
+This release introduced fixes to the creation of custom query schemas. For instance, the field ad_group_ad.ad.final_urls in the custom query has had its type changed from `{"type": "string"}` to `{"type": ["null", "array"], "items": {"type": "string"}}`. Users should refresh their schemas and data before updating.
diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md
@@ -230,6 +230,7 @@ Due to a limitation in the Google Ads API which does not allow getting performan
 
 | Version  | Date       | Pull Request                                             | Subject                                                                                                                              |
 |:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
+| `1.0.0`  | 2023-09-28  | [30705](https://github.com/airbytehq/airbyte/pull/30705) | Fix schemas for custom queries                                                                                                       |
 | `0.11.1` | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found                                                                              |
 | `0.11.0` | 2023-09-23 | [30704](https://github.com/airbytehq/airbyte/pull/30704) | Update error handling                                                                                                                |
 | `0.10.0` | 2023-09-19 | [30091](https://github.com/airbytehq/airbyte/pull/30091) | Fix schemas for correct primary and foreign keys                                                                                     |