diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index e4756124eb18..657afb9af570 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.11.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index f23f5d1aacda..ece1b5a39d8a 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -15,6 +15,8 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.11.1" # schemas for custom queries are fixed in 1.0.0 basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml index 795ec1358fa0..a50f1bbc9ff6 100644 --- a/airbyte-integrations/connectors/source-google-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-ads/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 - dockerImageTag: 0.11.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-google-ads githubIssueLabel: source-google-ads icon: google-adwords.svg @@ -36,6 +36,11 @@ data: - display_keyword_performance_report - shopping_performance_report - display_topics_performance_report + releases: + breakingChanges: + 1.0.0: + message: This release introduces fix to custom query schemas creation. User should refresh their schemas and data before update. + upgradeDeadline: "2023-10-31" documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads tags: - language:python diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index 2cf6066c4882..358290f5b906 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -52,6 +52,8 @@ def get_json_schema(self) -> Dict[str, Any]: "STRING": "string", "BOOLEAN": "boolean", "DATE": "string", + "MESSAGE": "string", + "ENUM": "string", } fields = list(self.config["query"].fields) if self.cursor_field: @@ -62,25 +64,16 @@ def get_json_schema(self) -> Dict[str, Any]: node = google_schema.get(field) # Data type return in enum format: "GoogleAdsFieldDataType." google_data_type = node.data_type.name + field_value = {"type": [google_datatype_mapping.get(google_data_type, "string"), "null"]} + + if google_data_type == "DATE": + field_value["format"] = "date" + if google_data_type == "ENUM": field_value = {"type": "string", "enum": list(node.enum_values)} - if node.is_repeated: - field_value = {"type": ["null", "array"], "items": field_value} - elif google_data_type == "MESSAGE": - # Represents protobuf message and could be anything, set custom - # attribute "protobuf_message" to convert it to a string (or - # array of strings) later. - # https://developers.google.com/google-ads/api/reference/rpc/v11/GoogleAdsFieldDataTypeEnum.GoogleAdsFieldDataType?hl=en#message - if node.is_repeated: - output_type = ["array", "null"] - else: - output_type = ["string", "null"] - field_value = {"type": output_type, "protobuf_message": True} - else: - output_type = [google_datatype_mapping.get(google_data_type, "string"), "null"] - field_value = {"type": output_type} - if google_data_type == "DATE": - field_value["format"] = "date" + + if node.is_repeated: + field_value = {"type": ["null", "array"], "items": field_value} local_json_schema["properties"][field] = field_value diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 80dd5aa3db0a..82ae0eb5afbf 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -178,17 +178,8 @@ def get_field_value(field_value: GoogleAdsRow, field: str, schema_type: Mapping[ # For example: # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en). # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en). - # - if not (isinstance(field_value, (list, int, float, str, bool, dict)) or field_value is None): + if not isinstance(field_value, (list, int, float, str, bool, dict)) and field_value is not None: field_value = str(field_value) - # In case of custom query field has MESSAGE type it represents protobuf - # message and could be anything, convert it to a string or array of - # string if it has "repeated" flag on metadata - if schema_type.get("protobuf_message"): - if "array" in schema_type.get("type"): - field_value = [str(field) for field in field_value] - else: - field_value = str(field_value) return field_value diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py index d83b8d52e42d..49fe305d3a0a 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py @@ -27,31 +27,29 @@ def __init__(self, **entries): def test_get_json_schema(): - query_object = MagicMock( - return_value={ - "a": Obj(data_type=Obj(name="ENUM"), is_repeated=False, enum_values=["a", "aa"]), - "b": Obj(data_type=Obj(name="ENUM"), is_repeated=True, enum_values=["b", "bb"]), - "c": Obj(data_type=Obj(name="MESSAGE"), is_repeated=False), - "d": Obj(data_type=Obj(name="MESSAGE"), is_repeated=True), - "e": Obj(data_type=Obj(name="STRING")), - "f": Obj(data_type=Obj(name="DATE")), - } - ) - instance = CustomQueryMixin(config={"query": Obj(fields=["a", "b", "c", "d", "e", "f"])}) + query_object = MagicMock(return_value={ + 'a': Obj(data_type=Obj(name='ENUM'), is_repeated=False, enum_values=['a', 'aa']), + 'b': Obj(data_type=Obj(name='ENUM'), is_repeated=True, enum_values=['b', 'bb']), + 'c': Obj(data_type=Obj(name='MESSAGE'), is_repeated=False), + 'd': Obj(data_type=Obj(name='MESSAGE'), is_repeated=True), + 'e': Obj(data_type=Obj(name='STRING'), is_repeated=False), + 'f': Obj(data_type=Obj(name='DATE'), is_repeated=False), + }) + instance = CustomQueryMixin(config={'query': Obj(fields=['a', 'b', 'c', 'd', 'e', 'f'])}) instance.cursor_field = None instance.google_ads_client = Obj(get_fields_metadata=query_object) schema = instance.get_json_schema() assert schema == { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": True, - "type": "object", - "properties": { - "a": {"type": "string", "enum": ["a", "aa"]}, - "b": {"type": ["null", "array"], "items": {"type": "string", "enum": ["b", "bb"]}}, - "c": {"type": ["string", "null"], "protobuf_message": True}, - "d": {"type": ["array", "null"], "protobuf_message": True}, - "e": {"type": ["string", "null"]}, - "f": {"type": ["string", "null"], "format": "date"}, - }, + '$schema': 'http://json-schema.org/draft-07/schema#', + 'additionalProperties': True, + 'type': 'object', + 'properties': { + 'a': {'type': 'string', 'enum': ['a', 'aa']}, + 'b': {'type': ['null', 'array'], 'items': {'type': 'string', 'enum': ['b', 'bb']}}, + 'c': {'type': ['string', 'null']}, + 'd': {'type': ['null', 'array'], 'items': {'type': ['string', 'null']}}, + 'e': {'type': ['string', 'null']}, + 'f': {'type': ['string', 'null'], 'format': 'date'}, + } } diff --git a/docs/integrations/sources/google-ads-migrations.md b/docs/integrations/sources/google-ads-migrations.md new file mode 100644 index 000000000000..8b92c1dc70f3 --- /dev/null +++ b/docs/integrations/sources/google-ads-migrations.md @@ -0,0 +1,5 @@ +# Google Ads Migration Guide + +## Upgrading to 1.0.0 + +This release introduced fixes to the creation of custom query schemas. For instance, the field ad_group_ad.ad.final_urls in the custom query has had its type changed from `{"type": "string"}` to `{"type": ["null", "array"], "items": {"type": "string"}}`. Users should refresh their schemas and data before updating. \ No newline at end of file diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index d24688f12356..beb0a32f2b7c 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -230,6 +230,7 @@ Due to a limitation in the Google Ads API which does not allow getting performan | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| +| `1.0.0` | 2023-09-28 | [30705](https://github.com/airbytehq/airbyte/pull/30705) | Fix schemas for custom queries | | `0.11.1` | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | | `0.11.0` | 2023-09-23 | [30704](https://github.com/airbytehq/airbyte/pull/30704) | Update error handling | | `0.10.0` | 2023-09-19 | [30091](https://github.com/airbytehq/airbyte/pull/30091) | Fix schemas for correct primary and foreign keys |