From 50a75286454de94b6494c9b2cebfba775a4b6da4 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 2 Feb 2024 14:56:22 -0800 Subject: [PATCH 1/5] rename streams --- .../schemas/{customer_label.json => account_labels.json} | 0 .../source_google_ads/schemas/{customer.json => accounts.json} | 0 .../schemas/{ad_group_ad_label.json => ad_group_ad_labels.json} | 0 .../schemas/{ad_group_ad_legacy.json => ad_group_ad_report.json} | 0 .../schemas/{ad_group_ad.json => ad_group_ads.json} | 0 ...oup_bidding_strategy.json => ad_group_bidding_strategies.json} | 0 ..._group_criterion_label.json => ad_group_criterion_labels.json} | 0 .../schemas/{ad_group_criterion.json => ad_group_criterions.json} | 0 .../schemas/{ad_group_label.json => ad_group_labels.json} | 0 .../source_google_ads/schemas/{ad_group.json => ad_groups.json} | 0 ...ting_group_criterion.json => ad_listing_group_criterions.json} | 0 ...ign_bidding_strategy.json => campaign_bidding_strategies.json} | 0 .../schemas/{campaign_label.json => campaign_labels.json} | 0 .../source_google_ads/schemas/{campaign.json => campaigns.json} | 0 ..._keyword_view.json => display_keyword_performance_report.json} | 0 .../schemas/{geographic_view.json => geographic_report.json} | 0 .../source_google_ads/schemas/{label.json => labels.json} | 0 ...ing_performance_view.json => shopping_performance_report.json} | 0 .../schemas/{topic_view.json => topic_performance_report.json} | 0 .../{user_location_view.json => user_location_report.json} | 0 20 files changed, 0 insertions(+), 0 deletions(-) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{customer_label.json => account_labels.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{customer.json => accounts.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_ad_label.json => ad_group_ad_labels.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_ad_legacy.json => ad_group_ad_report.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_ad.json => ad_group_ads.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_bidding_strategy.json => ad_group_bidding_strategies.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_criterion_label.json => ad_group_criterion_labels.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_criterion.json => ad_group_criterions.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group_label.json => ad_group_labels.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_group.json => ad_groups.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{ad_listing_group_criterion.json => ad_listing_group_criterions.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{campaign_bidding_strategy.json => campaign_bidding_strategies.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{campaign_label.json => campaign_labels.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{campaign.json => campaigns.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{display_keyword_view.json => display_keyword_performance_report.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{geographic_view.json => geographic_report.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{label.json => labels.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{shopping_performance_view.json => shopping_performance_report.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{topic_view.json => topic_performance_report.json} (100%) rename airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/{user_location_view.json => user_location_report.json} (100%) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_label.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/accounts.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/accounts.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_labels.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_label.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_labels.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_legacy.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_legacy.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategy.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategy.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_label.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_labels.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_label.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_labels.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_groups.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_groups.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategy.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategy.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_labels.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_label.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_labels.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_view.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_report.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_view.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_report.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/label.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_report.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_report.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_performance_report.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_view.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_performance_report.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_view.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json From ffefb855bef5be4096b1e709f49381ade479a598 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 2 Feb 2024 15:00:29 -0800 Subject: [PATCH 2/5] missed one --- .../schemas/keyword_view.json | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json deleted file mode 100644 index a4ade30ce77a..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "customer.id": { - "type": ["null", "integer"] - }, - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.keyword.text": { - "type": ["null", "string"] - }, - "ad_group_criterion.negative": { - "type": ["null", "boolean"] - }, - "ad_group_criterion.keyword.match_type": { - "type": ["null", "string"] - }, - "metrics.historical_quality_score": { - "type": ["null", "integer"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "campaign.bidding_strategy_type": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurability": { - "type": ["null", "number"] - }, - "metrics.active_view_measurable_cost_micros": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurable_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_viewability": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.interactions": { - "type": ["null", "integer"] - }, - "metrics.interaction_event_types": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "metrics.view_through_conversions": { - "type": ["null", "integer"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - } - } -} From 2abdd921935077998ec57e13652b31a04c1ef1a7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 2 Feb 2024 15:24:59 -0800 Subject: [PATCH 3/5] update the classes, the tests, and add a missing file --- .../schemas/keyword_report.json | 88 +++++++++++++++++++ .../source_google_ads/source.py | 84 +++++++++--------- .../source_google_ads/streams.py | 42 ++++----- .../unit_tests/test_errors.py | 6 +- .../custom_query/test_config.json | 19 +--- .../unit_tests/test_source.py | 6 +- .../unit_tests/test_streams.py | 10 +-- 7 files changed, 163 insertions(+), 92 deletions(-) create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json new file mode 100644 index 000000000000..a4ade30ce77a --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.id": { + "type": ["null", "integer"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.text": { + "type": ["null", "string"] + }, + "ad_group_criterion.negative": { + "type": ["null", "boolean"] + }, + "ad_group_criterion.keyword.match_type": { + "type": ["null", "string"] + }, + "metrics.historical_quality_score": { + "type": ["null", "integer"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "campaign.bidding_strategy_type": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index 2402cd18adbe..1e85e848a139 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -17,34 +17,34 @@ from .models import CustomerModel from .streams import ( AccountPerformanceReport, - AdGroup, - AdGroupAd, - AdGroupAdLabel, - AdGroupAdLegacy, - AdGroupBiddingStrategy, - AdGroupCriterion, - AdGroupCriterionLabel, - AdGroupLabel, - AdListingGroupCriterion, + AdGroups, + AdGroupAds, + AdGroupAdLabels, + AdGroupAdReport, + AdGroupBiddingStrategies, + AdGroupCriterions, + AdGroupCriterionLabels, + AdGroupLabels, + AdListingGroupCriterions, Audience, - Campaign, - CampaignBiddingStrategy, + Campaigns, + CampaignBiddingStrategies, CampaignBudget, CampaignCriterion, - CampaignLabel, + CampaignLabels, ClickView, - Customer, + Accounts, CustomerClient, - CustomerLabel, - DisplayKeywordView, - GeographicView, - KeywordView, - Label, + AccountLabels, + DisplayKeywordPerformanceReport, + GeographicReport, + KeywordReport, + Labels, ServiceAccounts, - ShoppingPerformanceView, - TopicView, + ShoppingPerformanceReport, + TopicPerformanceReport, UserInterest, - UserLocationView, + UserLocationReport, ) from .utils import GAQL @@ -224,38 +224,38 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: incremental_config = self.get_incremental_stream_config(google_api, config, customers) non_manager_incremental_config = self.get_incremental_stream_config(google_api, config, non_manager_accounts) streams = [ - AdGroup(**incremental_config), - AdGroupAd(**incremental_config), - AdGroupAdLabel(**default_config), - AdGroupBiddingStrategy(**incremental_config), - AdGroupCriterion(**default_config), - AdGroupCriterionLabel(**default_config), - AdGroupLabel(**default_config), - AdListingGroupCriterion(**default_config), + AdGroups(**incremental_config), + AdGroupAds(**incremental_config), + AdGroupAdLabels(**default_config), + AdGroupBiddingStrategies(**incremental_config), + AdGroupCriterions(**default_config), + AdGroupCriterionLabels(**default_config), + AdGroupLabels(**default_config), + AdListingGroupCriterions(**default_config), Audience(**default_config), - CampaignBiddingStrategy(**incremental_config), + CampaignBiddingStrategies(**incremental_config), CampaignCriterion(**default_config), - CampaignLabel(google_api, customers=customers), + CampaignLabels(google_api, customers=customers), ClickView(**incremental_config), - Customer(**incremental_config), - CustomerLabel(**default_config), - Label(**default_config), + Accounts(**incremental_config), + AccountLabels(**default_config), + Labels(**default_config), UserInterest(**default_config), ] # Metrics streams cannot be requested for a manager account. if non_manager_accounts: streams.extend( [ - Campaign(**non_manager_incremental_config), + Campaigns(**non_manager_incremental_config), CampaignBudget(**non_manager_incremental_config), - UserLocationView(**non_manager_incremental_config), + UserLocationReport(**non_manager_incremental_config), AccountPerformanceReport(**non_manager_incremental_config), - TopicView(**non_manager_incremental_config), - DisplayKeywordView(**non_manager_incremental_config), - ShoppingPerformanceView(**non_manager_incremental_config), - AdGroupAdLegacy(**non_manager_incremental_config), - GeographicView(**non_manager_incremental_config), - KeywordView(**non_manager_incremental_config), + TopicPerformanceReport(**non_manager_incremental_config), + DisplayKeywordPerformanceReport(**non_manager_incremental_config), + ShoppingPerformanceReport(**non_manager_incremental_config), + AdGroupAdReport(**non_manager_incremental_config), + GeographicReport(**non_manager_incremental_config), + KeywordReport(**non_manager_incremental_config), ] ) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 499bceca367e..5e557abf62e3 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -217,7 +217,7 @@ def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: return query -class Customer(IncrementalGoogleAdsStream): +class Accounts(IncrementalGoogleAdsStream): """ Customer stream: https://developers.google.com/google-ads/api/fields/v15/customer """ @@ -287,7 +287,7 @@ def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[s yield record -class CustomerLabel(GoogleAdsStream): +class AccountLabels(GoogleAdsStream): """ Customer Label stream: https://developers.google.com/google-ads/api/fields/v15/customer_label """ @@ -304,7 +304,7 @@ class ServiceAccounts(GoogleAdsStream): primary_key = ["customer.id"] -class Campaign(IncrementalGoogleAdsStream): +class Campaigns(IncrementalGoogleAdsStream): """ Campaign stream: https://developers.google.com/google-ads/api/fields/v15/campaign """ @@ -328,7 +328,7 @@ class CampaignBudget(IncrementalGoogleAdsStream): ] -class CampaignBiddingStrategy(IncrementalGoogleAdsStream): +class CampaignBiddingStrategies(IncrementalGoogleAdsStream): """ Campaign Bidding Strategy stream: https://developers.google.com/google-ads/api/fields/v15/campaign """ @@ -337,7 +337,7 @@ class CampaignBiddingStrategy(IncrementalGoogleAdsStream): primary_key = ["campaign.id", "bidding_strategy.id", "segments.date"] -class CampaignLabel(GoogleAdsStream): +class CampaignLabels(GoogleAdsStream): """ Campaign labels stream: https://developers.google.com/google-ads/api/fields/v15/campaign_label """ @@ -346,7 +346,7 @@ class CampaignLabel(GoogleAdsStream): primary_key = ["campaign.id", "label.id"] -class AdGroup(IncrementalGoogleAdsStream): +class AdGroups(IncrementalGoogleAdsStream): """ AdGroup stream: https://developers.google.com/google-ads/api/fields/v15/ad_group """ @@ -369,7 +369,7 @@ def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: return query -class AdGroupLabel(GoogleAdsStream): +class AdGroupLabels(GoogleAdsStream): """ Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_label """ @@ -378,7 +378,7 @@ class AdGroupLabel(GoogleAdsStream): primary_key = ["ad_group.id", "label.id"] -class AdGroupBiddingStrategy(IncrementalGoogleAdsStream): +class AdGroupBiddingStrategies(IncrementalGoogleAdsStream): """ Ad Group Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v15/ad_group """ @@ -387,7 +387,7 @@ class AdGroupBiddingStrategy(IncrementalGoogleAdsStream): primary_key = ["ad_group.id", "bidding_strategy.id", "segments.date"] -class AdGroupCriterionLabel(GoogleAdsStream): +class AdGroupCriterionLabels(GoogleAdsStream): """ Ad Group Criterion Label stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion_label """ @@ -396,7 +396,7 @@ class AdGroupCriterionLabel(GoogleAdsStream): primary_key = ["ad_group_criterion_label.resource_name"] -class AdGroupAd(IncrementalGoogleAdsStream): +class AdGroupAds(IncrementalGoogleAdsStream): """ Ad Group Ad stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad """ @@ -404,7 +404,7 @@ class AdGroupAd(IncrementalGoogleAdsStream): primary_key = ["ad_group.id", "ad_group_ad.ad.id", "segments.date"] -class AdGroupAdLabel(GoogleAdsStream): +class AdGroupAdLabels(GoogleAdsStream): """ Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ @@ -421,7 +421,7 @@ class AccountPerformanceReport(IncrementalGoogleAdsStream): primary_key = ["customer.id", "segments.date", "segments.ad_network_type", "segments.device"] -class AdGroupAdLegacy(IncrementalGoogleAdsStream): +class AdGroupAdReport(IncrementalGoogleAdsStream): """ AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#ad_performance @@ -430,7 +430,7 @@ class AdGroupAdLegacy(IncrementalGoogleAdsStream): primary_key = ["ad_group.id", "ad_group_ad.ad.id", "segments.date", "segments.ad_network_type"] -class DisplayKeywordView(IncrementalGoogleAdsStream): +class DisplayKeywordPerformanceReport(IncrementalGoogleAdsStream): """ DisplayKeywordView stream: https://developers.google.com/google-ads/api/fields/v15/display_keyword_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_keyword_performance @@ -445,7 +445,7 @@ class DisplayKeywordView(IncrementalGoogleAdsStream): ] -class TopicView(IncrementalGoogleAdsStream): +class TopicPerformanceReport(IncrementalGoogleAdsStream): """ DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v15/topic_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_topics_performance @@ -460,14 +460,14 @@ class TopicView(IncrementalGoogleAdsStream): ] -class ShoppingPerformanceView(IncrementalGoogleAdsStream): +class ShoppingPerformanceReport(IncrementalGoogleAdsStream): """ ShoppingPerformanceView stream: https://developers.google.com/google-ads/api/fields/v15/shopping_performance_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance """ -class UserLocationView(IncrementalGoogleAdsStream): +class UserLocationReport(IncrementalGoogleAdsStream): """ UserLocationView stream: https://developers.google.com/google-ads/api/fields/v15/user_location_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#geo_performance @@ -482,7 +482,7 @@ class UserLocationView(IncrementalGoogleAdsStream): ] -class GeographicView(IncrementalGoogleAdsStream): +class GeographicReport(IncrementalGoogleAdsStream): """ UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v15/geographic_view """ @@ -490,7 +490,7 @@ class GeographicView(IncrementalGoogleAdsStream): primary_key = ["customer.id", "geographic_view.country_criterion_id", "geographic_view.location_type", "segments.date"] -class KeywordView(IncrementalGoogleAdsStream): +class KeywordReport(IncrementalGoogleAdsStream): """ UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v15/keyword_view """ @@ -526,7 +526,7 @@ class Audience(GoogleAdsStream): primary_key = ["customer.id", "audience.id"] -class Label(GoogleAdsStream): +class Labels(GoogleAdsStream): """ Label stream: https://developers.google.com/google-ads/api/fields/v15/label """ @@ -808,7 +808,7 @@ def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: return query -class AdGroupCriterion(IncrementalEventsStream): +class AdGroupCriterions(IncrementalEventsStream): """ Ad Group Criterion stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion """ @@ -821,7 +821,7 @@ class AdGroupCriterion(IncrementalEventsStream): cursor_field = "change_status.last_change_date_time" -class AdListingGroupCriterion(AdGroupCriterion): +class AdListingGroupCriterions(AdGroupCriterions): """ Ad Listing Group Criterion stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion While this stream utilizes the same resource as the AdGroupCriterions, diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py index e71263296007..3b877cf7fcf0 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py @@ -12,7 +12,7 @@ from source_google_ads.google_ads import GoogleAds from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds -from source_google_ads.streams import AdGroupLabel, Label, ServiceAccounts +from source_google_ads.streams import AdGroupLabels, Labels, ServiceAccounts from .common import MockGoogleAdsClient, mock_google_ads_request_failure @@ -65,8 +65,8 @@ def test_expected_errors(mocker, config, exception, error_message): @pytest.mark.parametrize( ("cls", "raise_expected"), ( - (AdGroupLabel, False), - (Label, False), + (AdGroupLabels, False), + (Labels, False), (ServiceAccounts, True), ), ) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json index 2ce005d03ec0..3ed6a7f49f4f 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json @@ -1,18 +1 @@ -{ - "credentials": { - "developer_token": "developer_token", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token" - }, - "customer_id": "1234567890", - "start_date": "2023-09-04", - "conversion_window_days": 14, - "custom_queries": [ - { - "query": "SELECT campaign.name, metrics.clicks FROM campaign", - "primary_key": null, - "table_name": "test_query" - } - ] -} +{"credentials": {"developer_token": "developer_token", "client_id": "client_id", "client_secret": "client_secret", "refresh_token": "refresh_token"}, "customer_id": "1234567890", "start_date": "2023-09-04", "conversion_window_days": 14, "custom_queries": [{"query": "SELECT campaign.name, metrics.clicks FROM campaign", "primary_key": null, "table_name": "test_query"}]} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index 6394817edd99..eaa60dc09bea 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -16,7 +16,7 @@ from source_google_ads.google_ads import GoogleAds from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds -from source_google_ads.streams import AdGroupAdLegacy, chunk_date_range +from source_google_ads.streams import AdGroupAdReport, chunk_date_range from source_google_ads.utils import GAQL from .common import MockGoogleAdsClient @@ -35,7 +35,7 @@ def stream_mock(mocker, config, customers): def mock(latest_record): mocker.patch("source_google_ads.streams.GoogleAdsStream.read_records", Mock(return_value=[latest_record])) google_api = GoogleAds(credentials=config["credentials"]) - client = AdGroupAdLegacy( + client = AdGroupAdReport( start_date=config["start_date"], api=google_api, conversion_window_days=config["conversion_window_days"], customers=customers ) return client @@ -429,7 +429,7 @@ def test_end_date_is_not_in_the_future(customers): def test_stream_slices(config, customers): google_api = GoogleAds(credentials=config["credentials"]) - stream = AdGroupAdLegacy( + stream = AdGroupAdReport( start_date=config["start_date"], api=google_api, conversion_window_days=config["conversion_window_days"], diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index a171b869d3d2..a448e353ade2 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -14,7 +14,7 @@ from google.api_core.exceptions import DataLoss, InternalServerError, ResourceExhausted, TooManyRequests, Unauthenticated from grpc import RpcError from source_google_ads.google_ads import GoogleAds -from source_google_ads.streams import AdGroup, ClickView, Customer, CustomerLabel +from source_google_ads.streams import AdGroups, ClickView, Accounts, AccountLabels # EXPIRED_PAGE_TOKEN exception will be raised when page token has expired. exception = GoogleAdsException( @@ -255,7 +255,7 @@ def test_parse_response(mocker, customers, config): ) # Create an instance of the Customer class - accounts = Customer(**incremental_stream_config) + accounts = Accounts(**incremental_stream_config) # Use the parse_response method and get the output output = list(accounts.parse_response(response)) @@ -280,7 +280,7 @@ def test_read_records_unauthenticated(mocker, customers, config): api=api, customers=customers, ) - stream = CustomerLabel(**stream_config) + stream = AccountLabels(**stream_config) with pytest.raises(AirbyteTracedException) as exc_info: list(stream.read_records(SyncMode.full_refresh, {"customer_id": "customer_id", "login_customer_id": "default"})) @@ -293,8 +293,8 @@ def test_ad_group_stream_query_removes_metrics_field_for_manager(customers_manag credentials = config["credentials"] api = GoogleAds(credentials=credentials) stream_config = dict(api=api, customers=customers_manager, start_date="2020-01-01", conversion_window_days=10) - stream = AdGroup(**stream_config) + stream = AdGroups(**stream_config) assert "metrics" not in stream.get_query(stream_slice={"customer_id": "123"}) stream_config = dict(api=api, customers=customers, start_date="2020-01-01", conversion_window_days=10) - stream = AdGroup(**stream_config) + stream = AdGroups(**stream_config) assert "metrics" in stream.get_query(stream_slice={"customer_id": "123"}) From d38d36c9c066c37fd933421aaeca4ac753ed76e4 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 2 Feb 2024 16:06:13 -0800 Subject: [PATCH 4/5] update stream -> resource mapping --- .../source_google_ads/utils.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index 3085343c9278..8c82bc062d46 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -37,6 +37,33 @@ def get_resource_name(stream_name: str) -> str: "campaign_real_time_bidding_settings": "campaign", "campaign_bidding_strategy": "campaign", "service_accounts": "customer", + "accounts": "customer", + "account_labels": "customer_label", + "ad_group_ads": "ad_group_ad", + "ad_group_ad_labels": "ad_group_ad_label", + "ad_group_ad_report": "ad_group_ad", + "ad_groups": "ad_group", + "ad_group_bidding_strategies": "ad_group", + "ad_group_criterions": "ad_group_criterion", + "ad_group_criterion_labels": "ad_group_criterion_label", + "ad_group_labels": "ad_group_label", + "ad_listing_group_criterions": "ad_group_criterion", + "audience": "audience", + "campaigns": "campaign", + "campaign_criterion": "campaign_criterion", + "campaign_bidding_strategies": "campaign", + "campaign_budget": "campaign_budget", + "campaign_labels": "campaign_label", + "change_status": "change_status", + "click_view": "click_view", + "display_keyword_performance_report": "display_keyword_view", + "display_topics_performance_report": "topic_view", + "geographic_report": "geographic_view", + "keyword_report": "keyword_view", + "labels": "label", + "shopping_performance_report": "shopping_performance_view", + "user_interest": "user_interest", + "user_location_report": "user_location_view", } From d5efce4a2b3e14599fba80124d0399a496b6b1bb Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 2 Feb 2024 16:57:14 -0800 Subject: [PATCH 5/5] topic_view wasn't in the list --- .../connectors/source-google-ads/source_google_ads/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index 8c82bc062d46..1442b3796ae1 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -64,6 +64,7 @@ def get_resource_name(stream_name: str) -> str: "shopping_performance_report": "shopping_performance_view", "user_interest": "user_interest", "user_location_report": "user_location_view", + "topic_performance_report": "topic_view", }