Skip to content

Commit

Permalink
Add method to exclude the non-upsert fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@bhuvana-talend
bhuvana-talend committed Sep 25, 2023
1 parent 50d2251 commit 6e7e610
Showing 1 changed file with 211 additions and 0 deletions.
211 changes: 211 additions & 0 deletions tests/base_new_frmwrk.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,217 @@ def set_replication_methods(self, conn_id, catalogs, replication_methods):
connections.set_non_discoverable_metadata(
conn_id, catalog, menagerie.get_annotated_schema(conn_id, catalog['stream_id']), replication_md)

### Method to return the fields that are non-upsert to be excluded from the all fields test as these non-upsert fields are not replicated
def get_upsert_only_fields(self, selected_fields, stream=None):
non_upsert_fields = {
"ads_insights" : {
'outbound_clicks',
'conversions',
'cost_per_action_type',
'video_p100_watched_actions',
'video_p75_watched_actions',
'action_values',
'video_30_sec_watched_actions',
'canvas_avg_view_percent',
'video_p50_watched_actions',
'video_p25_watched_actions',
'video_play_curve_actions',
'conversion_values',
'unique_outbound_clicks',
'canvas_avg_view_time',
'cost_per_unique_action_type'
},
"campaigns" : {
'adlabels'
},
"adsets" : {
'adlabels',
'bid_info'
},
"adcreative" : {
'image_crops',
'product_set_id',
'url_tags',
'applink_treatment',
'object_id',
'link_og_id',
'template_url',
'template_url_spec',
'object_url',
'link_url',
'adlabels',
'instagram_story_id'
},
"ads_insights_country": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_age_and_gender": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_dma": {
'video_p75_watched_actions',
'conversions',
'cost_per_unique_click',
'inline_link_click_ctr',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_ctr',
'unique_outbound_clicks',
'unique_inline_link_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'cost_per_unique_inline_link_click',
'unique_link_clicks_ctr',
'video_p50_watched_actions',
'engagement_rate_ranking',
'unique_inline_link_click_ctr',
'video_p25_watched_actions',
'quality_ranking',
'cpp',
'video_play_curve_actions',
'canvas_avg_view_time',
'video_30_sec_watched_actions',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_region": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_hourly_advertiser": {
'conversions',
'cost_per_estimated_ad_recallers',
'cost_per_unique_click',
'cost_per_unique_outbound_click',
'frequency',
'conversion_values',
'canvas_avg_view_percent',
'cost_per_conversion',
'cost_per_thruplay',
'action_values',
'full_view_impressions',
'place_page_name',
'instant_experience_outbound_clicks',
'cost_per_unique_action_type',
'estimated_ad_recallers',
'outbound_clicks',
'social_spend',
'cost_per_unique_inline_link_click',
'instant_experience_clicks_to_start',
'attribution_setting',
'engagement_rate_ranking',
'purchase_roas',
'reach',
'cost_per_outbound_click',
'estimated_ad_recall_rate',
'quality_ranking',
'cpp',
'catalog_segment_value',
'canvas_avg_view_time',
'cost_per_action_type',
'outbound_clicks_ctr',
'qualifying_question_qualify_answer_rate',
'converted_product_quantity',
'converted_product_value',
'instant_experience_clicks_to_open',
'conversion_rate_ranking'
},
"ads": {
'bid_amount',
'bid_info',
'recommendations',
'adlabels'
},
"ads_insights_platform_and_device": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'conversion_rate_ranking',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'placement'
}
}
actual_expected = {}
non_upsert_streams = non_upsert_fields.keys()
for key, fields in selected_fields.items():
if key in non_upsert_streams:
actual_expected[key] = fields.difference(non_upsert_fields.get(key))
else:
actual_expected[key] = fields
if stream:
return actual_expected[stream]
return actual_expected

@classmethod
def setUpClass(cls,logging="Ensuring environment variables are sourced."):
super().setUpClass(logging=logging)
Expand Down

0 comments on commit 6e7e610

Please sign in to comment.