✨Source Snapchat Marketing: Add ability to break-down the Stats streams by report_dimensions query parameter

airbyte-integrations/connectors/source-harness/Dockerfile

@@ -4,17 +4,17 @@ FROM python:3.9.11-alpine3.15 as base
 FROM base as builder
 WORKDIR /airbyte/integration_code
 
-# upgrade pip to the latest version
+
 RUN apk --no-cache upgrade \
-    && pip install --upgrade pip \
+    && pip install --upgrade pip  \
     && apk --no-cache add tzdata build-base
 
 
 COPY setup.py ./
 # install necessary packages to a temporary folder
 RUN pip install --prefix=/install .
 
-# build a clean environment
+
 FROM base
 WORKDIR /airbyte/integration_code
 
@@ -34,5 +34,5 @@ COPY source_harness ./source_harness
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
-LABEL io.airbyte.name=airbyte/source-harness
+LABEL io.airbyte.version=0.3.0
+LABEL io.airbyte.name=airbyte/source-snapchat-marketing

airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml

@@ -8,7 +8,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b
-  dockerImageTag: 0.3.2
+  dockerImageTag: 0.3.3
   dockerRepository: airbyte/source-snapchat-marketing
   githubIssueLabel: source-snapchat-marketing
   icon: snapchat.svg

airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.3.2"
+version = "0.3.3"
 name = "source-snapchat-marketing"
 description = "Source implementation for Snapchat Marketing."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/basic_stats.json

@@ -227,6 +227,33 @@
     },
     "story_completes": {
       "type": ["null", "number"]
+    },
+    "country": {
+      "type": ["null", "string"]
+    },
+    "region": {
+      "type": ["null", "string"]
+    },
+    "dma": {
+      "type": ["null", "string"]
+    },
+    "gender": {
+      "type": ["null", "string"]
+    },
+    "age_bucket": {
+      "type": ["null", "string"]
+    },
+    "operating_system": {
+      "type": ["null", "string"]
+    },
+    "make": {
+      "type": ["null", "string"]
+    },
+    "interest_category_id": {
+      "type": ["null", "number"]
+    },
+    "interest_category_name": {
+      "type": ["null", "string"]
     }
   }
 }

airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py

@@ -99,6 +99,82 @@
     "earned_reach",
 ]
 
+# Maps which type of metric is supported by which dimension
+# Refer: https://marketingapi.snapchat.com/docs/#reporting-insights-and-dimensions
+DIMENSION_METRIC_DICT = {
+    "country": "Delivery/Conversion",
+    "country,os": "Delivery",
+    "os,country": "Delivery",
+    "region": "Delivery",
+    "dma": "Delivery",
+    "gender": "Delivery/Conversion",
+    "age": "Delivery/Conversion",
+    "age,gender": "Delivery/Conversion",
+    "gender,age": "Delivery/Conversion",
+    "os": "Delivery/Conversion",
+    "make": "Delivery",
+    "lifestyle_category": "Delivery"
+}
+
+# Refer: https://marketingapi.snapchat.com/docs/#reporting-insights-and-dimensions
+DELIVERY_METRICS = [
+    "impressions",
+    "swipes",
+    "quartile_1",
+    "quartile_2",
+    "quartile_3",
+    "view_completion",
+    "spend",
+    "video_views",
+    "attachment_quartile_1",
+    "attachment_quartile_2",
+    "attachment_quartile_3",
+    "attachment_view_completion",
+    "attachment_total_view_time_millis",
+    "attachment_video_views",
+    "story_opens",
+    "story_completes",
+    "shares",
+    "saves"
+]
+
+# Refer: https://marketingapi.snapchat.com/docs/#reporting-insights-and-dimensions
+CONVERSION_METRICS = [
+    "total_installs",
+    "conversion_purchases",
+    "conversion_purchases_value",
+    "conversion_save",
+    "conversion_start_checkout",
+    "conversion_add_cart",
+    "conversion_view_content",
+    "conversion_add_billing",
+    "conversion_sign_ups",
+    "conversion_searches",
+    "conversion_level_completes",
+    "conversion_app_opens",
+    "conversion_page_views",
+    "conversion_subscribe",
+    "conversion_ad_click",
+    "conversion_ad_view",
+    "conversion_complete_tutorial",
+    "conversion_invite",
+    "conversion_login",
+    "conversion_share",
+    "conversion_reserve",
+    "conversion_achievement_unlocked",
+    "conversion_add_to_wishlist",
+    "conversion_spend_credits",
+    "conversion_rate",
+    "conversion_start_trial",
+    "conversion_list_view",
+    "conversion_visit",
+    "custom_event_1",
+    "custom_event_2",
+    "custom_event_3",
+    "custom_event_4",
+    "custom_event_5"
+]
+
 logger = logging.getLogger("airbyte")
 
 
@@ -165,10 +241,11 @@ class SnapchatMarketingStream(HttpStream, ABC):
     primary_key = "id"
     raise_on_http_errors = True
 
-    def __init__(self, start_date, end_date, **kwargs):
+    def __init__(self, start_date, end_date, report_dimension="", **kwargs):
         super().__init__(**kwargs)
         self.start_date = start_date
         self.end_date = end_date
+        self.report_dimension = report_dimension
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         next_page_cursor = response.json().get("paging", False)
@@ -246,7 +323,7 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
         self.initial_state = stream_state.get(self.cursor_field) if stream_state else self.start_date
         self.max_state = self.initial_state
 
-        parent_stream = self.parent(authenticator=self.authenticator, start_date=self.start_date, end_date=self.end_date)
+        parent_stream = self.parent(authenticator=self.authenticator, start_date=self.start_date, end_date=self.end_date, report_dimension=self.report_dimension)
         stream_slices = get_parent_ids(parent_stream)
 
         if stream_slices:
@@ -368,7 +445,7 @@ def parent(self) -> SnapchatMarketingStream:
     def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
         """Each stream slice represents each entity id from parent stream"""
 
-        parent_stream = self.parent(authenticator=self.authenticator, start_date=self.start_date, end_date=self.end_date)
+        parent_stream = self.parent(authenticator=self.authenticator, start_date=self.start_date, end_date=self.end_date, report_dimension=self.report_dimension)
         self.parent_name = parent_stream.name
         stream_slices = get_parent_ids(parent_stream)
 
@@ -390,6 +467,19 @@ def request_params(
         params["granularity"] = self.granularity.value
         if self.metrics:
             params["fields"] = ",".join(self.metrics)
+        if self.report_dimension:
+            params["report_dimension"] = self.report_dimension
+            # Filter metrics based on the type supported by the provided report_dimension
+            new_metrics = []
+            supported_type = DIMENSION_METRIC_DICT.get(self.report_dimension)
+            for metric in self.metrics:
+                if supported_type == "Delivery/Conversion":
+                    if metric in DELIVERY_METRICS or metric in CONVERSION_METRICS:
+                        new_metrics.append(metric)
+                else:
+                    if metric in DELIVERY_METRICS:
+                        new_metrics.append(metric)
+            params["fields"] = ",".join(new_metrics)
 
         return params
 
@@ -407,7 +497,12 @@ def parse_response(
             response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
         ):
             # move all 'stats' metrics to root level
-            record.update(record.pop("stats", {}))
+            # if report dimension is specified, they are found under 'dimension_stats' instead of 'stats'
+            # Example for report dimension output here: https://marketingapi.snapchat.com/docs/#insights-by-demo
+            if self.report_dimension:
+                record.update(record.pop("dimension_stats", {}))
+            else:
+                record.update(record.pop("stats", {}))
 
             yield record
 
@@ -524,7 +619,12 @@ def parse_response(
                 # add common record identifiers
                 stat_item.update(record_identifiers)
                 # move all 'stats' metrics to root level
-                stat_item.update(stat_item.pop("stats", {}))
+                # if report dimension is specified, they are found under 'dimension_stats' instead of 'stats'
+                # Example for report dimension output here: https://marketingapi.snapchat.com/docs/#insights-by-demo
+                if self.report_dimension:
+                    stat_item.update(stat_item.pop("dimension_stats", {}))
+                else:
+                    stat_item.update(stat_item.pop("stats", {}))
 
                 # Update state for last record in the stream
                 self.update_state_after_last_record(stat_item)
@@ -808,6 +908,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             ),
             "start_date": config["start_date"],
             "end_date": config.get("end_date", default_end_date),
+            "report_dimension": config.get("report_dimension", "")
         }
 
         return [

airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json

@@ -45,6 +45,15 @@
         "examples": ["2022-01-30"],
         "order": 4,
         "format": "date"
+      },
+      "report_dimension":  {
+        "type": "string",
+        "enum": ["country", "country,os", "os,country", "region", "dma", "gender", "age", "age,gender", "gender", "age", "os", "make", "lifestyle_category"],
+        "title": "Report Dimension",
+        "description": "Used to get reporting insights based on different geographic, demographic, interest-based, and device-based dimensions.\n For usage refer: https://marketingapi.snapchat.com/docs/#reporting-insights-and-dimensions",
+        "pattern": "^(country|country,os|os,country|region|dma|gender|age|age,gender|gender,age|os|make|lifestyle_category)$",
+        "examples": ["country", "age", "os"],
+        "order": 5
       }
     }
   },

docs/integrations/sources/snapchat-marketing.md

@@ -113,7 +113,8 @@ Snapchat Marketing API has limitations to 1000 items per page.
 
 | Version | Date       | Pull Request                                             | Subject                                                       |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------|
-| 0.3.2 | 2024-02-12 | [35171](https://github.com/airbytehq/airbyte/pull/35171) | Manage dependencies with Poetry. |
+| 0.3.3   | 2024-02-12 | [30761](https://github.com/airbytehq/airbyte/pull/30761) | Add ability to break-down the Stats streams by the report_dimensions param |
+| 0.3.2   | 2024-02-12 | [35171](https://github.com/airbytehq/airbyte/pull/35171) | Manage dependencies with Poetry. |
 | 0.3.0   | 2023-05-22 | [26358](https://github.com/airbytehq/airbyte/pull/26358) | Remove deprecated authSpecification in favour of advancedAuth |
 | 0.2.0   | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce new field in the `Campaigns` stream schema          |
 | 0.1.16  | 2023-04-20 | [20897](https://github.com/airbytehq/airbyte/pull/20897) | Add missing fields to Basic Stats schema                      |