Source Marketo: fix json_schema for stream Leads (#30533)

airbytehq · Sep 19, 2023 · f5a4eec · f5a4eec
1 parent 2a870ee
commit f5a4eec
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 12 deletions.
diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile
@@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.2.0
+LABEL io.airbyte.version=1.2.1
 LABEL io.airbyte.name=airbyte/source-marketo
diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml
@@ -89,6 +89,18 @@ acceptance_tests:
             bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
           - name: "activities_scheduled_meetingin_dialogue"
             bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
+          - name: "activities_engagedwithan_agentin_dialogue"
+            bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
+          - name: "activities_reached_conversational_flow_goal"
+            bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
+          - name: "activities_engagedwithan_agentin_conversational_flow"
+            bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
+          - name: "activities_engagedwitha_conversational_flow"
+            bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
+          - name: "activities_scheduled_meetingin_conversational_flow"
+            bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
+          - name: "activities_interactedwith_documentin_conversational_flow"
+            bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
         # 52 streams, most of them use BULK API therefore it takes much time to run a sync
         timeout_seconds: 9000
         fail_on_extra_columns: false

diff --git a/airbyte-integrations/connectors/source-marketo/metadata.yaml b/airbyte-integrations/connectors/source-marketo/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 9e0556f4-69df-4522-a3fb-03264d36b348
-  dockerImageTag: 1.2.0
+  dockerImageTag: 1.2.1
   dockerRepository: airbyte/source-marketo
   githubIssueLabel: source-marketo
   icon: marketo.svg

diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py
@@ -342,7 +342,15 @@ def __init__(self, config: Mapping[str, Any]):
 
     @property
     def stream_fields(self):
-        return list(self.get_json_schema()["properties"].keys())
+        standard_properties = set(self.get_json_schema()["properties"])
+        resp = self._session.get(f"{self._url_base}rest/v1/leads/describe.json", headers=self.authenticator.get_auth_header())
+        available_fields = set(x.get("rest").get("name") for x in resp.json().get("result"))
+        return list(standard_properties & available_fields)
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        # TODO: make schema truly dynamic like in stream Activities
+        #  now blocked by https://github.com/airbytehq/airbyte/issues/30530 due to potentially > 500 fields in schema (can cause OOM)
+        return super().get_json_schema()
 
 
 class Activities(MarketoExportBase):

diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md
@@ -81,13 +81,19 @@ The Marketo source connector supports the following[ sync modes](https://docs.ai
 
 This connector can be used to sync the following tables from Marketo:
 
-- **activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means.
-- **activity_types.** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns.
-- **campaigns.** Contains info about your Marketo campaigns. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET).
-- **leads.** Contains info about your Marketo leads. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET).
-- **lists.** Contains info about your Marketo static lists. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET).
-- **programs.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET).
-- **segmentations.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET).
+- **Activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means.
+- **Activity types** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns.
+- **[Campaigns](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET)**: Contains info about your Marketo campaigns.
+- **[Leads](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET)**: Contains info about your Marketo leads.
+
+:::caution
+
+Available fields are limited by what is presented in the static schema.
+
+:::
+- **[Lists](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET)**: Contains info about your Marketo static lists.
+- **[Programs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET)**: Contains info about your Marketo programs.
+- **[Segmentations](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET)**: Contains info about your Marketo programs.
 
 ## Performance considerations
 
@@ -100,7 +106,7 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes                                                                           |
-| :--------------- | :----------- | :------------------------------------------------------------------------------ |
+|:-----------------|:-------------|:--------------------------------------------------------------------------------|
 | `array`          | `array`      | primitive arrays are converted into arrays of the types described in this table |
 | `int`, `long`    | `number`     |                                                                                 |
 | `object`         | `object`     |                                                                                 |
@@ -110,7 +116,8 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa
 ## Changelog
 
 | Version  | Date       | Pull Request                                             | Subject                                                                                          |
-| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------- |
+|:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------|
+| `1.2.1`  | 2023-09-18 | [30533](https://github.com/airbytehq/airbyte/pull/30533) | Fix `json_schema` for stream `Leads`                                                             |
 | `1.2.0`  | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2                                                                             |
 | `1.1.0`  | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream                                                                       |
 | `1.0.4`  | 2023-04-25 | [25481](https://github.com/airbytehq/airbyte/pull/25481) | Minor fix for bug caused by `<=` producing additional API call when there is a single date slice |