diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index d4276e9ecc21..aa8578b59f3a 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.2.0 +LABEL io.airbyte.version=1.2.1 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml index f5fa3a1faf4a..7fe08d8c848a 100644 --- a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml @@ -89,6 +89,18 @@ acceptance_tests: bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" - name: "activities_scheduled_meetingin_dialogue" bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwithan_agentin_dialogue" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_reached_conversational_flow_goal" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwithan_agentin_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwitha_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_scheduled_meetingin_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_interactedwith_documentin_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" # 52 streams, most of them use BULK API therefore it takes much time to run a sync timeout_seconds: 9000 fail_on_extra_columns: false diff --git a/airbyte-integrations/connectors/source-marketo/metadata.yaml b/airbyte-integrations/connectors/source-marketo/metadata.yaml index d0fc6429c308..b85246d7a187 100644 --- a/airbyte-integrations/connectors/source-marketo/metadata.yaml +++ b/airbyte-integrations/connectors/source-marketo/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 - dockerImageTag: 1.2.0 + dockerImageTag: 1.2.1 dockerRepository: airbyte/source-marketo githubIssueLabel: source-marketo icon: marketo.svg diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 06ad97f67491..fffc07af0974 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -342,7 +342,15 @@ def __init__(self, config: Mapping[str, Any]): @property def stream_fields(self): - return list(self.get_json_schema()["properties"].keys()) + standard_properties = set(self.get_json_schema()["properties"]) + resp = self._session.get(f"{self._url_base}rest/v1/leads/describe.json", headers=self.authenticator.get_auth_header()) + available_fields = set(x.get("rest").get("name") for x in resp.json().get("result")) + return list(standard_properties & available_fields) + + def get_json_schema(self) -> Mapping[str, Any]: + # TODO: make schema truly dynamic like in stream Activities + # now blocked by https://github.com/airbytehq/airbyte/issues/30530 due to potentially > 500 fields in schema (can cause OOM) + return super().get_json_schema() class Activities(MarketoExportBase): diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index b179e2515660..f121c9017d24 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -81,13 +81,19 @@ The Marketo source connector supports the following[ sync modes](https://docs.ai This connector can be used to sync the following tables from Marketo: -- **activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means. -- **activity_types.** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns. -- **campaigns.** Contains info about your Marketo campaigns. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET). -- **leads.** Contains info about your Marketo leads. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET). -- **lists.** Contains info about your Marketo static lists. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET). -- **programs.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET). -- **segmentations.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET). +- **Activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means. +- **Activity types** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns. +- **[Campaigns](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET)**: Contains info about your Marketo campaigns. +- **[Leads](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET)**: Contains info about your Marketo leads. + +:::caution + +Available fields are limited by what is presented in the static schema. + +::: +- **[Lists](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET)**: Contains info about your Marketo static lists. +- **[Programs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET)**: Contains info about your Marketo programs. +- **[Segmentations](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET)**: Contains info about your Marketo programs. ## Performance considerations @@ -100,7 +106,7 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Data type map | Integration Type | Airbyte Type | Notes | -| :--------------- | :----------- | :------------------------------------------------------------------------------ | +|:-----------------|:-------------|:--------------------------------------------------------------------------------| | `array` | `array` | primitive arrays are converted into arrays of the types described in this table | | `int`, `long` | `number` | | | `object` | `object` | | @@ -110,7 +116,8 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Changelog | Version | Date | Pull Request | Subject | -| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------- | +|:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------| +| `1.2.1` | 2023-09-18 | [30533](https://github.com/airbytehq/airbyte/pull/30533) | Fix `json_schema` for stream `Leads` | | `1.2.0` | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2 | | `1.1.0` | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream | | `1.0.4` | 2023-04-25 | [25481](https://github.com/airbytehq/airbyte/pull/25481) | Minor fix for bug caused by `<=` producing additional API call when there is a single date slice |