From f7f6b19bf90581dfaac104733d53a9207ce4f0e7 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Fri, 22 Sep 2023 15:59:46 +0400 Subject: [PATCH] Source Amplitude: prepare connector for certification (#30652) --- .../connectors/source-amplitude/Dockerfile | 2 +- .../acceptance-test-config.yml | 4 +++- .../integration_tests/expected_records.jsonl | 12 ++++------ .../connectors/source-amplitude/metadata.yaml | 8 ++++++- .../source_amplitude/spec.yaml | 1 + docs/integrations/sources/amplitude.md | 22 ++++++++++++++----- 6 files changed, 32 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile index 728f59891126..2624b8a13307 100644 --- a/airbyte-integrations/connectors/source-amplitude/Dockerfile +++ b/airbyte-integrations/connectors/source-amplitude/Dockerfile @@ -34,5 +34,5 @@ COPY source_amplitude ./source_amplitude ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.3.3 LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml index f2d615d750d4..c1d93a1a095a 100644 --- a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml @@ -6,6 +6,8 @@ acceptance_tests: spec: tests: - spec_path: "source_amplitude/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 0.3.2 # `start_date` format changed to format: date-time connection: tests: - config_path: "secrets/config.json" @@ -16,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: 0.1.24 # cursor field for stream events has been changed + disable_for_version: 0.3.2 # `start_date` format changed to format: date-time basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl index 27b00511b697..f87fbdcb8d61 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl @@ -2,14 +2,10 @@ {"stream": "active_users", "data": {"date": "2023-08-29", "statistics": {"(none)": 0}}, "emitted_at": 1694709513302} {"stream": "active_users", "data": {"date": "2023-08-30", "statistics": {"(none)": 0}}, "emitted_at": 1694709513303} {"stream": "active_users", "data": {"date": "2023-08-31", "statistics": {"(none)": 0}}, "emitted_at": 1694709513305} -{"stream": "average_session_length", "data": {"date": "2023-07-27", "length": 0}, "emitted_at": 1694709517086} -{"stream": "average_session_length", "data": {"date": "2023-07-28", "length": 0}, "emitted_at": 1694709517088} -{"stream": "average_session_length", "data": {"date": "2023-07-29", "length": 0}, "emitted_at": 1694709517090} -{"stream": "average_session_length", "data": {"date": "2023-07-30", "length": 0}, "emitted_at": 1694709517092} -{"stream": "average_session_length", "data": {"date": "2023-08-26", "length": 0}, "emitted_at": 1694709518827} -{"stream": "average_session_length", "data": {"date": "2023-08-27", "length": 0}, "emitted_at": 1694709518828} -{"stream": "average_session_length", "data": {"date": "2023-08-28", "length": 0}, "emitted_at": 1694709518830} -{"stream": "average_session_length", "data": {"date": "2023-08-29", "length": 0}, "emitted_at": 1694709518832} +{"stream": "average_session_length", "data": {"date": "2023-08-11", "length": 0}, "emitted_at": 1694709517092} +{"stream": "average_session_length", "data": {"date": "2023-08-18", "length": 0}, "emitted_at": 1694709517088} +{"stream": "average_session_length", "data": {"date": "2023-08-23", "length": 0}, "emitted_at": 1694709517090} +{"stream": "average_session_length", "data": {"date": "2023-08-27", "length": 0}, "emitted_at": 1694709517086} {"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-DESKTOP-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "user_properties_updated": true, "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 355175889, "event_properties": {"ad_metrics.cost": 0.528957, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "DESKTOP", "ad_metrics.impressions": 1535, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 0, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 0, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "37bcd2f0-2688-47d5-ba90-76b0bd13b0f8", "version_name": null}, "emitted_at": 1694709577929} {"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-MOBILE-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 604299598, "event_properties": {"ad_metrics.cost": 11.398659, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "MOBILE", "ad_metrics.impressions": 15084, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 28, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 28, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "60320805-6886-43d6-b2ea-d3a6eccefee2", "version_name": null}, "emitted_at": 1694709577931} {"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-TABLET-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 798716893, "event_properties": {"ad_metrics.cost": 0.644529, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "TABLET", "ad_metrics.impressions": 931, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 3, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 3, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "9ae0c1e5-3c39-4a4d-af8b-0902fe889410", "version_name": null}, "emitted_at": 1694709577932} diff --git a/airbyte-integrations/connectors/source-amplitude/metadata.yaml b/airbyte-integrations/connectors/source-amplitude/metadata.yaml index 5928851f0ccb..4275afa14991 100644 --- a/airbyte-integrations/connectors/source-amplitude/metadata.yaml +++ b/airbyte-integrations/connectors/source-amplitude/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396 - dockerImageTag: 0.3.2 + dockerImageTag: 0.3.3 dockerRepository: airbyte/source-amplitude githubIssueLabel: source-amplitude icon: amplitude.svg @@ -18,6 +18,12 @@ data: oss: enabled: true releaseStage: generally_available + suggestedStreams: + streams: + - events + - active_users + - annotations + - cohorts documentationUrl: https://docs.airbyte.com/integrations/sources/amplitude tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml index ee950f5e6658..78cd3163a673 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml @@ -33,6 +33,7 @@ connectionSpecification: airbyte_secret: true start_date: type: string + format: date-time title: Replication Start Date pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" description: diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md index 1d1cb424902d..db26cdcfe221 100644 --- a/docs/integrations/sources/amplitude.md +++ b/docs/integrations/sources/amplitude.md @@ -1,6 +1,6 @@ # Amplitude -This page guides you through setting up the Amplitude source connector to sync data for the [Amplitude API](https://developers.amplitude.com/docs/http-api-v2). +This page guides you through setting up the Amplitude source connector to sync data for the [Amplitude API](https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/). ## Prerequisite @@ -20,11 +20,11 @@ To set up the Amplitude source connector, you'll need your Amplitude [`API Key` The Amplitude source connector supports the following streams: -* [Active Users Counts](https://developers.amplitude.com/docs/dashboard-rest-api#active-and-new-user-counts) \(Incremental sync\) -* [Annotations](https://developers.amplitude.com/docs/chart-annotations-api#get-all-annotations) -* [Average Session Length](https://developers.amplitude.com/docs/dashboard-rest-api#average-session-length) \(Incremental sync\) -* [Cohorts](https://developers.amplitude.com/docs/behavioral-cohorts-api#listing-all-cohorts) -* [Events](https://developers.amplitude.com/docs/export-api#export-api---export-your-projects-event-data) \(Incremental sync\) +* [Active Users Counts](https://www.docs.developers.amplitude.com/analytics/apis/dashboard-rest-api/#get-active-and-new-user-counts) \(Incremental sync\) +* [Annotations](https://www.docs.developers.amplitude.com/analytics/apis/chart-annotations-api/#get-all-chart-annotations) +* [Average Session Length](https://www.docs.developers.amplitude.com/analytics/apis/dashboard-rest-api/#get-average-session-length) \(Incremental sync\) +* [Cohorts](https://www.docs.developers.amplitude.com/analytics/apis/behavioral-cohorts-api/#get-all-cohorts-response) +* [Events](https://www.docs.developers.amplitude.com/analytics/apis/export-api/#response-schema) \(Incremental sync\) If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) @@ -35,6 +35,15 @@ The Amplitude source connector supports the following [sync modes](https://docs. - Full Refresh - Incremental +## Connector-specific features + +There are two data region servers supported by Airbyte: + +- Standard Server +- EU Residency Server + +The `Standard Server` will be the default option until you change it in the Optional fields. + ## Performance considerations The Amplitude connector ideally should gracefully handle Amplitude API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues/new/choose) if you see any rate limit issues that are not automatically retried successfully. @@ -43,6 +52,7 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.3.3 | 2023-09-21 | [30652](https://github.com/airbytehq/airbyte/pull/30652) | Update spec: declare `start_date` type as `date-time` | | 0.3.2 | 2023-09-18 | [30525](https://github.com/airbytehq/airbyte/pull/30525) | Fix `KeyError` while getting `data_region` from config | | 0.3.1 | 2023-09-15 | [30471](https://github.com/airbytehq/airbyte/pull/30471) | Fix `Event` stream: Use `start_time` instead of cursor in the case of more recent | | 0.3.0 | 2023-09-13 | [30378](https://github.com/airbytehq/airbyte/pull/30378) | Switch to latest CDK version |