Source Amplitude: prepare connector for certification (#30652)

airbytehq · Sep 22, 2023 · f7f6b19 · f7f6b19
1 parent a104721
commit f7f6b19
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 17 deletions.
diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile
@@ -34,5 +34,5 @@ COPY source_amplitude ./source_amplitude
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.2
+LABEL io.airbyte.version=0.3.3
 LABEL io.airbyte.name=airbyte/source-amplitude
diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml
@@ -6,6 +6,8 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: "source_amplitude/spec.yaml"
+        backward_compatibility_tests_config:
+          disable_for_version: 0.3.2 # `start_date` format changed to format: date-time
   connection:
     tests:
       - config_path: "secrets/config.json"
@@ -16,7 +18,7 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         backward_compatibility_tests_config:
-          disable_for_version: 0.1.24 # cursor field for stream events has been changed
+          disable_for_version: 0.3.2 # `start_date` format changed to format: date-time
   basic_read:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl
@@ -2,14 +2,10 @@
 {"stream": "active_users", "data": {"date": "2023-08-29", "statistics": {"(none)": 0}}, "emitted_at": 1694709513302}
 {"stream": "active_users", "data": {"date": "2023-08-30", "statistics": {"(none)": 0}}, "emitted_at": 1694709513303}
 {"stream": "active_users", "data": {"date": "2023-08-31", "statistics": {"(none)": 0}}, "emitted_at": 1694709513305}
-{"stream": "average_session_length", "data": {"date": "2023-07-27", "length": 0}, "emitted_at": 1694709517086}
-{"stream": "average_session_length", "data": {"date": "2023-07-28", "length": 0}, "emitted_at": 1694709517088}
-{"stream": "average_session_length", "data": {"date": "2023-07-29", "length": 0}, "emitted_at": 1694709517090}
-{"stream": "average_session_length", "data": {"date": "2023-07-30", "length": 0}, "emitted_at": 1694709517092}
-{"stream": "average_session_length", "data": {"date": "2023-08-26", "length": 0}, "emitted_at": 1694709518827}
-{"stream": "average_session_length", "data": {"date": "2023-08-27", "length": 0}, "emitted_at": 1694709518828}
-{"stream": "average_session_length", "data": {"date": "2023-08-28", "length": 0}, "emitted_at": 1694709518830}
-{"stream": "average_session_length", "data": {"date": "2023-08-29", "length": 0}, "emitted_at": 1694709518832}
+{"stream": "average_session_length", "data": {"date": "2023-08-11", "length": 0}, "emitted_at": 1694709517092}
+{"stream": "average_session_length", "data": {"date": "2023-08-18", "length": 0}, "emitted_at": 1694709517088}
+{"stream": "average_session_length", "data": {"date": "2023-08-23", "length": 0}, "emitted_at": 1694709517090}
+{"stream": "average_session_length", "data": {"date": "2023-08-27", "length": 0}, "emitted_at": 1694709517086}
 {"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-DESKTOP-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "user_properties_updated": true, "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 355175889, "event_properties": {"ad_metrics.cost": 0.528957, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "DESKTOP", "ad_metrics.impressions": 1535, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 0, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 0, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "37bcd2f0-2688-47d5-ba90-76b0bd13b0f8", "version_name": null}, "emitted_at": 1694709577929}
 {"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-MOBILE-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 604299598, "event_properties": {"ad_metrics.cost": 11.398659, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "MOBILE", "ad_metrics.impressions": 15084, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 28, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 28, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "60320805-6886-43d6-b2ea-d3a6eccefee2", "version_name": null}, "emitted_at": 1694709577931}
 {"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-TABLET-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 798716893, "event_properties": {"ad_metrics.cost": 0.644529, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "TABLET", "ad_metrics.impressions": 931, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 3, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 3, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "9ae0c1e5-3c39-4a4d-af8b-0902fe889410", "version_name": null}, "emitted_at": 1694709577932}

diff --git a/airbyte-integrations/connectors/source-amplitude/metadata.yaml b/airbyte-integrations/connectors/source-amplitude/metadata.yaml
@@ -6,7 +6,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396
-  dockerImageTag: 0.3.2
+  dockerImageTag: 0.3.3
   dockerRepository: airbyte/source-amplitude
   githubIssueLabel: source-amplitude
   icon: amplitude.svg
@@ -18,6 +18,12 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  suggestedStreams:
+    streams:
+      - events
+      - active_users
+      - annotations
+      - cohorts
   documentationUrl: https://docs.airbyte.com/integrations/sources/amplitude
   tags:
     - language:low-code

diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml
@@ -33,6 +33,7 @@ connectionSpecification:
       airbyte_secret: true
     start_date:
       type: string
+      format: date-time
       title: Replication Start Date
       pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
       description:

diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md
@@ -1,6 +1,6 @@
 # Amplitude
 
-This page guides you through setting up the Amplitude source connector to sync data for the [Amplitude API](https://developers.amplitude.com/docs/http-api-v2).
+This page guides you through setting up the Amplitude source connector to sync data for the [Amplitude API](https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/).
 
 ## Prerequisite
 
@@ -20,11 +20,11 @@ To set up the Amplitude source connector, you'll need your Amplitude [`API Key`
 
 The Amplitude source connector supports the following streams:
 
-* [Active Users Counts](https://developers.amplitude.com/docs/dashboard-rest-api#active-and-new-user-counts) \(Incremental sync\)
-* [Annotations](https://developers.amplitude.com/docs/chart-annotations-api#get-all-annotations)
-* [Average Session Length](https://developers.amplitude.com/docs/dashboard-rest-api#average-session-length) \(Incremental sync\)
-* [Cohorts](https://developers.amplitude.com/docs/behavioral-cohorts-api#listing-all-cohorts)
-* [Events](https://developers.amplitude.com/docs/export-api#export-api---export-your-projects-event-data) \(Incremental sync\)
+* [Active Users Counts](https://www.docs.developers.amplitude.com/analytics/apis/dashboard-rest-api/#get-active-and-new-user-counts) \(Incremental sync\)
+* [Annotations](https://www.docs.developers.amplitude.com/analytics/apis/chart-annotations-api/#get-all-chart-annotations)
+* [Average Session Length](https://www.docs.developers.amplitude.com/analytics/apis/dashboard-rest-api/#get-average-session-length) \(Incremental sync\)
+* [Cohorts](https://www.docs.developers.amplitude.com/analytics/apis/behavioral-cohorts-api/#get-all-cohorts-response)
+* [Events](https://www.docs.developers.amplitude.com/analytics/apis/export-api/#response-schema) \(Incremental sync\)
 
 If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose)
 <!-- env:oss -->
@@ -35,6 +35,15 @@ The Amplitude source connector supports the following [sync modes](https://docs.
 - Full Refresh
 - Incremental
 
+## Connector-specific features
+
+There are two data region servers supported by Airbyte:
+
+- Standard Server
+- EU Residency Server
+
+The `Standard Server` will be the default option until you change it in the Optional fields.
+
 ## Performance considerations
 
 The Amplitude connector ideally should gracefully handle Amplitude API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues/new/choose) if you see any rate limit issues that are not automatically retried successfully.
@@ -43,6 +52,7 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                   |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------|
+| 0.3.3   | 2023-09-21 | [30652](https://github.com/airbytehq/airbyte/pull/30652) | Update spec: declare `start_date` type as `date-time`                                                     |
 | 0.3.2   | 2023-09-18 | [30525](https://github.com/airbytehq/airbyte/pull/30525) | Fix `KeyError` while getting `data_region` from config                                                    |
 | 0.3.1   | 2023-09-15 | [30471](https://github.com/airbytehq/airbyte/pull/30471) | Fix `Event` stream: Use `start_time` instead of cursor in the case of more recent                         |
 | 0.3.0   | 2023-09-13 | [30378](https://github.com/airbytehq/airbyte/pull/30378) | Switch to latest CDK version                                                                              |