From b984d59bf80cc3e3e77c8b50e73daf4838347fbc Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 17 Jan 2024 10:17:18 -0800 Subject: [PATCH] Rollback update to `destination_sync_mode` in `configured_catalog` --- .../acceptance-test-config.yml | 2 - .../integration_tests/configured_catalog.json | 48 ++++++++++--------- .../integration_tests/expected_records.jsonl | 6 ++- .../integration_tests/future_state.json | 7 +++ .../integration_tests/sample_state.json | 7 +++ .../source_chargebee/manifest.yaml | 7 +-- .../source_chargebee/spec.yaml | 2 +- 7 files changed, 50 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml index 45639765bfc0c..5c126b86ad5c9 100644 --- a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml @@ -54,8 +54,6 @@ acceptance_tests: future_state: future_state_path: "integration_tests/future_state.json" missing_streams: - - name: attached_item - bypass_reason: "This stream is Full-Refresh only" - name: contact bypass_reason: "This stream is Full-Refresh only" - name: quote_line_group diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json index b41a46a7eb1b2..ebfa1584a5e8a 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json @@ -10,7 +10,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -36,7 +36,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -58,7 +58,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -71,7 +71,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -84,7 +84,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -97,7 +97,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -109,8 +109,8 @@ "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append_dedup", + "sync_mode": "incremental", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -123,7 +123,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -136,7 +136,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -145,21 +145,25 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], + "default_cursor_field": ["created_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["created_at"] }, { "stream": { "name": "gift", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] }, { "stream": { @@ -171,7 +175,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -184,7 +188,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -197,7 +201,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -220,7 +224,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -238,7 +242,7 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_defined_cursor": ["migrated_at"], + "default_cursor_field": ["migrated_at"], "source_defined_primary_key": [["entity_id"]] }, "sync_mode": "incremental", @@ -268,7 +272,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, { @@ -281,7 +285,7 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "destination_sync_mode": "append", "cursor_field": ["updated_at"] } ] diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl index 9007c7adbdf3b..8de32cb8a83b9 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl @@ -18,7 +18,6 @@ {"stream":"promotional_credit","data":{"id":"pc_16CR0QTTZPx6wEbXR","customer_id":"Test-Custome-1","type":"decrement","amount":1000,"description":"Applied to the invoice # 39","credit_type":"general","closing_balance":26000,"created_at":1674298041,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220310} {"stream":"promotional_credit","data":{"id":"pc_16CZbuTTZGXtAFrxD","customer_id":"cbdemo_richard","type":"decrement","amount":1000,"description":"Applied to the invoice # 37","credit_type":"general","closing_balance":47000,"created_at":1674295799,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220318} {"stream":"promotional_credit","data":{"id":"pc_AzqD80TTZGFG7TiT6","customer_id":"cbdemo_tyler","type":"decrement","amount":1000,"description":"Applied to the invoice # 36","credit_type":"general","closing_balance":107000,"created_at":1674295727,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220325} -{"stream":"gift","data":{"id":"Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950","status":"unclaimed","scheduled_at":1674057609,"auto_claim":false,"updated_at":1674057613,"resource_version":1674057613941,"object":"gift","no_expiry":true,"gifter":{"customer_id":"Azz5jBTTJ96Mjlv5","invoice_id":"27","signature":"Airbyte","note":"Test gift","object":"gifter"},"gift_receiver":{"customer_id":"Azz5jBTTJ96UqlvE","subscription_id":"Azz5jBTTJ96Y2lvK","first_name":"Test","last_name":"2","email":"integration-tgest@airbyte.io","object":"gift_receiver"},"gift_timelines":[{"status":"unclaimed","occurred_at":1674057613,"object":"gift_timeline"},{"status":"scheduled","occurred_at":1674057604,"object":"gift_timeline"}],"custom_fields":[]},"emitted_at":1676569222624} {"stream": "coupon", "data": {"id": "TESTCOUPON4", "name": "Test Coupon 4", "invoice_name": "", "discount_type": "percentage", "discount_percentage": 12.0, "duration_type": "one_time", "valid_till": 1680332399, "status": "expired", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371290, "updated_at": 1676371290, "resource_version": 1676371290028, "object": "coupon", "redemptions": 1, "custom_fields": []}, "emitted_at": 1678971894181} {"stream": "coupon", "data": {"id": "TESTCOUPON5", "name": "Test Coupon 5", "invoice_name": "", "discount_type": "fixed_amount", "discount_amount": 300, "duration_type": "forever", "status": "active", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371313, "updated_at": 1676371313, "resource_version": 1676371313572, "object": "coupon", "redemptions": 1, "currency_code": "USD", "custom_fields": []}, "emitted_at": 1678971894193} {"stream": "coupon", "data": {"id": "TESTCOUPON6", "name": "Test Coupon 6", "invoice_name": "", "discount_type": "fixed_amount", "discount_amount": 500, "duration_type": "forever", "status": "active", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371327, "updated_at": 1676371327, "resource_version": 1676371327854, "object": "coupon", "redemptions": 1, "currency_code": "USD", "custom_fields": []}, "emitted_at": 1678971894207} @@ -51,3 +50,8 @@ {"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399378} {"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399498} {"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399634} +{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "integration-tgest@airbyte.io", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808513} +{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "integration-tgest@airbyte.io", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808659} +{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "integration-tgest@airbyte.io", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808802} +{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "integration-tgest@airbyte.io", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808935} +{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "integration-tgest@airbyte.io", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083809077} diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json index 32c87ffe2c6d2..6a12252e2465d 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json @@ -48,6 +48,13 @@ "stream_descriptor": { "name": "item" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "attached_item" } + } + }, { "type": "STREAM", "stream": { diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json index f8a43e7e1d9ab..49e8b43ee928a 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json @@ -48,6 +48,13 @@ "stream_descriptor": { "name": "item" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "attached_item" } + } + }, { "type": "STREAM", "stream": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml b/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml index d7a5edbc95d56..19a86e85902fe 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml @@ -88,7 +88,7 @@ definitions: path: "/addons" stream_cursor_field: "updated_at" attached_item_stream: - $ref: "#/definitions/base_stream" + $ref: "#/definitions/base_incremental_stream" retriever: $ref: "#/definitions/retriever" partition_router: @@ -96,12 +96,13 @@ definitions: parent_stream_configs: - type: ParentStreamConfig stream: "#/definitions/item_stream" - parent_key: id - partition_field: id + parent_key: "id" + partition_field: "id" $parameters: name: "attached_item" primary_key: "id" path: "/items/{{ stream_slice.id }}/attached_items" + stream_cursor_field: "updated_at" customer_stream: $ref: "#/definitions/base_incremental_stream" $parameters: diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml b/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml index 7b059e780ced1..34e151cf6c6ca 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml @@ -26,7 +26,7 @@ connectionSpecification: type: string format: date-time title: Start Date - description: UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated. + description: UTC date and time in the format 2017-01-25T00:00:00.000Z. Any data before this date will not be replicated. pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ examples: - "2021-01-25T00:00:00Z"