Skip to content

Commit

Permalink
Rollback update to destination_sync_mode in configured_catalog
Browse files Browse the repository at this point in the history
  • Loading branch information
@pnilan
pnilan committed Jan 17, 2024
1 parent eafbeac commit b984d59
Show file tree
Hide file tree
Showing 7 changed files with 50 additions and 29 deletions.
2 changes: 0 additions & 2 deletions airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@ acceptance_tests:
future_state:
future_state_path: "integration_tests/future_state.json"
missing_streams:
- name: attached_item
bypass_reason: "This stream is Full-Refresh only"
- name: contact
bypass_reason: "This stream is Full-Refresh only"
- name: quote_line_group
Expand Down
48 changes: 26 additions & 22 deletions airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -36,7 +36,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -58,7 +58,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -71,7 +71,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -84,7 +84,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -97,7 +97,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -109,8 +109,8 @@
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append_dedup",
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -123,7 +123,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -136,7 +136,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -145,21 +145,25 @@
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"default_cursor_field": ["created_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["created_at"]
},
{
"stream": {
"name": "gift",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
"stream": {
Expand All @@ -171,7 +175,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -184,7 +188,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -197,7 +201,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -220,7 +224,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -238,7 +242,7 @@
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_defined_cursor": ["migrated_at"],
"default_cursor_field": ["migrated_at"],
"source_defined_primary_key": [["entity_id"]]
},
"sync_mode": "incremental",
Expand Down Expand Up @@ -268,7 +272,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
Expand All @@ -281,7 +285,7 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append_dedup",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
}
]
Expand Down
6 changes: 5 additions & 1 deletion airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
{"stream":"promotional_credit","data":{"id":"pc_16CR0QTTZPx6wEbXR","customer_id":"Test-Custome-1","type":"decrement","amount":1000,"description":"Applied to the invoice # 39","credit_type":"general","closing_balance":26000,"created_at":1674298041,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220310}
{"stream":"promotional_credit","data":{"id":"pc_16CZbuTTZGXtAFrxD","customer_id":"cbdemo_richard","type":"decrement","amount":1000,"description":"Applied to the invoice # 37","credit_type":"general","closing_balance":47000,"created_at":1674295799,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220318}
{"stream":"promotional_credit","data":{"id":"pc_AzqD80TTZGFG7TiT6","customer_id":"cbdemo_tyler","type":"decrement","amount":1000,"description":"Applied to the invoice # 36","credit_type":"general","closing_balance":107000,"created_at":1674295727,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220325}
{"stream":"gift","data":{"id":"Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950","status":"unclaimed","scheduled_at":1674057609,"auto_claim":false,"updated_at":1674057613,"resource_version":1674057613941,"object":"gift","no_expiry":true,"gifter":{"customer_id":"Azz5jBTTJ96Mjlv5","invoice_id":"27","signature":"Airbyte","note":"Test gift","object":"gifter"},"gift_receiver":{"customer_id":"Azz5jBTTJ96UqlvE","subscription_id":"Azz5jBTTJ96Y2lvK","first_name":"Test","last_name":"2","email":"[email protected]","object":"gift_receiver"},"gift_timelines":[{"status":"unclaimed","occurred_at":1674057613,"object":"gift_timeline"},{"status":"scheduled","occurred_at":1674057604,"object":"gift_timeline"}],"custom_fields":[]},"emitted_at":1676569222624}
{"stream": "coupon", "data": {"id": "TESTCOUPON4", "name": "Test Coupon 4", "invoice_name": "", "discount_type": "percentage", "discount_percentage": 12.0, "duration_type": "one_time", "valid_till": 1680332399, "status": "expired", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371290, "updated_at": 1676371290, "resource_version": 1676371290028, "object": "coupon", "redemptions": 1, "custom_fields": []}, "emitted_at": 1678971894181}
{"stream": "coupon", "data": {"id": "TESTCOUPON5", "name": "Test Coupon 5", "invoice_name": "", "discount_type": "fixed_amount", "discount_amount": 300, "duration_type": "forever", "status": "active", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371313, "updated_at": 1676371313, "resource_version": 1676371313572, "object": "coupon", "redemptions": 1, "currency_code": "USD", "custom_fields": []}, "emitted_at": 1678971894193}
{"stream": "coupon", "data": {"id": "TESTCOUPON6", "name": "Test Coupon 6", "invoice_name": "", "discount_type": "fixed_amount", "discount_amount": 500, "duration_type": "forever", "status": "active", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371327, "updated_at": 1676371327, "resource_version": 1676371327854, "object": "coupon", "redemptions": 1, "currency_code": "USD", "custom_fields": []}, "emitted_at": 1678971894207}
Expand Down Expand Up @@ -51,3 +50,8 @@
{"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399378}
{"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399498}
{"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399634}
{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "[email protected]", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808513}
{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "[email protected]", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808659}
{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "[email protected]", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808802}
{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "[email protected]", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808935}
{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "[email protected]", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083809077}
7 changes: 7 additions & 0 deletions airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@
"stream_descriptor": { "name": "item" }
}
},
{
"type": "STREAM",
"stream": {
"stream_state": { "updated_at": 2147483647 },
"stream_descriptor": { "name": "attached_item" }
}
},
{
"type": "STREAM",
"stream": {
Expand Down
7 changes: 7 additions & 0 deletions airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@
"stream_descriptor": { "name": "item" }
}
},
{
"type": "STREAM",
"stream": {
"stream_state": { "updated_at": 1625596058 },
"stream_descriptor": { "name": "attached_item" }
}
},
{
"type": "STREAM",
"stream": {
Expand Down
7 changes: 4 additions & 3 deletions airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,20 +88,21 @@ definitions:
path: "/addons"
stream_cursor_field: "updated_at"
attached_item_stream:
$ref: "#/definitions/base_stream"
$ref: "#/definitions/base_incremental_stream"
retriever:
$ref: "#/definitions/retriever"
partition_router:
type: SubstreamPartitionRouter
parent_stream_configs:
- type: ParentStreamConfig
stream: "#/definitions/item_stream"
parent_key: id
partition_field: id
parent_key: "id"
partition_field: "id"
$parameters:
name: "attached_item"
primary_key: "id"
path: "/items/{{ stream_slice.id }}/attached_items"
stream_cursor_field: "updated_at"
customer_stream:
$ref: "#/definitions/base_incremental_stream"
$parameters:
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ connectionSpecification:
type: string
format: date-time
title: Start Date
description: UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated.
description: UTC date and time in the format 2017-01-25T00:00:00.000Z. Any data before this date will not be replicated.
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
examples:
- "2021-01-25T00:00:00Z"
Expand Down

0 comments on commit b984d59

Please sign in to comment.