From f33ddf6cb975b2f8f8f8e1097d6160dd47537fb3 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Mon, 18 Sep 2023 08:36:16 -0700 Subject: [PATCH] source-stripe: fix property of `invoices.lines` property (#30494) Co-authored-by: Denys Davydov --- .../connectors/source-stripe/Dockerfile | 2 +- .../source-stripe/acceptance-test-config.yml | 2 + .../integration_tests/expected_records.jsonl | 1 - .../connectors/source-stripe/metadata.yaml | 2 +- .../source_stripe/schemas/invoices.json | 6 +-- docs/integrations/sources/stripe.md | 37 ++++++++++--------- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 5cec47b28a5a..bd488734f77d 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=4.1.0 +LABEL io.airbyte.version=4.1.1 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 4a555acc838f..a25b6928b6f7 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -42,6 +42,8 @@ acceptance_tests: bypass_reason: "this stream can't be seeded in our sandbox account" - name: "transactions" bypass_reason: "This stream can't be seeded in our sandbox account" + - name: "events" + bypass_reason: "Data expires every 30 days." expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index ac87f8e34e6e..d699385e28e1 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -15,7 +15,6 @@ {"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MWIPLEcXtiJtvvhLnQYjVCj", "object": "customer_balance_transaction", "amount": 50000.0, "created": 1675166031, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": "Test credit balance", "ending_balance": 50000.0, "invoice": null, "livemode": false, "metadata": {}, "type": "adjustment"}, "emitted_at": 1691675200932} {"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1646998902}, "emitted_at": 1691675201702} {"stream": "disputes", "data": {"id": "dp_1MSI78EcXtiJtvvhxC77m2kh", "object": "dispute", "amount": 700, "balance_transaction": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "balance_transactions": [{"id": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "object": "balance_transaction", "amount": -700, "available_on": 1674518400, "created": 1674211590, "currency": "usd", "description": "Chargeback withdrawal for ch_3MSI77EcXtiJtvvh1GzoukUC", "exchange_rate": null, "fee": 1500, "fee_details": [{"amount": 1500, "application": null, "currency": "usd", "description": "Dispute fee", "type": "stripe_fee"}], "net": -2200, "reporting_category": "dispute", "source": "dp_1MSI78EcXtiJtvvhxC77m2kh", "status": "available", "type": "adjustment"}], "charge": "ch_3MSI77EcXtiJtvvh1GzoukUC", "created": 1674211590, "currency": "usd", "evidence": {"access_activity_log": null, "billing_address": "12345", "cancellation_policy": null, "cancellation_policy_disclosure": null, "cancellation_rebuttal": null, "customer_communication": null, "customer_email_address": null, "customer_name": null, "customer_purchase_ip": null, "customer_signature": null, "duplicate_charge_documentation": null, "duplicate_charge_explanation": null, "duplicate_charge_id": null, "product_description": null, "receipt": null, "refund_policy": null, "refund_policy_disclosure": null, "refund_refusal_explanation": null, "service_date": null, "service_documentation": null, "shipping_address": null, "shipping_carrier": null, "shipping_date": null, "shipping_documentation": null, "shipping_tracking_number": null, "uncategorized_file": null, "uncategorized_text": null}, "evidence_details": {"due_by": 1675036799.0, "has_evidence": false, "past_due": false, "submission_count": 0}, "is_charge_refundable": false, "livemode": false, "metadata": {}, "payment_intent": "pi_3MSI77EcXtiJtvvh1glmQd8s", "payment_method_details": {"card": {"brand": "visa", "network_reason_code": "83"}, "type": "card"}, "reason": "fraudulent", "status": "lost", "updated": 1674211590}, "emitted_at": 1692348303326} -{"stream": "events", "data": {"id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", "object": "event", "api_version": "2020-08-27", "created": 1691629292, "data": {"object": {"object": "balance", "available": [{"amount": 515674, "currency": "usd", "source_types": {"card": 515674}}], "connect_reserved": [{"amount": 0, "currency": "usd"}], "issuing": {"available": [{"amount": 150000, "currency": "usd"}]}, "livemode": false, "pending": [{"amount": 0, "currency": "usd", "source_types": {"card": 0}}]}}, "livemode": false, "pending_webhooks": 0, "request": {"id": null, "idempotency_key": null}, "type": "balance.available"}, "emitted_at": 1692093539344} {"stream": "invoice_items", "data": {"id": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "object": "invoiceitem", "amount": 8400, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "date": 1640123817, "description": "a box of parsnips", "discountable": true, "discounts": [], "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "quantity": 1, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 8400, "unit_amount_decimal": "8400", "updated": 1640123817}, "emitted_at": 1691675215106} {"stream": "invoice_items", "data": {"id": "ii_1MX384EcXtiJtvvhguyn3iYb", "object": "invoiceitem", "amount": 6000, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345628, "description": "Test Product 1", "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice": "in_1MX37hEcXtiJtvvhRSl1KbQm", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "quantity": 3, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 2000, "unit_amount_decimal": "2000", "updated": 1675345628}, "emitted_at": 1691675215403} {"stream": "invoice_items", "data": {"id": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "object": "invoiceitem", "amount": 25200, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345045, "description": "edgao-test-product", "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "quantity": 2, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 12600, "unit_amount_decimal": "12600", "updated": 1675345045}, "emitted_at": 1691675215404} diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index f760619bc92c..d0f8dde30900 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 4.1.0 + dockerImageTag: 4.1.1 dockerRepository: airbyte/source-stripe githubIssueLabel: source-stripe icon: stripe.svg diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json index c4fedda2b9ff..3bd0ccc77778 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json @@ -209,11 +209,7 @@ "type": ["null", "integer"] }, "lines": { - "type": ["null", "array", "object"], - "items": { - "type": ["null", "string"] - }, - "properties": {} + "type": ["null", "object"] }, "forgiven": { "type": ["null", "boolean"] diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 3cf76f358220..a5617add9fab 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -32,18 +32,19 @@ For more information on Stripe API Keys, see the [Stripe documentation](https:// 7. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. 8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments. This applies only to streams that do not support event-based incremental syncs, please see the list below. - - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. - - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours. - - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. + - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. + - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours. + - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. 9. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process. - For example, if you are replicating three years worth of data: + For example, if you are replicating three years worth of data: - - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. - - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. + - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. + - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. + + If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. - If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. 10. Click **Set up source** and wait for the tests to complete. ## Supported sync modes @@ -54,16 +55,17 @@ The Stripe source connector supports the following [sync modes](https://docs.air - Incremental :::note -Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the Events API under the hood to implement incremental syncs and export data based on its update date. +Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the Events API under the hood to implement incremental syncs and export data based on its update date. However, not all the entities are supported by the Events API, so the Stripe connector uses the `created` field to query for new data in your Stripe account. These are the entities synced based on the date of creation: - - `CheckoutSessionLineItems` - - `Events` - - `SetupAttempts` - - `ShippingRates` - - `BalanceTransactions` - - `Files` - - `FileLinks` -::: + +- `CheckoutSessionLineItems` +- `Events` +- `SetupAttempts` +- `ShippingRates` +- `BalanceTransactions` +- `Files` +- `FileLinks` + ::: ## Supported streams @@ -138,7 +140,8 @@ The Stripe connector should not run into Stripe API limitations under normal usa ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | +| 4.1.1 | 2023-09-15 | [30494](https://github.com/airbytehq/airbyte/pull/30494) | Fix datatype of invoices.lines property | | 4.1.0 | 2023-08-29 | [29950](https://github.com/airbytehq/airbyte/pull/29950) | Implement incremental deletes, add suggested streams | | 4.0.1 | 2023-09-07 | [30254](https://github.com/airbytehq/airbyte/pull/30254) | Fix cursorless incremental streams | | 4.0.0 | 2023-08-15 | [29330](https://github.com/airbytehq/airbyte/pull/29330) | Implement incremental syncs based on date of update |