diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index 74cb5b27390a..a2bb6f0da2d5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -107,11 +107,11 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_incremental.json" - timeout_seconds: 3600 + timeout_seconds: 7200 future_state: future_state_path: "integration_tests/future_state.json" full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 7200 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 02034bf962cc..9e9094d8e5df 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -15,7 +15,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 4.4.6 + dockerImageTag: 4.4.7 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner erdUrl: https://dbdocs.io/airbyteio/source-amazon-seller-partner?view=relationships diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml b/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml index a22084d1d039..542236d1dfa5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml @@ -3,7 +3,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.4.6" +version = "4.4.7" name = "source-amazon-seller-partner" description = "Source implementation for Amazon Seller Partner." authors = ["Airbyte "] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index f24974868a78..d2e1fda1a0c0 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -107,7 +107,7 @@ def _get_stream_kwargs(config: Mapping[str, Any]) -> Mapping[str, Any]: "authenticator": auth, "replication_start_date": start_date, "marketplace_id": marketplace_id, - "period_in_days": config.get("period_in_days", 30), + "period_in_days": config.get("period_in_days", 365), "replication_end_date": end_date, } return stream_kwargs diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 1715c5b2ccba..b65b7eb9a845 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -21,7 +21,6 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin, package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer @@ -232,7 +231,7 @@ def __init__( self._replication_start_date = replication_start_date self._replication_end_date = replication_end_date self.marketplace_id = marketplace_id - self.period_in_days = max(period_in_days, self.replication_start_date_limit_in_days) # ensure old configs work + self.period_in_days = min(period_in_days, self.replication_start_date_limit_in_days) # ensure old configs work self._report_options = report_options self._http_method = "GET" self._stream_name = stream_name @@ -575,6 +574,7 @@ class FlatFileOrdersReports(IncrementalReportsAmazonSPStream): report_name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" primary_key = "amazon-order-id" cursor_field = "last-updated-date" + replication_start_date_limit_in_days = 30 class FbaStorageFeesReports(IncrementalReportsAmazonSPStream): diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py index 1006cbc9080d..d57b88a9f907 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py @@ -141,6 +141,7 @@ def test_get_updated_state(self, report_init_kwargs, current_stream_state, lates def test_stream_slices(self, report_init_kwargs, start_date, end_date, stream_state, fixed_period_in_days, expected_slices): report_init_kwargs["replication_start_date"] = start_date report_init_kwargs["replication_end_date"] = end_date + report_init_kwargs["period_in_days"] = 365 stream = SomeIncrementalAnalyticsStream(**report_init_kwargs) stream.fixed_period_in_days = fixed_period_in_days with patch("pendulum.now", return_value=pendulum.parse("2023-09-09T00:00:00Z")): diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index 54c8bc037277..cae628254c61 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -85,6 +85,7 @@ def test_report_data(self, report_init_kwargs): def test_stream_slices(self, report_init_kwargs, start_date, end_date, expected_slices): report_init_kwargs["replication_start_date"] = start_date report_init_kwargs["replication_end_date"] = end_date + report_init_kwargs["period_in_days"] = 365 stream = SomeReportStream(**report_init_kwargs) with patch("pendulum.now", return_value=pendulum.parse("2023-01-01T00:00:00Z")): @@ -301,6 +302,7 @@ class TestVendorFulfillment: def test_stream_slices(self, init_kwargs, start_date, end_date, stream_state, expected_slices): init_kwargs["replication_start_date"] = start_date init_kwargs["replication_end_date"] = end_date + init_kwargs["period_in_days"] = 365 stream = VendorDirectFulfillmentShipping(**init_kwargs) with patch("pendulum.now", return_value=pendulum.parse("2022-09-05T00:00:00Z")): diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 4453d6a1639d..fbc098f77cc3 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -162,8 +162,8 @@ Report options can be assigned on a per-stream basis that alter the behavior whe For the full list, refer to Amazon’s report type values [documentation](https://developer-docs.amazon.com/sp-api/docs/report-type-values). Certain report types have required parameters that must be defined. -For `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` and `GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE` streams maximum value for `period_in_days` 30 days and 60 days. -So, for any value that exceeds the limit, the `period_in_days` will be automatically reduced to the limit for the stream. +For the `GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL`, `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL`, and `GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE` streams, the maximum allowable value for `period_in_days` is 30 days, 30 days, and 60 days, respectively. +If the specified `period_in_days` exceeds these limits, it will be automatically adjusted to the maximum value for the respective stream, or set to 365 days if not provided. For the Vendor Forecasting Report, we have two streams - `GET_VENDOR_FORECASTING_FRESH_REPORT` and `GET_VENDOR_FORECASTING_RETAIL_REPORT` which use the same `GET_VENDOR_FORECASTING_REPORT` Amazon's report, but with different options for the `sellingProgram` parameter - `FRESH` and `RETAIL` respectively. @@ -175,7 +175,7 @@ Information about rate limits you may find [here](https://developer-docs.amazon. ## Data type map | Integration Type | Airbyte Type | -| :----------------------- | :----------- | +|:-------------------------|:-------------| | `string` | `string` | | `int`, `float`, `number` | `number` | | `date` | `date` | @@ -228,24 +228,25 @@ Create a separate connection for streams which usually fail with error above "Fa | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 4.4.6 | 2024-11-25 | [48644](https://github.com/airbytehq/airbyte/pull/48644) | Update dependencies | -| 4.4.5 | 2024-11-04 | [47049](https://github.com/airbytehq/airbyte/pull/47049) | Update dependencies | -| 4.4.4 | 2024-10-12 | [46817](https://github.com/airbytehq/airbyte/pull/46817) | Update dependencies | -| 4.4.3 | 2024-10-05 | [46473](https://github.com/airbytehq/airbyte/pull/46473) | Update dependencies | -| 4.4.2 | 2024-09-28 | [44748](https://github.com/airbytehq/airbyte/pull/44748) | Update dependencies | -| 4.4.1 | 2024-08-17 | [43739](https://github.com/airbytehq/airbyte/pull/43739) | Update dependencies | -| 4.4.0 | 2024-07-17 | [42052](https://github.com/airbytehq/airbyte/pull/42052) | Add waiting between requests logic to avoid failed report requests | -| 4.3.11 | 2024-07-13 | [41873](https://github.com/airbytehq/airbyte/pull/41873) | Update dependencies | -| 4.3.10 | 2024-07-10 | [41345](https://github.com/airbytehq/airbyte/pull/41345) | Update dependencies | -| 4.3.9 | 2024-07-09 | [41158](https://github.com/airbytehq/airbyte/pull/41158) | Update dependencies | -| 4.3.8 | 2024-07-08 | [40751](https://github.com/airbytehq/airbyte/pull/40751) | Improve error messaging and turning on alerting | -| 4.3.7 | 2024-07-06 | [40990](https://github.com/airbytehq/airbyte/pull/40990) | Update dependencies | -| 4.3.6 | 2024-07-01 | [40590](https://github.com/airbytehq/airbyte/pull/40590) | Add log message when data only accessible to seller accounts, add report id in log message for fatal report status, add check for start date. | -| 4.3.5 | 2024-06-27 | [40215](https://github.com/airbytehq/airbyte/pull/40215) | Replaced deprecated AirbyteLogger with logging.Logger | -| 4.3.4 | 2024-06-25 | [40384](https://github.com/airbytehq/airbyte/pull/40384) | Update dependencies | -| 4.3.3 | 2024-06-22 | [40008](https://github.com/airbytehq/airbyte/pull/40008) | Update dependencies | -| 4.3.2 | 2024-06-13 | [39441](https://github.com/airbytehq/airbyte/pull/39441) | Update state handling for incremental streams | -| 4.3.1 | 2024-06-04 | [38969](https://github.com/airbytehq/airbyte/pull/38969) | [autopull] Upgrade base image to v1.2.1 | +| 4.4.7 | 2024-11-14 | [47691](https://github.com/airbytehq/airbyte/pull/47691) | Fix `period_in_days` definition | +| 4.4.6 | 2024-11-25 | [48644](https://github.com/airbytehq/airbyte/pull/48644) | Update dependencies | +| 4.4.5 | 2024-11-04 | [47049](https://github.com/airbytehq/airbyte/pull/47049) | Update dependencies | +| 4.4.4 | 2024-10-12 | [46817](https://github.com/airbytehq/airbyte/pull/46817) | Update dependencies | +| 4.4.3 | 2024-10-05 | [46473](https://github.com/airbytehq/airbyte/pull/46473) | Update dependencies | +| 4.4.2 | 2024-09-28 | [44748](https://github.com/airbytehq/airbyte/pull/44748) | Update dependencies | +| 4.4.1 | 2024-08-17 | [43739](https://github.com/airbytehq/airbyte/pull/43739) | Update dependencies | +| 4.4.0 | 2024-07-17 | [42052](https://github.com/airbytehq/airbyte/pull/42052) | Add waiting between requests logic to avoid failed report requests | +| 4.3.11 | 2024-07-13 | [41873](https://github.com/airbytehq/airbyte/pull/41873) | Update dependencies | +| 4.3.10 | 2024-07-10 | [41345](https://github.com/airbytehq/airbyte/pull/41345) | Update dependencies | +| 4.3.9 | 2024-07-09 | [41158](https://github.com/airbytehq/airbyte/pull/41158) | Update dependencies | +| 4.3.8 | 2024-07-08 | [40751](https://github.com/airbytehq/airbyte/pull/40751) | Improve error messaging and turning on alerting | +| 4.3.7 | 2024-07-06 | [40990](https://github.com/airbytehq/airbyte/pull/40990) | Update dependencies | +| 4.3.6 | 2024-07-01 | [40590](https://github.com/airbytehq/airbyte/pull/40590) | Add log message when data only accessible to seller accounts, add report id in log message for fatal report status, add check for start date. | +| 4.3.5 | 2024-06-27 | [40215](https://github.com/airbytehq/airbyte/pull/40215) | Replaced deprecated AirbyteLogger with logging.Logger | +| 4.3.4 | 2024-06-25 | [40384](https://github.com/airbytehq/airbyte/pull/40384) | Update dependencies | +| 4.3.3 | 2024-06-22 | [40008](https://github.com/airbytehq/airbyte/pull/40008) | Update dependencies | +| 4.3.2 | 2024-06-13 | [39441](https://github.com/airbytehq/airbyte/pull/39441) | Update state handling for incremental streams | +| 4.3.1 | 2024-06-04 | [38969](https://github.com/airbytehq/airbyte/pull/38969) | [autopull] Upgrade base image to v1.2.1 | | 4.3.0 | 2024-05-24 | [#38657](https://github.com/airbytehq/airbyte/pull/38657) | Extend the report_options spec config with a `stream_name` attribute | | 4.2.4 | 2024-05-15 | [#38210](https://github.com/airbytehq/airbyte/pull/38210) | Fix `GET_VENDOR_TRAFFIC_REPORT` stream with report option `reportPeriod=DAY` | | 4.2.3 | 2024-05-09 | [#38078](https://github.com/airbytehq/airbyte/pull/38078) | Hide OSS-only streams in report options config for cloud users |