From 8892a19cd94a4c29617f56eb82e4ab3defae779c Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Tue, 28 May 2024 13:31:19 -0600 Subject: [PATCH 1/8] Source instagram: Test PR for Live Tests, checking control and target urls --- .../connectors/source-instagram/metadata.yaml | 2 +- .../connectors/source-instagram/poetry.lock | 31 ++++---- .../source-instagram/pyproject.toml | 4 +- docs/integrations/sources/instagram.md | 73 ++++++++++--------- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 0c67e20e04dd..ce0f3bc5e010 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 3.0.9 + dockerImageTag: 3.0.10 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/poetry.lock b/airbyte-integrations/connectors/source-instagram/poetry.lock index 8f89472680bb..3960161d8abc 100644 --- a/airbyte-integrations/connectors/source-instagram/poetry.lock +++ b/airbyte-integrations/connectors/source-instagram/poetry.lock @@ -447,13 +447,13 @@ test = ["pytest (>=6)"] [[package]] name = "facebook-business" -version = "19.0.3" +version = "18.0.5" description = "Facebook Business SDK" optional = false python-versions = "*" files = [ - {file = "facebook_business-19.0.3-py3-none-any.whl", hash = "sha256:43613bdee5da5a86418f2858cd45cdaf1aa2c45597fc1d8a6da001c7702eec08"}, - {file = "facebook_business-19.0.3.tar.gz", hash = "sha256:a2bfb4a22237a53abd923505edb1d96a5f81d565e8e27d6b3bb66a158d4b4d13"}, + {file = "facebook_business-18.0.5-py3-none-any.whl", hash = "sha256:5b84043a9b67ea31ef2ccb95254c883d65f6f4a585bf077bfc2decb4896ed615"}, + {file = "facebook_business-18.0.5.tar.gz", hash = "sha256:7435860e1787b6d0c6fb29b43d8c2d07f503a53c8ef77202232bea4b68a33992"}, ] [package.dependencies] @@ -1146,13 +1146,13 @@ files = [ [[package]] name = "requests" -version = "2.32.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.0-py3-none-any.whl", hash = "sha256:f2c3881dddb70d056c5bd7600a4fae312b2a300e39be6a118d30b90bd27262b5"}, - {file = "requests-2.32.0.tar.gz", hash = "sha256:fa5490319474c82ef1d2c9bc459d3652e3ae4ef4c4ebdd18a21145a47ca4b6b8"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -1214,19 +1214,18 @@ fixture = ["fixtures"] [[package]] name = "setuptools" -version = "69.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, - {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -1252,13 +1251,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, ] [[package]] @@ -1491,4 +1490,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "a6f8a7375136624bdd9521de303d6e2d152ea5adf588c101e2d91be78bda7e25" +content-hash = "c0fcc4fb5b27e9690a32a8354eea01bbd5663cb3351a8157cde6510b73ccc207" diff --git a/airbyte-integrations/connectors/source-instagram/pyproject.toml b/airbyte-integrations/connectors/source-instagram/pyproject.toml index eb675428c2e2..fe88a1b2f6da 100644 --- a/airbyte-integrations/connectors/source-instagram/pyproject.toml +++ b/airbyte-integrations/connectors/source-instagram/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.0.9" +version = "3.0.10" name = "source-instagram" description = "Source implementation for Instagram." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_instagram" [tool.poetry.dependencies] python = "^3.9,<3.12" -facebook-business = "==19.0.3" +facebook-business = "==18.0.5" airbyte-cdk = "0.80.0" cached-property = "==1.5.2" diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index d90e9debd173..c58ebcf562ed 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -112,41 +112,42 @@ Instagram limits the number of requests that can be made at a time. See Facebook ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | -| 3.0.9 | 2024-05-21 | [38554](https://github.com/airbytehq/airbyte/pull/38554) | Upgrade to API v19.0 | -| 3.0.8 | 2024-05-20 | [38268](https://github.com/airbytehq/airbyte/pull/38268) | Replace AirbyteLogger with logging.Logger | -| 3.0.7 | 2024-04-19 | [36643](https://github.com/airbytehq/airbyte/pull/36643) | Updating to 0.80.0 CDK | -| 3.0.6 | 2024-04-12 | [36643](https://github.com/airbytehq/airbyte/pull/36643) | Schema descriptions | -| 3.0.5 | 2024-03-20 | [36314](https://github.com/airbytehq/airbyte/pull/36314) | Unpin CDK version | -| 3.0.4 | 2024-03-07 | [35875](https://github.com/airbytehq/airbyte/pull/35875) | Remove `total_interactions` from the `MediaInsights` queries. | -| 3.0.3 | 2024-02-12 | [35177](https://github.com/airbytehq/airbyte/pull/35177) | Manage dependencies with Poetry | -| 3.0.2 | 2024-01-15 | [34254](https://github.com/airbytehq/airbyte/pull/34254) | Prepare for airbyte-lib | -| 3.0.1 | 2024-01-08 | [33989](https://github.com/airbytehq/airbyte/pull/33989) | Remove metrics from video feed | -| 3.0.0 | 2024-01-05 | [33930](https://github.com/airbytehq/airbyte/pull/33930) | Upgrade to API v18.0 | -| 2.0.1 | 2024-01-03 | [33889](https://github.com/airbytehq/airbyte/pull/33889) | Change requested metrics for stream `media_insights` | -| 2.0.0 | 2023-11-17 | [32500](https://github.com/airbytehq/airbyte/pull/32500) | Add primary keys for UserLifetimeInsights and UserInsights; add airbyte_type to timestamp fields | -| 1.0.16 | 2023-11-17 | [32627](https://github.com/airbytehq/airbyte/pull/32627) | Fix start_date type; fix docs | -| 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | -| 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | -| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | -| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | -| 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | -| 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | -| 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | -| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights` | -| 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth` | -| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | -| 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | -| 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | -| 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream | -| 1.0.2 | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow | -| 1.0.1 | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values | -| 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | -| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | -| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | +|:--------| :--------- | :------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------| +| 3.0.10 | 2024-05-21 | [38554](https://github.com/airbytehq/airbyte/pull/38554) | Fake downgrade to API v18.0 for testing purposes, do not merge! | +| 3.0.9 | 2024-05-21 | [38554](https://github.com/airbytehq/airbyte/pull/38554) | Upgrade to API v19.0 | +| 3.0.8 | 2024-05-20 | [38268](https://github.com/airbytehq/airbyte/pull/38268) | Replace AirbyteLogger with logging.Logger | +| 3.0.7 | 2024-04-19 | [36643](https://github.com/airbytehq/airbyte/pull/36643) | Updating to 0.80.0 CDK | +| 3.0.6 | 2024-04-12 | [36643](https://github.com/airbytehq/airbyte/pull/36643) | Schema descriptions | +| 3.0.5 | 2024-03-20 | [36314](https://github.com/airbytehq/airbyte/pull/36314) | Unpin CDK version | +| 3.0.4 | 2024-03-07 | [35875](https://github.com/airbytehq/airbyte/pull/35875) | Remove `total_interactions` from the `MediaInsights` queries. | +| 3.0.3 | 2024-02-12 | [35177](https://github.com/airbytehq/airbyte/pull/35177) | Manage dependencies with Poetry | +| 3.0.2 | 2024-01-15 | [34254](https://github.com/airbytehq/airbyte/pull/34254) | Prepare for airbyte-lib | +| 3.0.1 | 2024-01-08 | [33989](https://github.com/airbytehq/airbyte/pull/33989) | Remove metrics from video feed | +| 3.0.0 | 2024-01-05 | [33930](https://github.com/airbytehq/airbyte/pull/33930) | Upgrade to API v18.0 | +| 2.0.1 | 2024-01-03 | [33889](https://github.com/airbytehq/airbyte/pull/33889) | Change requested metrics for stream `media_insights` | +| 2.0.0 | 2023-11-17 | [32500](https://github.com/airbytehq/airbyte/pull/32500) | Add primary keys for UserLifetimeInsights and UserInsights; add airbyte_type to timestamp fields | +| 1.0.16 | 2023-11-17 | [32627](https://github.com/airbytehq/airbyte/pull/32627) | Fix start_date type; fix docs | +| 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | +| 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | +| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | +| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | +| 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | +| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights` | +| 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth` | +| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | +| 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | +| 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | +| 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream | +| 1.0.2 | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow | +| 1.0.1 | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values | +| 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | +| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | +| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | From 10bae378710fae3a05663a4f374d6f62ad5832f5 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Wed, 29 May 2024 21:52:39 +0100 Subject: [PATCH 2/8] Live tests: don't look for connection_id.txt for the control image --- .../live-tests/src/live_tests/commons/models.py | 3 ++- .../live-tests/src/live_tests/commons/utils.py | 11 +++++++---- .../src/live_tests/regression_tests/conftest.py | 13 +++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py index 6b0a6b406a28..c8375657683f 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py @@ -178,9 +178,10 @@ async def from_image_name( cls: type[ConnectorUnderTest], dagger_client: dagger.Client, image_name: str, + is_ci: bool, target_or_control: TargetOrControl, ) -> ConnectorUnderTest: - container = await get_connector_container(dagger_client, image_name) + container = await get_connector_container(dagger_client, image_name, is_ci, target_or_control == target_or_control.TARGET) return cls(image_name, container, target_or_control) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py index a30342f926cf..867f3eaa54ae 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py @@ -84,12 +84,14 @@ async def get_container_from_dockerhub_image(dagger_client: dagger.Client, docke pytest.exit(f"Failed to import connector image from DockerHub: {e}") -async def get_connector_container(dagger_client: dagger.Client, image_name_with_tag: str) -> dagger.Container: +async def get_connector_container(dagger_client: dagger.Client, image_name_with_tag: str, is_ci: bool, is_target: bool) -> dagger.Container: """Get a dagger container for the connector image to test. Args: dagger_client (dagger.Client): The dagger client to use to import the connector image image_name_with_tag (str): The docker image name and tag of the connector image to test + is_ci (bool): Whether the tests are being run with airbyte-ci (locally or in GHA) + is_target (bool): Whether the container to get is the target container Returns: dagger.Container: The dagger container for the connector image to test @@ -97,9 +99,10 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ # If a container_id.txt file is available, we'll use it to load the connector container # We use a txt file as container ids can be too long to be passed as env vars # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation - container_id_path = Path("/tmp/container_id.txt") - if container_id_path.exists(): - return await get_container_from_id(dagger_client, container_id_path.read_text()) + if is_ci and is_target: + container_id_path = Path("/tmp/container_id.txt") + if container_id_path.exists(): + return await get_container_from_id(dagger_client, container_id_path.read_text()) # If the CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH env var is set, we'll use it to import the connector image from the tarball if connector_image_tarball_path := os.environ.get("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"): diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py index d375d49ffe94..e4b841f44fa9 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py @@ -387,13 +387,18 @@ async def dagger_client( @pytest.fixture(scope="session") -async def control_connector(dagger_client: dagger.Client, connector_image: str, control_version: str) -> ConnectorUnderTest: - return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{control_version}", TargetOrControl.CONTROL) +async def is_ci(pytestconfig: Config) -> bool: + return pytestconfig.stash.get(stash_keys.RUN_IN_AIRBYTE_CI, False) or pytestconfig.stash.get(stash_keys.IS_PRODUCTION_CI, False) @pytest.fixture(scope="session") -async def target_connector(dagger_client: dagger.Client, connector_image: str, target_version: str) -> ConnectorUnderTest: - return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{target_version}", TargetOrControl.TARGET) +async def control_connector(dagger_client: dagger.Client, is_ci: bool, connector_image: str, control_version: str) -> ConnectorUnderTest: + return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{control_version}", is_ci, TargetOrControl.CONTROL) + + +@pytest.fixture(scope="session") +async def target_connector(dagger_client: dagger.Client, is_ci: bool, connector_image: str, target_version: str) -> ConnectorUnderTest: + return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{target_version}", is_ci, TargetOrControl.TARGET) @pytest.fixture(scope="session") From 5001e75a15c996e25ad8e05c32ec6ad9f358d1ee Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Wed, 29 May 2024 22:44:51 +0100 Subject: [PATCH 3/8] bump package --- airbyte-ci/connectors/live-tests/README.md | 4 ++++ airbyte-ci/connectors/live-tests/pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/live-tests/README.md b/airbyte-ci/connectors/live-tests/README.md index b3b55f3ad890..4661baa50d11 100644 --- a/airbyte-ci/connectors/live-tests/README.md +++ b/airbyte-ci/connectors/live-tests/README.md @@ -275,6 +275,10 @@ The traffic recorded on the control connector is passed to the target connector ## Changelog +### 0.17.4 + +Fix control image when running tests in CI. + ### 0.17.3 Pin requests dependency. diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml index 3227735d4fff..99dc280f24e1 100644 --- a/airbyte-ci/connectors/live-tests/pyproject.toml +++ b/airbyte-ci/connectors/live-tests/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "live-tests" -version = "0.17.2" +version = "0.17.4" description = "Contains utilities for testing connectors against live data." authors = ["Airbyte "] license = "MIT" From 8953ea0ee0fb5b7f3a9c90d596b67086e1988dbf Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 30 May 2024 13:49:12 +0100 Subject: [PATCH 4/8] CR comments --- .../live-tests/src/live_tests/commons/models.py | 3 +-- .../live-tests/src/live_tests/commons/utils.py | 9 ++++----- .../src/live_tests/regression_tests/conftest.py | 9 ++------- .../pipelines/airbyte_ci/connectors/test/steps/common.py | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py index c8375657683f..6b0a6b406a28 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py @@ -178,10 +178,9 @@ async def from_image_name( cls: type[ConnectorUnderTest], dagger_client: dagger.Client, image_name: str, - is_ci: bool, target_or_control: TargetOrControl, ) -> ConnectorUnderTest: - container = await get_connector_container(dagger_client, image_name, is_ci, target_or_control == target_or_control.TARGET) + container = await get_connector_container(dagger_client, image_name) return cls(image_name, container, target_or_control) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py index 867f3eaa54ae..9eb039c904bd 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py @@ -84,7 +84,7 @@ async def get_container_from_dockerhub_image(dagger_client: dagger.Client, docke pytest.exit(f"Failed to import connector image from DockerHub: {e}") -async def get_connector_container(dagger_client: dagger.Client, image_name_with_tag: str, is_ci: bool, is_target: bool) -> dagger.Container: +async def get_connector_container(dagger_client: dagger.Client, image_name_with_tag: str) -> dagger.Container: """Get a dagger container for the connector image to test. Args: @@ -99,10 +99,9 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ # If a container_id.txt file is available, we'll use it to load the connector container # We use a txt file as container ids can be too long to be passed as env vars # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation - if is_ci and is_target: - container_id_path = Path("/tmp/container_id.txt") - if container_id_path.exists(): - return await get_container_from_id(dagger_client, container_id_path.read_text()) + container_id_path = Path(f"/tmp/{image_name_with_tag}_container_id.txt") + if container_id_path.exists(): + return await get_container_from_id(dagger_client, container_id_path.read_text()) # If the CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH env var is set, we'll use it to import the connector image from the tarball if connector_image_tarball_path := os.environ.get("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"): diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py index e4b841f44fa9..c35121cb7768 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py @@ -386,19 +386,14 @@ async def dagger_client( yield client -@pytest.fixture(scope="session") -async def is_ci(pytestconfig: Config) -> bool: - return pytestconfig.stash.get(stash_keys.RUN_IN_AIRBYTE_CI, False) or pytestconfig.stash.get(stash_keys.IS_PRODUCTION_CI, False) - - @pytest.fixture(scope="session") async def control_connector(dagger_client: dagger.Client, is_ci: bool, connector_image: str, control_version: str) -> ConnectorUnderTest: - return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{control_version}", is_ci, TargetOrControl.CONTROL) + return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{control_version}", TargetOrControl.CONTROL) @pytest.fixture(scope="session") async def target_connector(dagger_client: dagger.Client, is_ci: bool, connector_image: str, target_version: str) -> ConnectorUnderTest: - return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{target_version}", is_ci, TargetOrControl.TARGET) + return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{target_version}", TargetOrControl.TARGET) @pytest.fixture(scope="session") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index f54aaf2abf4c..5bcb77b5de45 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -456,7 +456,7 @@ async def _build_regression_test_container(self, target_container_id: str) -> Co # regression tests. The connector can be found if you know the container ID, so we write the container ID to a file and put # it in the regression test container. This way regression tests will use the already-built connector instead of trying to # build their own. - .with_new_file("/tmp/container_id.txt", contents=str(target_container_id)) + .with_new_file(f"/tmp/{self.context.docker_image}_container_id.txt", contents=str(target_container_id)) ) if self.context.is_ci: From b798a15d8ab456d250ac1e5a23d66924f4248e1a Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 30 May 2024 16:51:48 +0100 Subject: [PATCH 5/8] Live tests: fix control image when running tests in CI Also fixes --selected-streams --- .../connectors/live-tests/src/live_tests/commons/utils.py | 4 +--- .../live-tests/src/live_tests/regression_tests/conftest.py | 4 ++-- .../pipelines/airbyte_ci/connectors/test/steps/common.py | 6 ++++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py index 9eb039c904bd..9e1d016f0e4d 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py @@ -90,8 +90,6 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ Args: dagger_client (dagger.Client): The dagger client to use to import the connector image image_name_with_tag (str): The docker image name and tag of the connector image to test - is_ci (bool): Whether the tests are being run with airbyte-ci (locally or in GHA) - is_target (bool): Whether the container to get is the target container Returns: dagger.Container: The dagger container for the connector image to test @@ -99,7 +97,7 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ # If a container_id.txt file is available, we'll use it to load the connector container # We use a txt file as container ids can be too long to be passed as env vars # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation - container_id_path = Path(f"/tmp/{image_name_with_tag}_container_id.txt") + container_id_path = Path(f"/tmp/{image_name_with_tag.split(':')[1]}_container_id.txt") if container_id_path.exists(): return await get_container_from_id(dagger_client, container_id_path.read_text()) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py index c35121cb7768..d375d49ffe94 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py @@ -387,12 +387,12 @@ async def dagger_client( @pytest.fixture(scope="session") -async def control_connector(dagger_client: dagger.Client, is_ci: bool, connector_image: str, control_version: str) -> ConnectorUnderTest: +async def control_connector(dagger_client: dagger.Client, connector_image: str, control_version: str) -> ConnectorUnderTest: return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{control_version}", TargetOrControl.CONTROL) @pytest.fixture(scope="session") -async def target_connector(dagger_client: dagger.Client, is_ci: bool, connector_image: str, target_version: str) -> ConnectorUnderTest: +async def target_connector(dagger_client: dagger.Client, connector_image: str, target_version: str) -> ConnectorUnderTest: return await ConnectorUnderTest.from_image_name(dagger_client, f"{connector_image}:{target_version}", TargetOrControl.TARGET) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 5bcb77b5de45..50d9a0063788 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -344,6 +344,7 @@ def regression_tests_command(self) -> List[str]: (See https://docs.dagger.io/manuals/developer/python/328492/services/ and https://cloud.google.com/sql/docs/postgres/sql-proxy#cloud-sql-auth-proxy-docker-image) """ run_proxy = "./cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --credentials-file /tmp/credentials.json" + selected_streams = ["--stream", self.selected_streams] if self.selected_streams else [] run_pytest = " ".join( [ "poetry", @@ -364,7 +365,7 @@ def regression_tests_command(self) -> List[str]: self.run_id or "", "--should-read-with-state", str(self.should_read_with_state), - ] + ] + selected_streams ) run_pytest_with_proxy = dedent( f""" @@ -398,6 +399,7 @@ def __init__(self, context: ConnectorContext) -> None: self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest") self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev") self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", True) + self.selected_streams = self.context.run_step_options.get_item_or_default(options, "selected-streams", None) self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time())) async def _run(self, connector_under_test_container: Container) -> StepResult: @@ -456,7 +458,7 @@ async def _build_regression_test_container(self, target_container_id: str) -> Co # regression tests. The connector can be found if you know the container ID, so we write the container ID to a file and put # it in the regression test container. This way regression tests will use the already-built connector instead of trying to # build their own. - .with_new_file(f"/tmp/{self.context.docker_image}_container_id.txt", contents=str(target_container_id)) + .with_new_file(f"/tmp/{self.target_version}_container_id.txt", contents=str(target_container_id)) ) if self.context.is_ci: From fd204ce0181578de391c5a86c7a243326b053102 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 30 May 2024 17:04:08 +0100 Subject: [PATCH 6/8] formatting --- .../pipelines/airbyte_ci/connectors/test/steps/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 50d9a0063788..8431a7c7c3f8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -365,7 +365,8 @@ def regression_tests_command(self) -> List[str]: self.run_id or "", "--should-read-with-state", str(self.should_read_with_state), - ] + selected_streams + ] + + selected_streams ) run_pytest_with_proxy = dedent( f""" From 21110c1f3be52827159565a2e661cae776238fc4 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 30 May 2024 17:20:08 +0100 Subject: [PATCH 7/8] use slugify --- airbyte-ci/connectors/live-tests/poetry.lock | 30 +++++++++++++++- .../connectors/live-tests/pyproject.toml | 1 + .../src/live_tests/commons/utils.py | 3 +- .../connectors/test/steps/common.py | 5 ++- airbyte-ci/connectors/pipelines/poetry.lock | 34 +++++++++++++++++-- .../connectors/pipelines/pyproject.toml | 1 + 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/live-tests/poetry.lock b/airbyte-ci/connectors/live-tests/poetry.lock index 7d1349a9671c..8be39be3eb8b 100644 --- a/airbyte-ci/connectors/live-tests/poetry.lock +++ b/airbyte-ci/connectors/live-tests/poetry.lock @@ -2884,6 +2884,23 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-slugify" +version = "8.0.4" +description = "A Python slugify application that also handles Unicode" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856"}, + {file = "python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8"}, +] + +[package.dependencies] +text-unidecode = ">=1.3" + +[package.extras] +unidecode = ["Unidecode (>=1.1.1)"] + [[package]] name = "pytz" version = "2024.1" @@ -3384,6 +3401,17 @@ files = [ [package.extras] tests = ["pytest", "pytest-cov"] +[[package]] +name = "text-unidecode" +version = "1.3" +description = "The most basic Text::Unidecode port" +optional = false +python-versions = "*" +files = [ + {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, + {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -3773,4 +3801,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10,<3.12" -content-hash = "409a21ec0ced14b78a136ff08a07efc9bac2eee3b7249f24ceb7cf16475ac624" +content-hash = "02d813aa0511a7a4d2ebcec7ac1f430d15b10cfd5e268b357374b86db8ae7661" diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml index 99dc280f24e1..d0e0ff7166da 100644 --- a/airbyte-ci/connectors/live-tests/pyproject.toml +++ b/airbyte-ci/connectors/live-tests/pyproject.toml @@ -39,6 +39,7 @@ pyyaml = "^6.0.1" dpath = "^2.1.6" genson = "^1.2.2" segment-analytics-python = "^2.3.2" +python-slugify = ">=8.0.4" [tool.poetry.scripts] live-tests = "live_tests.cli:live_tests" diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py index 9e1d016f0e4d..eba71fc30486 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py @@ -13,6 +13,7 @@ import pytest from mitmproxy import http, io # type: ignore from mitmproxy.addons.savehar import SaveHar # type: ignore +from slugify import slugify async def get_container_from_id(dagger_client: dagger.Client, container_id: str) -> dagger.Container: @@ -97,7 +98,7 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ # If a container_id.txt file is available, we'll use it to load the connector container # We use a txt file as container ids can be too long to be passed as env vars # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation - container_id_path = Path(f"/tmp/{image_name_with_tag.split(':')[1]}_container_id.txt") + container_id_path = Path(f"/tmp/{slugify(image_name_with_tag)}_container_id.txt") if container_id_path.exists(): return await get_container_from_id(dagger_client, container_id_path.read_text()) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 8431a7c7c3f8..4115254806fe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -27,6 +27,7 @@ from pipelines.helpers.utils import METADATA_FILE_NAME, get_exec_result from pipelines.models.secrets import Secret from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus +from slugify import slugify class VersionCheck(Step, ABC): @@ -459,7 +460,9 @@ async def _build_regression_test_container(self, target_container_id: str) -> Co # regression tests. The connector can be found if you know the container ID, so we write the container ID to a file and put # it in the regression test container. This way regression tests will use the already-built connector instead of trying to # build their own. - .with_new_file(f"/tmp/{self.target_version}_container_id.txt", contents=str(target_container_id)) + .with_new_file( + f"/tmp/{slugify(self.connector_image + ':' + self.target_version)}_container_id.txt", contents=str(target_container_id) + ) ) if self.context.is_ci: diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index f845a2c2cf58..7d7a6c3fbd6a 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -447,7 +447,7 @@ url = "../common_utils" [[package]] name = "connector-ops" -version = "0.4.0" +version = "0.5.0" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" @@ -1578,6 +1578,7 @@ files = [ {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, @@ -2070,6 +2071,23 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-slugify" +version = "8.0.4" +description = "A Python slugify application that also handles Unicode" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856"}, + {file = "python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8"}, +] + +[package.dependencies] +text-unidecode = ">=1.3" + +[package.extras] +unidecode = ["Unidecode (>=1.1.1)"] + [[package]] name = "pytz" version = "2024.1" @@ -2140,6 +2158,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2496,6 +2515,17 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "text-unidecode" +version = "1.3" +description = "The most basic Text::Unidecode port" +optional = false +python-versions = "*" +files = [ + {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, + {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, +] + [[package]] name = "toml" version = "0.10.2" @@ -2822,4 +2852,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "867cf46af77c6613e30bafb67e728c86d3e4b16edadd3624cc66ae45f3575d57" +content-hash = "5a120ee01ce9fdfd60108ce2214dc5039f79a3bf5440f22f1a2bcfbd77d3e89d" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 572daa680a18..030178401da3 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -38,6 +38,7 @@ google-cloud-secret-manager = "^2.20.0" google-auth = "^2.29.0" pygithub = "^2.3.0" pydash = "6.0.2" +python-slugify = ">=8.0.4" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" From d7b9e21d661f9a4cfe0354f4254515f0e18cfcc3 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 30 May 2024 17:49:47 +0100 Subject: [PATCH 8/8] bump airbyte-ci version --- airbyte-ci/connectors/pipelines/README.md | 6 +++++- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index a660c4581066..ab67144e5823 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -748,7 +748,11 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| 4.15.4 | [#38646](https://github.com/airbytehq/airbyte/pull/38646) | Make airbyte-ci able to test external repos. | +---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| 4.15.7 | [#38772](https://github.com/airbytehq/airbyte/pull/38772) | Fix regression test connector image retrieval. | +| 4.15.6 | [#38783](https://github.com/airbytehq/airbyte/pull/38783) | Fix a variable access error with `repo_dir` in the `bump_version` command. | +| 4.15.5 | [#38732](https://github.com/airbytehq/airbyte/pull/38732) | Update metadata deploy pipeline to 3.10 | +| 4.15.4 | [#38646](https://github.com/airbytehq/airbyte/pull/38646) | Make airbyte-ci able to test external repos. | | 4.15.3 | [#38645](https://github.com/airbytehq/airbyte/pull/38645) | Fix typo preventing correct secret mounting on Python connectors integration tests. | | 4.15.2 | [#38628](https://github.com/airbytehq/airbyte/pull/38628) | Introduce ConnectorTestContext to avoid trying fetching connector secret in the PublishContext. | | 4.15.1 | [#38615](https://github.com/airbytehq/airbyte/pull/38615) | Do not eagerly fetch connector secrets. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 030178401da3..6c103be448e5 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.15.4" +version = "4.15.7" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]