From 8496a7ddc3165b5e5e6883a42abe8311fcc53dfc Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Tue, 3 Oct 2023 10:30:33 -0700 Subject: [PATCH 1/5] try no timeout --- .../connectors/pipelines/pipelines/tests/python_connectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py index fbd89c4ee458..f98be68ab96d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py @@ -59,7 +59,7 @@ class ConnectorPackageInstall(Step): """A step to install the Python connector package in a container.""" title = "Connector package install" - max_duration = timedelta(minutes=20) + # max_duration = timedelta(minutes=20) max_retries = 3 async def _run(self) -> StepResult: From 4b679800eae70f31567bf484d5fdbe1c6b9f67f4 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Tue, 3 Oct 2023 10:44:21 -0700 Subject: [PATCH 2/5] rm docker test reqs --- airbyte-integrations/connectors/source-file/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index d3ae04ed60c6..4177cb3edae7 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -24,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1", "docker-compose"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_file", From 108a0945d66375334e28686b9c1d3b41bfd8cb32 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 6 Oct 2023 15:26:39 -0700 Subject: [PATCH 3/5] bump dep, add docker to test container --- .../connectors/pipelines/pipelines/actions/environments.py | 3 ++- airbyte-integrations/connectors/source-file/setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py index 16cdd59098cb..ae05550b078e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py @@ -57,7 +57,7 @@ def with_python_base(context: PipelineContext, python_version: str = "3.10") -> sh_dash_c( [ "apt-get update", - "apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 git", + "apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 git curl", "pip install pip==23.1.2", ] ) @@ -82,6 +82,7 @@ def with_testing_dependencies(context: PipelineContext) -> Container: return ( python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) .with_file(f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file) .with_file(f"/{LICENSE_SHORT_FILE_PATH}", license_short_file) ) diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 4177cb3edae7..23fe07d1084f 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -24,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=2.0", "pytest-mock~=3.6.1"] setup( name="source_file", From ea8ea319636ff92d0ed80516807ba9d1033c19b2 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 6 Oct 2023 15:30:17 -0700 Subject: [PATCH 4/5] re-add timeout --- .../connectors/pipelines/pipelines/tests/python_connectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py index f98be68ab96d..fbd89c4ee458 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py @@ -59,7 +59,7 @@ class ConnectorPackageInstall(Step): """A step to install the Python connector package in a container.""" title = "Connector package install" - # max_duration = timedelta(minutes=20) + max_duration = timedelta(minutes=20) max_retries = 3 async def _run(self) -> StepResult: From 63b8aaf3e5a8be96bb7a99152739dd011816a5ac Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 6 Oct 2023 15:33:17 -0700 Subject: [PATCH 5/5] bump airbyte-ci version --- airbyte-ci/connectors/pipelines/README.md | 11 ++++++----- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 796e16c12c61..e15fd7d29364 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -396,8 +396,9 @@ This command runs the Python tests for a airbyte-ci poetry package. `airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests` ## Changelog -| Version | PR | Description | -|---------| --------------------------------------------------------- |-----------------------------------------------------------------------------------------------------------| +| Version | PR | Description | +|---------| -------------------------------------------------------- |-----------------------------------------------------------------------------------------------------------| +| 1.5.0 |[#30984](https://github.com/airbytehq/airbyte/pull/30984) | Adds docker to python test containers | | 1.4.6 |[ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date | | 1.4.5 | [#31133](https://github.com/airbytehq/airbyte/pull/31133) | Fix bug when building containers using `with_integration_base_java_and_normalization`. | | 1.4.4 | [#30743](https://github.com/airbytehq/airbyte/pull/30743) | Add `--disable-report-auto-open` and `--use-host-gradle-dist-tar` to allow gradle integration. | @@ -405,8 +406,8 @@ This command runs the Python tests for a airbyte-ci poetry package. | 1.4.2 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Remove directory name requirement | | 1.4.1 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Load base migration guide into QA Test container for strict encrypt variants | | 1.4.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add support for pyproject.toml as the prefered entry point for a connector package | -| 1.3.0 | [#30461](https://github.com/airbytehq/airbyte/pull/30461) | Add `--use-local-cdk` flag to all connectors commands | -| 1.2.3 | [#30477](https://github.com/airbytehq/airbyte/pull/30477) | Fix a test regression introduced the previous version. | +| 1.3.0 | [#30461](https://github.com/airbytehq/airbyte/pull/30461) | Add `--use-local-cdk` flag to all connectors commands | +| 1.2.3 | [#30477](https://github.com/airbytehq/airbyte/pull/30477) | Fix a test regression introduced the previous version. | | 1.2.2 | [#30438](https://github.com/airbytehq/airbyte/pull/30438) | Add workaround to always stream logs properly with --is-local. | | 1.2.1 | [#30384](https://github.com/airbytehq/airbyte/pull/30384) | Java connector test performance fixes. | | 1.2.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add `--metadata-query` option to connectors command | @@ -432,7 +433,7 @@ This command runs the Python tests for a airbyte-ci poetry package. | 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | | 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | | 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | -| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | +| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | ## More info This project is owned by the Connectors Operations team. diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index d2a286ac8f30..52fe64041d9c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.4.6" +version = "1.5.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]