From 1eca3fcf7feed105861b5786382c7f5cbc274581 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 30 May 2024 13:57:42 -0400 Subject: [PATCH] Fix live test image when running in CI (#38772) --- airbyte-ci/connectors/live-tests/README.md | 4 +++ airbyte-ci/connectors/live-tests/poetry.lock | 30 +++++++++++++++- .../connectors/live-tests/pyproject.toml | 3 +- .../src/live_tests/commons/utils.py | 3 +- airbyte-ci/connectors/pipelines/README.md | 1 + .../connectors/test/steps/common.py | 8 ++++- airbyte-ci/connectors/pipelines/poetry.lock | 34 +++++++++++++++++-- .../connectors/pipelines/pyproject.toml | 3 +- 8 files changed, 79 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/live-tests/README.md b/airbyte-ci/connectors/live-tests/README.md index b3b55f3ad890..4661baa50d11 100644 --- a/airbyte-ci/connectors/live-tests/README.md +++ b/airbyte-ci/connectors/live-tests/README.md @@ -275,6 +275,10 @@ The traffic recorded on the control connector is passed to the target connector ## Changelog +### 0.17.4 + +Fix control image when running tests in CI. + ### 0.17.3 Pin requests dependency. diff --git a/airbyte-ci/connectors/live-tests/poetry.lock b/airbyte-ci/connectors/live-tests/poetry.lock index 7d1349a9671c..8be39be3eb8b 100644 --- a/airbyte-ci/connectors/live-tests/poetry.lock +++ b/airbyte-ci/connectors/live-tests/poetry.lock @@ -2884,6 +2884,23 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-slugify" +version = "8.0.4" +description = "A Python slugify application that also handles Unicode" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856"}, + {file = "python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8"}, +] + +[package.dependencies] +text-unidecode = ">=1.3" + +[package.extras] +unidecode = ["Unidecode (>=1.1.1)"] + [[package]] name = "pytz" version = "2024.1" @@ -3384,6 +3401,17 @@ files = [ [package.extras] tests = ["pytest", "pytest-cov"] +[[package]] +name = "text-unidecode" +version = "1.3" +description = "The most basic Text::Unidecode port" +optional = false +python-versions = "*" +files = [ + {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, + {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -3773,4 +3801,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10,<3.12" -content-hash = "409a21ec0ced14b78a136ff08a07efc9bac2eee3b7249f24ceb7cf16475ac624" +content-hash = "02d813aa0511a7a4d2ebcec7ac1f430d15b10cfd5e268b357374b86db8ae7661" diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml index 3227735d4fff..d0e0ff7166da 100644 --- a/airbyte-ci/connectors/live-tests/pyproject.toml +++ b/airbyte-ci/connectors/live-tests/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "live-tests" -version = "0.17.2" +version = "0.17.4" description = "Contains utilities for testing connectors against live data." authors = ["Airbyte "] license = "MIT" @@ -39,6 +39,7 @@ pyyaml = "^6.0.1" dpath = "^2.1.6" genson = "^1.2.2" segment-analytics-python = "^2.3.2" +python-slugify = ">=8.0.4" [tool.poetry.scripts] live-tests = "live_tests.cli:live_tests" diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py index a30342f926cf..eba71fc30486 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py @@ -13,6 +13,7 @@ import pytest from mitmproxy import http, io # type: ignore from mitmproxy.addons.savehar import SaveHar # type: ignore +from slugify import slugify async def get_container_from_id(dagger_client: dagger.Client, container_id: str) -> dagger.Container: @@ -97,7 +98,7 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ # If a container_id.txt file is available, we'll use it to load the connector container # We use a txt file as container ids can be too long to be passed as env vars # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation - container_id_path = Path("/tmp/container_id.txt") + container_id_path = Path(f"/tmp/{slugify(image_name_with_tag)}_container_id.txt") if container_id_path.exists(): return await get_container_from_id(dagger_client, container_id_path.read_text()) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 12ee7be280f5..fe09349c55b0 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -748,6 +748,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | |---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| 4.15.7 | [#38772](https://github.com/airbytehq/airbyte/pull/38772) | Fix regression test connector image retrieval. | | 4.15.6 | [#38783](https://github.com/airbytehq/airbyte/pull/38783) | Fix a variable access error with `repo_dir` in the `bump_version` command. | | 4.15.5 | [#38732](https://github.com/airbytehq/airbyte/pull/38732) | Update metadata deploy pipeline to 3.10 | | 4.15.4 | [#38646](https://github.com/airbytehq/airbyte/pull/38646) | Make airbyte-ci able to test external repos. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index f54aaf2abf4c..4115254806fe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -27,6 +27,7 @@ from pipelines.helpers.utils import METADATA_FILE_NAME, get_exec_result from pipelines.models.secrets import Secret from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus +from slugify import slugify class VersionCheck(Step, ABC): @@ -344,6 +345,7 @@ def regression_tests_command(self) -> List[str]: (See https://docs.dagger.io/manuals/developer/python/328492/services/ and https://cloud.google.com/sql/docs/postgres/sql-proxy#cloud-sql-auth-proxy-docker-image) """ run_proxy = "./cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --credentials-file /tmp/credentials.json" + selected_streams = ["--stream", self.selected_streams] if self.selected_streams else [] run_pytest = " ".join( [ "poetry", @@ -365,6 +367,7 @@ def regression_tests_command(self) -> List[str]: "--should-read-with-state", str(self.should_read_with_state), ] + + selected_streams ) run_pytest_with_proxy = dedent( f""" @@ -398,6 +401,7 @@ def __init__(self, context: ConnectorContext) -> None: self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest") self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev") self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", True) + self.selected_streams = self.context.run_step_options.get_item_or_default(options, "selected-streams", None) self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time())) async def _run(self, connector_under_test_container: Container) -> StepResult: @@ -456,7 +460,9 @@ async def _build_regression_test_container(self, target_container_id: str) -> Co # regression tests. The connector can be found if you know the container ID, so we write the container ID to a file and put # it in the regression test container. This way regression tests will use the already-built connector instead of trying to # build their own. - .with_new_file("/tmp/container_id.txt", contents=str(target_container_id)) + .with_new_file( + f"/tmp/{slugify(self.connector_image + ':' + self.target_version)}_container_id.txt", contents=str(target_container_id) + ) ) if self.context.is_ci: diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index f845a2c2cf58..7d7a6c3fbd6a 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -447,7 +447,7 @@ url = "../common_utils" [[package]] name = "connector-ops" -version = "0.4.0" +version = "0.5.0" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" @@ -1578,6 +1578,7 @@ files = [ {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, @@ -2070,6 +2071,23 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-slugify" +version = "8.0.4" +description = "A Python slugify application that also handles Unicode" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856"}, + {file = "python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8"}, +] + +[package.dependencies] +text-unidecode = ">=1.3" + +[package.extras] +unidecode = ["Unidecode (>=1.1.1)"] + [[package]] name = "pytz" version = "2024.1" @@ -2140,6 +2158,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2496,6 +2515,17 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "text-unidecode" +version = "1.3" +description = "The most basic Text::Unidecode port" +optional = false +python-versions = "*" +files = [ + {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, + {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, +] + [[package]] name = "toml" version = "0.10.2" @@ -2822,4 +2852,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "867cf46af77c6613e30bafb67e728c86d3e4b16edadd3624cc66ae45f3575d57" +content-hash = "5a120ee01ce9fdfd60108ce2214dc5039f79a3bf5440f22f1a2bcfbd77d3e89d" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 3c1732f8f5a0..6c103be448e5 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.15.6" +version = "4.15.7" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] @@ -38,6 +38,7 @@ google-cloud-secret-manager = "^2.20.0" google-auth = "^2.29.0" pygithub = "^2.3.0" pydash = "6.0.2" +python-slugify = ">=8.0.4" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2"