Fix live test image when running in CI (#38772)

airbyte-ci/connectors/live-tests/README.md

@@ -275,6 +275,10 @@ The traffic recorded on the control connector is passed to the target connector
 
 ## Changelog
 
+### 0.17.4
+
+Fix control image when running tests in CI.
+
 ### 0.17.3
 
 Pin requests dependency.

airbyte-ci/connectors/live-tests/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "live-tests"
-version = "0.17.2"
+version = "0.17.4"
 description = "Contains utilities for testing connectors against live data."
 authors = ["Airbyte <[email protected]>"]
 license = "MIT"
@@ -39,6 +39,7 @@ pyyaml = "^6.0.1"
 dpath = "^2.1.6"
 genson = "^1.2.2"
 segment-analytics-python = "^2.3.2"
+python-slugify = ">=8.0.4"
 
 [tool.poetry.scripts]
 live-tests = "live_tests.cli:live_tests"

airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py

@@ -13,6 +13,7 @@
 import pytest
 from mitmproxy import http, io  # type: ignore
 from mitmproxy.addons.savehar import SaveHar  # type: ignore
+from slugify import slugify
 
 
 async def get_container_from_id(dagger_client: dagger.Client, container_id: str) -> dagger.Container:
@@ -97,7 +98,7 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_
     # If a container_id.txt file is available, we'll use it to load the connector container
     # We use a txt file as container ids can be too long to be passed as env vars
     # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation
-    container_id_path = Path("/tmp/container_id.txt")
+    container_id_path = Path(f"/tmp/{slugify(image_name_with_tag)}_container_id.txt")
     if container_id_path.exists():
         return await get_container_from_id(dagger_client, container_id_path.read_text())
 

airbyte-ci/connectors/pipelines/README.md

@@ -748,6 +748,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 |---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
+| 4.15.7  | [#38772](https://github.com/airbytehq/airbyte/pull/38772)      | Fix regression test connector image retrieval.                                                                               |
 | 4.15.6  | [#38783](https://github.com/airbytehq/airbyte/pull/38783)  | Fix a variable access error with `repo_dir` in the `bump_version` command.                                                   |
 | 4.15.5  | [#38732](https://github.com/airbytehq/airbyte/pull/38732)  | Update metadata deploy pipeline to 3.10                                                                                      |
 | 4.15.4  | [#38646](https://github.com/airbytehq/airbyte/pull/38646)  | Make airbyte-ci able to test external repos.                                                                                 |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

@@ -27,6 +27,7 @@
 from pipelines.helpers.utils import METADATA_FILE_NAME, get_exec_result
 from pipelines.models.secrets import Secret
 from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus
+from slugify import slugify
 
 
 class VersionCheck(Step, ABC):
@@ -344,6 +345,7 @@ def regression_tests_command(self) -> List[str]:
         (See https://docs.dagger.io/manuals/developer/python/328492/services/ and https://cloud.google.com/sql/docs/postgres/sql-proxy#cloud-sql-auth-proxy-docker-image)
         """
         run_proxy = "./cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --credentials-file /tmp/credentials.json"
+        selected_streams = ["--stream", self.selected_streams] if self.selected_streams else []
         run_pytest = " ".join(
             [
                 "poetry",
@@ -365,6 +367,7 @@ def regression_tests_command(self) -> List[str]:
                 "--should-read-with-state",
                 str(self.should_read_with_state),
             ]
+            + selected_streams
         )
         run_pytest_with_proxy = dedent(
             f"""
@@ -398,6 +401,7 @@ def __init__(self, context: ConnectorContext) -> None:
         self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest")
         self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev")
         self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", True)
+        self.selected_streams = self.context.run_step_options.get_item_or_default(options, "selected-streams", None)
         self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time()))
 
     async def _run(self, connector_under_test_container: Container) -> StepResult:
@@ -456,7 +460,9 @@ async def _build_regression_test_container(self, target_container_id: str) -> Co
             # regression tests. The connector can be found if you know the container ID, so we write the container ID to a file and put
             # it in the regression test container. This way regression tests will use the already-built connector instead of trying to
             # build their own.
-            .with_new_file("/tmp/container_id.txt", contents=str(target_container_id))
+            .with_new_file(
+                f"/tmp/{slugify(self.connector_image + ':' + self.target_version)}_container_id.txt", contents=str(target_container_id)
+            )
         )
 
         if self.context.is_ci:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.15.6"
+version = "4.15.7"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 
@@ -38,6 +38,7 @@ google-cloud-secret-manager = "^2.20.0"
 google-auth = "^2.29.0"
 pygithub = "^2.3.0"
 pydash = "6.0.2"
+python-slugify = ">=8.0.4"
 
 [tool.poetry.group.dev.dependencies]
 freezegun = "^1.2.2"