From adad694da152258111f92f6d867d5ace599c5094 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Wed, 11 Dec 2024 12:25:37 -0800 Subject: [PATCH 01/11] fix: use root for python installs --- .../pipelines/pipelines/dagger/actions/python/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index e021092a281b3..86ce487d21e1a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -215,6 +215,8 @@ async def with_installed_python_package( has_requirements_txt = await check_path_in_workdir(container, "requirements.txt") has_pyproject_toml = await check_path_in_workdir(container, "pyproject.toml") + container = container.with_user("root") + # All of these will require root access to install dependencies if has_pyproject_toml: container = with_poetry_cache(container, context.dagger_client) container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package) @@ -225,6 +227,8 @@ async def with_installed_python_package( container = with_pip_cache(container, context.dagger_client) container = _install_python_dependencies_from_requirements_txt(container) + container = container.with_user(user) + return container From f7b23509482cf55d4167bee6a3b8290b3a1ce566 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Wed, 11 Dec 2024 22:16:26 -0800 Subject: [PATCH 02/11] chore: version bump and changelog --- airbyte-ci/connectors/pipelines/README.md | 1 + airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index dd5194efa033f..deb6c5a777eaf 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -854,6 +854,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only | Version | PR | Description | | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.45.4 | [#49136](https://github.com/airbytehq/airbyte/pull/49136) | Fix failed install of python components due to non-root permissions. | | 4.45.3 | [#48927](https://github.com/airbytehq/airbyte/pull/48927) | Fix bug in determine_changelog_entry_comment | | 4.45.2 | [#48868](https://github.com/airbytehq/airbyte/pull/48868) | Fix ownership issues while using `--use-local-cdk` | | 4.45.1 | [#48872](https://github.com/airbytehq/airbyte/pull/48872) | Make the `connectors list` command write its output to a JSON file. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 72baddb0dc729..da393172a7499 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.45.3" +version = "4.45.4" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 346b63437e37942d199cfbd55ed93fc1f3a6dfc6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 14:29:48 +0100 Subject: [PATCH 03/11] more root --- .../test/steps/python_connectors.py | 24 ++++++++++--- .../pipelines/dagger/actions/python/common.py | 34 +++++++------------ .../pipelines/dagger/actions/secrets.py | 5 +-- .../pipelines/dagger/containers/python.py | 12 +++---- .../pipelines/pipelines/helpers/utils.py | 13 +++++++ 5 files changed, 53 insertions(+), 35 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 9caf07e22d595..5f38935d67e41 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -20,6 +20,7 @@ from pipelines.dagger.actions import secrets from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun +from pipelines.helpers.utils import raise_if_not_user from pipelines.models.steps import STEP_PARAMS, Step, StepResult # Pin the PyAirbyte version to avoid updates from breaking CI @@ -77,6 +78,7 @@ async def _run(self, connector_under_test: Container) -> StepResult: test_environment = await self.install_testing_environment( connector_under_test, test_config_file_name, test_config_file, self.extra_dependencies_names ) + pytest_command = self.get_pytest_command(test_config_file_name) if self.bind_to_docker_host: @@ -149,8 +151,9 @@ async def install_testing_environment( Returns: Container: The container with the test environment installed. """ - secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets", self.secrets) user = await BuildConnectorImages.get_image_user(built_connector_container) + secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets", self.secrets, owner=user) + container_with_test_deps = ( # Install the connector python package in /test_environment with the extra dependencies await pipelines.dagger.actions.python.common.with_python_connector_installed( @@ -163,15 +166,26 @@ async def install_testing_environment( ) ) if self.common_test_dependencies: - container_with_test_deps = container_with_test_deps.with_exec(["pip", "install", f'{" ".join(self.common_test_dependencies)}']) - return ( + container_with_test_deps = container_with_test_deps.with_user("root").with_exec( + ["pip", "install", f'{" ".join(self.common_test_dependencies)}'] + ) + + container_with_test_deps = ( container_with_test_deps # Mount the test config file - .with_mounted_file(test_config_file_name, test_config_file) + .with_mounted_file(test_config_file_name, test_config_file, owner=user) # Mount the secrets - .with_(secret_mounting_function).with_env_variable("PYTHONPATH", ".") + .with_(secret_mounting_function) + .with_env_variable("PYTHONPATH", ".") + # Make sure all files that were created or mounted under /airbyte are owned by the user + .with_user("root") + .with_exec(["chown", "-R", f"{user}:{user}", "/airbyte"]) + .with_user(user) ) + await raise_if_not_user(container_with_test_deps, user) + return container_with_test_deps + class UnitTests(PytestStep): """A step to run the connector unit tests with Pytest.""" diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 86ce487d21e1a..141dcf2c0570d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -11,8 +11,8 @@ from pipelines import hacks from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import PATH_TO_LOCAL_CDK -from pipelines.dagger.containers.python import with_pip_cache, with_poetry_cache, with_python_base, with_testing_dependencies -from pipelines.helpers.utils import check_path_in_workdir, get_file_contents +from pipelines.dagger.containers.python import with_pip_cache, with_poetry_cache, with_python_base +from pipelines.helpers.utils import check_path_in_workdir, get_file_contents, raise_if_not_user def with_python_package( @@ -21,6 +21,7 @@ def with_python_package( package_source_code_path: str, exclude: Optional[List] = None, include: Optional[List] = None, + owner: str = "root", ) -> Container: """Load a python package source code to a python environment container. @@ -30,13 +31,16 @@ def with_python_package( package_source_code_path (str): The local path to the package source code. additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. exclude (Optional[List]): A list of file or directory to exclude from the python package source code. - + include (Optional[List]): A list of file or directory to include from the python package source code. + owner (str): The user to use in the container. Defaults to "root". Returns: Container: A python environment container with the python package source code. """ package_source_code_directory: Directory = context.get_repo_dir(package_source_code_path, exclude=exclude, include=include) work_dir_path = f"/{package_source_code_path}" - container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory).with_workdir(work_dir_path) + container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory, owner=owner).with_workdir( + work_dir_path + ) return container @@ -109,7 +113,6 @@ async def find_local_python_dependencies( package_source_code_path (str): The local path to the python package source code. search_dependencies_in_setup_py (bool, optional): Whether to search for local dependencies in the setup.py file. Defaults to True. search_dependencies_in_requirements_txt (bool, optional): Whether to search for local dependencies in the requirements.txt file. Defaults to True. - Returns: List[str]: Paths to the local dependencies relative to the airbyte repo. """ @@ -205,7 +208,7 @@ async def with_installed_python_package( Container: A python environment container with the python package installed. """ - container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include) + container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include, owner=user) local_dependencies = await find_local_python_dependencies(context, package_source_code_path) for dependency_directory in local_dependencies: @@ -217,8 +220,9 @@ async def with_installed_python_package( container = container.with_user("root") # All of these will require root access to install dependencies + # Dependencies are installed at the system level, if the user is not root it is not allowed to install system level dependencies if has_pyproject_toml: - container = with_poetry_cache(container, context.dagger_client) + container = with_poetry_cache(container, context.dagger_client, owner=user) container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package) elif has_setup_py: container = with_pip_cache(container, context.dagger_client) @@ -232,20 +236,6 @@ async def with_installed_python_package( return container -def with_python_connector_source(context: ConnectorContext) -> Container: - """Load an airbyte connector source code in a testing environment. - - Args: - context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. - Returns: - Container: A python environment container (with the connector source code). - """ - connector_source_path = str(context.connector.code_directory) - testing_environment: Container = with_testing_dependencies(context) - - return with_python_package(context, testing_environment, connector_source_path) - - def apply_python_development_overrides(context: ConnectorContext, connector_container: Container, current_user: str) -> Container: # Run the connector using the local cdk if flag is set if context.use_local_cdk: @@ -328,7 +318,7 @@ async def with_python_connector_installed( ) container = await apply_python_development_overrides(context, container, user) - + await raise_if_not_user(container, user) return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index 8c2eb2e29cfdf..b36933f923908 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -49,7 +49,7 @@ async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GC async def mounted_connector_secrets( - context: ConnectorContext, secret_directory_path: str, connector_secrets: List[Secret] + context: ConnectorContext, secret_directory_path: str, connector_secrets: List[Secret], owner: str = "root" ) -> Callable[[Container], Container]: """Returns an argument for a dagger container's with_ method which mounts all connector secrets in it. @@ -57,6 +57,7 @@ async def mounted_connector_secrets( context (ConnectorContext): The context providing a connector object and its secrets. secret_directory_path (str): Container directory where the secrets will be mounted, as files. connector_secrets (List[secrets]): List of secrets to mount to the connector container. + owner (str): The owner of the mounted secrets. Returns: fn (Callable[[Container], Container]): A function to pass as argument to the connector container's with_ method. """ @@ -73,7 +74,7 @@ def with_secrets_mounted_as_dagger_secrets(container: Container) -> Container: for secret in connector_secrets: if secret.file_name: container = container.with_mounted_secret( - f"{secret_directory_path}/{secret.file_name}", secret.as_dagger_secret(context.dagger_client) + f"{secret_directory_path}/{secret.file_name}", secret.as_dagger_secret(context.dagger_client), owner=owner ) return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 8fd842253a877..133323ba6ed56 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -67,25 +67,25 @@ def with_testing_dependencies(context: PipelineContext) -> Container: ) -def with_pip_cache(container: Container, dagger_client: Client) -> Container: +def with_pip_cache(container: Container, dagger_client: Client, owner="root") -> Container: """Mounts the pip cache in the container. Args: container (Container): A container with python installed - + owner (str, optional): The owner of the cache. Defaults to "root". Returns: Container: A container with the pip cache mounted. """ pip_cache_volume = dagger_client.cache_volume(PIP_CACHE_VOLUME_NAME) - return container.with_mounted_cache(PIP_CACHE_PATH, pip_cache_volume, sharing=CacheSharingMode.SHARED) + return container.with_mounted_cache(PIP_CACHE_PATH, pip_cache_volume, sharing=CacheSharingMode.SHARED, owner=owner) -def with_poetry_cache(container: Container, dagger_client: Client) -> Container: +def with_poetry_cache(container: Container, dagger_client: Client, owner: str = "root") -> Container: """Mounts the poetry cache in the container. Args: container (Container): A container with python installed - + owner (str, optional): The owner of the cache. Defaults to "root". Returns: Container: A container with the poetry cache mounted. """ poetry_cache_volume = dagger_client.cache_volume(POETRY_CACHE_VOLUME_NAME) - return container.with_mounted_cache(POETRY_CACHE_PATH, poetry_cache_volume, sharing=CacheSharingMode.SHARED) + return container.with_mounted_cache(POETRY_CACHE_PATH, poetry_cache_volume, sharing=CacheSharingMode.SHARED, owner=owner) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index a33bc56864c45..ae93155a2a1cd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -374,3 +374,16 @@ def dagger_directory_as_zip_file(dagger_client: Client, directory: Directory, di .with_exec(["zip", "-r", "/zipped.zip", f"/{directory_name}"]) .file("/zipped.zip") ) + + +async def raise_if_not_user(container: Container, user: str) -> Container: + """Raise an error if the container is not running as the specified user. + + Args: + container (Container): The container to check. + user (str): The user to check. + + Returns: + Container: The container. + """ + assert (await container.with_exec(["whoami"]).stdout()).strip() == user, f"Container is not running as {user}." From caf547ac2d1e04e255ba6d68f0ca9b159462ed2f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 15:11:58 +0100 Subject: [PATCH 04/11] more root --- .../pipelines/pipelines/dagger/actions/python/common.py | 4 ++-- .../pipelines/pipelines/dagger/actions/secrets.py | 4 ++-- .../pipelines/pipelines/dagger/containers/python.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 141dcf2c0570d..59418b7c18263 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -21,7 +21,7 @@ def with_python_package( package_source_code_path: str, exclude: Optional[List] = None, include: Optional[List] = None, - owner: str = "root", + owner: str | None = None, ) -> Container: """Load a python package source code to a python environment container. @@ -32,7 +32,7 @@ def with_python_package( additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. exclude (Optional[List]): A list of file or directory to exclude from the python package source code. include (Optional[List]): A list of file or directory to include from the python package source code. - owner (str): The user to use in the container. Defaults to "root". + owner (str, optional): The owner of the mounted directory. Defaults to None. Returns: Container: A python environment container with the python package source code. """ diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index b36933f923908..81773fd7ea4a5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -49,7 +49,7 @@ async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GC async def mounted_connector_secrets( - context: ConnectorContext, secret_directory_path: str, connector_secrets: List[Secret], owner: str = "root" + context: ConnectorContext, secret_directory_path: str, connector_secrets: List[Secret], owner: str | None = None ) -> Callable[[Container], Container]: """Returns an argument for a dagger container's with_ method which mounts all connector secrets in it. @@ -57,7 +57,7 @@ async def mounted_connector_secrets( context (ConnectorContext): The context providing a connector object and its secrets. secret_directory_path (str): Container directory where the secrets will be mounted, as files. connector_secrets (List[secrets]): List of secrets to mount to the connector container. - owner (str): The owner of the mounted secrets. + owner (str, optional): The owner of the mounted secrets. Defaults to None. Returns: fn (Callable[[Container], Container]): A function to pass as argument to the connector container's with_ method. """ diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 133323ba6ed56..245190f66ede4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -67,11 +67,11 @@ def with_testing_dependencies(context: PipelineContext) -> Container: ) -def with_pip_cache(container: Container, dagger_client: Client, owner="root") -> Container: +def with_pip_cache(container: Container, dagger_client: Client, owner: str | None = None) -> Container: """Mounts the pip cache in the container. Args: container (Container): A container with python installed - owner (str, optional): The owner of the cache. Defaults to "root". + owner (str, optional): The owner of the cache. Defaults to None. Returns: Container: A container with the pip cache mounted. """ @@ -79,11 +79,11 @@ def with_pip_cache(container: Container, dagger_client: Client, owner="root") -> return container.with_mounted_cache(PIP_CACHE_PATH, pip_cache_volume, sharing=CacheSharingMode.SHARED, owner=owner) -def with_poetry_cache(container: Container, dagger_client: Client, owner: str = "root") -> Container: +def with_poetry_cache(container: Container, dagger_client: Client, owner: str | None = None) -> Container: """Mounts the poetry cache in the container. Args: container (Container): A container with python installed - owner (str, optional): The owner of the cache. Defaults to "root". + owner (str, optional): The owner of the cache. Defaults to None. Returns: Container: A container with the poetry cache mounted. """ From ae37fe0ec0f596339ce78bbaad55ad4438b5419f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 15:12:33 +0100 Subject: [PATCH 05/11] poe lint --- .../airbyte_ci/connectors/test/steps/manifest_only_connectors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py index 690ecf4ea73c4..8c8b74e15511e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py @@ -14,7 +14,6 @@ from pipelines.airbyte_ci.connectors.test.steps.python_connectors import PytestStep from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun -from pipelines.models.steps import StepResult def get_test_steps(context: ConnectorTestContext) -> STEP_TREE: From c7086f8886e7e5c876e7fdc7c14fa588a3c94a3a Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 15:13:32 +0100 Subject: [PATCH 06/11] poe type_check --- airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index ae93155a2a1cd..a7e0546f2fd61 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -376,14 +376,11 @@ def dagger_directory_as_zip_file(dagger_client: Client, directory: Directory, di ) -async def raise_if_not_user(container: Container, user: str) -> Container: +async def raise_if_not_user(container: Container, user: str) -> None: """Raise an error if the container is not running as the specified user. Args: container (Container): The container to check. user (str): The user to check. - - Returns: - Container: The container. """ assert (await container.with_exec(["whoami"]).stdout()).strip() == user, f"Container is not running as {user}." From 62997131edbed77761d497c2321e7666e6344685 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 15:19:50 +0100 Subject: [PATCH 07/11] disable pat rate limit check on connector test: probably useless --- .github/workflows/connectors_tests.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index f31ca6d88e297..97a6fc03f06fd 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -83,10 +83,6 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v4 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_MAINTENANCE_OSS }} - name: Extract branch name [WORKFLOW DISPATCH] shell: bash if: github.event_name == 'workflow_dispatch' @@ -112,7 +108,7 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ steps.extract_branch.outputs.branch }} git_revision: ${{ steps.fetch_last_commit_id_wd.outputs.commit_id }} - github_token: ${{ env.PAT }} + github_token: ${{ github.token }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" @@ -129,7 +125,7 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ github.head_ref }} git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - github_token: ${{ env.PAT }} + github_token: ${{ github.token }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds) From cde1087056147d5d5d23fad095f8ff50698211a9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 17:27:44 +0100 Subject: [PATCH 08/11] revert me: use airbyte-ci installed from source --- .github/actions/install-airbyte-ci/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml index d0563331c805f..bd23cd59560aa 100644 --- a/.github/actions/install-airbyte-ci/action.yml +++ b/.github/actions/install-airbyte-ci/action.yml @@ -38,8 +38,8 @@ runs: echo "install-mode=source" >> $GITHUB_OUTPUT echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV else - echo "install-mode=binary" >> $GITHUB_OUTPUT - echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV + echo "install-mode=source" >> $GITHUB_OUTPUT + echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV fi - name: Install Airbyte CI from binary From 445ae2731caef72fcf291bd0b1562f0d019d44c9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 17:47:34 +0100 Subject: [PATCH 09/11] fix setup tests --- .../tests/test_dagger/test_actions/test_python/test_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py index a6a5c35afd754..f504b27654ae6 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -72,6 +72,7 @@ async def test_with_python_connector_installed_from_setup(context_with_setup, py # Uninstall and reinstall the latest cdk version cdk_install_latest_output = ( await container.with_env_variable("CACHEBUSTER", datetime.datetime.now().isoformat()) + .with_user("root") .with_exec(["pip", "uninstall", "-y", f"airbyte-cdk=={latest_cdk_version}"]) .with_exec(["pip", "install", f"airbyte-cdk=={latest_cdk_version}"]) .stdout() From 0c8b2252084bb0556ae81185959a518b36036aeb Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 17:47:48 +0100 Subject: [PATCH 10/11] Revert "revert me: use airbyte-ci installed from source" This reverts commit cde1087056147d5d5d23fad095f8ff50698211a9. --- .github/actions/install-airbyte-ci/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml index bd23cd59560aa..d0563331c805f 100644 --- a/.github/actions/install-airbyte-ci/action.yml +++ b/.github/actions/install-airbyte-ci/action.yml @@ -38,8 +38,8 @@ runs: echo "install-mode=source" >> $GITHUB_OUTPUT echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV else - echo "install-mode=source" >> $GITHUB_OUTPUT - echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV + echo "install-mode=binary" >> $GITHUB_OUTPUT + echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV fi - name: Install Airbyte CI from binary From c9ad51258ec67f5f107dc0e5d7b9f248bf607592 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 12 Dec 2024 17:48:00 +0100 Subject: [PATCH 11/11] Revert "disable pat rate limit check on connector test: probably useless" This reverts commit 62997131edbed77761d497c2321e7666e6344685. --- .github/workflows/connectors_tests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 97a6fc03f06fd..f31ca6d88e297 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -83,6 +83,10 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v4 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_MAINTENANCE_OSS }} - name: Extract branch name [WORKFLOW DISPATCH] shell: bash if: github.event_name == 'workflow_dispatch' @@ -108,7 +112,7 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ steps.extract_branch.outputs.branch }} git_revision: ${{ steps.fetch_last_commit_id_wd.outputs.commit_id }} - github_token: ${{ github.token }} + github_token: ${{ env.PAT }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" @@ -125,7 +129,7 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ github.head_ref }} git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - github_token: ${{ github.token }} + github_token: ${{ env.PAT }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)