diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 982a59c5ab48..de1c9ee81e5c 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -854,6 +854,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only | Version | PR | Description | | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.46.2 | [#49136](https://github.com/airbytehq/airbyte/pull/49136) | Fix failed install of python components due to non-root permissions. | | 4.46.1 | [#49146](https://github.com/airbytehq/airbyte/pull/49146) | Update `crane` image address as the one we were using has been deleted by the maintainer. | | 4.46.0 | [#48790](https://github.com/airbytehq/airbyte/pull/48790) | Add unit tests step for manifest-only connectors | | 4.45.3 | [#48927](https://github.com/airbytehq/airbyte/pull/48927) | Fix bug in determine_changelog_entry_comment | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py index 690ecf4ea73c..8c8b74e15511 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py @@ -14,7 +14,6 @@ from pipelines.airbyte_ci.connectors.test.steps.python_connectors import PytestStep from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun -from pipelines.models.steps import StepResult def get_test_steps(context: ConnectorTestContext) -> STEP_TREE: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 9caf07e22d59..5f38935d67e4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -20,6 +20,7 @@ from pipelines.dagger.actions import secrets from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun +from pipelines.helpers.utils import raise_if_not_user from pipelines.models.steps import STEP_PARAMS, Step, StepResult # Pin the PyAirbyte version to avoid updates from breaking CI @@ -77,6 +78,7 @@ async def _run(self, connector_under_test: Container) -> StepResult: test_environment = await self.install_testing_environment( connector_under_test, test_config_file_name, test_config_file, self.extra_dependencies_names ) + pytest_command = self.get_pytest_command(test_config_file_name) if self.bind_to_docker_host: @@ -149,8 +151,9 @@ async def install_testing_environment( Returns: Container: The container with the test environment installed. """ - secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets", self.secrets) user = await BuildConnectorImages.get_image_user(built_connector_container) + secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets", self.secrets, owner=user) + container_with_test_deps = ( # Install the connector python package in /test_environment with the extra dependencies await pipelines.dagger.actions.python.common.with_python_connector_installed( @@ -163,15 +166,26 @@ async def install_testing_environment( ) ) if self.common_test_dependencies: - container_with_test_deps = container_with_test_deps.with_exec(["pip", "install", f'{" ".join(self.common_test_dependencies)}']) - return ( + container_with_test_deps = container_with_test_deps.with_user("root").with_exec( + ["pip", "install", f'{" ".join(self.common_test_dependencies)}'] + ) + + container_with_test_deps = ( container_with_test_deps # Mount the test config file - .with_mounted_file(test_config_file_name, test_config_file) + .with_mounted_file(test_config_file_name, test_config_file, owner=user) # Mount the secrets - .with_(secret_mounting_function).with_env_variable("PYTHONPATH", ".") + .with_(secret_mounting_function) + .with_env_variable("PYTHONPATH", ".") + # Make sure all files that were created or mounted under /airbyte are owned by the user + .with_user("root") + .with_exec(["chown", "-R", f"{user}:{user}", "/airbyte"]) + .with_user(user) ) + await raise_if_not_user(container_with_test_deps, user) + return container_with_test_deps + class UnitTests(PytestStep): """A step to run the connector unit tests with Pytest.""" diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index e021092a281b..59418b7c1826 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -11,8 +11,8 @@ from pipelines import hacks from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import PATH_TO_LOCAL_CDK -from pipelines.dagger.containers.python import with_pip_cache, with_poetry_cache, with_python_base, with_testing_dependencies -from pipelines.helpers.utils import check_path_in_workdir, get_file_contents +from pipelines.dagger.containers.python import with_pip_cache, with_poetry_cache, with_python_base +from pipelines.helpers.utils import check_path_in_workdir, get_file_contents, raise_if_not_user def with_python_package( @@ -21,6 +21,7 @@ def with_python_package( package_source_code_path: str, exclude: Optional[List] = None, include: Optional[List] = None, + owner: str | None = None, ) -> Container: """Load a python package source code to a python environment container. @@ -30,13 +31,16 @@ def with_python_package( package_source_code_path (str): The local path to the package source code. additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. exclude (Optional[List]): A list of file or directory to exclude from the python package source code. - + include (Optional[List]): A list of file or directory to include from the python package source code. + owner (str, optional): The owner of the mounted directory. Defaults to None. Returns: Container: A python environment container with the python package source code. """ package_source_code_directory: Directory = context.get_repo_dir(package_source_code_path, exclude=exclude, include=include) work_dir_path = f"/{package_source_code_path}" - container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory).with_workdir(work_dir_path) + container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory, owner=owner).with_workdir( + work_dir_path + ) return container @@ -109,7 +113,6 @@ async def find_local_python_dependencies( package_source_code_path (str): The local path to the python package source code. search_dependencies_in_setup_py (bool, optional): Whether to search for local dependencies in the setup.py file. Defaults to True. search_dependencies_in_requirements_txt (bool, optional): Whether to search for local dependencies in the requirements.txt file. Defaults to True. - Returns: List[str]: Paths to the local dependencies relative to the airbyte repo. """ @@ -205,7 +208,7 @@ async def with_installed_python_package( Container: A python environment container with the python package installed. """ - container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include) + container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include, owner=user) local_dependencies = await find_local_python_dependencies(context, package_source_code_path) for dependency_directory in local_dependencies: @@ -215,8 +218,11 @@ async def with_installed_python_package( has_requirements_txt = await check_path_in_workdir(container, "requirements.txt") has_pyproject_toml = await check_path_in_workdir(container, "pyproject.toml") + container = container.with_user("root") + # All of these will require root access to install dependencies + # Dependencies are installed at the system level, if the user is not root it is not allowed to install system level dependencies if has_pyproject_toml: - container = with_poetry_cache(container, context.dagger_client) + container = with_poetry_cache(container, context.dagger_client, owner=user) container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package) elif has_setup_py: container = with_pip_cache(container, context.dagger_client) @@ -225,21 +231,9 @@ async def with_installed_python_package( container = with_pip_cache(container, context.dagger_client) container = _install_python_dependencies_from_requirements_txt(container) - return container - - -def with_python_connector_source(context: ConnectorContext) -> Container: - """Load an airbyte connector source code in a testing environment. + container = container.with_user(user) - Args: - context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. - Returns: - Container: A python environment container (with the connector source code). - """ - connector_source_path = str(context.connector.code_directory) - testing_environment: Container = with_testing_dependencies(context) - - return with_python_package(context, testing_environment, connector_source_path) + return container def apply_python_development_overrides(context: ConnectorContext, connector_container: Container, current_user: str) -> Container: @@ -324,7 +318,7 @@ async def with_python_connector_installed( ) container = await apply_python_development_overrides(context, container, user) - + await raise_if_not_user(container, user) return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index 8c2eb2e29cfd..81773fd7ea4a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -49,7 +49,7 @@ async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GC async def mounted_connector_secrets( - context: ConnectorContext, secret_directory_path: str, connector_secrets: List[Secret] + context: ConnectorContext, secret_directory_path: str, connector_secrets: List[Secret], owner: str | None = None ) -> Callable[[Container], Container]: """Returns an argument for a dagger container's with_ method which mounts all connector secrets in it. @@ -57,6 +57,7 @@ async def mounted_connector_secrets( context (ConnectorContext): The context providing a connector object and its secrets. secret_directory_path (str): Container directory where the secrets will be mounted, as files. connector_secrets (List[secrets]): List of secrets to mount to the connector container. + owner (str, optional): The owner of the mounted secrets. Defaults to None. Returns: fn (Callable[[Container], Container]): A function to pass as argument to the connector container's with_ method. """ @@ -73,7 +74,7 @@ def with_secrets_mounted_as_dagger_secrets(container: Container) -> Container: for secret in connector_secrets: if secret.file_name: container = container.with_mounted_secret( - f"{secret_directory_path}/{secret.file_name}", secret.as_dagger_secret(context.dagger_client) + f"{secret_directory_path}/{secret.file_name}", secret.as_dagger_secret(context.dagger_client), owner=owner ) return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 8fd842253a87..245190f66ede 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -67,25 +67,25 @@ def with_testing_dependencies(context: PipelineContext) -> Container: ) -def with_pip_cache(container: Container, dagger_client: Client) -> Container: +def with_pip_cache(container: Container, dagger_client: Client, owner: str | None = None) -> Container: """Mounts the pip cache in the container. Args: container (Container): A container with python installed - + owner (str, optional): The owner of the cache. Defaults to None. Returns: Container: A container with the pip cache mounted. """ pip_cache_volume = dagger_client.cache_volume(PIP_CACHE_VOLUME_NAME) - return container.with_mounted_cache(PIP_CACHE_PATH, pip_cache_volume, sharing=CacheSharingMode.SHARED) + return container.with_mounted_cache(PIP_CACHE_PATH, pip_cache_volume, sharing=CacheSharingMode.SHARED, owner=owner) -def with_poetry_cache(container: Container, dagger_client: Client) -> Container: +def with_poetry_cache(container: Container, dagger_client: Client, owner: str | None = None) -> Container: """Mounts the poetry cache in the container. Args: container (Container): A container with python installed - + owner (str, optional): The owner of the cache. Defaults to None. Returns: Container: A container with the poetry cache mounted. """ poetry_cache_volume = dagger_client.cache_volume(POETRY_CACHE_VOLUME_NAME) - return container.with_mounted_cache(POETRY_CACHE_PATH, poetry_cache_volume, sharing=CacheSharingMode.SHARED) + return container.with_mounted_cache(POETRY_CACHE_PATH, poetry_cache_volume, sharing=CacheSharingMode.SHARED, owner=owner) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index a33bc56864c4..a7e0546f2fd6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -374,3 +374,13 @@ def dagger_directory_as_zip_file(dagger_client: Client, directory: Directory, di .with_exec(["zip", "-r", "/zipped.zip", f"/{directory_name}"]) .file("/zipped.zip") ) + + +async def raise_if_not_user(container: Container, user: str) -> None: + """Raise an error if the container is not running as the specified user. + + Args: + container (Container): The container to check. + user (str): The user to check. + """ + assert (await container.with_exec(["whoami"]).stdout()).strip() == user, f"Container is not running as {user}." diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 2d7bb93f8564..ca78405e886d 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.46.1" +version = "4.46.2" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py index a6a5c35afd75..f504b27654ae 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -72,6 +72,7 @@ async def test_with_python_connector_installed_from_setup(context_with_setup, py # Uninstall and reinstall the latest cdk version cdk_install_latest_output = ( await container.with_env_variable("CACHEBUSTER", datetime.datetime.now().isoformat()) + .with_user("root") .with_exec(["pip", "uninstall", "-y", f"airbyte-cdk=={latest_cdk_version}"]) .with_exec(["pip", "install", f"airbyte-cdk=={latest_cdk_version}"]) .stdout()