diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py index 0c9116d592da..99dc17e0385f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py @@ -46,6 +46,7 @@ async def _build_from_base_image(self, platform: Platform) -> Container: # Mount manifest file base_container = self._get_base_container(platform) + base_container = hacks.set_python_userspace(base_container) user = await self.get_image_user(base_container) base_container = base_container.with_file( f"source_declarative_manifest/{MANIFEST_FILE_PATH}", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py index 262fdbc780d3..fc9bc0bbd31d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py @@ -6,6 +6,7 @@ from typing import Any from dagger import Container, Platform +from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -66,6 +67,8 @@ async def _build_from_base_image(self, platform: Platform) -> Container: self.logger.info(f"Building connector from base image in metadata for {platform}") base = self._get_base_container(platform) user = await self.get_image_user(base) + # TODO: remove this hack when we've correctly set the userspace in the base image + base = await hacks.set_python_userspace(base) customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger) main_file_name = build_customization.get_main_file_name(self.context.connector) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py index 690ecf4ea73c..4ac7ba6226df 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py @@ -83,10 +83,6 @@ async def install_testing_environment( ["ln", "-s", "/source-declarative-manifest", connector_path] ) - # Specify the root user for installation of dependencies. - # This is necessary to install poetry dependencies in rootless containers. - test_environment = test_environment.with_user("root") - return await super().install_testing_environment( test_environment, test_config_file_name, diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 86ce487d21e1..e021092a281b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -215,8 +215,6 @@ async def with_installed_python_package( has_requirements_txt = await check_path_in_workdir(container, "requirements.txt") has_pyproject_toml = await check_path_in_workdir(container, "pyproject.toml") - container = container.with_user("root") - # All of these will require root access to install dependencies if has_pyproject_toml: container = with_poetry_cache(container, context.dagger_client) container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package) @@ -227,8 +225,6 @@ async def with_installed_python_package( container = with_pip_cache(container, context.dagger_client) container = _install_python_dependencies_from_requirements_txt(container) - container = container.with_user(user) - return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index 76c968236947..8ed28a099953 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -14,6 +14,7 @@ from pipelines import consts from pipelines.airbyte_ci.steps.base_image import UpdateBaseImageMetadata from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, is_automerge_pull_request, update_commit_status_check +from pipelines.helpers.utils import sh_dash_c if TYPE_CHECKING: from dagger import Container @@ -141,3 +142,16 @@ def determine_changelog_entry_comment(upgrade_base_image_in_metadata_result: Ste ): return "Starting with this version, the Docker image is now rootless. Please note that this and future versions will not be compatible with Airbyte versions earlier than 0.64" return default_comment + + +async def set_python_userspace(container: Container) -> Container: + """ + Set the PYTHONUSERBASE environment variable to /airbyte when the current user is airbyte and the airbyte directory exists. + This is to allow user pip installs in the airbyte directory. + This hack should be temporary until the base image is updated to set the PYTHONUSERBASE environment variable. + """ + airbyte_user_exists = "airbyte" in (await container.with_exec(["cut", "-d:", "-f1", "/etc/passwd"]).stdout()).split("\n") + slash_airbyte_exists = "airbyte" in (await container.with_exec(["ls", "/"]).stdout()).split() + if airbyte_user_exists and slash_airbyte_exists: + container = container.with_env_variable("PYTHONUSERBASE", "/airbyte") + return container