set PYTHONUSERSPACE env var at build time

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py

@@ -46,6 +46,7 @@ async def _build_from_base_image(self, platform: Platform) -> Container:
 
         # Mount manifest file
         base_container = self._get_base_container(platform)
+        base_container = hacks.set_python_userspace(base_container)
         user = await self.get_image_user(base_container)
         base_container = base_container.with_file(
             f"source_declarative_manifest/{MANIFEST_FILE_PATH}",

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py

@@ -6,6 +6,7 @@
 from typing import Any
 
 from dagger import Container, Platform
+from pipelines import hacks
 from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
 from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
@@ -66,6 +67,8 @@ async def _build_from_base_image(self, platform: Platform) -> Container:
         self.logger.info(f"Building connector from base image in metadata for {platform}")
         base = self._get_base_container(platform)
         user = await self.get_image_user(base)
+        # TODO: remove this hack when we've correctly set the userspace in the base image
+        base = await hacks.set_python_userspace(base)
         customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger)
         main_file_name = build_customization.get_main_file_name(self.context.connector)
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py

@@ -83,10 +83,6 @@ async def install_testing_environment(
             ["ln", "-s", "/source-declarative-manifest", connector_path]
         )
 
-        # Specify the root user for installation of dependencies.
-        # This is necessary to install poetry dependencies in rootless containers.
-        test_environment = test_environment.with_user("root")
-
         return await super().install_testing_environment(
             test_environment,
             test_config_file_name,

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py

@@ -215,8 +215,6 @@ async def with_installed_python_package(
     has_requirements_txt = await check_path_in_workdir(container, "requirements.txt")
     has_pyproject_toml = await check_path_in_workdir(container, "pyproject.toml")
 
-    container = container.with_user("root")
-    # All of these will require root access to install dependencies
     if has_pyproject_toml:
         container = with_poetry_cache(container, context.dagger_client)
         container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package)
@@ -227,8 +225,6 @@ async def with_installed_python_package(
         container = with_pip_cache(container, context.dagger_client)
         container = _install_python_dependencies_from_requirements_txt(container)
 
-    container = container.with_user(user)
-
     return container
 
 

airbyte-ci/connectors/pipelines/pipelines/hacks.py

@@ -14,6 +14,7 @@
 from pipelines import consts
 from pipelines.airbyte_ci.steps.base_image import UpdateBaseImageMetadata
 from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, is_automerge_pull_request, update_commit_status_check
+from pipelines.helpers.utils import sh_dash_c
 
 if TYPE_CHECKING:
     from dagger import Container
@@ -141,3 +142,16 @@ def determine_changelog_entry_comment(upgrade_base_image_in_metadata_result: Ste
         ):
             return "Starting with this version, the Docker image is now rootless. Please note that this and future versions will not be compatible with Airbyte versions earlier than 0.64"
     return default_comment
+
+
+async def set_python_userspace(container: Container) -> Container:
+    """
+    Set the PYTHONUSERBASE environment variable to /airbyte when the current user is airbyte and the airbyte directory exists.
+    This is to allow user pip installs in the airbyte directory.
+    This hack should be temporary until the base image is updated to set the PYTHONUSERBASE environment variable.
+    """
+    airbyte_user_exists = "airbyte" in (await container.with_exec(["cut", "-d:", "-f1", "/etc/passwd"]).stdout()).split("\n")
+    slash_airbyte_exists = "airbyte" in (await container.with_exec(["ls", "/"]).stdout()).split()
+    if airbyte_user_exists and slash_airbyte_exists:
+        container = container.with_env_variable("PYTHONUSERBASE", "/airbyte")
+    return container