airbytehq · alafanechere · Oct 11, 2023 · Sep 15, 2023
@@ -492,6 +492,10 @@ def normalization_tag(self) -> Optional[str]:
         if self.supports_normalization:
             return f"{self.metadata['normalizationConfig']['normalizationTag']}"
 
+    @property
+    def is_using_poetry(self) -> bool:
+        return Path(self.code_directory / "pyproject.toml").exists()
+
     def get_secret_manager(self, gsm_credentials: str):
         return SecretsManager(connector_name=self.technical_name, gsm_credentials=gsm_credentials)
 

@@ -395,6 +395,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 1.6.0   | [#30474](https://github.com/airbytehq/airbyte/pull/30474)  | Test connector inside their containers.                                                                  |
 | 1.5.1   | [#31227](https://github.com/airbytehq/airbyte/pull/31227)  | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'.     |
 | 1.5.0   | [#30456](https://github.com/airbytehq/airbyte/pull/30456)  | Start building Python connectors using our base images.                                                   |
 | 1.4.6   | [ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date                                                            |

@@ -333,7 +333,7 @@ def _install_python_dependencies_from_poetry(
 ) -> Container:
     pip_install_poetry_cmd = ["pip", "install", "poetry"]
     poetry_disable_virtual_env_cmd = ["poetry", "config", "virtualenvs.create", "false"]
-    poetry_install_no_venv_cmd = ["poetry", "install", "--no-root"]
+    poetry_install_no_venv_cmd = ["poetry", "install"]
     if additional_dependency_groups:
         for group in additional_dependency_groups:
             poetry_install_no_venv_cmd += ["--with", group]
@@ -373,7 +373,7 @@ async def with_installed_python_package(
     has_pyproject_toml = await check_path_in_workdir(container, "pyproject.toml")
 
     if has_pyproject_toml:
-        container = _install_python_dependencies_from_poetry(container)
+        container = _install_python_dependencies_from_poetry(container, additional_dependency_groups)
     elif has_setup_py:
         container = _install_python_dependencies_from_setup_py(container, additional_dependency_groups)
     elif has_requirements_txt:
@@ -438,38 +438,6 @@ async def with_python_connector_installed(
     return container
 
 
-async def with_test_python_connector_installed(context: ConnectorContext) -> Container:
-    """Install an airbyte connector python package in a testing environment.
-
-    Args:
-        context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled.
-    Returns:
-        Container: A python environment container (with the connector installed).
-    """
-    connector_source_path = str(context.connector.code_directory)
-    testing_environment: Container = with_testing_dependencies(context)
-    exclude = [
-        f"{context.connector.code_directory}/{item}"
-        for item in [
-            "secrets",
-            "metadata.yaml",
-            "bootstrap.md",
-            "icon.svg",
-            "README.md",
-            "Dockerfile",
-            "acceptance-test-docker.sh",
-            "build.gradle",
-            ".hypothesis",
-            ".dockerignore",
-        ]
-    ]
-    container = await with_python_connector_installed(
-        context, testing_environment, connector_source_path, additional_dependency_groups=["dev", "tests", "main"], exclude=exclude
-    )
-
-    return container
-
-
 async def with_ci_credentials(context: PipelineContext, gsm_secret: Secret) -> Container:
     """Install the ci_credentials package in a python environment.
 
@@ -1032,15 +1000,15 @@ async def mounted_connector_secrets(context: PipelineContext, secret_directory_p
             contents[secret_file_name] = await secret.plaintext()
 
         def with_secrets_mounted_as_regular_files(container: Container) -> Container:
-            container = container.with_exec(["mkdir", secret_directory_path], skip_entrypoint=True)
+            container = container.with_exec(["mkdir", "-p", secret_directory_path], skip_entrypoint=True)
             for secret_file_name, secret_content_str in contents.items():
                 container = container.with_new_file(f"{secret_directory_path}/{secret_file_name}", secret_content_str, permissions=0o600)
             return container
 
         return with_secrets_mounted_as_regular_files
 
     def with_secrets_mounted_as_dagger_secrets(container: Container) -> Container:
-        container = container.with_exec(["mkdir", secret_directory_path], skip_entrypoint=True)
+        container = container.with_exec(["mkdir", "-p", secret_directory_path], skip_entrypoint=True)
         for secret_file_name, secret in context.connector_secrets.items():
             container = container.with_mounted_secret(f"{secret_directory_path}/{secret_file_name}", secret)
         return container

@@ -23,8 +23,8 @@
 from jinja2 import Environment, PackageLoader, select_autoescape
 from pipelines import sentry_utils
 from pipelines.actions import remote_storage
-from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT, PYPROJECT_TOML_FILE_PATH
-from pipelines.utils import METADATA_FILE_NAME, check_path_in_workdir, format_duration, get_exec_result
+from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT
+from pipelines.utils import METADATA_FILE_NAME, format_duration, get_exec_result
 from rich.console import Group
 from rich.panel import Panel
 from rich.style import Style
@@ -276,42 +276,6 @@ def _get_timed_out_step_result(self) -> StepResult:
         )
 
 
-class PytestStep(Step, ABC):
-    """An abstract class to run pytest tests and evaluate success or failure according to pytest logs."""
-
-    skipped_exit_code = 5
-
-    async def _run_tests_in_directory(self, connector_under_test: Container, test_directory: str) -> StepResult:
-        """Run the pytest tests in the test_directory that was passed.
-
-        A StepStatus.SKIPPED is returned if no tests were discovered.
-
-        Args:
-            connector_under_test (Container): The connector under test container.
-            test_directory (str): The directory in which the python test modules are declared
-
-        Returns:
-            Tuple[StepStatus, Optional[str], Optional[str]]: Tuple of StepStatus, stderr and stdout.
-        """
-        test_config = "pytest.ini" if await check_path_in_workdir(connector_under_test, "pytest.ini") else "/" + PYPROJECT_TOML_FILE_PATH
-        if await check_path_in_workdir(connector_under_test, test_directory):
-            tester = connector_under_test.with_exec(
-                [
-                    "python",
-                    "-m",
-                    "pytest",
-                    "-s",
-                    test_directory,
-                    "-c",
-                    test_config,
-                ]
-            )
-            return await self.get_step_result(tester)
-
-        else:
-            return StepResult(self, StepStatus.SKIPPED)
-
-
 class NoOpStep(Step):
     """A step that does nothing."""
 

@@ -91,7 +91,7 @@ async def _build_from_dockerfile(self, platform: Platform) -> Container:
             Container: The connector container built from its Dockerfile.
         """
         self.logger.warn(
-            "This connector is built from its Dockerfile. This is now deprecated. Please set connectorBuildOptions.baseImage metadata field to use or new build process."
+            "This connector is built from its Dockerfile. This is now deprecated. Please set connectorBuildOptions.baseImage metadata field to use our new build process."
         )
         container = self.dagger_client.container(platform=platform).build(await self.context.get_connector_dir())
         container = await apply_python_development_overrides(self.context, container)

@@ -46,7 +46,7 @@ async def run_test(poetry_package_path: str, test_directory: str) -> bool:
     logger = logging.getLogger(f"{poetry_package_path}.tests")
     logger.info(f"Running tests for {poetry_package_path}")
     # The following directories are always mounted because a lot of tests rely on them
-    directories_to_always_mount = [".git", "airbyte-integrations", "airbyte-ci", "airbyte-cdk"]
+    directories_to_always_mount = [".git", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"]
     directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount]))
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client:
         try:

@@ -17,7 +17,7 @@
 from dagger import Container, Directory, File
 from pipelines import hacks
 from pipelines.actions import environments
-from pipelines.bases import CIContext, PytestStep, Step, StepResult, StepStatus
+from pipelines.bases import CIContext, Step, StepResult, StepStatus
 from pipelines.utils import METADATA_FILE_NAME
 
 
@@ -175,12 +175,13 @@ async def _run(self) -> StepResult:
         return await self.get_step_result(qa_checks)
 
 
-class AcceptanceTests(PytestStep):
+class AcceptanceTests(Step):
     """A step to run acceptance tests for a connector if it has an acceptance test config file."""
 
     title = "Acceptance tests"
     CONTAINER_TEST_INPUT_DIRECTORY = "/test_input"
     CONTAINER_SECRETS_DIRECTORY = "/test_input/secrets"
+    skipped_exit_code = 5
 
     @property
     def base_cat_command(self) -> List[str]: