PoC: Enable manifest-only connector unit tests (#48790)

Co-authored-by: Octavia Squidington III <[email protected]>

airbyte-ci/connectors/pipelines/README.md

@@ -854,7 +854,8 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
-| 4.45.3  | [#48927](https://github.com/airbytehq/airbyte/pull/48927)      | Fix bug in determine_changelog_entry_comment                                                                                 |
+| 4.46.0  | [#48790](https://github.com/airbytehq/airbyte/pull/48790)  | Add unit tests step for manifest-only connectors                                                                             |
+| 4.45.3  | [#48927](https://github.com/airbytehq/airbyte/pull/48927)      | Fix bug in determine_changelog_entry_comment                                                                             |
 | 4.45.2  | [#48868](https://github.com/airbytehq/airbyte/pull/48868)  | Fix ownership issues while using `--use-local-cdk`                                                                           |
 | 4.45.1  | [#48872](https://github.com/airbytehq/airbyte/pull/48872)  | Make the `connectors list` command write its output to a JSON file.                                                          |
 | 4.45.0  | [#48866](https://github.com/airbytehq/airbyte/pull/48866)  | Adds `--rc` option to `bump-version` command                                                                                 |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py

@@ -4,12 +4,17 @@
 
 """This module groups steps made to run tests for a specific manifest only connector given a test context."""
 
+from typing import List, Sequence, Tuple
+
+from dagger import Container, File
 from pipelines.airbyte_ci.connectors.build_image.steps.manifest_only_connectors import BuildConnectorImages
 from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID
 from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext
 from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, IncrementalAcceptanceTests, LiveTests
+from pipelines.airbyte_ci.connectors.test.steps.python_connectors import PytestStep
 from pipelines.consts import LOCAL_BUILD_PLATFORM
 from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun
+from pipelines.models.steps import StepResult
 
 
 def get_test_steps(context: ConnectorTestContext) -> STEP_TREE:
@@ -20,6 +25,12 @@ def get_test_steps(context: ConnectorTestContext) -> STEP_TREE:
     return [
         [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD, step=BuildConnectorImages(context))],
         [
+            StepToRun(
+                id=CONNECTOR_TEST_STEP_ID.UNIT,
+                step=ManifestOnlyConnectorUnitTests(context),
+                args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]},
+                depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
+            ),
             StepToRun(
                 id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE,
                 step=AcceptanceTests(
@@ -46,3 +57,61 @@ def get_test_steps(context: ConnectorTestContext) -> STEP_TREE:
             )
         ],
     ]
+
+
+class ManifestOnlyConnectorUnitTests(PytestStep):
+    """A step to run unit tests for a manifest-only connector"""
+
+    title = "Manifest-only unit tests"
+    test_directory_name = "unit_tests"
+    common_test_dependencies = ["pytest"]
+
+    async def install_testing_environment(
+        self,
+        built_connector_container: Container,
+        test_config_file_name: str,
+        test_config_file: File,
+        extra_dependencies_names: Sequence[str],
+    ) -> Container:
+        """Install the testing environment for manifest-only connectors."""
+
+        connector_name = self.context.connector.technical_name
+        connector_path = f"/airbyte-integrations/connectors/{connector_name}"
+
+        # Create a symlink between the local connector's directory and the built container
+        test_environment = built_connector_container.with_workdir(f"{connector_path}/unit_tests").with_exec(
+            ["ln", "-s", "/source-declarative-manifest", connector_path]
+        )
+
+        # Specify the root user for installation of dependencies.
+        # This is necessary to install poetry dependencies in rootless containers.
+        test_environment = test_environment.with_user("root")
+
+        return await super().install_testing_environment(
+            test_environment,
+            test_config_file_name,
+            test_config_file,
+            extra_dependencies_names,
+        )
+
+    async def get_config_file_name_and_file(self) -> Tuple[str, File]:
+        """
+        Get the config file name and file to use for pytest.
+        For manifest-only connectors, we expect the poetry config to be found in the unit_tests directory.
+        """
+        connector_name = self.context.connector.technical_name
+        connector_dir = await self.context.get_connector_dir()
+        unit_tests_dir = connector_dir.directory("unit_tests")
+        unit_tests_entries = await unit_tests_dir.entries()
+        if self.PYPROJECT_FILE_NAME in unit_tests_entries:
+            config_file_name = self.PYPROJECT_FILE_NAME
+            test_config = unit_tests_dir.file(self.PYPROJECT_FILE_NAME)
+            self.logger.info(f"Found {self.PYPROJECT_FILE_NAME} in the unit_tests directory for {connector_name}, using it for testing.")
+            return config_file_name, test_config
+        else:
+            raise FileNotFoundError(f"Could not find {self.PYPROJECT_FILE_NAME} in the unit_tests directory for {connector_name}.")
+
+    def get_pytest_command(self, test_config_file_name: str) -> List[str]:
+        """Get the pytest command to run."""
+        cmd = ["pytest", "-s", self.test_directory_name, "-c", test_config_file_name] + self.params_as_cli_options
+        return ["poetry", "run"] + cmd

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.45.3"
+version = "4.46.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml

@@ -3,9 +3,10 @@ data:
     ql: 100
     sl: 100
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/source-declarative-manifest:6.9.2@sha256:ea8087899b36a891ce16e47035b10de8d52b0fb041b593b18e53ed2d699e3b46
+    baseImage: docker.io/airbyte/source-declarative-manifest:6.10.0@sha256:58722e84dbd06bb2af9250e37d24d1c448e247fc3a84d75ee4407d52771b6f03
   connectorSubtype: api
   connectorTestSuitesOptions:
+    - suite: unitTests
     - suite: liveTests
       testConnections:
         - name: the-guardian-api_config_dev_null
@@ -19,7 +20,7 @@ data:
             type: GSM
   connectorType: source
   definitionId: d42bd69f-6bf0-4d0b-9209-16231af07a92
-  dockerImageTag: 0.2.3
+  dockerImageTag: 0.2.4
   dockerRepository: airbyte/source-the-guardian-api
   documentationUrl: https://docs.airbyte.com/integrations/sources/the-guardian-api
   githubIssueLabel: source-the-guardian-api

airbyte-integrations/connectors/source-the-guardian-api/unit_tests/conftest.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+pytest_plugins = ["airbyte_cdk.test.utils.manifest_only_fixtures"]