From 48dd82d722b289b793e0898073a11e6f2cbe38b3 Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 11 Oct 2023 16:09:26 +0200 Subject: [PATCH] Revert "[airbyte-ci] implement pre/post build hooks" (#31280) --- airbyte-ci/connectors/pipelines/README.md | 1 - .../pipelines/builds/build_customization.py | 88 ------------------- .../pipelines/builds/python_connectors.py | 21 ++--- .../connectors/pipelines/pyproject.toml | 2 +- .../test_builds/dummy_build_customization.py | 35 -------- .../test_builds/test_python_connectors.py | 54 ++---------- .../connectors/source-zendesk-chat/Dockerfile | 20 +++++ .../build_customization.py | 23 ----- .../source-zendesk-chat/metadata.yaml | 4 +- docs/integrations/sources/zendesk-chat.md | 1 - 10 files changed, 38 insertions(+), 211 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py delete mode 100644 airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/Dockerfile delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/build_customization.py diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index f087303eb3dad..b18881e6f33aa 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -395,7 +395,6 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 1.7.0 | [#30526](https://github.com/airbytehq/airbyte/pull/30526) | Implement pre/post install hooks support. | | 1.6.0 | [#30474](https://github.com/airbytehq/airbyte/pull/30474) | Test connector inside their containers. | | 1.5.1 | [#31227](https://github.com/airbytehq/airbyte/pull/31227) | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'. | | 1.5.0 | [#30456](https://github.com/airbytehq/airbyte/pull/30456) | Start building Python connectors using our base images. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py b/airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py deleted file mode 100644 index 03d6f13f9757c..0000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -import importlib -from logging import Logger -from types import ModuleType -from typing import List, Optional - -from connector_ops.utils import Connector -from dagger import Container - -BUILD_CUSTOMIZATION_MODULE_NAME = "build_customization" -BUILD_CUSTOMIZATION_SPEC_NAME = f"{BUILD_CUSTOMIZATION_MODULE_NAME}.py" -DEFAULT_MAIN_FILE_NAME = "main.py" - - -def get_build_customization_module(connector: Connector) -> Optional[ModuleType]: - """Import the build_customization.py file from the connector directory if it exists. - Returns: - Optional[ModuleType]: The build_customization.py module if it exists, None otherwise. - """ - build_customization_spec_path = connector.code_directory / BUILD_CUSTOMIZATION_SPEC_NAME - if not build_customization_spec_path.exists(): - return None - build_customization_spec = importlib.util.spec_from_file_location( - f"{connector.code_directory.name}_{BUILD_CUSTOMIZATION_MODULE_NAME}", build_customization_spec_path - ) - build_customization_module = importlib.util.module_from_spec(build_customization_spec) - build_customization_spec.loader.exec_module(build_customization_module) - return build_customization_module - - -def get_main_file_name(connector: Connector) -> str: - """Get the main file name from the build_customization.py module if it exists, DEFAULT_MAIN_FILE_NAME otherwise. - - Args: - connector (Connector): The connector to build. - - Returns: - str: The main file name. - """ - build_customization_module = get_build_customization_module(connector) - if hasattr(build_customization_module, "MAIN_FILE_NAME"): - return build_customization_module.MAIN_FILE_NAME - return DEFAULT_MAIN_FILE_NAME - - -def get_entrypoint(connector: Connector) -> List[str]: - main_file_name = get_main_file_name(connector) - return ["python", f"/airbyte/integration_code/{main_file_name}"] - - -async def pre_install_hooks(connector: Connector, base_container: Container, logger: Logger) -> Container: - """Run the pre_connector_install hook if it exists in the build_customization.py module. - It will mutate the base_container and return it. - - Args: - connector (Connector): The connector to build. - base_container (Container): The base container to mutate. - logger (Logger): The logger to use. - - Returns: - Container: The mutated base_container. - """ - build_customization_module = get_build_customization_module(connector) - if hasattr(build_customization_module, "pre_connector_install"): - base_container = await build_customization_module.pre_connector_install(base_container) - logger.info(f"Connector {connector.technical_name} pre install hook executed.") - return base_container - - -async def post_install_hooks(connector: Connector, connector_container: Container, logger: Logger) -> Container: - """Run the post_connector_install hook if it exists in the build_customization.py module. - It will mutate the connector_container and return it. - - Args: - connector (Connector): The connector to build. - connector_container (Container): The connector container to mutate. - logger (Logger): The logger to use. - - Returns: - Container: The mutated connector_container. - """ - build_customization_module = get_build_customization_module(connector) - if hasattr(build_customization_module, "post_connector_install"): - connector_container = await build_customization_module.post_connector_install(connector_container) - logger.info(f"Connector {connector.technical_name} post install hook executed.") - return connector_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py index 7c97ca3981f62..c12bbb91f5f16 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py @@ -6,7 +6,6 @@ from dagger import Container, Platform from pipelines.actions.environments import apply_python_development_overrides, with_python_connector_installed from pipelines.bases import StepResult -from pipelines.builds import build_customization from pipelines.builds.common import BuildConnectorImagesBase from pipelines.contexts import ConnectorContext @@ -17,6 +16,7 @@ class BuildConnectorImages(BuildConnectorImagesBase): A spec command is run on the container to validate it was built successfully. """ + DEFAULT_ENTRYPOINT = ["python", "/airbyte/integration_code/main.py"] PATH_TO_INTEGRATION_CODE = "/airbyte/integration_code" async def _build_connector(self, platform: Platform): @@ -35,6 +35,8 @@ def _get_base_container(self, platform: Platform) -> Container: async def _create_builder_container(self, base_container: Container) -> Container: """Pre install the connector dependencies in a builder container. + If a python connectors depends on another local python connector, we need to mount its source in the container + This occurs for the source-file-secure connector for example, which depends on source-file Args: base_container (Container): The base container to use to build the connector. @@ -60,11 +62,7 @@ async def _build_from_base_image(self, platform: Platform) -> Container: """ self.logger.info("Building connector from base image in metadata") base = self._get_base_container(platform) - customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger) - entrypoint = build_customization.get_entrypoint(self.context.connector) - main_file_name = build_customization.get_main_file_name(self.context.connector) - - builder = await self._create_builder_container(customized_base) + builder = await self._create_builder_container(base) # The snake case name of the connector corresponds to the python package name of the connector # We want to mount it to the container under PATH_TO_INTEGRATION_CODE/connector_snake_case_name @@ -72,20 +70,19 @@ async def _build_from_base_image(self, platform: Platform) -> Container: connector_container = ( # copy python dependencies from builder to connector container - customized_base.with_directory("/usr/local", builder.directory("/usr/local")) + base.with_directory("/usr/local", builder.directory("/usr/local")) .with_workdir(self.PATH_TO_INTEGRATION_CODE) - .with_file(main_file_name, (await self.context.get_connector_dir(include=main_file_name)).file(main_file_name)) + .with_file("main.py", (await self.context.get_connector_dir(include="main.py")).file("main.py")) .with_directory( connector_snake_case_name, (await self.context.get_connector_dir(include=connector_snake_case_name)).directory(connector_snake_case_name), ) - .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) - .with_entrypoint(entrypoint) + .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(self.DEFAULT_ENTRYPOINT)) + .with_entrypoint(self.DEFAULT_ENTRYPOINT) .with_label("io.airbyte.version", self.context.connector.metadata["dockerImageTag"]) .with_label("io.airbyte.name", self.context.connector.metadata["dockerRepository"]) ) - customized_connector = await build_customization.post_install_hooks(self.context.connector, connector_container, self.logger) - return customized_connector + return connector_container async def _build_from_dockerfile(self, platform: Platform) -> Container: """Build the connector container using its Dockerfile. diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 4967da65b2187..1ae1e3f5e9632 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.7.0" +version = "1.6.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py b/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py deleted file mode 100644 index 70840b06ce0bd..0000000000000 --- a/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - """This function will run before the connector installation. - It can mutate the base image container. - - Args: - base_image_container (Container): The base image container to mutate. - - Returns: - Container: The mutated base image container. - """ - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - - -async def post_connector_install(connector_container: Container) -> Container: - """This function will run after the connector installation during the build process. - It can mutate the connector container. - - Args: - connector_container (Container): The connector container to mutate. - - Returns: - Container: The mutated connector container. - """ - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") diff --git a/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py index d8b042491fb30..576726b54011b 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py @@ -2,11 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from pathlib import Path - import pytest from pipelines.bases import StepStatus -from pipelines.builds import build_customization, python_connectors +from pipelines.builds import python_connectors from pipelines.contexts import ConnectorContext pytestmark = [ @@ -30,41 +28,17 @@ def test_context_with_connector_without_base_image(self, test_context): return test_context @pytest.fixture - def connector_with_base_image_no_build_customization(self, all_connectors): + def connector_with_base_image(self, all_connectors): for connector in all_connectors: if connector.metadata and connector.metadata.get("connectorBuildOptions", {}).get("baseImage"): - if not (connector.code_directory / "build_customization.py").exists(): - return connector + return connector pytest.skip("No connector with a connectorBuildOptions.baseImage metadata found") @pytest.fixture - def connector_with_base_image_with_build_customization(self, connector_with_base_image_no_build_customization): - dummy_build_customization = (Path(__file__).parent / "dummy_build_customization.py").read_text() - (connector_with_base_image_no_build_customization.code_directory / "build_customization.py").write_text(dummy_build_customization) - yield connector_with_base_image_no_build_customization - (connector_with_base_image_no_build_customization.code_directory / "build_customization.py").unlink() - - @pytest.fixture - def test_context_with_real_connector_using_base_image(self, connector_with_base_image_no_build_customization, dagger_client): - context = ConnectorContext( - pipeline_name="test build", - connector=connector_with_base_image_no_build_customization, - git_branch="test", - git_revision="test", - report_output_prefix="test", - is_local=True, - use_remote_secrets=True, - ) - context.dagger_client = dagger_client - return context - - @pytest.fixture - def test_context_with_real_connector_using_base_image_with_build_customization( - self, connector_with_base_image_with_build_customization, dagger_client - ): + def test_context_with_real_connector_using_base_image(self, connector_with_base_image, dagger_client): context = ConnectorContext( pipeline_name="test build", - connector=connector_with_base_image_with_build_customization, + connector=connector_with_base_image, git_branch="test", git_revision="test", report_output_prefix="test", @@ -113,11 +87,9 @@ async def test_building_from_base_image_for_real(self, test_context_with_real_co step_result = await step._run() step_result.status is StepStatus.SUCCESS built_container = step_result.output_artifact[current_platform] - assert await built_container.env_variable("AIRBYTE_ENTRYPOINT") == " ".join( - build_customization.get_entrypoint(step.context.connector) - ) + assert await built_container.env_variable("AIRBYTE_ENTRYPOINT") == " ".join(step.DEFAULT_ENTRYPOINT) assert await built_container.workdir() == step.PATH_TO_INTEGRATION_CODE - assert await built_container.entrypoint() == build_customization.get_entrypoint(step.context.connector) + assert await built_container.entrypoint() == step.DEFAULT_ENTRYPOINT assert ( await built_container.label("io.airbyte.version") == test_context_with_real_connector_using_base_image.connector.metadata["dockerImageTag"] @@ -127,18 +99,6 @@ async def test_building_from_base_image_for_real(self, test_context_with_real_co == test_context_with_real_connector_using_base_image.connector.metadata["dockerRepository"] ) - async def test_building_from_base_image_with_customization_for_real( - self, test_context_with_real_connector_using_base_image_with_build_customization, current_platform - ): - step = python_connectors.BuildConnectorImages( - test_context_with_real_connector_using_base_image_with_build_customization, current_platform - ) - step_result = await step._run() - step_result.status is StepStatus.SUCCESS - built_container = step_result.output_artifact[current_platform] - assert await built_container.env_variable("MY_PRE_BUILD_ENV_VAR") == "my_pre_build_env_var_value" - assert await built_container.env_variable("MY_POST_BUILD_ENV_VAR") == "my_post_build_env_var_value" - async def test__run_using_base_dockerfile_with_mocks(self, mocker, test_context_with_connector_without_base_image, current_platform): container_built_from_dockerfile = mocker.AsyncMock() mocker.patch.object( diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile new file mode 100644 index 0000000000000..c7764ec2d2e5b --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.9-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="source_zendesk_chat" +ENV AIRBYTE_IMPL_MODULE="source_zendesk_chat" +ENV AIRBYTE_IMPL_PATH="SourceZendeskChat" +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main_dev.py" + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY main_dev.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] + +LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py deleted file mode 100644 index b974ca15ee594..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from dagger import Container - -MAIN_FILE_NAME = "main_dev.py" - - -async def pre_connector_install(base_image_container: Container) -> Container: - """This function will run before the connector installation. - We set these environment variable to match what was originally in the Dockerfile. - Disclaimer: I have no idea if these env vars are actually needed. - """ - return await ( - base_image_container.with_env_variable("AIRBYTE_IMPL_MODULE", "source_zendesk_chat").with_env_variable( - "AIRBYTE_IMPL_PATH", "SourceZendeskChat" - ) - ) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 18185bf76a273..5d9c006e08538 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -2,12 +2,10 @@ data: allowedHosts: hosts: - zopim.com - connectorBuildOptions: - baseImage: airbyte/python-connector-base:1.0.0 connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 0.2.0 + dockerImageTag: 0.1.14 dockerRepository: airbyte/source-zendesk-chat githubIssueLabel: source-zendesk-chat icon: zendesk-chat.svg diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index 31cfcba59e2c9..eab821953fd88 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,7 +80,6 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | -| 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | | 0.1.14 | 2023-02-10 | [24190](https://github.com/airbytehq/airbyte/pull/24190) | Fix remove too high min/max from account stream | | 0.1.13 | 2023-02-10 | [22819](https://github.com/airbytehq/airbyte/pull/22819) | Specified date formatting in specification | | 0.1.12 | 2023-01-27 | [22026](https://github.com/airbytehq/airbyte/pull/22026) | Set `AvailabilityStrategy` for streams explicitly to `None` |