diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index c9206c9b9009..297fc7145936 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -853,9 +853,10 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only ## Changelog | Version | PR | Description | -|---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | -| 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | +| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.47.0 | [#49832](https://github.com/airbytehq/airbyte/pull/49462) | Build java connectors from the base image declared in `metadata.yaml`. | +| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | +| 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | | 4.46.3 | [#49465](https://github.com/airbytehq/airbyte/pull/49465) | Fix `--use-local-cdk` on rootless connectors. | | 4.46.2 | [#49136](https://github.com/airbytehq/airbyte/pull/49136) | Fix failed install of python components due to non-root permissions. | | 4.46.1 | [#49146](https://github.com/airbytehq/airbyte/pull/49146) | Update `crane` image address as the one we were using has been deleted by the maintainer. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index ad7fb7e4bcdf..47bbe7822b21 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -9,9 +9,10 @@ from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions.connector.hooks import finalize_build from pipelines.dagger.actions.connector.normalization import DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, with_normalization -from pipelines.helpers.utils import sh_dash_c +from pipelines.helpers.utils import deprecated, sh_dash_c +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container: return ( context.dagger_client.container(platform=build_platform) @@ -24,6 +25,7 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) -> ) +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container: integration_base = with_integration_base(context, build_platform) yum_packages_to_install = [ @@ -72,6 +74,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor ) +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base_java_and_normalization(context: ConnectorContext, build_platform: Platform) -> Container: yum_packages_to_install = [ "python3", @@ -158,22 +161,31 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ ) ) ) - - if ( + # TODO: remove the condition below once all connectors have a base image declared in their metadata. + if "connectorBuildOptions" in context.connector.metadata and "baseImage" in context.connector.metadata["connectorBuildOptions"]: + base_image_address = context.connector.metadata["connectorBuildOptions"]["baseImage"] + context.logger.info(f"Using base image {base_image_address} from connector metadata to build connector.") + base = context.dagger_client.container(platform=build_platform).from_(base_image_address) + elif ( context.connector.supports_normalization and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"] ): - base = with_integration_base_java_and_normalization(context, build_platform) - entrypoint = ["/airbyte/run_with_normalization.sh"] + context.logger.warn( + f"Connector {context.connector.technical_name} has in-connector normalization enabled. This is supposed to be deprecated. " + f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." + ) + base = with_integration_base_java_and_normalization(context, build_platform).with_entrypoint(["/airbyte/run_with_normalization.sh"]) else: - base = with_integration_base_java(context, build_platform) - entrypoint = ["/airbyte/base.sh"] + context.logger.warn( + f"Connector {context.connector.technical_name} does not declare a base image in its connector metadata. " + f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." + ) + base = with_integration_base_java(context, build_platform).with_entrypoint(["/airbyte/base.sh"]) connector_container = ( base.with_workdir("/airbyte") .with_env_variable("APPLICATION", application) .with_mounted_directory("built_artifacts", build_stage.directory("/airbyte")) .with_exec(sh_dash_c(["mv built_artifacts/* ."])) - .with_entrypoint(entrypoint) ) return await finalize_build(context, connector_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index 8ab32e8754e3..ca397153becd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -7,10 +7,12 @@ import contextlib import datetime +import functools import os import re import sys import unicodedata +import warnings import xml.sax.saxutils from io import TextIOWrapper from pathlib import Path @@ -388,3 +390,15 @@ async def raise_if_not_user(container: Container, expected_user: str) -> None: assert ( actual_user == expected_user ), f"Container is not running as the expected user '{expected_user}', it is running as '{actual_user}'." + + +def deprecated(reason: str) -> Callable: + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + warnings.warn(f"{func.__name__} is deprecated: {reason}", DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index dc32ae4df374..468b9b0e925c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.46.5" +version = "4.47.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]