airbyte-ci: use the base image to build java connectors (#49832)

airbyte-ci/connectors/pipelines/README.md

@@ -853,9 +853,10 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 ## Changelog
 
 | Version | PR                                                         | Description                                                                                                                  |
-|---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
-| 4.46.5  | [#49835](https://github.com/airbytehq/airbyte/pull/49835)  | Fix connector language discovery for projects with Kotlin Gradle build scripts.                                             |
-| 4.46.4  | [#49462](https://github.com/airbytehq/airbyte/pull/49462)  | Support Kotlin Gradle build scripts in connectors.                                                                            |
+| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.47.0  | [#49832](https://github.com/airbytehq/airbyte/pull/49462)  | Build java connectors from the base image declared in `metadata.yaml`.                                                       |
+| 4.46.5  | [#49835](https://github.com/airbytehq/airbyte/pull/49835)  | Fix connector language discovery for projects with Kotlin Gradle build scripts.                                              |
+| 4.46.4  | [#49462](https://github.com/airbytehq/airbyte/pull/49462)  | Support Kotlin Gradle build scripts in connectors.                                                                           |
 | 4.46.3  | [#49465](https://github.com/airbytehq/airbyte/pull/49465)  | Fix `--use-local-cdk` on rootless connectors.                                                                                |
 | 4.46.2  | [#49136](https://github.com/airbytehq/airbyte/pull/49136)  | Fix failed install of python components due to non-root permissions.                                                         |
 | 4.46.1  | [#49146](https://github.com/airbytehq/airbyte/pull/49146)  | Update `crane` image address as the one we were using has been deleted by the maintainer.                                    |

airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py

@@ -9,9 +9,10 @@
 from pipelines.consts import AMAZONCORRETTO_IMAGE
 from pipelines.dagger.actions.connector.hooks import finalize_build
 from pipelines.dagger.actions.connector.normalization import DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, with_normalization
-from pipelines.helpers.utils import sh_dash_c
+from pipelines.helpers.utils import deprecated, sh_dash_c
 
 
+@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.")
 def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container:
     return (
         context.dagger_client.container(platform=build_platform)
@@ -24,6 +25,7 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) ->
     )
 
 
+@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.")
 def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container:
     integration_base = with_integration_base(context, build_platform)
     yum_packages_to_install = [
@@ -72,6 +74,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor
     )
 
 
+@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.")
 def with_integration_base_java_and_normalization(context: ConnectorContext, build_platform: Platform) -> Container:
     yum_packages_to_install = [
         "python3",
@@ -158,22 +161,31 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_
             )
         )
     )
-
-    if (
+    # TODO: remove the condition below once all connectors have a base image declared in their metadata.
+    if "connectorBuildOptions" in context.connector.metadata and "baseImage" in context.connector.metadata["connectorBuildOptions"]:
+        base_image_address = context.connector.metadata["connectorBuildOptions"]["baseImage"]
+        context.logger.info(f"Using base image {base_image_address} from connector metadata to build connector.")
+        base = context.dagger_client.container(platform=build_platform).from_(base_image_address)
+    elif (
         context.connector.supports_normalization
         and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"]
     ):
-        base = with_integration_base_java_and_normalization(context, build_platform)
-        entrypoint = ["/airbyte/run_with_normalization.sh"]
+        context.logger.warn(
+            f"Connector {context.connector.technical_name} has in-connector normalization enabled. This is supposed to be deprecated. "
+            f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)."
+        )
+        base = with_integration_base_java_and_normalization(context, build_platform).with_entrypoint(["/airbyte/run_with_normalization.sh"])
     else:
-        base = with_integration_base_java(context, build_platform)
-        entrypoint = ["/airbyte/base.sh"]
+        context.logger.warn(
+            f"Connector {context.connector.technical_name} does not declare a base image in its connector metadata. "
+            f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)."
+        )
+        base = with_integration_base_java(context, build_platform).with_entrypoint(["/airbyte/base.sh"])
 
     connector_container = (
         base.with_workdir("/airbyte")
         .with_env_variable("APPLICATION", application)
         .with_mounted_directory("built_artifacts", build_stage.directory("/airbyte"))
         .with_exec(sh_dash_c(["mv built_artifacts/* ."]))
-        .with_entrypoint(entrypoint)
     )
     return await finalize_build(context, connector_container)

airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py

@@ -7,10 +7,12 @@
 
 import contextlib
 import datetime
+import functools
 import os
 import re
 import sys
 import unicodedata
+import warnings
 import xml.sax.saxutils
 from io import TextIOWrapper
 from pathlib import Path
@@ -388,3 +390,15 @@ async def raise_if_not_user(container: Container, expected_user: str) -> None:
     assert (
         actual_user == expected_user
     ), f"Container is not running as the expected user '{expected_user}', it is running as '{actual_user}'."
+
+
+def deprecated(reason: str) -> Callable:
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            warnings.warn(f"{func.__name__} is deprecated: {reason}", DeprecationWarning, stacklevel=2)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.46.5"
+version = "4.47.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]