From fcdf2f2826fad6d350e9e0213fd83bb2851f8907 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 17 Dec 2024 10:18:20 +0100 Subject: [PATCH 1/4] airbyte-ci: use the base image to build java connectors --- airbyte-ci/connectors/pipelines/README.md | 7 ++++--- .../pipelines/dagger/containers/java.py | 21 ++++++++++++++++--- .../pipelines/pipelines/helpers/utils.py | 14 +++++++++++++ .../connectors/pipelines/pyproject.toml | 2 +- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index c9206c9b9009..297fc7145936 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -853,9 +853,10 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only ## Changelog | Version | PR | Description | -|---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | -| 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | +| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.47.0 | [#49832](https://github.com/airbytehq/airbyte/pull/49462) | Build java connectors from the base image declared in `metadata.yaml`. | +| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | +| 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | | 4.46.3 | [#49465](https://github.com/airbytehq/airbyte/pull/49465) | Fix `--use-local-cdk` on rootless connectors. | | 4.46.2 | [#49136](https://github.com/airbytehq/airbyte/pull/49136) | Fix failed install of python components due to non-root permissions. | | 4.46.1 | [#49146](https://github.com/airbytehq/airbyte/pull/49146) | Update `crane` image address as the one we were using has been deleted by the maintainer. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index ad7fb7e4bcdf..44f35e40da15 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -9,9 +9,10 @@ from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions.connector.hooks import finalize_build from pipelines.dagger.actions.connector.normalization import DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, with_normalization -from pipelines.helpers.utils import sh_dash_c +from pipelines.helpers.utils import deprecated, sh_dash_c +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container: return ( context.dagger_client.container(platform=build_platform) @@ -24,6 +25,7 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) -> ) +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container: integration_base = with_integration_base(context, build_platform) yum_packages_to_install = [ @@ -72,6 +74,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor ) +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base_java_and_normalization(context: ConnectorContext, build_platform: Platform) -> Container: yum_packages_to_install = [ "python3", @@ -158,14 +161,26 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ ) ) ) - - if ( + # TODO: remove the condition below once all connectors have a base image declared in their metadata. + if "connectorBuildOptions" in context.connector.metadata and "baseImage" in context.connector.metadata["connectorBuildOptions"]: + base_image_address = context.connector.metadata["connectorBuildOptions"]["baseImage"] + context.logger.info(f"Using base image {base_image_address} from connector metadata to build connector.") + base = context.dagger_client.container(platform=build_platform).from_(base_image_address) + elif ( context.connector.supports_normalization and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"] ): + context.logger.warn( + f"Connector {context.connector.technical_name} has in-connector normalization enabled. This is supposed to be deprecated. " + f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." + ) base = with_integration_base_java_and_normalization(context, build_platform) entrypoint = ["/airbyte/run_with_normalization.sh"] else: + context.logger.warn( + f"Connector {context.connector.technical_name} does not declare a base image in its connector metadata. " + f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." + ) base = with_integration_base_java(context, build_platform) entrypoint = ["/airbyte/base.sh"] diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index 8ab32e8754e3..ca397153becd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -7,10 +7,12 @@ import contextlib import datetime +import functools import os import re import sys import unicodedata +import warnings import xml.sax.saxutils from io import TextIOWrapper from pathlib import Path @@ -388,3 +390,15 @@ async def raise_if_not_user(container: Container, expected_user: str) -> None: assert ( actual_user == expected_user ), f"Container is not running as the expected user '{expected_user}', it is running as '{actual_user}'." + + +def deprecated(reason: str) -> Callable: + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + warnings.warn(f"{func.__name__} is deprecated: {reason}", DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index dc32ae4df374..468b9b0e925c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.46.5" +version = "4.47.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 8b31d07eeffb67b511ec10d1815a992844746934 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 17 Dec 2024 10:33:02 +0100 Subject: [PATCH 2/4] to revert: make source-postgres use the RC java base image --- airbyte-integrations/connectors/source-postgres/metadata.yaml | 4 +++- docs/integrations/sources/postgres.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index ad14df0875a6..120f12791417 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.6.24 + dockerImageTag: 3.6.25 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres @@ -26,6 +26,8 @@ data: supportLevel: certified tags: - language:java + connectorBuildOptions: + baseImage: docker.io/airbyte/java-connector-base:1.0.0-rc.2@sha256:fca66e81b4d2e4869a03b57b1b34beb048e74f5d08deb2046c3bb9919e7e2273 connectorTestSuitesOptions: - suite: unitTests - suite: integrationTests diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index ebab3c84a6e0..1fe09abd45b4 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -329,6 +329,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.6.25 | 2024-12-17 | [49832](https://github.com/airbytehq/airbyte/pull/49832) | Use `airbyte/java-connector-base` as a base image. | | 3.6.24 | 2024-12-16 | [49469](https://github.com/airbytehq/airbyte/pull/49469) | Simplify CTID_TABLE_BLOCK_SIZE query for Postgres integration | | 3.6.23 | 2024-11-13 | [\#48482](https://github.com/airbytehq/airbyte/pull/48482) | Convert large integer typed using NUMERIC(X, 0) into a BigInteger. l | 3.6.22 | 2024-10-02 | [46900](https://github.com/airbytehq/airbyte/pull/46900) | Fixed a bug where source docs won't render on Airbyte 1.1 | From e1f9da6ee7d07c1108c35f0722f283dc5c632c25 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 17 Dec 2024 10:48:53 +0100 Subject: [PATCH 3/4] fix entrypoint --- .../pipelines/pipelines/dagger/containers/java.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index 44f35e40da15..47bbe7822b21 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -174,21 +174,18 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ f"Connector {context.connector.technical_name} has in-connector normalization enabled. This is supposed to be deprecated. " f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." ) - base = with_integration_base_java_and_normalization(context, build_platform) - entrypoint = ["/airbyte/run_with_normalization.sh"] + base = with_integration_base_java_and_normalization(context, build_platform).with_entrypoint(["/airbyte/run_with_normalization.sh"]) else: context.logger.warn( f"Connector {context.connector.technical_name} does not declare a base image in its connector metadata. " f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." ) - base = with_integration_base_java(context, build_platform) - entrypoint = ["/airbyte/base.sh"] + base = with_integration_base_java(context, build_platform).with_entrypoint(["/airbyte/base.sh"]) connector_container = ( base.with_workdir("/airbyte") .with_env_variable("APPLICATION", application) .with_mounted_directory("built_artifacts", build_stage.directory("/airbyte")) .with_exec(sh_dash_c(["mv built_artifacts/* ."])) - .with_entrypoint(entrypoint) ) return await finalize_build(context, connector_container) From b647e438e00955046fe5e1b08142c833c13732a0 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 17 Dec 2024 11:59:11 +0100 Subject: [PATCH 4/4] Revert "to revert: make source-postgres use the RC java base image" This reverts commit c01c2904a79ceaeb4a432e8056df0f31949afce4. --- airbyte-integrations/connectors/source-postgres/metadata.yaml | 4 +--- docs/integrations/sources/postgres.md | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index 120f12791417..ad14df0875a6 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.6.25 + dockerImageTag: 3.6.24 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres @@ -26,8 +26,6 @@ data: supportLevel: certified tags: - language:java - connectorBuildOptions: - baseImage: docker.io/airbyte/java-connector-base:1.0.0-rc.2@sha256:fca66e81b4d2e4869a03b57b1b34beb048e74f5d08deb2046c3bb9919e7e2273 connectorTestSuitesOptions: - suite: unitTests - suite: integrationTests diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 1fe09abd45b4..ebab3c84a6e0 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -329,7 +329,6 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.6.25 | 2024-12-17 | [49832](https://github.com/airbytehq/airbyte/pull/49832) | Use `airbyte/java-connector-base` as a base image. | | 3.6.24 | 2024-12-16 | [49469](https://github.com/airbytehq/airbyte/pull/49469) | Simplify CTID_TABLE_BLOCK_SIZE query for Postgres integration | | 3.6.23 | 2024-11-13 | [\#48482](https://github.com/airbytehq/airbyte/pull/48482) | Convert large integer typed using NUMERIC(X, 0) into a BigInteger. l | 3.6.22 | 2024-10-02 | [46900](https://github.com/airbytehq/airbyte/pull/46900) | Fixed a bug where source docs won't render on Airbyte 1.1 |