From 152d9c33b64d1498e5b137b77c9445537327d84e Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 25 Apr 2024 12:22:31 -0700 Subject: [PATCH 01/11] exclude zookeeper --- airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle | 2 +- .../connectors/destination-databricks/build.gradle | 1 + airbyte-integrations/connectors/destination-s3/build.gradle | 1 + .../connectors/destination-starburst-galaxy/build.gradle | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index cc81388e4707..914d5f191ffb 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -27,7 +27,7 @@ dependencies { api 'org.apache.commons:commons-csv:1.10.0' api 'org.apache.commons:commons-text:1.11.0' api ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' } - api 'org.apache.hadoop:hadoop-common:3.3.6' + api ('org.apache.hadoop:hadoop-common:3.3.6') { exclude group: 'org.apache.zookeeper', module: 'zookeeper' } api 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6' api 'org.apache.parquet:parquet-avro:1.13.1' runtimeOnly 'com.hadoop.gplcompression:hadoop-lzo:0.4.20' diff --git a/airbyte-integrations/connectors/destination-databricks/build.gradle b/airbyte-integrations/connectors/destination-databricks/build.gradle index 28be23138318..e42535196e2e 100644 --- a/airbyte-integrations/connectors/destination-databricks/build.gradle +++ b/airbyte-integrations/connectors/destination-databricks/build.gradle @@ -47,6 +47,7 @@ dependencies { implementation ('org.apache.hadoop:hadoop-common:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' } implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index e02dbe7c386f..85b649d3dbe2 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -27,6 +27,7 @@ dependencies { implementation ('org.apache.hadoop:hadoop-common:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' } implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle index ffe2bf71cf6b..83392eead160 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation 'org.apache.iceberg:iceberg-aws:1.1.0' implementation 'org.apache.iceberg:iceberg-parquet:1.1.0' - implementation 'org.apache.hadoop:hadoop-common:3.3.3' + implementation ('org.apache.hadoop:hadoop-common:3.3.3') { exclude group: 'org.apache.zookeeper', module: 'zookeeper' } implementation "org.apache.hadoop:hadoop-aws:3.3.2" implementation 'software.amazon.awssdk:bundle:2.20.20' From fca6cd1642c71416fb7dbe5644586a5a2e76ed48 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 25 Apr 2024 14:53:20 -0700 Subject: [PATCH 02/11] ignore sbg --- .../connectors/destination-starburst-galaxy/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle index 83392eead160..ffe2bf71cf6b 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation 'org.apache.iceberg:iceberg-aws:1.1.0' implementation 'org.apache.iceberg:iceberg-parquet:1.1.0' - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { exclude group: 'org.apache.zookeeper', module: 'zookeeper' } + implementation 'org.apache.hadoop:hadoop-common:3.3.3' implementation "org.apache.hadoop:hadoop-aws:3.3.2" implementation 'software.amazon.awssdk:bundle:2.20.20' From 7b1b136dc72db939d3768ff4b032733ce87678ef Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 25 Apr 2024 15:57:16 -0700 Subject: [PATCH 03/11] Bigquery maybe builds? --- .../core/src/main/resources/version.properties | 2 +- .../java/airbyte-cdk/s3-destinations/build.gradle | 12 ++++++++++-- .../connectors/destination-bigquery/build.gradle | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 7208b1456165..229d3800320f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.30.11 +version=0.31.0 diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index 914d5f191ffb..c7d326e57baf 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -27,8 +27,16 @@ dependencies { api 'org.apache.commons:commons-csv:1.10.0' api 'org.apache.commons:commons-text:1.11.0' api ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' } - api ('org.apache.hadoop:hadoop-common:3.3.6') { exclude group: 'org.apache.zookeeper', module: 'zookeeper' } - api 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6' + api ('org.apache.hadoop:hadoop-common:3.3.6') { + exclude group: 'org.apache.zookeeper' + exclude group: 'org.apache.hadoop', module: 'hadoop-auth' + exclude group: 'org.apache.hadoop', module: 'hadoop-yarn-common' + } + api ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6') { + exclude group: 'org.apache.zookeeper' + exclude group: 'org.apache.hadoop', module: 'hadoop-auth' + exclude group: 'org.apache.hadoop', module: 'hadoop-yarn-common' + } api 'org.apache.parquet:parquet-avro:1.13.1' runtimeOnly 'com.hadoop.gplcompression:hadoop-lzo:0.4.20' diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index cdbcc1b97dee..8ed2da4e7e37 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -3,7 +3,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.30.11' + cdkVersionRequired = '0.31.0' features = [ 'db-destinations', 'datastore-bigquery', @@ -11,7 +11,7 @@ airbyteJavaConnector { 'gcs-destinations', 'core', ] - useLocalCdk = false + useLocalCdk = true } java { From 54b32740c58bcb52191d027afcdaa27afedb41b6 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 25 Apr 2024 15:58:09 -0700 Subject: [PATCH 04/11] just one destination at a time --- .../connectors/destination-databricks/build.gradle | 1 - airbyte-integrations/connectors/destination-s3/build.gradle | 1 - 2 files changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-databricks/build.gradle b/airbyte-integrations/connectors/destination-databricks/build.gradle index e42535196e2e..28be23138318 100644 --- a/airbyte-integrations/connectors/destination-databricks/build.gradle +++ b/airbyte-integrations/connectors/destination-databricks/build.gradle @@ -47,7 +47,6 @@ dependencies { implementation ('org.apache.hadoop:hadoop-common:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' - exclude group: 'org.apache.zookeeper', module: 'zookeeper' } implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index 85b649d3dbe2..e02dbe7c386f 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -27,7 +27,6 @@ dependencies { implementation ('org.apache.hadoop:hadoop-common:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j' - exclude group: 'org.apache.zookeeper', module: 'zookeeper' } implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { From 5eeeb7b6a1a280ad275c57366c65456957d2b57f Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 25 Apr 2024 18:04:57 -0700 Subject: [PATCH 05/11] allow hadoop auth --- airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index c7d326e57baf..893868766092 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -29,12 +29,10 @@ dependencies { api ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' } api ('org.apache.hadoop:hadoop-common:3.3.6') { exclude group: 'org.apache.zookeeper' - exclude group: 'org.apache.hadoop', module: 'hadoop-auth' exclude group: 'org.apache.hadoop', module: 'hadoop-yarn-common' } api ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6') { exclude group: 'org.apache.zookeeper' - exclude group: 'org.apache.hadoop', module: 'hadoop-auth' exclude group: 'org.apache.hadoop', module: 'hadoop-yarn-common' } api 'org.apache.parquet:parquet-avro:1.13.1' From cb23c2ceb7ac9afe59e1cec838655618915b11fb Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 26 Apr 2024 08:52:50 -0700 Subject: [PATCH 06/11] Empty-Commit From f812e086cca1702b45b0462e47002225f20637d5 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 26 Apr 2024 08:54:52 -0700 Subject: [PATCH 07/11] bump connector version --- .../connectors/destination-bigquery/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 3b80791db3b6..2a41f20ee0a0 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.4.13 + dockerImageTag: 2.4.14 dockerRepository: airbyte/destination-bigquery documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery githubIssueLabel: destination-bigquery diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 1918b306b1cc..60a4dfed8f56 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -209,8 +209,9 @@ tutorials: ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.4.13 | 2024-02-25 | [36899](https://github.com/airbytehq/airbyte/pull/36899) | adopt latest CDK | +| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.4.13 | 2024-02-25 | [37584](https://github.com/airbytehq/airbyte/pull/37584) | Remove unused insecure dependencies from CDK | +| 2.4.13 | 2024-02-25 | [36899](https://github.com/airbytehq/airbyte/pull/36899) | adopt latest CDK | | 2.4.12 | 2024-03-04 | [35315](https://github.com/airbytehq/airbyte/pull/35315) | Adopt CDK 0.23.11 | | 2.4.11 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | | 2.4.10 | 2024-02-15 | [35240](https://github.com/airbytehq/airbyte/pull/35240) | Adopt CDK 0.20.9 | From 02aadcfad66be3f5062fd46ec277bdfe853bf636 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 26 Apr 2024 08:55:12 -0700 Subject: [PATCH 08/11] fix docs version --- docs/integrations/destinations/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 60a4dfed8f56..4b791f4050c0 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -210,7 +210,7 @@ tutorials: | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2.4.13 | 2024-02-25 | [37584](https://github.com/airbytehq/airbyte/pull/37584) | Remove unused insecure dependencies from CDK | +| 2.4.14 | 2024-02-25 | [37584](https://github.com/airbytehq/airbyte/pull/37584) | Remove unused insecure dependencies from CDK | | 2.4.13 | 2024-02-25 | [36899](https://github.com/airbytehq/airbyte/pull/36899) | adopt latest CDK | | 2.4.12 | 2024-03-04 | [35315](https://github.com/airbytehq/airbyte/pull/35315) | Adopt CDK 0.23.11 | | 2.4.11 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | From ec1b4a30800583ee95d173b5177aef0988c7b0b3 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 26 Apr 2024 12:19:58 -0700 Subject: [PATCH 09/11] bump changelog --- airbyte-cdk/java/airbyte-cdk/README.md | 127 +++++++++++++++---------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 31a25528b5ec..1a2d29e77b1b 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -2,22 +2,22 @@ This page will walk through the process of developing with the Java CDK. -* [Developing with the Java CDK](#developing-with-the-java-cdk) - * [Intro to the Java CDK](#intro-to-the-java-cdk) - * [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) - * [How is the CDK published?](#how-is-the-cdk-published) - * [Using the Java CDK](#using-the-java-cdk) - * [Building the CDK](#building-the-cdk) - * [Bumping the CDK version](#bumping-the-cdk-version) - * [Publishing the CDK](#publishing-the-cdk) - * [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) - * [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) - * [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) - * [Publishing the CDK and switching to a pinned CDK reference](#publishing-the-cdk-and-switching-to-a-pinned-cdk-reference) - * [Troubleshooting CDK Dependency Caches](#troubleshooting-cdk-dependency-caches) - * [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) - * [Changelog](#changelog) - * [Java CDK](#java-cdk) +- [Developing with the Java CDK](#developing-with-the-java-cdk) + - [Intro to the Java CDK](#intro-to-the-java-cdk) + - [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) + - [How is the CDK published?](#how-is-the-cdk-published) + - [Using the Java CDK](#using-the-java-cdk) + - [Building the CDK](#building-the-cdk) + - [Bumping the CDK version](#bumping-the-cdk-version) + - [Publishing the CDK](#publishing-the-cdk) + - [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) + - [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) + - [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) + - [Publishing the CDK and switching to a pinned CDK reference](#publishing-the-cdk-and-switching-to-a-pinned-cdk-reference) + - [Troubleshooting CDK Dependency Caches](#troubleshooting-cdk-dependency-caches) + - [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) + - [Changelog](#changelog) + - [Java CDK](#java-cdk) ## Intro to the Java CDK @@ -31,15 +31,23 @@ The java CDK is comprised of separate modules, among which: Each CDK submodule may contain these elements: -- `src/main` - (Required.) The classes that will ship with the connector, providing capabilities to the connectors. -- `src/test` - (Required.) These are unit tests that run as part of every build of the CDK. They help ensure that CDK `main` code is in a healthy state. -- `src/testFixtures` - (Optional.) These shared classes are exported for connectors for use in the connectors' own test implementations. Connectors will have access to these classes within their unit and integration tests, but the classes will not be shipped with connectors when they are published. +- `src/main` - (Required.) The classes that will ship with the connector, providing capabilities to + the connectors. +- `src/test` - (Required.) These are unit tests that run as part of every build of the CDK. They + help ensure that CDK `main` code is in a healthy state. +- `src/testFixtures` - (Optional.) These shared classes are exported for connectors for use in the + connectors' own test implementations. Connectors will have access to these classes within their + unit and integration tests, but the classes will not be shipped with connectors when they are + published. ### How is the CDK published? -The CDK is published as a set of jar files sharing a version number. Every submodule generates one runtime jar for the main classes. If the submodule contains test fixtures, a second jar will be published with the test fixtures classes. +The CDK is published as a set of jar files sharing a version number. Every submodule generates one +runtime jar for the main classes. If the submodule contains test fixtures, a second jar will be +published with the test fixtures classes. -Note: Connectors do not have to manage which jars they should depend on, as this is handled automatically by the `airbyte-java-connector` plugin. See example below. +Note: Connectors do not have to manage which jars they should depend on, as this is handled +automatically by the `airbyte-java-connector` plugin. See example below. ## Using the Java CDK @@ -55,7 +63,8 @@ To build and test the Java CDK, execute the following: You will need to bump this version manually whenever you are making changes to code inside the CDK. -While under development, the next version number for the CDK is tracked in the file: `airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties`. +While under development, the next version number for the CDK is tracked in the file: +`airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties`. If the CDK is not being modified, this file will contain the most recently published version number. @@ -63,9 +72,11 @@ If the CDK is not being modified, this file will contain the most recently publi _⚠️ These steps should only be performed after all testing and approvals are in place on the PR. ⚠️_ -The CDK can be published with a GitHub Workflow and a slash command which can be run by Airbyte personnel. +The CDK can be published with a GitHub Workflow and a slash command which can be run by Airbyte +personnel. -To invoke via slash command (recommended), use the following syntax in a comment on the PR that contains your changes: +To invoke via slash command (recommended), use the following syntax in a comment on the PR that +contains your changes: ```bash /publish-java-cdk # Run with the defaults (dry-run=false, force=false) @@ -77,12 +88,18 @@ Note: - Remember to **document your changes** in the Changelog section below. - After you publish the CDK, remember to toggle `useLocalCdk` back to `false` in all connectors. -- Unless you specify `force=true`, the pipeline should fail if the version you are trying to publish already exists. -- By running the publish with `dry-run=true`, you can confirm the process is working as expected, without actually publishing the changes. -- In dry-run mode, you can also view and download the jars that are generated. To do so, navigate to the job status in GitHub Actions and navigate to the 'artifacts' section. -- You can also invoke manually in the GitHub Web UI. To do so: go to `Actions` tab, select the `Publish Java CDK` workflow, and click `Run workflow`. -- You can view and administer published CDK versions here: https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/cdk -- The public endpoint for published CDK versions is here: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/cdk/ +- Unless you specify `force=true`, the pipeline should fail if the version you are trying to publish + already exists. +- By running the publish with `dry-run=true`, you can confirm the process is working as expected, + without actually publishing the changes. +- In dry-run mode, you can also view and download the jars that are generated. To do so, navigate to + the job status in GitHub Actions and navigate to the 'artifacts' section. +- You can also invoke manually in the GitHub Web UI. To do so: go to `Actions` tab, select the + `Publish Java CDK` workflow, and click `Run workflow`. +- You can view and administer published CDK versions here: + https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/cdk +- The public endpoint for published CDK versions is here: + https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/cdk/ ## Developing Connectors with the Java CDK @@ -104,20 +121,26 @@ airbyteJavaConnector { ``` -Replace `0.1.0` with the CDK version you are working with. If you're actively developing the CDK and want to use the latest version locally, use the `useLocalCdk` flag to use the live CDK code during builds and tests. +Replace `0.1.0` with the CDK version you are working with. If you're actively developing the CDK and +want to use the latest version locally, use the `useLocalCdk` flag to use the live CDK code during +builds and tests. ### Developing a connector alongside the CDK -You can iterate on changes in the CDK local and test them in the connector without needing to publish the CDK changes publicly. +You can iterate on changes in the CDK local and test them in the connector without needing to +publish the CDK changes publicly. When modifying the CDK and a connector in the same PR or branch, please use the following steps: -1. Set the version of the CDK in `version.properties` to the next appropriate version number and add a description in the `Changelog` at the bottom of this readme file. +1. Set the version of the CDK in `version.properties` to the next appropriate version number and add + a description in the `Changelog` at the bottom of this readme file. 2. Modify your connector's build.gradle file as follows: - 1. Set `useLocalCdk` to `true` in the connector you are working on. This will ensure the connector always uses the local CDK definitions instead of the published version. + 1. Set `useLocalCdk` to `true` in the connector you are working on. This will ensure the + connector always uses the local CDK definitions instead of the published version. 2. Set `cdkVersionRequired` to use the new _to-be-published_ CDK version. -After the above, you can build and test your connector as usual. Gradle will automatically use the local CDK code files while you are working on the connector. +After the above, you can build and test your connector as usual. Gradle will automatically use the +local CDK code files while you are working on the connector. ### Publishing the CDK and switching to a pinned CDK reference @@ -128,30 +151,38 @@ Once you are done developing and testing your CDK changes: ### Troubleshooting CDK Dependency Caches -Note: after switching between a local and a pinned CDK reference, you may need to refresh dependency caches in Gradle and/or your IDE. +Note: after switching between a local and a pinned CDK reference, you may need to refresh dependency +caches in Gradle and/or your IDE. -In Gradle, you can use the CLI arg `--refresh-dependencies` the next time you build or test your connector, which will ensure that the correct version of the CDK is used after toggling the `useLocalCdk` value. +In Gradle, you can use the CLI arg `--refresh-dependencies` the next time you build or test your +connector, which will ensure that the correct version of the CDK is used after toggling the +`useLocalCdk` value. ### Developing a connector against a pinned CDK version -You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`. +You can always pin your connector to a prior stable version of the CDK, which may not match what is +the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` +while the latest version may be `0.2.0`. -Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your connector, and you can use your local IDE to browse the prior version of the codebase that corresponds to that version. +Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your +connector, and you can use your local IDE to browse the prior version of the codebase that +corresponds to that version. ## Changelog ### Java CDK | Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.30.11 | 2024-04-25 | [\#36899](https://github.com/airbytehq/airbyte/pull/36899) | changes for bigQuery destination. | -| 0.30.10 | 2024-04-24 | [\#37541](https://github.com/airbytehq/airbyte/pull/37541) | remove excessive logging | -| 0.30.9 | 2024-04-24 | [\#37477](https://github.com/airbytehq/airbyte/pull/37477) | remove unnecessary logs -| 0.30.7 | 2024-04-23 | [\#37477](https://github.com/airbytehq/airbyte/pull/37477) | fix kotlin warnings in core CDK submodule -| 0.30.7 | 2024-04-23 | [\#37484](https://github.com/airbytehq/airbyte/pull/37484) | fix kotlin warnings in dependencies CDK submodule | -| 0.30.7 | 2024-04-23 | [\#37479](https://github.com/airbytehq/airbyte/pull/37479) | fix kotlin warnings in azure-destination, datastore-{bigquery,mongo,postgres} CDK submodules | -| 0.30.7 | 2024-04-23 | [\#37481](https://github.com/airbytehq/airbyte/pull/37481) | fix kotlin warnings in destination CDK submodules | -| 0.30.7 | 2024-04-23 | [\#37482](https://github.com/airbytehq/airbyte/pull/37482) | fix kotlin warnings in db-sources CDK submodule | +| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.31.0 | 2024-04-26 | [\#37584](https://github.com/airbytehq/airbyte/pull/37584) | Update S# destination deps to exclude zookeeper and hadoop-yarn-common | +| 0.30.11 | 2024-04-25 | [\#36899](https://github.com/airbytehq/airbyte/pull/36899) | changes for bigQuery destination. | +| 0.30.10 | 2024-04-24 | [\#37541](https://github.com/airbytehq/airbyte/pull/37541) | remove excessive logging | +| 0.30.9 | 2024-04-24 | [\#37477](https://github.com/airbytehq/airbyte/pull/37477) | remove unnecessary logs | +| 0.30.7 | 2024-04-23 | [\#37477](https://github.com/airbytehq/airbyte/pull/37477) | fix kotlin warnings in core CDK submodule | +| 0.30.7 | 2024-04-23 | [\#37484](https://github.com/airbytehq/airbyte/pull/37484) | fix kotlin warnings in dependencies CDK submodule | +| 0.30.7 | 2024-04-23 | [\#37479](https://github.com/airbytehq/airbyte/pull/37479) | fix kotlin warnings in azure-destination, datastore-{bigquery,mongo,postgres} CDK submodules | +| 0.30.7 | 2024-04-23 | [\#37481](https://github.com/airbytehq/airbyte/pull/37481) | fix kotlin warnings in destination CDK submodules | +| 0.30.7 | 2024-04-23 | [\#37482](https://github.com/airbytehq/airbyte/pull/37482) | fix kotlin warnings in db-sources CDK submodule | | 0.30.6 | 2024-04-19 | [\#37442](https://github.com/airbytehq/airbyte/pull/37442) | Destinations: Rename File format related classes to be agnostic of S3 | | 0.30.3 | 2024-04-12 | [\#37106](https://github.com/airbytehq/airbyte/pull/37106) | Destinations: Simplify constructors in `AsyncStreamConsumer` | | 0.30.2 | 2024-04-12 | [\#36926](https://github.com/airbytehq/airbyte/pull/36926) | Destinations: Remove `JdbcSqlOperations#formatData`; misc changes for java interop | From 02611126466154f77aefc04b98d46368925782bd Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 26 Apr 2024 12:20:59 -0700 Subject: [PATCH 10/11] typo --- airbyte-cdk/java/airbyte-cdk/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 1a2d29e77b1b..e4de26ec6039 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -174,7 +174,7 @@ corresponds to that version. | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0.31.0 | 2024-04-26 | [\#37584](https://github.com/airbytehq/airbyte/pull/37584) | Update S# destination deps to exclude zookeeper and hadoop-yarn-common | +| 0.31.0 | 2024-04-26 | [\#37584](https://github.com/airbytehq/airbyte/pull/37584) | Update S3 destination deps to exclude zookeeper and hadoop-yarn-common | | 0.30.11 | 2024-04-25 | [\#36899](https://github.com/airbytehq/airbyte/pull/36899) | changes for bigQuery destination. | | 0.30.10 | 2024-04-24 | [\#37541](https://github.com/airbytehq/airbyte/pull/37541) | remove excessive logging | | 0.30.9 | 2024-04-24 | [\#37477](https://github.com/airbytehq/airbyte/pull/37477) | remove unnecessary logs | From 40348e0069afb6b416ce19c300ec76a50fc0346d Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 26 Apr 2024 13:14:20 -0700 Subject: [PATCH 11/11] useLocalCdk=false --- .../connectors/destination-bigquery/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 8ed2da4e7e37..71cd37427777 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -11,7 +11,7 @@ airbyteJavaConnector { 'gcs-destinations', 'core', ] - useLocalCdk = true + useLocalCdk = false } java {