merge

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.25
+current_version = 0.50.30
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/pull_request_template.md

@@ -81,7 +81,7 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
 - Issue acceptance criteria met
 - PR name follows [PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/issues-and-pull-requests#pull-request-title-convention)
 - If adding a new generator, add it to the [list of scaffold modules being tested](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connector-templates/generator/build.gradle#L41)
-- The generator test modules (all connectors with `-scaffold` in their name) have been updated with the latest scaffold by running `./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates` then checking in your changes
+- The generator test modules (all connectors with `-scaffold` in their name) have been updated with the latest scaffold by running `./gradlew :airbyte-integrations:connector-templates:generator:generateScaffolds` then checking in your changes
 - Documentation which references the generator is updated as needed
 
 </details>

.github/workflows/airbyte-ci-tests.yml

@@ -1,4 +1,4 @@
-name: Airbyte CI pipeline tests
+name: Connector Ops CI - Pipeline Unit Test
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/cat-tests.yml

@@ -1,4 +1,4 @@
-name: CAT unit tests
+name: Connector Ops CI - CAT Unit Tests
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/connectors_nightly_build.yml

@@ -1,4 +1,4 @@
-name: Connectors nightly build
+name: Connector Ops CI - Connectors Nightly Tests
 
 on:
   schedule:

.github/workflows/connectors_tests.yml

@@ -1,4 +1,4 @@
-name: Connectors tests
+name: Connector Ops CI - Connectors Acceptance Tests
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/connectors_weekly_build.yml

@@ -1,4 +1,4 @@
-name: Connectors weekly build
+name: Connector Ops CI - Connectors Weekly Tests
 
 on:
   schedule:
@@ -41,4 +41,4 @@ jobs:
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           git_branch: ${{ steps.extract_branch.outputs.branch }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=3 --support-level=community' }} test"
+          subcommand: "--show-dagger-logs connectors ${{ inputs.test-connectors-options || '--concurrency=3 --metadata-query=\"(data.ab_internal.ql > 100) & (data.ab_internal.sl < 200)\"' }} test"

.github/workflows/gradle.yml

@@ -102,10 +102,11 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Cache Build Artifacts
-        uses: ./.github/actions/cache-build-artifacts
-        with:
-          cache-key: ${{ secrets.CACHE_VERSION }}-format
+      # Caching causes occasional failure of the checkPython step which can't find the venv
+      # - name: Cache Build Artifacts
+      #   uses: ./.github/actions/cache-build-artifacts
+      #   with:
+      #     cache-key: ${{ secrets.CACHE_VERSION }}-format
 
       - uses: actions/setup-java@v3
         with:
@@ -141,12 +142,12 @@ jobs:
       - name: Remove any files that have been gitignored
         run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
 
-      - name: Commit Formatting Changes
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          commit_message: Automated Commit - Formatting Changes
-          commit_user_name: Octavia Squidington III
-          commit_user_email: [email protected]
+      # - name: Commit Formatting Changes
+      #   uses: stefanzweifel/git-auto-commit-action@v4
+      #   with:
+      #     commit_message: Automated Commit - Formatting Changes
+      #     commit_user_name: Octavia Squidington III
+      #     commit_user_email: [email protected]
 
   ## BUILDS
   octavia-cli-build:
@@ -398,7 +399,7 @@ jobs:
       - name: Generate Template scaffold
         uses: Wandalen/[email protected]
         with:
-          command: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
+          command: ./gradlew :airbyte-integrations:connector-templates:generator:generateScaffolds --scan
           attempt_limit: 3
           attempt_delay: 5000 # in ms
 

.github/workflows/metadata_service_deploy_orchestrator_dagger.yml

@@ -1,4 +1,4 @@
-name: Connector metadata service deploy orchestrator
+name: Connector Ops CI - Metadata Service Deploy Orchestrator
 
 on:
   workflow_dispatch:

.github/workflows/metadata_service_tests_dagger.yml

@@ -1,4 +1,4 @@
-name: Connector metadata service CI
+name: Connector Ops CI -  Metadata Service Tests
 
 on:
   workflow_dispatch:

.github/workflows/publish_connectors.yml

@@ -1,4 +1,4 @@
-name: Publish connectors on merge to master
+name: Connector Ops CI - Publish Connectors
 
 on:
   push:

.github/workflows/run-qa-engine.yml

@@ -1,4 +1,4 @@
-name: Run QA Engine
+name: Connector Ops CI - QA Engine
 
 on:
   workflow_dispatch:

.github/workflows/slash-commands.yml

@@ -19,7 +19,7 @@ jobs:
         id: scd
         uses: peter-evans/slash-command-dispatch@v2
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           permission: write
           commands: |
             test

.github/workflows/upload-metadata-files.yml

@@ -1,4 +1,4 @@
-name: "Upload any Changed Metadata Files [Exceptional Use!]"
+name: "Connector Ops CI - Upload Changed Metadata Files [Emergency Use!]"
 
 on:
   workflow_dispatch:

.gitignore

@@ -102,3 +102,6 @@ tools/ci_connector_ops/pipeline_reports/
 
 # ignore local build scan uri output
 scan-journal.log
+
+# connectors' cache
+*.sqlite

airbyte-api/build.gradle

@@ -7,8 +7,10 @@ plugins {
 
 def specFile = "$projectDir/src/main/openapi/config.yaml"
 
+def generate = tasks.register('generate')
+
 // Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer
-task generateApiServerLegacy(type: GenerateTask) {
+def generateApiServerLegacy = tasks.register('generateApiServerLegacy', GenerateTask) {
     def serverOutputDir = "$buildDir/generated/api/server"
 
     inputs.file specFile
@@ -22,7 +24,7 @@ task generateApiServerLegacy(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.invoker.generated"
     modelPackage = "io.airbyte.api.model.generated"
 
-    schemaMappings = [
+    schemaMappings.set([
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -31,11 +33,11 @@ task generateApiServerLegacy(type: GenerateTask) {
             'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
             'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
             'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ]
+    ])
 
     generateApiDocumentation = false
 
-    configOptions = [
+    configOptions.set([
             dateLibrary  : "java8",
             generatePom  : "false",
             interfaceOnly: "true",
@@ -47,10 +49,13 @@ task generateApiServerLegacy(type: GenerateTask) {
             Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details.
             */
             additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)",
-    ]
+    ])
+}
+generate.configure {
+    dependsOn generateApiServerLegacy
 }
 
-task generateApiServer(type: GenerateTask) {
+def generateApiServer = tasks.register('generateApiServer', GenerateTask) {
     def serverOutputDir = "$buildDir/generated/api/server"
 
     inputs.file specFile
@@ -64,7 +69,7 @@ task generateApiServer(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.invoker.generated"
     modelPackage = "io.airbyte.api.model.generated"
 
-    schemaMappings = [
+    schemaMappings.set([
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -73,11 +78,11 @@ task generateApiServer(type: GenerateTask) {
             'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
             'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
             'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ]
+    ])
 
     generateApiDocumentation = false
 
-    configOptions = [
+    configOptions.set([
             dateLibrary  : "java8",
             generatePom  : "false",
             interfaceOnly: "true",
@@ -92,12 +97,13 @@ task generateApiServer(type: GenerateTask) {
 
             // Generate separate classes for each endpoint "domain"
             useTags: "true"
-    ]
+    ])
+}
+generate.configure {
+    dependsOn generateApiServer
 }
 
-compileJava.dependsOn tasks.generateApiServerLegacy, tasks.generateApiServer
-
-task generateApiClient(type: GenerateTask) {
+def generateApiClient = tasks.register('generateApiClient', GenerateTask) {
     def clientOutputDir = "$buildDir/generated/api/client"
 
     inputs.file specFile
@@ -111,7 +117,7 @@ task generateApiClient(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.client.invoker.generated"
     modelPackage = "io.airbyte.api.client.model.generated"
 
-    schemaMappings = [
+    schemaMappings.set([
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -120,21 +126,23 @@ task generateApiClient(type: GenerateTask) {
             'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
             'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
             'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ]
+    ])
 
     library = "native"
 
     generateApiDocumentation = false
 
-    configOptions = [
+    configOptions.set([
             dateLibrary  : "java8",
             generatePom  : "false",
             interfaceOnly: "true"
-    ]
+    ])
+}
+generate.configure {
+    dependsOn generateApiClient
 }
-compileJava.dependsOn tasks.generateApiClient
 
-task generateApiDocs(type: GenerateTask) {
+def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) {
     def docsOutputDir = "$buildDir/generated/api/docs"
 
     generatorName = "html"
@@ -145,7 +153,7 @@ task generateApiDocs(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.client.invoker.generated"
     modelPackage = "io.airbyte.api.client.model.generated"
 
-    schemaMappings = [
+    schemaMappings.set([
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -154,15 +162,15 @@ task generateApiDocs(type: GenerateTask) {
             'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
             'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
             'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ]
+    ])
 
     generateApiDocumentation = false
 
-    configOptions = [
+    configOptions.set([
             dateLibrary  : "java8",
             generatePom  : "false",
             interfaceOnly: "true"
-    ]
+    ])
 
     doLast {
         def target = file(rootProject.file("docs/reference/api/generated-api-html"))
@@ -176,7 +184,9 @@ task generateApiDocs(type: GenerateTask) {
         }
     }
 }
-compileJava.dependsOn tasks.generateApiDocs
+generate.configure {
+    dependsOn generateApiDocs
+}
 
 dependencies {
     implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310'
@@ -201,4 +211,6 @@ sourceSets {
     }
 }
 
-Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)
+tasks.named('compileJava').configure {
+    dependsOn generate
+}