Github Action: Add format.yml workflow (#30604)

Co-authored-by: bnchrch <[email protected]>
Co-authored-by: octavia-approvington <[email protected]>

.devcontainer/destination-duckdb/devcontainer.json

@@ -23,21 +23,21 @@
   "customizations": {
     "vscode": {
       "extensions": [
-				// Python extensions:
-				"charliermarsh.ruff",
-				"matangover.mypy",
-				"ms-python.black-formatter",
-				"ms-python.python",
-				"ms-python.vscode-pylance",
+        // Python extensions:
+        "charliermarsh.ruff",
+        "matangover.mypy",
+        "ms-python.black-formatter",
+        "ms-python.python",
+        "ms-python.vscode-pylance",
 
-				// Toml support
-				"tamasfe.even-better-toml",
-
-				// Yaml and JSON Schema support:
-				"redhat.vscode-yaml",
+        // Toml support
+        "tamasfe.even-better-toml",
 
-				// Contributing:
-				"GitHub.vscode-pull-request-github"
+        // Yaml and JSON Schema support:
+        "redhat.vscode-yaml",
+
+        // Contributing:
+        "GitHub.vscode-pull-request-github"
       ],
       "settings": {
         "extensions.ignoreRecommendations": true,
@@ -53,8 +53,8 @@
           "."
         ],
         "[python]": {
-            "editor.defaultFormatter": "ms-python.black-formatter"
-        }    
+          "editor.defaultFormatter": "ms-python.black-formatter"
+        }
       }
     }
   },

.github/workflows/connectors_tests.yml

@@ -1,7 +1,15 @@
 name: Connector Ops CI - Connectors Acceptance Tests
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
+  #
+  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
+  #  ensuring that only one run per pull request is active at a time.
+  #
+  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
+  # This allows us to run multiple workflow dispatches in parallel.
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 
 on:
   workflow_dispatch:

.github/workflows/format.yml

@@ -0,0 +1,72 @@
+name: Format Code (Python + Java)
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  format-and-commit:
+    runs-on: ubuntu-latest
+    name: "Apply All Formatting Rules"
+    timeout-minutes: 20
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          # Important that this is set so that CI checks are triggered again
+          # Without this we would be be forever waiting on required checks to pass
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m \
+            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Format
+        uses: Wandalen/[email protected]
+        with:
+          command: ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+
+      # This is helpful in the case that we change a previously committed generated file to be ignored by git.
+      - name: Remove any files that have been gitignored
+        run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
+
+      - name: Commit Formatting Changes (PR)
+        uses: stefanzweifel/git-auto-commit-action@v4
+        # do not commit if master branch
+        if: github.ref != 'refs/heads/master'
+        with:
+          commit_message: Automated Commit - Formatting Changes
+          commit_user_name: Octavia Squidington III
+          commit_user_email: [email protected]
+
+      - name: "Fail on Formatting Changes (Master)"
+        if: github.ref == 'refs/heads/master'
+        run: git --no-pager diff && test -z "$(git --no-pager diff)"
+

.github/workflows/gradle.yml

@@ -89,71 +89,10 @@ jobs:
   #      - run: |
   #          echo '${{ toJSON(needs) }}'
 
-  format:
-    needs: changes
-    runs-on: ubuntu-latest
-    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
-    if: needs.changes.outputs.any_change == 'true' || (always() && github.ref == 'refs/heads/master')
-    name: "Apply All Formatting Rules"
-    timeout-minutes: 20
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-
-      # Caching causes occasional failure of the checkPython step which can't find the venv
-      # - name: Cache Build Artifacts
-      #   uses: ./.github/actions/cache-build-artifacts
-      #   with:
-      #     cache-key: ${{ secrets.CACHE_VERSION }}-format
-
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: "17"
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-
-      - name: Set up CI Gradle Properties
-        run: |
-          mkdir -p ~/.gradle/
-          cat > ~/.gradle/gradle.properties <<EOF
-          org.gradle.jvmargs=-Xmx8g -Xss4m \
-            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-          org.gradle.workers.max=8
-          org.gradle.vfs.watch=false
-          EOF
-
-      - name: Format
-        uses: Wandalen/[email protected]
-        with:
-          command: ./gradlew format --scan --info --stacktrace
-          attempt_limit: 3
-          attempt_delay: 5000 # in ms
-
-      # This is helpful in the case that we change a previously committed generated file to be ignored by git.
-      - name: Remove any files that have been gitignored
-        run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
-
-      # - name: Commit Formatting Changes
-      #   uses: stefanzweifel/git-auto-commit-action@v4
-      #   with:
-      #     commit_message: Automated Commit - Formatting Changes
-      #     commit_user_name: Octavia Squidington III
-      #     commit_user_email: [email protected]
-
   ## BUILDS
   octavia-cli-build:
     needs:
       - changes
-      - format
     runs-on: ubuntu-latest
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -212,7 +151,6 @@ jobs:
   python-cdk-build:
     needs:
       - changes
-      - format
     runs-on: ubuntu-latest
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.python_cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -267,7 +205,6 @@ jobs:
   java-cdk-build:
     needs:
       - changes
-      - format
     runs-on: ubuntu-latest
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.java_cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -314,7 +251,6 @@ jobs:
     name: "Connectors Base: Start Build EC2 Runner"
     needs:
       - changes
-      - format
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: |
       needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors_base == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -399,21 +335,14 @@ jobs:
       - name: Generate Template scaffold
         uses: Wandalen/[email protected]
         with:
-          command: ./gradlew :airbyte-integrations:connector-templates:generator:generateScaffolds --scan
+          command: CI=true ./gradlew :airbyte-integrations:connector-templates:generator:generateScaffolds --scan
           attempt_limit: 3
           attempt_delay: 5000 # in ms
 
-      # Verify that the only committed file changes will be those made by the build, formatter or processResources
-      - name: Ensure no file change from code formatting
+      # Verify that the the scaffold generation did not change any files. Meaning that the generated files are up to date.
+      - name: Ensure no file change from scaffold generation
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
-      - name: Format
-        uses: Wandalen/[email protected]
-        with:
-          command: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
-          attempt_limit: 3
-          attempt_delay: 5000 # in ms
-
       - name: Build
         uses: Wandalen/[email protected]
         with:
@@ -428,17 +357,6 @@ jobs:
           attempt_limit: 3
           attempt_delay: 5000 # in ms
 
-      # This is helpful in the case that we change a previously commited generated file to be ignored by git.
-      - name: Remove any files that have been gitignored
-        run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
-
-      - name: Commit Changes
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          commit_message: Automated Commit - Format and Process Resources Changes
-          commit_user_name: Octavia Squidington III
-          commit_user_email: [email protected]
-
       - name: Publish Connectors Base Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
         id: connectors-test-results

airbyte-cdk/python/airbyte_cdk/entrypoint.py

@@ -7,14 +7,14 @@
 import ipaddress
 import logging
 import os.path
-import requests
 import socket
 import sys
 import tempfile
 from functools import wraps
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib.parse import urlparse
 
+import requests
 from airbyte_cdk.connector import TConfig
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
 from airbyte_cdk.logger import init_logger