From 0439ce819b4e80bba5aba7fbcdcbe6269aae0d87 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 14:21:18 -0800 Subject: [PATCH 01/10] format pre-commit yaml --- .pre-commit-config.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d89105f08ce..3d178a53bbbf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,18 +4,19 @@ exclude: | ^.*/__init__\.py$| ^.*?/\.venv/.*$| ^.*?/node_modules/.*$| - + ^.*?/charts/.*$| ^airbyte-integrations/bases/base-normalization/.*$| ^.*?/normalization_test_output/.*$| - + ^.*?/pnpm-lock\.yaml$| ^.*?/source-amplitude/unit_tests/api_data/zipped\.json$| - + # Generated/test files ^airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/.*$| ^.*?/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$| ^airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| + ^.*?/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| ^.*?/airbyte-integrations/connectors/destination-.*/expected-spec\.json$ ) @@ -25,8 +26,7 @@ repos: hooks: - id: black language_version: python3.10 - verbose: true - args: ["--verbose"] + args: [--config=pyproject.toml] - repo: https://github.com/pycqa/isort rev: 5.12.0 @@ -34,6 +34,7 @@ repos: - id: isort files: \.py$ require_serial: true + # args: [--settings-file=pyproject.toml] - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.3 From a11e9cce54890ab5a081e26cfc8b8c1bf31f32e3 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 14:32:28 -0800 Subject: [PATCH 02/10] exclude airbyte_ci non-formatted java from spotless formatter --- spotless-maven-pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spotless-maven-pom.xml b/spotless-maven-pom.xml index 63de55c6766d..0a24e538b566 100644 --- a/spotless-maven-pom.xml +++ b/spotless-maven-pom.xml @@ -26,6 +26,9 @@ **/*.java + + **/non_formatted_code/* + 4.21 From b163c3fbc6e69c9e5644d135a1dfbdf94cd20d77 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 17 Dec 2024 14:37:56 -0800 Subject: [PATCH 03/10] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d178a53bbbf..4bad2be24587 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,6 @@ repos: - id: isort files: \.py$ require_serial: true - # args: [--settings-file=pyproject.toml] - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.3 From ac119805100016a187d2059f32f6054cf831bbb0 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 14:53:41 -0800 Subject: [PATCH 04/10] ci: use pre-commit in format-check --- .github/workflows/format_check.yml | 57 ++++++++---------------------- 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 4bb83c118312..4e41f4fe9454 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -10,53 +10,21 @@ on: jobs: format-check: - # IMPORTANT: This name must match the require check name on the branch protection settings name: "Check for formatting errors" - # Do not run this job on forks - # Forked PRs are handled by the community_ci.yml workflow - if: github.event.pull_request.head.repo.fork != true - runs-on: tooling-test-small + runs-on: ubuntu-latest steps: - - name: Checkout Airbyte (with credentials) - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5.3.0 + - uses: pre-commit/action@v3.0.1 + id: format-check with: - ref: ${{ github.head_ref }} - token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - fetch-depth: 1 - - name: Run airbyte-ci format check [MASTER] - id: airbyte_ci_format_check_all_master - if: github.ref == 'refs/heads/master' - uses: ./.github/actions/run-airbyte-ci - continue-on-error: true - with: - context: "master" - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - subcommand: "format check all" - - - name: Run airbyte-ci format check [PULL REQUEST] - id: airbyte_ci_format_check_all_pr - if: github.event_name == 'pull_request' - uses: ./.github/actions/run-airbyte-ci - continue-on-error: false - with: - context: "pull_request" - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - subcommand: "format check all" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }} - - - name: Run airbyte-ci format check [WORKFLOW DISPATCH] - id: airbyte_ci_format_check_all_manual - if: github.event_name == 'workflow_dispatch' - uses: ./.github/actions/run-airbyte-ci - continue-on-error: false - with: - context: "manual" - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - subcommand: "format check all" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }} + extra_args: --all-files - name: Match GitHub User to Slack User [MASTER] - if: github.ref == 'refs/heads/master' + if: > + always() && steps.format-check.outcome == 'failure' && + github.ref == 'refs/heads/master' && + github.event.pull_request.head.repo.fork == false id: match-github-to-slack-user uses: ./.github/actions/match-github-to-slack-user env: @@ -64,7 +32,10 @@ jobs: GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Format Failure on Master Slack Channel [MASTER] - if: steps.airbyte_ci_format_check_all_master.outcome == 'failure' && github.ref == 'refs/heads/master' + if: > + always() && steps.format-check.outcome == 'failure' && + github.ref == 'refs/heads/master' && + github.event.pull_request.head.repo.fork == false uses: abinoda/slack-action@master env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} From 7c6ee8030c3569864df126c03ac201f8a43912ba Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 15:11:01 -0800 Subject: [PATCH 05/10] ci: use newer ubuntu --- .github/workflows/format_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 4e41f4fe9454..1e44846e0cf3 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -11,7 +11,7 @@ on: jobs: format-check: name: "Check for formatting errors" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5.3.0 From 4764973e9cddd22094495dd9dbedd2137c9f48ae Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 15:15:13 -0800 Subject: [PATCH 06/10] ci: explicit java and python setup --- .github/workflows/format_check.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 1e44846e0cf3..efc5b5275c22 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -14,8 +14,17 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5.3.0 - - uses: pre-commit/action@v3.0.1 + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "21" + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 id: format-check with: extra_args: --all-files From 168aad8b02ba356611e01cdaf49df52333bbcd25 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 15:27:59 -0800 Subject: [PATCH 07/10] update ignore --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bad2be24587..c772dc97ae45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,8 +15,9 @@ exclude: | # Generated/test files ^airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/.*$| ^.*?/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$| - ^airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| + ^airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$| ^.*?/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| + ^airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| ^.*?/airbyte-integrations/connectors/destination-.*/expected-spec\.json$ ) From 3bce6b3b10a7de1cfd8df052aae13ba18ff7ba28 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 15:41:13 -0800 Subject: [PATCH 08/10] ci: update format-fix --- .github/workflows/format-fix-command.yml | 27 ++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/format-fix-command.yml b/.github/workflows/format-fix-command.yml index 03988c66d38f..1c46f61c4010 100644 --- a/.github/workflows/format-fix-command.yml +++ b/.github/workflows/format-fix-command.yml @@ -29,8 +29,8 @@ concurrency: jobs: format-fix: - name: "Run airbyte-ci format fix all" - runs-on: ubuntu-latest + name: "Run pre-commit fix" + runs-on: ubuntu-24.04 steps: - name: Get job variables id: job-vars @@ -67,16 +67,21 @@ jobs: [1]: ${{ steps.job-vars.outputs.run-url }} - - name: Run airbyte-ci format fix all - uses: ./.github/actions/run-airbyte-ci - continue-on-error: true + # Compare the below to the `format_check.yml` workflow + - uses: actions/checkout@v4 + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "21" + - name: Setup Python + uses: actions/setup-python@v5 with: - context: "manual" - gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format fix all" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }} + python-version: "3.10" + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + continue-on-error: true + id: format-fix # This is helpful in the case that we change a previously committed generated file to be ignored by git. - name: Remove any files that have been gitignored From 4ff1e85b2b17c78f4550544deb292a9227ceaad8 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 17 Dec 2024 15:49:52 -0800 Subject: [PATCH 09/10] dummy change to break formatting --- .../destination-chroma/destination_chroma/destination.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py index 8dda62704cc3..26290af953d9 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py @@ -25,7 +25,6 @@ BATCH_SIZE = 128 - class DestinationChroma(Destination): indexer: Indexer From aad2e236aa393941eeadd4e4bf1da3bc2d6df8b3 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 17 Dec 2024 23:55:55 +0000 Subject: [PATCH 10/10] chore: auto-fix lint and format issues --- .../destination-chroma/destination_chroma/destination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py index 26290af953d9..8dda62704cc3 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py @@ -25,6 +25,7 @@ BATCH_SIZE = 128 + class DestinationChroma(Destination): indexer: Indexer