From 2e3fef4c27e0a6ce01c8eb3f8bfcb8a6fb941cf2 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 17 Dec 2024 17:21:58 -0800 Subject: [PATCH] Patch: AJ's pre-commit updates (#49846) Co-authored-by: Octavia Squidington III --- .github/workflows/format-fix-command.yml | 27 ++++++----- .github/workflows/format_check.yml | 62 ++++++++---------------- .pre-commit-config.yaml | 11 +++-- spotless-maven-pom.xml | 3 ++ 4 files changed, 46 insertions(+), 57 deletions(-) diff --git a/.github/workflows/format-fix-command.yml b/.github/workflows/format-fix-command.yml index 03988c66d38f..1c46f61c4010 100644 --- a/.github/workflows/format-fix-command.yml +++ b/.github/workflows/format-fix-command.yml @@ -29,8 +29,8 @@ concurrency: jobs: format-fix: - name: "Run airbyte-ci format fix all" - runs-on: ubuntu-latest + name: "Run pre-commit fix" + runs-on: ubuntu-24.04 steps: - name: Get job variables id: job-vars @@ -67,16 +67,21 @@ jobs: [1]: ${{ steps.job-vars.outputs.run-url }} - - name: Run airbyte-ci format fix all - uses: ./.github/actions/run-airbyte-ci - continue-on-error: true + # Compare the below to the `format_check.yml` workflow + - uses: actions/checkout@v4 + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "21" + - name: Setup Python + uses: actions/setup-python@v5 with: - context: "manual" - gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format fix all" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }} + python-version: "3.10" + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + continue-on-error: true + id: format-fix # This is helpful in the case that we change a previously committed generated file to be ignored by git. - name: Remove any files that have been gitignored diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 4bb83c118312..efc5b5275c22 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -10,53 +10,30 @@ on: jobs: format-check: - # IMPORTANT: This name must match the require check name on the branch protection settings name: "Check for formatting errors" - # Do not run this job on forks - # Forked PRs are handled by the community_ci.yml workflow - if: github.event.pull_request.head.repo.fork != true - runs-on: tooling-test-small + runs-on: ubuntu-24.04 steps: - - name: Checkout Airbyte (with credentials) - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - name: Setup Java + uses: actions/setup-java@v3 with: - ref: ${{ github.head_ref }} - token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - fetch-depth: 1 - - name: Run airbyte-ci format check [MASTER] - id: airbyte_ci_format_check_all_master - if: github.ref == 'refs/heads/master' - uses: ./.github/actions/run-airbyte-ci - continue-on-error: true + distribution: "zulu" + java-version: "21" + - name: Setup Python + uses: actions/setup-python@v5 with: - context: "master" - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - subcommand: "format check all" - - - name: Run airbyte-ci format check [PULL REQUEST] - id: airbyte_ci_format_check_all_pr - if: github.event_name == 'pull_request' - uses: ./.github/actions/run-airbyte-ci - continue-on-error: false - with: - context: "pull_request" - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - subcommand: "format check all" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }} - - - name: Run airbyte-ci format check [WORKFLOW DISPATCH] - id: airbyte_ci_format_check_all_manual - if: github.event_name == 'workflow_dispatch' - uses: ./.github/actions/run-airbyte-ci - continue-on-error: false + python-version: "3.10" + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + id: format-check with: - context: "manual" - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - subcommand: "format check all" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }} + extra_args: --all-files - name: Match GitHub User to Slack User [MASTER] - if: github.ref == 'refs/heads/master' + if: > + always() && steps.format-check.outcome == 'failure' && + github.ref == 'refs/heads/master' && + github.event.pull_request.head.repo.fork == false id: match-github-to-slack-user uses: ./.github/actions/match-github-to-slack-user env: @@ -64,7 +41,10 @@ jobs: GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Format Failure on Master Slack Channel [MASTER] - if: steps.airbyte_ci_format_check_all_master.outcome == 'failure' && github.ref == 'refs/heads/master' + if: > + always() && steps.format-check.outcome == 'failure' && + github.ref == 'refs/heads/master' && + github.event.pull_request.head.repo.fork == false uses: abinoda/slack-action@master env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d89105f08ce..c772dc97ae45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,17 +4,19 @@ exclude: | ^.*/__init__\.py$| ^.*?/\.venv/.*$| ^.*?/node_modules/.*$| - + ^.*?/charts/.*$| ^airbyte-integrations/bases/base-normalization/.*$| ^.*?/normalization_test_output/.*$| - + ^.*?/pnpm-lock\.yaml$| ^.*?/source-amplitude/unit_tests/api_data/zipped\.json$| - + # Generated/test files ^airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/.*$| ^.*?/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$| + ^airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$| + ^.*?/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| ^airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$| ^.*?/airbyte-integrations/connectors/destination-.*/expected-spec\.json$ ) @@ -25,8 +27,7 @@ repos: hooks: - id: black language_version: python3.10 - verbose: true - args: ["--verbose"] + args: [--config=pyproject.toml] - repo: https://github.com/pycqa/isort rev: 5.12.0 diff --git a/spotless-maven-pom.xml b/spotless-maven-pom.xml index 63de55c6766d..0a24e538b566 100644 --- a/spotless-maven-pom.xml +++ b/spotless-maven-pom.xml @@ -26,6 +26,9 @@ **/*.java + + **/non_formatted_code/* + 4.21