Bump shandy-sqlfmt[jinjafmt] from 0.23.2 to 0.24.0 #189
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci_pipeline | |
| on: | |
| pull_request_target: | |
| branches: | |
| - stg | |
| - prd | |
| env: | |
| DBT_CICD_RUN: "true" | |
| DESTINATION_BRANCH: "${{ github.event.pull_request.base.ref }}" | |
| DBT_DATASET: "ci_${{ github.event.number }}_${{ github.run_number }}_${{ github.sha }}" | |
| DBT_PROFILES_DIR: "." | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| ci_pipeline: | |
| permissions: | |
| pull-requests: write | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: Save stg GCP credentials to runner | |
| run: | | |
| echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_STG_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json && \ | |
| export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json" | |
| if: ${{ github.event.pull_request.base.ref == 'stg' }} | |
| - name: Save prd GCP credentials to runner | |
| run: | | |
| echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_PRD_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json && \ | |
| export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json" | |
| if: ${{ github.event.pull_request.base.ref == 'prd' }} | |
| - name: Authenticate with GCP | |
| run: | | |
| gcloud auth activate-service-account dbt-cicd@beyond-basics-$DESTINATION_BRANCH.iam.gserviceaccount.com --key-file=./service_account.json && \ | |
| gcloud config set project beyond-basics-$DESTINATION_BRANCH | |
| - name: Install python packages | |
| run: pip install -r requirements.txt -r requirements_dev.txt | |
| - name: dbt compile | |
| run: dbt compile --target $DESTINATION_BRANCH | |
| - run: pytest ./tests/pytest -m no_deps -n 5 | |
| - run: dbt deps | |
| - name: dbt debug | |
| run: dbt debug --target $DESTINATION_BRANCH | |
| # Create objects in BigQuery and run unit tests | |
| - name: dbt seed | |
| run: dbt seed --target $DESTINATION_BRANCH | |
| - name: dbt run --empty | |
| run : dbt run --empty --target $DESTINATION_BRANCH | |
| - name: Run unit tests | |
| run: dbt test --select "test_type:unit" --target $DESTINATION_BRANCH | |
| # Needs to be run early in the CI pipeline to allow `dbt docs generate` to succeed (which is a dependency of pre-commit and dbt-coverage) | |
| - name: dbt build | |
| run: dbt --warn-error build --fail-fast --full-refresh --exclude "test_type:unit" --target $DESTINATION_BRANCH | |
| - name: dbt docs generate | |
| run: dbt docs generate --target $DESTINATION_BRANCH | |
| - name: Run `dbt-bouncer` | |
| run: dbt-bouncer | |
| - name: pre-commit run -a | |
| run: SKIP=autoflake,dbt-compile,dbt-docs-generate pre-commit run -a | |
| - name: Generate docs coverage report | |
| id: dbt-docs-coverage-report | |
| run: | | |
| dbt-coverage compute doc --cov-report coverage-doc.json --cov-format markdown >> coverage-doc.md | |
| sed -i '/# Coverage report/c\# Doc coverage report.' coverage-doc.md | |
| - uses: mshick/add-pr-comment@v2 | |
| with: | |
| message-path: coverage-doc.md | |
| message-id: dbt-docs-coverage-report | |
| refresh-message-position: true | |
| - name: Generate test coverage report | |
| id: dbt-test-coverage-report | |
| run: | | |
| dbt-coverage compute test --cov-report coverage-test.json --cov-format markdown >> coverage-test.md | |
| sed -i '/# Coverage report/c\# Test coverage report.' coverage-test.md | |
| - uses: mshick/add-pr-comment@v2 | |
| with: | |
| message-path: coverage-test.md | |
| message-id: dbt-test-coverage-report | |
| refresh-message-position: true | |
| - run: pytest ./tests/pytest -m run_results_json -n 5 | |
| - name: dbt source freshness | |
| run: dbt source freshness --target $DESTINATION_BRANCH || true # source freshness is allowed to fail in CI due to stale sources, the next command runs pytest on sources.json to validate the generated SQL (the prupose of this step) | |
| - run: pytest ./tests/pytest -m sources_json -n 5 | |
| - name: dbt build incremental models | |
| run: dbt --warn-error build --fail-fast --select config.materialized:incremental --exclude "test_type:unit" --target $DESTINATION_BRANCH | |
| # Mart Monitor | |
| - run: if [ "$DESTINATION_BRANCH" == "stg" ]; then DBT_CICD_RUN="false" dbt compile --target $DESTINATION_BRANCH --threads 64; else echo "Only runs for PRs to stg"; fi # Need manifest.json as exists during $DESTINATION_BRANCH runs and not during CI runs | |
| - run: if [ "$DESTINATION_BRANCH" == "stg" ]; then python ./scripts/mart_monitor_commenter.py --dbt_dataset $DBT_DATASET --pull_request_id ${{ github.event.number }} --target_branch $DESTINATION_BRANCH; else echo "Only runs for PRs to stg"; fi | |
| dev_container: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build container and run dbt command | |
| uses: devcontainers/[email protected] | |
| env: | |
| DBT_DATASET: "cicd_${{ github.event.number }}_${{ github.run_number }}_${{ github.sha }}_dev_container" | |
| DBT_PROFILES_DIR: "." | |
| DESTINATION_BRANCH: "${{ github.event.pull_request.base.ref }}" | |
| with: | |
| runCmd: | | |
| echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_STG_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json | |
| export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json"cloud auth activate-service-account dbt-cicd@beyond-basics-$DESTINATION_BRANCH.iam.gserviceaccount.com --key-file=./service_account.json | |
| gcloud config set project beyond-basics-$DESTINATION_BRANCH | |
| dbt compile --target $DESTINATION_BRANCH | |
| env: | | |
| DBT_DATASET | |
| DBT_PROFILES_DIR | |
| DESTINATION_BRANCH |