Migrating dbt to 1.6 and programmatic invocations #38
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci_pipeline | |
| on: | |
| pull_request: | |
| branches: | |
| - stg | |
| - prd | |
| env: | |
| DBT_CICD_RUN: "true" | |
| DESTINATION_BRANCH: "${{ github.event.pull_request.base.ref }}" | |
| DBT_DATASET: "ci_${{ github.event.number }}_${{ github.run_number }}_${{ github.sha }}" | |
| DBT_PROFILES_DIR: "." | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| ci_pipeline: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Save stg GCP credentials to runner | |
| run: | | |
| echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_STG_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json && \ | |
| export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json" | |
| if: ${{ github.event.pull_request.base.ref == 'stg' }} | |
| - name: Save prd GCP credentials to runner | |
| run: | | |
| echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_PRD_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json && \ | |
| export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json" | |
| if: ${{ github.event.pull_request.base.ref == 'prd' }} | |
| - name: Authenticate with GCP | |
| run: | | |
| gcloud auth activate-service-account dbt-cicd@beyond-basics-$DESTINATION_BRANCH.iam.gserviceaccount.com --key-file=./service_account.json && \ | |
| gcloud config set project beyond-basics-$DESTINATION_BRANCH | |
| - name: Install python packages | |
| run: pip install -r requirements.txt -r requirements_dev.txt | |
| - name: dbt compile | |
| run: dbt compile --target $DESTINATION_BRANCH | |
| - name: dbt docs generate | |
| run: dbt docs generate --target $DESTINATION_BRANCH | |
| - name: pre-commit run -a | |
| run: SKIP=dbt-compile,dbt-docs-generate pre-commit run -a | |
| - run: pytest ./tests/pytest -m no_deps -n 5 | |
| - run: dbt deps | |
| - name: dbt debug | |
| run: dbt debug --target $DESTINATION_BRANCH | |
| # Run dbt | |
| - name: dbt build | |
| run: dbt --warn-error build --fail-fast --target $DESTINATION_BRANCH | |
| - run: pytest ./tests/pytest -m run_results_json -n 5 | |
| - name: dbt source freshness | |
| run: dbt source freshness --target $DESTINATION_BRANCH || true # source freshness is allowed to fail in CI due to stale sources, the next command runs pytest on sources.json to validate the generated SQL (the prupose of this step) | |
| - run: pytest ./tests/pytest -m sources_json -n 5 | |
| - name: dbt build incremental models | |
| run: dbt --warn-error build --fail-fast --select config.materialized:incremental --target $DESTINATION_BRANCH | |
| # Mart Monitor | |
| - run: if [ "$DESTINATION_BRANCH" == "stg" ]; then DBT_CICD_RUN="false" dbt compile --target $DESTINATION_BRANCH --threads 64; else echo "Only runs for PRs to stg"; fi # Need manifest.json as exists during $DESTINATION_BRANCH runs and not during CI runs | |
| - run: if [ "$DESTINATION_BRANCH" == "stg" ]; then python ./scripts/mart_monitor_commenter.py --dbt_dataset $DBT_DATASET --pull_request_id ${{ github.event.number }} --target_branch $DESTINATION_BRANCH; else echo "Only runs for PRs to stg"; fi | |
| dev_container: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Build container and run dbt command | |
| uses: devcontainers/[email protected] | |
| env: | |
| DBT_DATASET: "cicd_${{ github.event.number }}_${{ github.run_number }}_${{ github.sha }}_dev_container" | |
| DBT_PROFILES_DIR: "." | |
| DESTINATION_BRANCH: "${{ github.event.pull_request.base.ref }}" | |
| with: | |
| runCmd: | | |
| echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_STG_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json | |
| export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json"cloud auth activate-service-account dbt-cicd@beyond-basics-$DESTINATION_BRANCH.iam.gserviceaccount.com --key-file=./service_account.json | |
| gcloud config set project beyond-basics-$DESTINATION_BRANCH | |
| dbt compile --target $DESTINATION_BRANCH | |
| env: | | |
| DBT_DATASET | |
| DBT_PROFILES_DIR | |
| DESTINATION_BRANCH |