NTD: bringing safety and security tables into the warehouse as staging and mart #1202
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test, visualize, and build dbt project | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| paths: | |
| - '.github/workflows/build-warehouse-image.yml' | |
| - 'warehouse/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/build-warehouse-image.yml' | |
| - 'warehouse/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| check: | |
| name: check python | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: sudo apt-get install -y libgraphviz-dev graphviz-dev | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - run: curl -sSL https://install.python-poetry.org | python - | |
| - run: cd warehouse && poetry install && poetry run mypy scripts | |
| compile: | |
| name: check dbt | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: sudo apt-get install -y libgraphviz-dev graphviz graphviz-dev | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - run: curl -sSL https://install.python-poetry.org | python - | |
| - uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| - uses: google-github-actions/setup-gcloud@v2 | |
| - name: Compile dbt project | |
| working-directory: warehouse | |
| run: | | |
| poetry install | |
| poetry run dbt deps | |
| poetry run dbt compile --target prod --full-refresh | |
| poetry run dbt docs generate --target prod --no-compile | |
| - uses: 'google-github-actions/upload-cloud-storage@v1' | |
| with: | |
| path: './warehouse/logs/' | |
| destination: 'calitp-ci-artifacts/${{github.workflow}}/run_id=${{github.run_id}}/job=${{github.job}}/logs/' | |
| - uses: 'google-github-actions/upload-cloud-storage@v1' | |
| with: | |
| path: './warehouse/target/' | |
| glob: '*.json' | |
| destination: 'calitp-ci-artifacts/${{github.workflow}}/run_id=${{github.run_id}}/job=${{github.job}}/target/' | |
| # Only do visualization if we actually changed models and we are merging against main | |
| - uses: tj-actions/changed-files@v41 | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: changed-files-warehouse | |
| with: | |
| files: 'warehouse/models/**/*.sql' | |
| # install a specific version of node before cml https://github.com/iterative/cml/issues/1377 | |
| - uses: actions/setup-node@v1 | |
| with: | |
| node-version: '16' | |
| - uses: iterative/setup-cml@v1 | |
| if: steps.changed-files-warehouse.outputs.any_changed == 'true' | |
| - name: Create GitHub comment | |
| if: steps.changed-files-warehouse.outputs.any_changed == 'true' | |
| env: | |
| REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cd warehouse | |
| gsutil cp -r gs://calitp-dbt-artifacts/latest/ . | |
| poetry run python scripts/visualize.py ci-report | |
| cml comment update target/report.md | |
| build_push: | |
| name: package warehouse image | |
| runs-on: ubuntu-latest | |
| needs: [check] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: tj-actions/changed-files@v41 | |
| with: | |
| files: | | |
| warehouse/Dockerfile | |
| warehouse/packages.yml | |
| warehouse/poetry.lock | |
| warehouse/pyproject.yml | |
| warehouse/requirements.txt | |
| - uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v4 | |
| if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || steps.changed-files-specific.outputs.any_changed == 'true' }} | |
| with: | |
| context: warehouse | |
| push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| tags: ghcr.io/${{github.repository}}/warehouse:latest |