ci.yaml

#
# Providence
# CI Workflow
#

name: Providence CI
on:
  push:
    branches-ignore: 
    # temporary branches created by pipeline github Integration to run tasks on github actions
    - "workflow-*"
env:
  CONTAINER_REGISTRY: ghcr.io
  PYTHON_VERSION: 3.10.6
# add permissions needed to push containers to GHCR with GITHUB_TOKEN & test 
# pipelines github integration.
permissions:
  actions: write
  contents: write
  packages: write
jobs:
  dbt-transform:
    name: "DBT Transform"
    runs-on: ubuntu-22.04
    env:
      B2_ACCOUNT_ID: ${{ secrets.B2_ACCOUNT_ID }}
      B2_APP_KEY: ${{ secrets.B2_APP_KEY }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "${{ env.PYTHON_VERSION }}"
      - name: Setup dependencies
        run: make deps-dbt
      - name: Lint
        run: make lint-dbt
      - name: Build
        run: make build-dbt

  dbt-docs:
    name: "DBT Docs"
    runs-on: ubuntu-22.04
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "${{ env.PYTHON_VERSION }}"
      - name: Setup dependencies
        run: make deps-dbt
      - name: Generate docs
        env:
          DBT_TARGET: prod
          B2_ACCOUNT_ID: ${{ secrets.B2_ACCOUNT_ID }}
          B2_APP_KEY: ${{ secrets.B2_APP_KEY }}
          motherduck_token: ${{ secrets.motherduck_token }}
        working-directory: transforms/dbt
        run: |
          set -ex -o pipefail
          # generate docs into 'target' dir
          dbt docs generate
          # create 'docs' dir containing only the files needed for dbt docs
          mkdir docs
          cp target/{index.html,manifest.json,catalog.json} -t docs
      - name: Publish docs
        uses: peaceiris/actions-gh-pages@v3
        if: ${{ github.ref == 'refs/heads/main' }}
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: transforms/dbt/docs
          user_name: "github-actions[bot]"
          user_email: "github-actions[bot]@users.noreply.github.com"
          full_commit_message: "doc(transform): update dbt docs for ${{ github.sha }}"

  pipelines:
    name: "Pipelines: Lint, Test & Publish"
    runs-on: ubuntu-22.04
    env:
      PVD_LAKE_BUCKET: mrzzy-co-data-lake
      YNAB_BUDGET_ID: f3f15316-e48c-4235-8d5d-1aa3191b3b8c
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/python
        env:
          B2_ACCOUNT_ID: ${{ secrets.B2_ACCOUNT_ID }}
          B2_APP_KEY: ${{ secrets.B2_APP_KEY }}
          SIMPLYGO_SRC_USERNAME: ${{ secrets.SIMPLYGO_SRC_USERNAME }}
          SIMPLYGO_SRC_PASSWORD: ${{ secrets.SIMPLYGO_SRC_PASSWORD }}
          YNAB_ACCESS_TOKEN: ${{ secrets.YNAB_ACCESS_TOKEN }}
          # hack to workaround: https://github.com/PrefectHQ/prefect/issues/13349
          # by directing prefect to retrieve blocks from prefect cloud instead.
          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
          motherduck_token: ${{ secrets.motherduck_token }}
        with:
          make-suffix: pipelines
          python-version: "${{ env.PYTHON_VERSION }}"
      - name: Publish Container
        uses: ./.github/actions/ghcr
        with:
          username: "${{ github.actor }}"
          password: "${{ secrets.GITHUB_TOKEN }}"
          name: "pvd-pipeline"
          dockerfile: "infra/Dockerfile.pipeline"
      - name: Authenticate with Prefect Cloud
        uses: PrefectHQ/actions-prefect-auth@v1
        with:
          prefect-api-key: ${{ secrets.PREFECT_API_KEY }}
          prefect-workspace: mrzzy/default
      - name: Deploy Pipelines
        if: ${{ github.ref == 'refs/heads/main' }}
        working-directory: pipelines
        env:
          PREFECT_WORK_POOL: azure-container-instances
        run: python deploy.py