bigquery_cleanup.yml

name: bigquery_cleanup

on:
  schedule:
    - cron: "0 1 * * *"

jobs:
  cleanup_stg:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
        with:
          ref: stg

      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Save stg GCP credentials to runner
        run: |
          echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_STG_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json && \
          export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json"

      - name: Authenticate with GCP
        run: |
          gcloud auth activate-service-account dbt-cicd@beyond-basics-stg.iam.gserviceaccount.com --key-file=./service_account.json && \
          gcloud config set project beyond-basics-stg

      - name: Install python packages
        run: pip install -r requirements.txt -r requirements_dev.txt

      - run: python ./scripts/drop_unused_bq_resources.py --dataset_pattern "cicd_" --environment stg

  cleanup_prd:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
        with:
          ref: prd

      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Save prd GCP credentials to runner
        run: |
          echo "${{ secrets.DBT_CICD_SA_SECRET_KEY_PRD_BASE64_ENCODED }}" | base64 --decode >> ./service_account.json && \
          export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/service_account.json"

      - name: Authenticate with GCP
        run: |
          gcloud auth activate-service-account dbt-cicd@beyond-basics-prd.iam.gserviceaccount.com --key-file=./service_account.json && \
          gcloud config set project beyond-basics-prd

      - name: Install python packages
        run: pip install -r requirements.txt -r requirements_dev.txt

      - run: python ./scripts/drop_unused_bq_resources.py --dataset_pattern "cicd_" --environment prd