test_dbt_runner.yml

name: test dbt runner

on:
  pull_request:
    branches:
      - master
      - devel
  workflow_dispatch:

env:
  # all credentials must be present to be passed to dbt runner
  DESTINATION__POSTGRES__CREDENTIALS: postgres://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data
  DESTINATION__REDSHIFT__CREDENTIALS: postgres://loader@3.73.90.3:5439/chat_analytics_rasa_ci
  CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
  CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
  CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
  CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
  CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token

jobs:

  run_dbt:
    name: Tests dbt runner
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}

    steps:

      - name: Check out
        uses: actions/checkout@master

      - name: Setup Python
        uses: actions/setup-python@v1
        with:
          python-version: "3.10.x"

      - name: Install Poetry without dbt
        uses: snok/install-poetry@v1.3.2
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          # path: ${{ steps.pip-cache.outputs.dir }}
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-dbt-runner

      - name: Install dependencies
        # install dlt with postgres support
        run: poetry install --no-interaction -E postgres -E dbt

      - run: |
          RUNTIME__LOG_LEVEL=ERROR poetry run pytest tests --ignore=tests/cli --ignore=tests/common --ignore=tests/normalize --ignore=tests/load -k '(not venv)'
        if: runner.os != 'Windows'
        name: Run dbt runner with venv - Linux/MAC
      - run: |
          poetry run pytest tests --ignore=tests/cli --ignore=tests/common --ignore=tests/normalize --ignore=tests/load -m "not forked" -k "(not venv)"
        if: runner.os == 'Windows'
        name: Run dbt runner with venv - Windows
        shell: cmd

      - name: Remove dbt-core
        # install dlt with postgres support
        run: poetry run pip uninstall dbt-core

      - run: |
          RUNTIME__LOG_LEVEL=ERROR poetry run pytest tests --ignore=tests/cli --ignore=tests/common --ignore=tests/normalize --ignore=tests/load --ignore=tests/dbt_runner/local -k '(not local)'
        if: runner.os != 'Windows'
        name: Run dbt runner with venv - Linux/MAC
      - run: |
          poetry run pytest tests --ignore=tests/cli --ignore=tests/common --ignore=tests/normalize --ignore=tests/load --ignore=tests/dbt_runner/local -m "not forked" -k "(not local)"
        if: runner.os == 'Windows'
        name: Run dbt runner with venv - Windows
        shell: cmd