Skip to content

simplifies cloning resources #1986

simplifies cloning resources

simplifies cloning resources #1986

Workflow file for this run

name: test dbt runner
on:
pull_request:
branches:
- master
- devel
workflow_dispatch:
env:
# all credentials must be present to be passed to dbt runner
DESTINATION__POSTGRES__CREDENTIALS: postgresql://[email protected]:5432/dlt_data
DESTINATION__REDSHIFT__CREDENTIALS: postgresql://[email protected]:5439/dlt_ci
DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE
DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
RUNTIME__LOG_LEVEL: ERROR
jobs:
get_docs_changes:
uses: ./.github/workflows/get_docs_changes.yml
if: ${{ !github.event.pull_request.head.repo.fork }}
run_dbt:
name: Tests dbt runner
needs: get_docs_changes
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest"]
# os: ["ubuntu-latest", "macos-latest", "windows-latest"]
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
steps:
- name: Check out
uses: actions/checkout@master
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10.x"
- name: Install Poetry without dbt
uses: snok/[email protected]
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
# path: ${{ steps.pip-cache.outputs.dir }}
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-dbt-runner
- name: Install dependencies
# install dlt with postgres support
run: poetry install --no-interaction -E postgres -E dbt
- run: |
poetry run pytest tests/helpers/dbt_tests -k '(not venv)'
if: runner.os != 'Windows'
name: Run dbt runner - Linux/MAC
- run: |
poetry run pytest tests/helpers/dbt_tests -k "(not venv)"
if: runner.os == 'Windows'
name: Run dbt runner - Windows
shell: cmd
- name: Remove dbt-core
# install dlt with postgres support
run: poetry run pip uninstall dbt-core -y
- run: |
poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
if: runner.os != 'Windows'
name: Run dbt runner with venv - Linux/MAC
- run: |
poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)"
if: runner.os == 'Windows'
name: Run dbt runner with venv - Windows
shell: cmd
matrix_job_required_check:
name: dbt runner tests
needs: run_dbt
runs-on: ubuntu-latest
if: always()
steps:
- name: Check matrix job results
if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
run: |
echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1