From 0168db0f073ef74c168b3b9d3049745748355a42 Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 4 Oct 2023 17:50:45 +0200 Subject: [PATCH] add synapse test workflow --- .../workflows/test_destination_synapse.yml | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/test_destination_synapse.yml diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml new file mode 100644 index 0000000000..8111f76ae0 --- /dev/null +++ b/.github/workflows/test_destination_synapse.yml @@ -0,0 +1,79 @@ +name: test synapse + +on: + workflow_dispatch: + +env: + DESTINATION__SYNAPSE__CREDENTIALS: ${{ secrets.SYNAPSE_CREDENTIALS }} + DESTINATION__SYNAPSE__CREDENTIALS__PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }} + + RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752 + RUNTIME__LOG_LEVEL: ERROR + + ACTIVE_DESTINATIONS: "[\"synapse\"]" + ALL_FILESYSTEM_DRIVERS: "[\"memory\"]" + +jobs: + + run_loader: + name: Tests Synapse loader + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + defaults: + run: + shell: bash + runs-on: ${{ matrix.os }} + + steps: + + - name: Check out + uses: actions/checkout@master + + - name: Install ODBC driver for SQL Server + run: | + sudo ACCEPT_EULA=Y apt-get install --yes msodbcsql18 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10.x" + + - name: Install Poetry + uses: snok/install-poetry@v1.3.2 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp + + - name: Install dependencies + run: poetry install --no-interaction -E synapse -E s3 -E gs -E az + + - run: | + poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py + if: runner.os != 'Windows' + name: Run tests Linux/MAC + - run: | + poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py + if: runner.os == 'Windows' + name: Run tests Windows + shell: cmd + + matrix_job_required_check: + name: Synapse loader tests + needs: run_loader + runs-on: ubuntu-latest + if: always() + steps: + - name: Check matrix job results + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 \ No newline at end of file