diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c9c7e888..06649856 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,13 +1,11 @@ name: Integration Tests + on: + # we use pull_request_target to run the CI also for forks + pull_request_target: + types: [opened, reopened, synchronize, labeled] push: - branches: - - "main" - - "*.latest" - - "releases/*" - paths-ignore: - - "**.MD" - - "**.md" + branches: [main] permissions: id-token: write @@ -22,8 +20,10 @@ defaults: shell: bash jobs: - integ: - name: integration test / python ${{ matrix.python-version }} + # workflow that is invoked when for PRs with labels 'enable-functional-tests' + functional-tests-pr: + name: Functional Tests - PR / python ${{ matrix.python-version }} + if: contains(github.event.pull_request.labels.*.name, 'enable-functional-tests') runs-on: ubuntu-latest timeout-minutes: 60 @@ -87,3 +87,71 @@ jobs: with: name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv path: unit_results.csv + + # workflow that is invoked when a push to main happens + functional-tests-main: + name: Functional Tests - main / python ${{ matrix.python-version }} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] # Use single version to avoid resource conflicts in an AWS account + + env: + TOXENV: "integration" + PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" + DBT_AWS_ACCOUNT: ${{ secrets.DBT_AWS_ACCOUNT }} + DBT_GLUE_ROLE_ARN: ${{ secrets.DBT_GLUE_ROLE_ARN }} + DBT_GLUE_REGION: ${{ secrets.DBT_GLUE_REGION }} + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + sudo apt-get update + sudo apt-get install libsasl2-dev + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: Generate session name + id: session + run: | + repo="${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" + echo "name=${repo}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}" + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-session-name: ${{ steps.session.outputs.name }} + role-to-assume: arn:aws:iam::${{ secrets.DBT_AWS_ACCOUNT }}:role/dbt-glue + aws-region: ${{ secrets.DBT_GLUE_REGION }} + mask-aws-account-id: true + + - name: Run tox + run: | + export DBT_S3_LOCATION=${{ secrets.DBT_S3_LOCATION }}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${{ matrix.python-version }} + tox + + - name: Get current date + if: always() + id: date + run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv + path: unit_results.csv \ No newline at end of file diff --git a/tox.ini b/tox.ini index f4c81be6..f30a666e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = True -envlist = unit, flake8, integration-spark-thrift +envlist = unit, integration [testenv:{unit}] allowlist_externals = @@ -16,7 +16,7 @@ deps = [testenv:{integration}] allowlist_externals = /bin/bash -commands = /bin/bash -c '{envpython} -m pytest -v {posargs}' +commands = /bin/bash -c '{envpython} -m pytest -v {posargs} tests/functional/adapter/' passenv = DBT_* PYTEST_ADDOPTS