Merge pull request #258 from moomindani/main

Add tests and CI

.github/workflows/integration.yml

@@ -0,0 +1,85 @@
+name: Integration Tests
+on:
+  push:
+    branches:
+      - "main"
+      - "*.latest"
+      - "releases/*"
+    paths-ignore:
+      - "**.MD"
+      - "**.md"
+
+permissions:
+  id-token: write
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  integ:
+    name: integration test / python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"] # Use single version to avoid resource conflicts in an AWS account
+
+    env:
+      TOXENV: "integration"
+      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
+      DBT_AWS_ACCOUNT: ${{ secrets.DBT_AWS_ACCOUNT }}
+      DBT_GLUE_ROLE_ARN: ${{ secrets.DBT_GLUE_ROLE_ARN }}
+      DBT_GLUE_REGION: ${{ secrets.DBT_GLUE_REGION }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Generate session name
+        id: session
+        run: |
+          repo="${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}"
+          echo "name=${repo}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-session-name: ${{ steps.session.outputs.name }}
+          role-to-assume: arn:aws:iam::${{ secrets.DBT_AWS_ACCOUNT }}:role/dbt-glue
+          aws-region: ${{ secrets.DBT_GLUE_REGION }}
+          mask-aws-account-id: true
+
+      - name: Run tox
+        run: |
+          export DBT_S3_LOCATION=${{ secrets.DBT_S3_LOCATION }}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${{ matrix.python-version }}
+          tox
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: unit_results.csv

.github/workflows/main.yml

@@ -0,0 +1,218 @@
+# **what?**
+# Runs code quality checks, unit tests, and verifies python build on
+# all code commited to the repository. This workflow should not
+# require any secrets since it runs for PRs from forked repos.
+# By default, secrets are not passed to workflows running from
+# a forked repo.
+
+# **why?**
+# Ensure code for dbt meets a certain quality standard.
+
+# **when?**
+# This will run for all PRs, when code is pushed to a release
+# branch, and when manually triggered.
+
+name: Tests and Code Checks
+
+on:
+  push:
+    branches:
+      - "main"
+      - "*.latest"
+      - "releases/*"
+    paths-ignore:
+      - "**.MD"
+      - "**.md"
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+#  code-quality:
+#    name: code-quality
+#
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 10
+#
+#    steps:
+#      - name: Check out the repository
+#        uses: actions/checkout@v3
+#        with:
+#          persist-credentials: false
+#
+#      - name: Set up Python
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: '3.8'
+#
+#      - name: Install python dependencies
+#        run: |
+#          sudo apt-get update
+#          sudo apt-get install libsasl2-dev
+#          python -m pip install --user --upgrade pip
+#          python -m pip --version
+#          python -m pip install pre-commit
+#          pre-commit --version
+#          python -m pip install mypy==0.942
+#          python -m pip install types-requests
+#          mypy --version
+#          python -m pip install -r dev-requirements.txt
+#          dbt --version
+#
+#      - name: Run pre-commit hooks
+#        run: pre-commit run --all-files --show-diff-on-failure
+
+  unit:
+    name: unit test / python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    env:
+      TOXENV: "unit"
+      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Run tox
+        run: tox
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: unit_results.csv
+
+
+  build:
+    name: build packages
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
+          python -m pip --version
+
+      - name: Build distributions
+        run: ./scripts/build-dist.sh
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Check distribution descriptions
+        run: |
+          twine check dist/*
+      - name: Check wheel contents
+        run: |
+          check-wheel-contents dist/*.whl --ignore W007,W008
+
+      - name: Check if this is an alpha version
+        id: check-is-alpha
+        run: |
+          export is_alpha=0
+          if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi
+          echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist/
+
+  test-build:
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    if: needs.build.outputs.is_alpha == 0
+
+    needs: build
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip install --upgrade wheel
+          python -m pip --version
+      - uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist/
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Install wheel distributions
+        run: |
+          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+      - name: Check wheel distributions
+        run: |
+          dbt --version
+      - name: Install source distributions
+        run: |
+          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+      - name: Check source distributions
+        run: |
+          dbt --version

.pre-commit-config.yaml

@@ -0,0 +1,63 @@
+# For more on configuring pre-commit hooks (see https://pre-commit.com/)
+
+# Force all unspecified python hooks to run python 3.8
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+        args: [--unsafe]
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-case-conflict
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+        additional_dependencies: ['click~=8.1']
+        args:
+          - "--line-length=99"
+          - "--target-version=py38"
+      - id: black
+        alias: black-check
+        stages: [manual]
+        additional_dependencies: ['click~=8.1']
+        args:
+          - "--line-length=99"
+          - "--target-version=py38"
+          - "--check"
+          - "--diff"
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+      - id: flake8
+        alias: flake8-check
+        stages: [manual]
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.2.0
+    hooks:
+      - id: mypy
+        # N.B.: Mypy is... a bit fragile.
+        #
+        # By using `language: system` we run this hook in the local
+        # environment instead of a pre-commit isolated one.  This is needed
+        # to ensure mypy correctly parses the project.
+
+        # It may cause trouble in that it adds environmental variables out
+        # of our control to the mix.  Unfortunately, there's nothing we can
+        # do about per pre-commit's author.
+        # See https://github.com/pre-commit/pre-commit/issues/730 for details.
+        args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases, --warn-unused-ignores, --disallow-untyped-defs]
+        files: ^dbt/adapters/.*
+        language: system
+      - id: mypy
+        alias: mypy-check
+        stages: [manual]
+        args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases]
+        files: ^dbt/adapters
+        language: system

dbt/adapters/glue/__init__.py

@@ -9,5 +9,5 @@
     adapter=GlueAdapter,
     credentials=GlueCredentials,
     include_path=glue.PACKAGE_PATH,
-    dependencies = ["spark"],
+    dependencies=["spark"],
 )

dev-requirements.txt

@@ -18,6 +18,13 @@ pytest-xdist
 pytest-dotenv
 pytest-csv
 flaky
-dbt-tests-adapter==1.7.0
 mypy==1.6.1
 black==23.10.1
+
+# Adapter specific dependencies
+waiter
+boto3
+
+dbt-core==1.7.0
+dbt-tests-adapter==1.7.0
+dbt-spark==1.7.0