Added CI

.github/workflows/main.yml

@@ -0,0 +1,232 @@
+# **what?**
+# Runs code quality checks, unit tests, and verifies python build on
+# all code commited to the repository. This workflow should not
+# require any secrets since it runs for PRs from forked repos.
+# By default, secrets are not passed to workflows running from
+# a forked repo.
+
+# **why?**
+# Ensure code for dbt meets a certain quality standard.
+
+# **when?**
+# This will run for all PRs, when code is pushed to a release
+# branch, and when manually triggered.
+
+name: Tests and Code Checks
+
+on:
+  push:
+    branches:
+      - "main"
+      - "*.latest"
+      - "releases/*"
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+#  code-quality:
+#    name: code-quality
+#
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 10
+#
+#    steps:
+#      - name: Check out the repository
+#        uses: actions/checkout@v3
+#        with:
+#          persist-credentials: false
+#
+#      - name: Set up Python
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: '3.8'
+#
+#      - name: Install python dependencies
+#        run: |
+#          sudo apt-get update
+#          sudo apt-get install libsasl2-dev
+#          python -m pip install --user --upgrade pip
+#          python -m pip --version
+#          python -m pip install pre-commit
+#          pre-commit --version
+#          python -m pip install mypy==0.942
+#          python -m pip install types-requests
+#          mypy --version
+#          python -m pip install -r dev-requirements.txt
+#          dbt --version
+#
+#      - name: Run pre-commit hooks
+#        run: pre-commit run --all-files --show-diff-on-failure
+
+  unit:
+    name: unit test / python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    env:
+      TOXENV: "unit"
+      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Generate session name
+        id: session
+        run: |
+          repo="${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}"
+          echo "name=${repo}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-session-name: ${{ steps.session.outputs.name }}
+          role-to-assume: arn:aws:iam::421396179189:role/dbt-glue
+          aws-region: us-east-1
+
+      - name: Run tox
+        run: |
+          export DBT_AWS_ACCOUNT=421396179189
+          export DBT_S3_LOCATION=s3://dbt-glue-ci/data/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${{ matrix.python-version }}/
+          export DBT_GLUE_ROLE_ARN=arn:aws:iam::421396179189:role/GlueServiceRole
+          export DBT_GLUE_REGION=us-east-1
+          tox
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: unit_results.csv
+
+  build:
+    name: build packages
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
+          python -m pip --version
+
+      - name: Build distributions
+        run: ./scripts/build-dist.sh
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Check distribution descriptions
+        run: |
+          twine check dist/*
+      - name: Check wheel contents
+        run: |
+          check-wheel-contents dist/*.whl --ignore W007,W008
+
+      - name: Check if this is an alpha version
+        id: check-is-alpha
+        run: |
+          export is_alpha=0
+          if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi
+          echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist/
+
+  test-build:
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    if: needs.build.outputs.is_alpha == 0
+
+    needs: build
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip install --upgrade wheel
+          python -m pip --version
+      - uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist/
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Install wheel distributions
+        run: |
+          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+      - name: Check wheel distributions
+        run: |
+          dbt --version
+      - name: Install source distributions
+        run: |
+          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+      - name: Check source distributions
+        run: |
+          dbt --version

.pre-commit-config.yaml

@@ -0,0 +1,63 @@
+# For more on configuring pre-commit hooks (see https://pre-commit.com/)
+
+# Force all unspecified python hooks to run python 3.8
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+        args: [--unsafe]
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-case-conflict
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+        additional_dependencies: ['click~=8.1']
+        args:
+          - "--line-length=99"
+          - "--target-version=py38"
+      - id: black
+        alias: black-check
+        stages: [manual]
+        additional_dependencies: ['click~=8.1']
+        args:
+          - "--line-length=99"
+          - "--target-version=py38"
+          - "--check"
+          - "--diff"
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+      - id: flake8
+        alias: flake8-check
+        stages: [manual]
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.2.0
+    hooks:
+      - id: mypy
+        # N.B.: Mypy is... a bit fragile.
+        #
+        # By using `language: system` we run this hook in the local
+        # environment instead of a pre-commit isolated one.  This is needed
+        # to ensure mypy correctly parses the project.
+
+        # It may cause trouble in that it adds environmental variables out
+        # of our control to the mix.  Unfortunately, there's nothing we can
+        # do about per pre-commit's author.
+        # See https://github.com/pre-commit/pre-commit/issues/730 for details.
+        args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases, --warn-unused-ignores, --disallow-untyped-defs]
+        files: ^dbt/adapters/.*
+        language: system
+      - id: mypy
+        alias: mypy-check
+        stages: [manual]
+        args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases]
+        files: ^dbt/adapters
+        language: system

dev-requirements.txt

@@ -18,6 +18,13 @@ pytest-xdist
 pytest-dotenv
 pytest-csv
 flaky
-dbt-tests-adapter==1.7.0
 mypy==1.6.1
 black==23.10.1
+
+# Adapter specific dependencies
+waiter
+boto3
+
+dbt-core==1.7.0
+dbt-tests-adapter==1.7.0
+dbt-spark==1.7.0

scripts/build-dist.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eo pipefail
+
+DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )"
+
+PYTHON_BIN=${PYTHON_BIN:-python}
+
+echo "$PYTHON_BIN"
+
+set -x
+
+rm -rf "$DBT_PATH"/dist
+rm -rf "$DBT_PATH"/build
+mkdir -p "$DBT_PATH"/dist
+
+cd "$DBT_PATH"
+$PYTHON_BIN setup.py sdist bdist_wheel
+
+set +x

tests/conftest.py

@@ -19,7 +19,7 @@ def dbt_profile_target():
         'worker_type': 'G.1X',
         'schema': 'dbt_functional_test_01',
         'database': 'dbt_functional_test_01',
-        'session_provisioning_timeout_in_seconds': 120,
+        'session_provisioning_timeout_in_seconds': 300,
         'location': os.getenv('DBT_S3_LOCATION'),
         'datalake_formats': 'delta',
         'conf': "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog --conf spark.sql.legacy.allowNonEmptyLocationInCTAS=true",

tests/unit/conftest.py

@@ -47,7 +47,7 @@ def credentials():
         database=None,
         schema="airbotinigo",
         worker_type="G.1X",
-        session_provisioning_timeout_in_seconds=120,
+        session_provisioning_timeout_in_seconds=300,
         workers=3
     )
 

tests/unit/test_glue_session.py

@@ -1,6 +1,8 @@
 from dbt.adapters.glue.gluedbapi import GlueConnection, GlueCursor
 import boto3
 import uuid
+import string
+import random
 from tests.util import get_account_id, get_s3_location
 
 
@@ -62,7 +64,8 @@ def __test_query_with_comments(session):
 def test_create_database(session, region):
     client = boto3.client("glue", region_name=region)
     schema = "testdb111222333"
-    table_name = "test123"
+    table_suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=4))
+    table_name = f"test123_{table_suffix}"
     try:
         response = client.create_database(
             DatabaseInput={

tox.ini

@@ -0,0 +1,15 @@
+[tox]
+skipsdist = True
+envlist = unit, flake8, integration-spark-thrift
+
+[testenv:{unit,py38,py39,py310,py}]
+allowlist_externals =
+    /bin/bash
+commands = /bin/bash -c '{envpython} -m pytest -v {posargs} tests/unit'
+passenv =
+    DBT_*
+    PYTEST_ADDOPTS
+    AWS_*
+deps =
+  -rdev-requirements.txt
+  -e.