Initial commit

.github/workflows/cd_prod.yml

@@ -0,0 +1,55 @@
+name: dbt Cloud Deploy Prod
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud Deploy Prod Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 409009
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud Deploy Prod BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 553247
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/cd_staging.yml

@@ -0,0 +1,55 @@
+name: dbt Cloud Deploy Staging
+
+on:
+  push:
+    branches:
+      - staging
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud Deploy Staging Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 565266
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud Deploy Staging BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 560539
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/ci.yml

@@ -0,0 +1,56 @@
+name: dbt Cloud CI
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+jobs:
+  run_snowflake:
+    name: dbt Cloud PR CI Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 552843
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud PR CI BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 561096
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/scripts/dbt_cloud_run_job.py

@@ -0,0 +1,134 @@
+import os
+import time
+import requests
+
+# ------------------------------------------------------------------------------
+# get environment variables
+# ------------------------------------------------------------------------------
+api_base = os.getenv(
+    "DBT_URL", "https://cloud.getdbt.com"
+)  # default to multitenant url
+job_cause = os.getenv(
+    "DBT_JOB_CAUSE", "API-triggered job"
+)  # default to generic message
+git_branch = os.getenv("DBT_JOB_BRANCH", None)  # default to None
+schema_override = os.getenv("DBT_JOB_SCHEMA_OVERRIDE", None)  # default to None
+api_key = os.environ[
+    "DBT_API_KEY"
+]  # no default here, just throw an error here if key not provided
+account_id = os.environ[
+    "DBT_ACCOUNT_ID"
+]  # no default here, just throw an error here if id not provided
+project_id = os.environ[
+    "DBT_PROJECT_ID"
+]  # no default here, just throw an error here if id not provided
+job_id = os.environ[
+    "DBT_PR_JOB_ID"
+]  # no default here, just throw an error here if id not provided
+
+print(f"""
+Configuration:
+api_base: {api_base}
+job_cause: {job_cause}
+git_branch: {git_branch}
+schema_override: {schema_override}
+account_id: {account_id}
+project_id: {project_id}
+job_id: {job_id}
+""")
+
+req_auth_header = {"Authorization": f"Token {api_key}"}
+req_job_url = f"{api_base}/api/v2/accounts/{account_id}/jobs/{job_id}/run/"
+run_status_map = {  # dbt run statuses are encoded as integers. This map provides a human-readable status
+    1: "Queued",
+    2: "Starting",
+    3: "Running",
+    10: "Success",
+    20: "Error",
+    30: "Cancelled",
+}
+
+type AuthHeader = dict[str, str]
+
+
+def run_job(
+    url: str,
+    headers: AuthHeader,
+    cause: str,
+    branch: str | None = None,
+    schema_override: str | None = None,
+) -> int:
+    """
+    Runs a dbt job
+    """
+
+    # build payload
+    req_payload = {"cause": cause}
+    if branch and not branch.startswith(
+        "$("
+    ):  # starts with '$(' indicates a valid branch name was not provided
+        req_payload["git_branch"] = branch.replace("refs/heads/", "")
+    if schema_override:
+        req_payload["schema_override"] = schema_override.replace("-", "_").replace(
+            "/", "_"
+        )
+
+    # trigger job
+    print(f"Triggering job:\n\turl: {url}\n\tpayload: {req_payload}")
+
+    response = requests.post(url, headers=headers, json=req_payload)
+    run_id: int = response.json()["data"]["id"]
+    return run_id
+
+
+def get_run_status(url: str, headers: AuthHeader) -> str:
+    """
+    gets the status of a running dbt job
+    """
+    # get status
+    response = requests.get(url, headers=headers)
+    run_status_code: int = response.json()["data"]["status"]
+    run_status = run_status_map[run_status_code]
+    return run_status
+
+
+def main():
+    print("Beginning request for job run...")
+
+    # run job
+    run_id: int = 0
+    try:
+        run_id = run_job(
+            req_job_url, req_auth_header, job_cause, git_branch, schema_override
+        )
+    except Exception as e:
+        print(f"ERROR! - Could not trigger job:\n {e}")
+        raise
+
+    # build status check url and run status link
+    req_status_url = f"{api_base}/api/v2/accounts/{account_id}/runs/{run_id}/"
+    run_status_link = (
+        f"{api_base}/deploy/{account_id}/projects/{project_id}/runs/{run_id}/"
+    )
+
+    # update user with status link
+    print(f"Job running! See job status at {run_status_link}")
+
+    # check status indefinitely with an initial wait period
+    time.sleep(30)
+    while True:
+        status = get_run_status(req_status_url, req_auth_header)
+        print(f"Run status -> {status}")
+
+        if status in ["Error", "Cancelled"]:
+            raise Exception(f"Run failed or canceled. See why at {run_status_link}")
+
+        if status == "Success":
+            print(f"Job completed successfully! See details at {run_status_link}")
+            return
+
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -0,0 +1,16 @@
+.venv
+venv
+.env
+env
+
+target/
+dbt_packages/
+logs/
+profiles.yml
+
+.DS_Store
+
+.user.yml
+
+.ruff_cache
+__pycache__

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: requirements-txt-fixer
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.3.4
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format

.sqlfluff

@@ -0,0 +1,40 @@
+[sqlfluff]
+templater = dbt
+dialect = snowflake
+runaway_limit = 10
+max_line_length = 80
+indent_unit = space
+
+[sqlfluff:templater:dbt]
+profiles_dir = .
+
+[sqlfluff:indentation]
+tab_space_size = 4
+
+[sqlfluff:layout:type:comma]
+spacing_before = touch
+line_position = trailing
+
+[sqlfluff:rules:capitalisation.keywords]
+capitalisation_policy = lower
+
+[sqlfluff:rules:aliasing.table]
+aliasing = explicit
+
+[sqlfluff:rules:aliasing.column]
+aliasing = explicit
+
+[sqlfluff:rules:aliasing.expression]
+allow_scalar = False
+
+[sqlfluff:rules:capitalisation.identifiers]
+extended_capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.functions]
+capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.literals]
+capitalisation_policy = lower
+
+[sqlfluff:rules:ambiguous.column_references]  # Number in group by
+group_by_and_order_by_style = implicit

.sqlfluffignore

@@ -0,0 +1,4 @@
+target
+dbt_packages
+macros
+.venv