Catch staging up to main (#22)

We still haven't implemented the full W-A-P flow yet so `staging` is
behind `main`, fixing that and then we'll transition to the proper flow.

---------

Co-authored-by: Winnie Winship <[email protected]>
Co-authored-by: dave-connors-3 <[email protected]>

.github/workflows/cd.yml

@@ -0,0 +1,55 @@
+name: Run dbt Cloud Deploy to Prod
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud Deploy Prod Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 409009
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud Deploy Prod BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 553247
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: Run dbt Cloud CI job
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud PR CI Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 552843
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud PR CI BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 561096
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/scripts/dbt_cloud_run_job.py

@@ -0,0 +1,134 @@
+import os
+import time
+import requests
+
+# ------------------------------------------------------------------------------
+# get environment variables
+# ------------------------------------------------------------------------------
+api_base = os.getenv(
+    "DBT_URL", "https://cloud.getdbt.com"
+)  # default to multitenant url
+job_cause = os.getenv(
+    "DBT_JOB_CAUSE", "API-triggered job"
+)  # default to generic message
+git_branch = os.getenv("DBT_JOB_BRANCH", None)  # default to None
+schema_override = os.getenv("DBT_JOB_SCHEMA_OVERRIDE", None)  # default to None
+api_key = os.environ[
+    "DBT_API_KEY"
+]  # no default here, just throw an error here if key not provided
+account_id = os.environ[
+    "DBT_ACCOUNT_ID"
+]  # no default here, just throw an error here if id not provided
+project_id = os.environ[
+    "DBT_PROJECT_ID"
+]  # no default here, just throw an error here if id not provided
+job_id = os.environ[
+    "DBT_PR_JOB_ID"
+]  # no default here, just throw an error here if id not provided
+
+print(f"""
+Configuration:
+api_base: {api_base}
+job_cause: {job_cause}
+git_branch: {git_branch}
+schema_override: {schema_override}
+account_id: {account_id}
+project_id: {project_id}
+job_id: {job_id}
+""")
+
+req_auth_header = {"Authorization": f"Token {api_key}"}
+req_job_url = f"{api_base}/api/v2/accounts/{account_id}/jobs/{job_id}/run/"
+run_status_map = {  # dbt run statuses are encoded as integers. This map provides a human-readable status
+    1: "Queued",
+    2: "Starting",
+    3: "Running",
+    10: "Success",
+    20: "Error",
+    30: "Cancelled",
+}
+
+type AuthHeader = dict[str, str]
+
+
+def run_job(
+    url: str,
+    headers: AuthHeader,
+    cause: str,
+    branch: str | None = None,
+    schema_override: str | None = None,
+) -> int:
+    """
+    Runs a dbt job
+    """
+
+    # build payload
+    req_payload = {"cause": cause}
+    if branch and not branch.startswith(
+        "$("
+    ):  # starts with '$(' indicates a valid branch name was not provided
+        req_payload["git_branch"] = branch.replace("refs/heads/", "")
+    if schema_override:
+        req_payload["schema_override"] = schema_override.replace("-", "_").replace(
+            "/", "_"
+        )
+
+    # trigger job
+    print(f"Triggering job:\n\turl: {url}\n\tpayload: {req_payload}")
+
+    response = requests.post(url, headers=headers, json=req_payload)
+    run_id: int = response.json()["data"]["id"]
+    return run_id
+
+
+def get_run_status(url: str, headers: AuthHeader) -> str:
+    """
+    gets the status of a running dbt job
+    """
+    # get status
+    response = requests.get(url, headers=headers)
+    run_status_code: int = response.json()["data"]["status"]
+    run_status = run_status_map[run_status_code]
+    return run_status
+
+
+def main():
+    print("Beginning request for job run...")
+
+    # run job
+    run_id: int = 0
+    try:
+        run_id = run_job(
+            req_job_url, req_auth_header, job_cause, git_branch, schema_override
+        )
+    except Exception as e:
+        print(f"ERROR! - Could not trigger job:\n {e}")
+        raise
+
+    # build status check url and run status link
+    req_status_url = f"{api_base}/api/v2/accounts/{account_id}/runs/{run_id}/"
+    run_status_link = (
+        f"{api_base}/deploy/{account_id}/projects/{project_id}/runs/{run_id}/"
+    )
+
+    # update user with status link
+    print(f"Job running! See job status at {run_status_link}")
+
+    # check status indefinitely with an initial wait period
+    time.sleep(30)
+    while True:
+        status = get_run_status(req_status_url, req_auth_header)
+        print(f"Run status -> {status}")
+
+        if status in ["Error", "Cancelled"]:
+            raise Exception(f"Run failed or canceled. See why at {run_status_link}")
+
+        if status == "Success":
+            print(f"Job completed successfully! See details at {run_status_link}")
+            return
+
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -6,7 +6,11 @@ env
 target/
 dbt_packages/
 logs/
+profiles.yml
 
 .DS_Store
 
 .user.yml
+
+.ruff_cache
+__pycache__

.pre-commit-config.yaml

@@ -7,26 +7,17 @@ repos:
       - id: trailing-whitespace
       - id: requirements-txt-fixer
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.3.3
+    rev: v0.3.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
   - repo: https://github.com/sqlfluff/sqlfluff
-    rev: "3.0.1"
+    rev: "3.0.3"
     hooks:
-      - id: sqlfluff-lint
-        additional_dependencies:
-          [
-            "dbt-metricflow[duckdb,snowflake,postgres]~=0.6.0",
-            "sqlfluff-templater-dbt~=3.0.1",
-          ]
       - id: sqlfluff-fix
         additional_dependencies:
           [
-            "dbt-metricflow[duckdb,snowflake,postgres]~=0.6.0",
-            "sqlfluff-templater-dbt~=3.0.1",
+            "dbt-metricflow[snowflake,bigquery,postgres]~=0.6.0",
+            "sqlfluff-templater-dbt~=3.0.3",
           ]
-  - repo: https://github.com/psf/black
-    rev: "24.3.0"
-    hooks:
-      - id: black

Taskfile.yml

@@ -12,11 +12,11 @@ tasks:
 
   build:
     cmds:
-      - source .venv/bin/activate && dbt deps
-      - source .venv/bin/activate && dbt seed
+      - dbt deps
+      - dbt seed
       - rm -rf jaffle-data
-      - source .venv/bin/activate && dbt run
-      - source .venv/bin/activate && dbt test
+      - dbt run
+      - dbt test
 
   setup:
     cmds:

dbt_project.yml

@@ -2,11 +2,15 @@ config-version: 2
 
 name: "jaffle_shop"
 version: "3.0.0"
-require-dbt-version: ">=1.7.1"
+require-dbt-version: ">=1.5.0"
 
 dbt-cloud:
   project-id: 283328 # Put your project id here
 
+# If you want to run SQLFluff pre-commit hooks you'll need
+# to set up a working profile it can use and list it below
+profile: default
+
 model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["data-tests"]
@@ -20,7 +24,7 @@ clean-targets:
   - "dbt_packages"
 
 vars:
-  truncate_timespan_to: "cast({{ dbt_date.now() }} as datetime)"
+  truncate_timespan_to: "{{ dbt.cast(dbt.current_timestamp(), 'datetime') }}"
   "dbt_date:time_zone": "America/Los_Angeles"
 
 seeds: