Add manual GHA CI/CD for BQ and SF (#19)

There is a race condition using multiple warehouses for CI jobs on the
same repo. This adds GitHub Actions to run the CI jobs so we can check
against all of them.

It also adds slim CD jobs that run on merge to main.

.github/workflows/cd.yml

@@ -0,0 +1,55 @@
+name: Run dbt Cloud Deploy to Prod
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud Deploy Prod Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 409009
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud Deploy Prod BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 553247
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: Run dbt Cloud CI job
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud PR CI Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 552843
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud PR CI BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 561096
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/scripts/dbt_cloud_run_job.py

@@ -0,0 +1,134 @@
+import os
+import time
+import requests
+
+# ------------------------------------------------------------------------------
+# get environment variables
+# ------------------------------------------------------------------------------
+api_base = os.getenv(
+    "DBT_URL", "https://cloud.getdbt.com"
+)  # default to multitenant url
+job_cause = os.getenv(
+    "DBT_JOB_CAUSE", "API-triggered job"
+)  # default to generic message
+git_branch = os.getenv("DBT_JOB_BRANCH", None)  # default to None
+schema_override = os.getenv("DBT_JOB_SCHEMA_OVERRIDE", None)  # default to None
+api_key = os.environ[
+    "DBT_API_KEY"
+]  # no default here, just throw an error here if key not provided
+account_id = os.environ[
+    "DBT_ACCOUNT_ID"
+]  # no default here, just throw an error here if id not provided
+project_id = os.environ[
+    "DBT_PROJECT_ID"
+]  # no default here, just throw an error here if id not provided
+job_id = os.environ[
+    "DBT_PR_JOB_ID"
+]  # no default here, just throw an error here if id not provided
+
+print(f"""
+Configuration:
+api_base: {api_base}
+job_cause: {job_cause}
+git_branch: {git_branch}
+schema_override: {schema_override}
+account_id: {account_id}
+project_id: {project_id}
+job_id: {job_id}
+""")
+
+req_auth_header = {"Authorization": f"Token {api_key}"}
+req_job_url = f"{api_base}/api/v2/accounts/{account_id}/jobs/{job_id}/run/"
+run_status_map = {  # dbt run statuses are encoded as integers. This map provides a human-readable status
+    1: "Queued",
+    2: "Starting",
+    3: "Running",
+    10: "Success",
+    20: "Error",
+    30: "Cancelled",
+}
+
+type AuthHeader = dict[str, str]
+
+
+def run_job(
+    url: str,
+    headers: AuthHeader,
+    cause: str,
+    branch: str | None = None,
+    schema_override: str | None = None,
+) -> int:
+    """
+    Runs a dbt job
+    """
+
+    # build payload
+    req_payload = {"cause": cause}
+    if branch and not branch.startswith(
+        "$("
+    ):  # starts with '$(' indicates a valid branch name was not provided
+        req_payload["git_branch"] = branch.replace("refs/heads/", "")
+    if schema_override:
+        req_payload["schema_override"] = schema_override.replace("-", "_").replace(
+            "/", "_"
+        )
+
+    # trigger job
+    print(f"Triggering job:\n\turl: {url}\n\tpayload: {req_payload}")
+
+    response = requests.post(url, headers=headers, json=req_payload)
+    run_id: int = response.json()["data"]["id"]
+    return run_id
+
+
+def get_run_status(url: str, headers: AuthHeader) -> str:
+    """
+    gets the status of a running dbt job
+    """
+    # get status
+    response = requests.get(url, headers=headers)
+    run_status_code: int = response.json()["data"]["status"]
+    run_status = run_status_map[run_status_code]
+    return run_status
+
+
+def main():
+    print("Beginning request for job run...")
+
+    # run job
+    run_id: int = 0
+    try:
+        run_id = run_job(
+            req_job_url, req_auth_header, job_cause, git_branch, schema_override
+        )
+    except Exception as e:
+        print(f"ERROR! - Could not trigger job:\n {e}")
+        raise
+
+    # build status check url and run status link
+    req_status_url = f"{api_base}/api/v2/accounts/{account_id}/runs/{run_id}/"
+    run_status_link = (
+        f"{api_base}/deploy/{account_id}/projects/{project_id}/runs/{run_id}/"
+    )
+
+    # update user with status link
+    print(f"Job running! See job status at {run_status_link}")
+
+    # check status indefinitely with an initial wait period
+    time.sleep(30)
+    while True:
+        status = get_run_status(req_status_url, req_auth_header)
+        print(f"Run status -> {status}")
+
+        if status in ["Error", "Cancelled"]:
+            raise Exception(f"Run failed or canceled. See why at {run_status_link}")
+
+        if status == "Success":
+            print(f"Job completed successfully! See details at {run_status_link}")
+            return
+
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -10,3 +10,4 @@ logs/
 .DS_Store
 
 .user.yml
+*.hurl

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
   - repo: https://github.com/sqlfluff/sqlfluff
     rev: "3.0.1"
     hooks:
@@ -26,7 +27,3 @@ repos:
             "dbt-metricflow[duckdb,snowflake,postgres]~=0.6.0",
             "sqlfluff-templater-dbt~=3.0.1",
           ]
-  - repo: https://github.com/psf/black
-    rev: "24.3.0"
-    hooks:
-      - id: black

requirements.in

@@ -1 +1,2 @@
 pre-commit~=3.6.0
+requests~=2.31.0

requirements.txt

@@ -1,21 +1,30 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile requirements.in -o requirements.txt
+certifi==2024.2.2
+    # via requests
 cfgv==3.4.0
     # via pre-commit
+charset-normalizer==3.3.2
+    # via requests
 distlib==0.3.8
     # via virtualenv
 filelock==3.13.1
     # via virtualenv
 identify==2.5.35
     # via pre-commit
+idna==3.6
+    # via requests
 nodeenv==1.8.0
     # via pre-commit
 platformdirs==4.2.0
     # via virtualenv
 pre-commit==3.6.2
 pyyaml==6.0.1
     # via pre-commit
+requests==2.31.0
 setuptools==69.2.0
     # via nodeenv
+urllib3==2.2.1
+    # via requests
 virtualenv==20.25.1
     # via pre-commit