diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..461d6543 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,55 @@ +name: Run dbt Cloud Deploy to Prod + +on: + push: + branches: + - main + +jobs: + run_snowflake: + name: dbt Cloud Deploy Prod Snowflake + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 283328 + DBT_PR_JOB_ID: 409009 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: main + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py + + run_bigquery: + name: dbt Cloud Deploy Prod BigQuery + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 275557 + DBT_PR_JOB_ID: 553247 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: main + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8dd9c014 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: Run dbt Cloud CI job + +on: + pull_request: + branches: + - main + +jobs: + run_snowflake: + name: dbt Cloud PR CI Snowflake + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 283328 + DBT_PR_JOB_ID: 552843 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: ${{ github.head_ref }} + DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}} + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py + + run_bigquery: + name: dbt Cloud PR CI BigQuery + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 275557 + DBT_PR_JOB_ID: 561096 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: ${{ github.head_ref }} + DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}} + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py diff --git a/.github/workflows/scripts/dbt_cloud_run_job.py b/.github/workflows/scripts/dbt_cloud_run_job.py new file mode 100644 index 00000000..efebceab --- /dev/null +++ b/.github/workflows/scripts/dbt_cloud_run_job.py @@ -0,0 +1,134 @@ +import os +import time +import requests + +# ------------------------------------------------------------------------------ +# get environment variables +# ------------------------------------------------------------------------------ +api_base = os.getenv( + "DBT_URL", "https://cloud.getdbt.com" +) # default to multitenant url +job_cause = os.getenv( + "DBT_JOB_CAUSE", "API-triggered job" +) # default to generic message +git_branch = os.getenv("DBT_JOB_BRANCH", None) # default to None +schema_override = os.getenv("DBT_JOB_SCHEMA_OVERRIDE", None) # default to None +api_key = os.environ[ + "DBT_API_KEY" +] # no default here, just throw an error here if key not provided +account_id = os.environ[ + "DBT_ACCOUNT_ID" +] # no default here, just throw an error here if id not provided +project_id = os.environ[ + "DBT_PROJECT_ID" +] # no default here, just throw an error here if id not provided +job_id = os.environ[ + "DBT_PR_JOB_ID" +] # no default here, just throw an error here if id not provided + +print(f""" +Configuration: +api_base: {api_base} +job_cause: {job_cause} +git_branch: {git_branch} +schema_override: {schema_override} +account_id: {account_id} +project_id: {project_id} +job_id: {job_id} +""") + +req_auth_header = {"Authorization": f"Token {api_key}"} +req_job_url = f"{api_base}/api/v2/accounts/{account_id}/jobs/{job_id}/run/" +run_status_map = { # dbt run statuses are encoded as integers. This map provides a human-readable status + 1: "Queued", + 2: "Starting", + 3: "Running", + 10: "Success", + 20: "Error", + 30: "Cancelled", +} + +type AuthHeader = dict[str, str] + + +def run_job( + url: str, + headers: AuthHeader, + cause: str, + branch: str | None = None, + schema_override: str | None = None, +) -> int: + """ + Runs a dbt job + """ + + # build payload + req_payload = {"cause": cause} + if branch and not branch.startswith( + "$(" + ): # starts with '$(' indicates a valid branch name was not provided + req_payload["git_branch"] = branch.replace("refs/heads/", "") + if schema_override: + req_payload["schema_override"] = schema_override.replace("-", "_").replace( + "/", "_" + ) + + # trigger job + print(f"Triggering job:\n\turl: {url}\n\tpayload: {req_payload}") + + response = requests.post(url, headers=headers, json=req_payload) + run_id: int = response.json()["data"]["id"] + return run_id + + +def get_run_status(url: str, headers: AuthHeader) -> str: + """ + gets the status of a running dbt job + """ + # get status + response = requests.get(url, headers=headers) + run_status_code: int = response.json()["data"]["status"] + run_status = run_status_map[run_status_code] + return run_status + + +def main(): + print("Beginning request for job run...") + + # run job + run_id: int = 0 + try: + run_id = run_job( + req_job_url, req_auth_header, job_cause, git_branch, schema_override + ) + except Exception as e: + print(f"ERROR! - Could not trigger job:\n {e}") + raise + + # build status check url and run status link + req_status_url = f"{api_base}/api/v2/accounts/{account_id}/runs/{run_id}/" + run_status_link = ( + f"{api_base}/deploy/{account_id}/projects/{project_id}/runs/{run_id}/" + ) + + # update user with status link + print(f"Job running! See job status at {run_status_link}") + + # check status indefinitely with an initial wait period + time.sleep(30) + while True: + status = get_run_status(req_status_url, req_auth_header) + print(f"Run status -> {status}") + + if status in ["Error", "Cancelled"]: + raise Exception(f"Run failed or canceled. See why at {run_status_link}") + + if status == "Success": + print(f"Job completed successfully! See details at {run_status_link}") + return + + time.sleep(10) + + +if __name__ == "__main__": + main() diff --git a/.gitignore b/.gitignore index 4d4b1afc..dc965695 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ logs/ .DS_Store .user.yml +*.hurl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6431b73f..8c305a40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/sqlfluff/sqlfluff rev: "3.0.1" hooks: @@ -26,7 +27,3 @@ repos: "dbt-metricflow[duckdb,snowflake,postgres]~=0.6.0", "sqlfluff-templater-dbt~=3.0.1", ] - - repo: https://github.com/psf/black - rev: "24.3.0" - hooks: - - id: black diff --git a/requirements.in b/requirements.in index 9f247ade..7c65c441 100644 --- a/requirements.in +++ b/requirements.in @@ -1 +1,2 @@ pre-commit~=3.6.0 +requests~=2.31.0 diff --git a/requirements.txt b/requirements.txt index 4e4a353e..fd6036b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,19 @@ # This file was autogenerated by uv via the following command: # uv pip compile requirements.in -o requirements.txt +certifi==2024.2.2 + # via requests cfgv==3.4.0 # via pre-commit +charset-normalizer==3.3.2 + # via requests distlib==0.3.8 # via virtualenv filelock==3.13.1 # via virtualenv identify==2.5.35 # via pre-commit +idna==3.6 + # via requests nodeenv==1.8.0 # via pre-commit platformdirs==4.2.0 @@ -15,7 +21,10 @@ platformdirs==4.2.0 pre-commit==3.6.2 pyyaml==6.0.1 # via pre-commit +requests==2.31.0 setuptools==69.2.0 # via nodeenv +urllib3==2.2.1 + # via requests virtualenv==20.25.1 # via pre-commit