From 558fe27bfc552fc8b58dc0dc7e70216d2eb485e1 Mon Sep 17 00:00:00 2001 From: winnie <91998347+gwenwindflower@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:17:01 -0500 Subject: [PATCH] Add manual GHA CI/CD for BQ and SF (#19) There is a race condition using multiple warehouses for CI jobs on the same repo. This adds GitHub Actions to run the CI jobs so we can check against all of them. It also adds slim CD jobs that run on merge to main. --- .github/workflows/cd.yml | 55 +++++++ .github/workflows/ci.yml | 57 ++++++++ .../workflows/scripts/dbt_cloud_run_job.py | 134 ++++++++++++++++++ .gitignore | 1 + .pre-commit-config.yaml | 5 +- requirements.in | 1 + requirements.txt | 9 ++ 7 files changed, 258 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/cd.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/scripts/dbt_cloud_run_job.py diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..461d6543 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,55 @@ +name: Run dbt Cloud Deploy to Prod + +on: + push: + branches: + - main + +jobs: + run_snowflake: + name: dbt Cloud Deploy Prod Snowflake + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 283328 + DBT_PR_JOB_ID: 409009 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: main + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py + + run_bigquery: + name: dbt Cloud Deploy Prod BigQuery + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 275557 + DBT_PR_JOB_ID: 553247 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: main + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8dd9c014 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: Run dbt Cloud CI job + +on: + pull_request: + branches: + - main + +jobs: + run_snowflake: + name: dbt Cloud PR CI Snowflake + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 283328 + DBT_PR_JOB_ID: 552843 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: ${{ github.head_ref }} + DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}} + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py + + run_bigquery: + name: dbt Cloud PR CI BigQuery + runs-on: macos-latest + + env: + DBT_ACCOUNT_ID: 188483 + DBT_PROJECT_ID: 275557 + DBT_PR_JOB_ID: 561096 + DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }} + DBT_JOB_CAUSE: "GitHub Actions Request" + DBT_JOB_BRANCH: ${{ github.head_ref }} + DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}} + + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: Install uv + run: python3 -m pip install uv + - name: Install deps + run: uv pip install -r requirements.txt --system + - name: Run dbt Cloud job + run: python3 .github/workflows/scripts/dbt_cloud_run_job.py diff --git a/.github/workflows/scripts/dbt_cloud_run_job.py b/.github/workflows/scripts/dbt_cloud_run_job.py new file mode 100644 index 00000000..efebceab --- /dev/null +++ b/.github/workflows/scripts/dbt_cloud_run_job.py @@ -0,0 +1,134 @@ +import os +import time +import requests + +# ------------------------------------------------------------------------------ +# get environment variables +# ------------------------------------------------------------------------------ +api_base = os.getenv( + "DBT_URL", "https://cloud.getdbt.com" +) # default to multitenant url +job_cause = os.getenv( + "DBT_JOB_CAUSE", "API-triggered job" +) # default to generic message +git_branch = os.getenv("DBT_JOB_BRANCH", None) # default to None +schema_override = os.getenv("DBT_JOB_SCHEMA_OVERRIDE", None) # default to None +api_key = os.environ[ + "DBT_API_KEY" +] # no default here, just throw an error here if key not provided +account_id = os.environ[ + "DBT_ACCOUNT_ID" +] # no default here, just throw an error here if id not provided +project_id = os.environ[ + "DBT_PROJECT_ID" +] # no default here, just throw an error here if id not provided +job_id = os.environ[ + "DBT_PR_JOB_ID" +] # no default here, just throw an error here if id not provided + +print(f""" +Configuration: +api_base: {api_base} +job_cause: {job_cause} +git_branch: {git_branch} +schema_override: {schema_override} +account_id: {account_id} +project_id: {project_id} +job_id: {job_id} +""") + +req_auth_header = {"Authorization": f"Token {api_key}"} +req_job_url = f"{api_base}/api/v2/accounts/{account_id}/jobs/{job_id}/run/" +run_status_map = { # dbt run statuses are encoded as integers. This map provides a human-readable status + 1: "Queued", + 2: "Starting", + 3: "Running", + 10: "Success", + 20: "Error", + 30: "Cancelled", +} + +type AuthHeader = dict[str, str] + + +def run_job( + url: str, + headers: AuthHeader, + cause: str, + branch: str | None = None, + schema_override: str | None = None, +) -> int: + """ + Runs a dbt job + """ + + # build payload + req_payload = {"cause": cause} + if branch and not branch.startswith( + "$(" + ): # starts with '$(' indicates a valid branch name was not provided + req_payload["git_branch"] = branch.replace("refs/heads/", "") + if schema_override: + req_payload["schema_override"] = schema_override.replace("-", "_").replace( + "/", "_" + ) + + # trigger job + print(f"Triggering job:\n\turl: {url}\n\tpayload: {req_payload}") + + response = requests.post(url, headers=headers, json=req_payload) + run_id: int = response.json()["data"]["id"] + return run_id + + +def get_run_status(url: str, headers: AuthHeader) -> str: + """ + gets the status of a running dbt job + """ + # get status + response = requests.get(url, headers=headers) + run_status_code: int = response.json()["data"]["status"] + run_status = run_status_map[run_status_code] + return run_status + + +def main(): + print("Beginning request for job run...") + + # run job + run_id: int = 0 + try: + run_id = run_job( + req_job_url, req_auth_header, job_cause, git_branch, schema_override + ) + except Exception as e: + print(f"ERROR! - Could not trigger job:\n {e}") + raise + + # build status check url and run status link + req_status_url = f"{api_base}/api/v2/accounts/{account_id}/runs/{run_id}/" + run_status_link = ( + f"{api_base}/deploy/{account_id}/projects/{project_id}/runs/{run_id}/" + ) + + # update user with status link + print(f"Job running! See job status at {run_status_link}") + + # check status indefinitely with an initial wait period + time.sleep(30) + while True: + status = get_run_status(req_status_url, req_auth_header) + print(f"Run status -> {status}") + + if status in ["Error", "Cancelled"]: + raise Exception(f"Run failed or canceled. See why at {run_status_link}") + + if status == "Success": + print(f"Job completed successfully! See details at {run_status_link}") + return + + time.sleep(10) + + +if __name__ == "__main__": + main() diff --git a/.gitignore b/.gitignore index 4d4b1afc..dc965695 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ logs/ .DS_Store .user.yml +*.hurl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6431b73f..8c305a40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/sqlfluff/sqlfluff rev: "3.0.1" hooks: @@ -26,7 +27,3 @@ repos: "dbt-metricflow[duckdb,snowflake,postgres]~=0.6.0", "sqlfluff-templater-dbt~=3.0.1", ] - - repo: https://github.com/psf/black - rev: "24.3.0" - hooks: - - id: black diff --git a/requirements.in b/requirements.in index 9f247ade..7c65c441 100644 --- a/requirements.in +++ b/requirements.in @@ -1 +1,2 @@ pre-commit~=3.6.0 +requests~=2.31.0 diff --git a/requirements.txt b/requirements.txt index 4e4a353e..fd6036b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,19 @@ # This file was autogenerated by uv via the following command: # uv pip compile requirements.in -o requirements.txt +certifi==2024.2.2 + # via requests cfgv==3.4.0 # via pre-commit +charset-normalizer==3.3.2 + # via requests distlib==0.3.8 # via virtualenv filelock==3.13.1 # via virtualenv identify==2.5.35 # via pre-commit +idna==3.6 + # via requests nodeenv==1.8.0 # via pre-commit platformdirs==4.2.0 @@ -15,7 +21,10 @@ platformdirs==4.2.0 pre-commit==3.6.2 pyyaml==6.0.1 # via pre-commit +requests==2.31.0 setuptools==69.2.0 # via nodeenv +urllib3==2.2.1 + # via requests virtualenv==20.25.1 # via pre-commit