Release stg to prd

.github/pull_request_template.md

@@ -0,0 +1,61 @@
+<!---
+Source: https://docs.getdbt.com/blog/analytics-pull-request-template
+-->
+
+
+<!---
+Provide a short summary in the Title above. Examples of good PR titles:
+* "Feature: add so-and-so models"
+* "Fix: deduplicate such-and-such"
+* "Update: dbt version 0.13.0"
+-->
+
+## Description & motivation
+<!---
+Describe your changes, and why you're making them. Is this linked to an open
+issue, a Trello card, or another pull request? Link it here.
+-->
+
+## To-do before merge
+<!---
+(Optional -- remove this section if not needed)
+Include any notes about things that need to happen before this PR is merged, e.g.:
+- [ ] Change the base branch
+- [ ] Update dbt Cloud jobs
+- [ ] Ensure PR #56 is merged
+-->
+
+## Screenshots:
+<!---
+Include a screenshot of the relevant section of the updated DAG. You can access
+your version of the DAG by running `dbt docs generate && dbt docs serve`.
+-->
+
+## Validation of models:
+<!---
+Include any output that confirms that the models do what is expected. This might
+be a link to an in-development dashboard in your BI tool, or a query that
+compares an existing model with a new one.
+-->
+
+## Changes to existing models:
+<!---
+Include this section if you are changing any existing models. Link any related
+pull requests on your BI tool, or instructions for merge (e.g. whether old
+models should be dropped after merge, or whether a full-refresh run is required)
+-->
+
+## Checklist:
+<!---
+This checklist is mostly useful as a reminder of small things that can easily be
+forgotten – it is meant as a helpful tool rather than hoops to jump through.
+Put an `x` in all the items that apply, make notes next to any that haven't been
+addressed, and remove any items that are not relevant to this PR.
+-->
+- [ ] My pull request represents one logical piece of work.
+- [ ] My commits are related to the pull request and look clean.
+- [ ] My SQL follows the [dbt Labs style guide](https://github.com/dbt-labs/corp/blob/master/dbt_style_guide.md).
+- [ ] I have materialized my models appropriately.
+- [ ] I have added appropriate tests and documentation to any new models.
+- [ ] I have updated the README file if applicable.
+- [ ] I have added partitioning an clustering fields where appropriate.

.github/workflows/bigquery_cleanup.yml

@@ -10,6 +10,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: stg
 
       - uses: actions/setup-python@v4
         with:
@@ -35,6 +37,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: prd
 
       - uses: actions/setup-python@v4
         with:

.github/workflows/ci_pipeline.yml

@@ -9,7 +9,7 @@ on:
 env:
   DBT_CICD_RUN: "true"
   DESTINATION_BRANCH: "${{ github.event.pull_request.base.ref }}"
-  DBT_DATASET: "cicd_${{ github.event.number }}_${{ github.run_number }}_${{ github.sha }}"
+  DBT_DATASET: "ci_${{ github.event.number }}_${{ github.run_number }}_${{ github.sha }}"
   DBT_PROFILES_DIR: "."
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -53,7 +53,7 @@ jobs:
       - name: pre-commit run -a
         run: SKIP=dbt-compile,dbt-docs-generate pre-commit run -a
 
-      - run: pytest ./tests/pytest -m no_deps
+      - run: pytest ./tests/pytest -m no_deps -n 5
       - run: dbt deps
       - name: dbt debug
         run: dbt debug --target $DESTINATION_BRANCH
@@ -62,12 +62,12 @@ jobs:
       - name: dbt build
         run: dbt --warn-error build --fail-fast --target $DESTINATION_BRANCH
 
-      - run: pytest ./tests/pytest -m run_results_json
+      - run: pytest ./tests/pytest -m run_results_json -n 5
 
       - name: dbt source freshness
         run: dbt source freshness --target $DESTINATION_BRANCH || true # source freshness is allowed to fail in CI due to stale sources, the next command runs pytest on sources.json to validate the generated SQL (the prupose of this step)
 
-      - run: pytest ./tests/pytest -m sources_json
+      - run: pytest ./tests/pytest -m sources_json -n 5
 
       - name: dbt build incremental models
         run: dbt --warn-error build --fail-fast --select config.materialized:incremental --target $DESTINATION_BRANCH

.github/workflows/daily_runs.yml

@@ -24,6 +24,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: stg
 
       - uses: actions/setup-python@v4
         with:
@@ -42,6 +44,11 @@ jobs:
       - name: Install python packages
         run: pip install -r requirements.txt
 
+      - name: dbt seed
+        run: dbt seed --target stg
+        env:
+          DBT_DATASET: "stg"
+
       - name: dbt build
         run: dbt build --target stg
         env:
@@ -53,6 +60,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: prd
 
       - uses: actions/setup-python@v4
         with:
@@ -71,6 +80,11 @@ jobs:
       - name: Install python packages
         run: pip install -r requirements.txt
 
+      - name: dbt seed
+        run: dbt seed --target prd
+        env:
+          DBT_DATASET: "prd"
+
       - name: dbt build
         run: dbt build --target prd
         env:

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 .state/
 .user.yml
 .vscode/
+*.Identifier
 *.ipynb
 dbt_packages/
 logs/

.pre-commit-config.yaml

@@ -42,7 +42,7 @@ repos:
     hooks:
       - id: pytest-no-deps
         name: pytest-no-deps
-        entry: pytest ./tests/pytest -m no_deps
+        entry: pytest ./tests/pytest -m no_deps -n 5
         language: system
         pass_filenames: false
         always_run: true
@@ -76,15 +76,15 @@ repos:
     hooks:
       - id: pytest-catalog-json
         name: pytest-catalog-json
-        entry: pytest ./tests/pytest -m catalog_json
+        entry: pytest ./tests/pytest -m catalog_json -n 5
         language: system
         pass_filenames: false
         always_run: true
   - repo: local
     hooks:
       - id: pytest-manifest-json
         name: pytest-manifest-json
-        entry: pytest ./tests/pytest -m manifest_json
+        entry: pytest ./tests/pytest -m manifest_json -n 5
         language: system
         pass_filenames: false
         always_run: true