Skip to content

Commit 00a7bb3

Browse files
erikamovohrite
erikamov
authored and
ohrite
committed
wip Run workflow for staging and production
1 parent be2d178 commit 00a7bb3

File tree

1 file changed

+29
-27
lines changed

1 file changed

+29
-27
lines changed

‎.github/workflows/deploy-dbt-docs.yml‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
name: Deploy dbt docs
22

33
on:
4-
# schedule:
5-
# - cron: "0 8 * * 1" # 8am UTC (1am PST) every Monday
4+
schedule:
5+
- cron: "0 8 * * 1" # 8am UTC (1am PST) every Monday
66
push:
77
branches:
88
- main
@@ -21,14 +21,6 @@ concurrency:
2121
env:
2222
PYTHON_VERSION: '3.11'
2323
POETRY_VERSION: '2.0.1'
24-
SERVICE_ACCOUNT: ${{ github.ref == 'refs/heads/main' && 'github-actions-service-account@cal-itp-data-infra.iam.gserviceaccount.com' || 'github-actions-service-account@cal-itp-data-infra-staging.iam.gserviceaccount.com' }}
25-
WORKLOAD_IDENTITY_PROVIDER: ${{ github.ref == 'refs/heads/main' && 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions/providers/data-infra' || 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra' }}
26-
PROJECT_ID: ${{ github.ref == 'refs/heads/main' && 'cal-itp-data-infra' || 'cal-itp-data-infra-staging' }}
27-
DBT_TARGET: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }}
28-
DBT_ARTIFACTS_BUCKET: ${{ github.ref == 'refs/heads/main' && 'calitp-dbt-artifacts' || 'calitp-staging-dbt-artifacts' }}
29-
DBT_DOCS_BUCKET: ${{ github.ref == 'refs/heads/main' && 'calitp-dbt-docs' || 'calitp-staging-dbt-docs' }}
30-
DBT_SOURCE_DATABASE: ${{ github.ref == 'refs/heads/main' && 'cal-itp-data-infra' || 'cal-itp-data-infra-staging' }}
31-
METABASE_DESTINATION_DATABASE: ${{ github.ref == 'refs/heads/main' && 'Data Marts (formerly Warehouse Views)' || '(Internal) Staging Warehouse Views' }}
3224

3325
jobs:
3426
compile:
@@ -39,6 +31,16 @@ jobs:
3931
contents: read
4032
id-token: write
4133

34+
strategy:
35+
matrix:
36+
include:
37+
- service_account: 'github-actions-service-account@cal-itp-data-infra-staging.iam.gserviceaccount.com'
38+
workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra'
39+
project_id: 'cal-itp-data-infra-staging'
40+
dbt_target: 'staging'
41+
dbt_artifacts_bucket: 'calitp-staging-dbt-artifacts'
42+
dbt_docs_bucket: 'calitp-staging-dbt-docs'
43+
4244
steps:
4345
- name: Checkout
4446
uses: actions/checkout@v4
@@ -47,9 +49,9 @@ jobs:
4749
uses: google-github-actions/auth@v2
4850
with:
4951
create_credentials_file: 'true'
50-
project_id: ${{ env.PROJECT_ID }}
51-
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
52-
service_account: ${{ env.SERVICE_ACCOUNT }}
52+
project_id: ${{ matrix.project_id }}
53+
workload_identity_provider: ${{ matrix.workload_identity_provider }}
54+
service_account: ${{ matrix.service_account }}
5355

5456
- name: Setup GCloud utilities
5557
uses: google-github-actions/setup-gcloud@v2
@@ -95,11 +97,11 @@ jobs:
9597

9698
- name: Print dbt environment
9799
working-directory: warehouse
98-
run: poetry run dbt debug --target ${{ env.DBT_TARGET }}
100+
run: poetry run dbt debug --target ${{ matrix.dbt_target }}
99101

100102
- name: Compile dbt
101103
working-directory: warehouse
102-
run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
104+
run: poetry run dbt compile --target ${{ matrix.dbt_target }} --full-refresh
103105

104106
- name: Create Metabase folder
105107
working-directory: warehouse/models
@@ -111,11 +113,11 @@ jobs:
111113

112114
- name: Compile dbt with exposures
113115
working-directory: warehouse
114-
run: poetry run dbt compile --target ${{ env.DBT_TARGET }} --full-refresh
116+
run: poetry run dbt compile --target ${{ matrix.dbt_target }} --full-refresh
115117

116118
- name: Generate dbt documentation
117119
working-directory: warehouse
118-
run: poetry run dbt docs generate --target ${{ env.DBT_TARGET }} --no-compile
120+
run: poetry run dbt docs generate --target ${{ matrix.dbt_target }} --no-compile
119121

120122
- name: Archive compilation artifacts
121123
uses: actions/upload-artifact@v4
@@ -157,9 +159,9 @@ jobs:
157159
uses: google-github-actions/auth@v2
158160
with:
159161
create_credentials_file: 'true'
160-
project_id: ${{ env.PROJECT_ID }}
161-
workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }}
162-
service_account: ${{ env.SERVICE_ACCOUNT }}
162+
project_id: ${{ matrix.project_id }}
163+
workload_identity_provider: ${{ matrix.workload_identity_provider }}
164+
service_account: ${{ matrix.service_account }}
163165

164166
- name: Setup GCloud utilities
165167
uses: google-github-actions/setup-gcloud@v2
@@ -185,7 +187,7 @@ jobs:
185187
glob: 'catalog.json'
186188
parent: false
187189
process_gcloudignore: false
188-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/catalog.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
190+
destination: "${{ matrix.dbt_artifacts_bucket }}/catalog.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
189191

190192
- name: Upload manifest.json
191193
uses: google-github-actions/upload-cloud-storage@v1
@@ -194,7 +196,7 @@ jobs:
194196
glob: 'manifest.json'
195197
parent: false
196198
process_gcloudignore: false
197-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/manifest.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
199+
destination: "${{ matrix.dbt_artifacts_bucket }}/manifest.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
198200

199201
- name: Upload run_results.json
200202
uses: google-github-actions/upload-cloud-storage@v1
@@ -203,7 +205,7 @@ jobs:
203205
glob: 'run_results.json'
204206
parent: false
205207
process_gcloudignore: false
206-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/run_results.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
208+
destination: "${{ matrix.dbt_artifacts_bucket }}/run_results.json/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
207209

208210
- name: Upload index.html
209211
uses: google-github-actions/upload-cloud-storage@v1
@@ -212,7 +214,7 @@ jobs:
212214
glob: 'index.html'
213215
parent: false
214216
process_gcloudignore: false
215-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/index.html/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
217+
destination: "${{ matrix.dbt_artifacts_bucket }}/index.html/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
216218

217219
- name: Upload artifacts to latest
218220
uses: google-github-actions/upload-cloud-storage@v1
@@ -221,15 +223,15 @@ jobs:
221223
glob: '{catalog.json,manifest.json,index.html,run_results.json}'
222224
parent: false
223225
process_gcloudignore: false
224-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/latest/"
226+
destination: "${{ matrix.dbt_artifacts_bucket }}/latest/"
225227

226228
- name: Upload Metabase exposures
227229
uses: google-github-actions/upload-cloud-storage@v1
228230
with:
229231
path: './warehouse/models/metabase'
230232
parent: false
231233
process_gcloudignore: false
232-
destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/exposures/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
234+
destination: "${{ matrix.dbt_artifacts_bucket }}/exposures/dt=${{ steps.current-date.outputs.formattedTime }}/ts=${{ steps.current-time.outputs.formattedTime }}/"
233235

234236
- name: Upload documentation
235237
uses: google-github-actions/upload-cloud-storage@v1
@@ -238,4 +240,4 @@ jobs:
238240
glob: '{catalog.json,manifest.json,index.html}'
239241
parent: false
240242
process_gcloudignore: false
241-
destination: ${{ env.DBT_DOCS_BUCKET }}
243+
destination: ${{ matrix.dbt_docs_bucket }}

0 commit comments

Comments
 (0)