Skip to content

Commit 75c403d

Browse files
committed
Use workflow identity federation in composer
1 parent ac9d450 commit 75c403d

File tree

23 files changed

+14
-132
lines changed

23 files changed

+14
-132
lines changed

airflow/dags/parse_and_validate_rt_v2/parse_rt_service_alerts.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,10 @@ env_vars:
2323
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2424
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2525
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
26-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2726
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2827
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
2928
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3029

31-
secrets:
32-
- deploy_type: volume
33-
deploy_target: /secrets/jobs-data/
34-
secret: jobs-data
35-
key: service_account.json
36-
3730
k8s_resources:
3831
request_memory: 2.0Gi
3932
request_cpu: 1

airflow/dags/parse_and_validate_rt_v2/parse_rt_trip_updates.yml

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ arguments:
1414
- "{{ execution_date.replace(minute=0, second=0).format('YYYY-MM-DDTHH:mm:ss') }}"
1515
- "--verbose"
1616

17-
1817
is_delete_operator_pod: true
1918
get_logs: true
2019

@@ -24,17 +23,10 @@ env_vars:
2423
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2524
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2625
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2826
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2927
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3028
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3129

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3830
k8s_resources:
3931
request_memory: 2.0Gi
4032
request_cpu: 1
@@ -44,6 +36,7 @@ tolerations:
4436
operator: Equal
4537
value: computetask
4638
effect: NoSchedule
39+
4740
affinity:
4841
nodeAffinity:
4942
requiredDuringSchedulingIgnoredDuringExecution:

airflow/dags/parse_and_validate_rt_v2/parse_rt_vehicle_positions.yml

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,10 @@ env_vars:
2323
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2424
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2525
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
26-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2726
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2827
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
2928
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3029

31-
secrets:
32-
- deploy_type: volume
33-
deploy_target: /secrets/jobs-data/
34-
secret: jobs-data
35-
key: service_account.json
36-
3730
k8s_resources:
3831
request_memory: 2.0Gi
3932
request_cpu: 1
@@ -43,6 +36,7 @@ tolerations:
4336
operator: Equal
4437
value: computetask
4538
effect: NoSchedule
39+
4640
affinity:
4741
nodeAffinity:
4842
requiredDuringSchedulingIgnoredDuringExecution:

airflow/dags/parse_and_validate_rt_v2/validate_rt_service_alerts.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,10 @@ env_vars:
2424
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2525
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2626
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2827
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2928
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3029
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3130

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3831
k8s_resources:
3932
request_memory: 5.0Gi
4033
request_cpu: 2

airflow/dags/parse_and_validate_rt_v2/validate_rt_trip_updates.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,10 @@ env_vars:
2424
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2525
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2626
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2827
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2928
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3029
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3130

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3831
k8s_resources:
3932
request_memory: 5.0Gi
4033
request_cpu: 2

airflow/dags/parse_and_validate_rt_v2/validate_rt_vehicle_positions.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,10 @@ env_vars:
2424
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2525
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2626
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2827
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2928
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3029
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3130

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3831
k8s_resources:
3932
request_memory: 5.0Gi
4033
request_cpu: 2

airflow/dags/publish_open_data/publish_california_open_data.yml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,10 @@ is_delete_operator_pod: true
1414
get_logs: true
1515

1616
env_vars:
17-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
1817
CALITP_BUCKET__DBT_ARTIFACTS: "{{ env_var('CALITP_BUCKET__DBT_ARTIFACTS') }}"
1918
CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
2019

2120
secrets:
22-
- deploy_type: volume
23-
deploy_target: /secrets/jobs-data/
24-
secret: jobs-data
25-
key: service-account.json
2621
- deploy_type: env
2722
deploy_target: CALITP_CKAN_GTFS_SCHEDULE_KEY
2823
secret: jobs-data

airflow/dags/transform_warehouse/dbt_run_and_upload_artifacts.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,13 @@ env_vars:
2222
GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
2323
CALITP_BUCKET__DBT_ARTIFACTS: "{{ env_var('CALITP_BUCKET__DBT_ARTIFACTS') }}"
2424
CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
25-
BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
2625
DBT_PROJECT_DIR: /app
2726
DBT_PROFILES_DIR: /app
2827
DBT_DATABASE: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
2928
DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
3029
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3130
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3231

33-
secrets:
34-
- deploy_type: volume
35-
deploy_target: /secrets/jobs-data/
36-
secret: jobs-data
37-
key: service-account.json
38-
3932
k8s_resources:
4033
request_memory: 2.0Gi
4134
request_cpu: 1

airflow/dags/transform_warehouse/dbt_test.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,20 +25,13 @@ get_logs: true
2525
env_vars:
2626
AIRFLOW_ENV: "{{ env_var('AIRFLOW_ENV') }}"
2727
GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
28-
BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
2928
DBT_PROJECT_DIR: /app
3029
DBT_PROFILES_DIR: /app
3130
DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
3231
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3332
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3433
CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
3534

36-
secrets:
37-
- deploy_type: volume
38-
deploy_target: /secrets/jobs-data/
39-
secret: jobs-data
40-
key: service-account.json
41-
4235
k8s_resources:
4336
request_memory: 2.0Gi
4437
request_cpu: 1

airflow/dags/transform_warehouse_full_refresh/dbt_run_and_upload_artifacts.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,13 @@ env_vars:
2323
GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
2424
CALITP_BUCKET__DBT_ARTIFACTS: "{{ env_var('CALITP_BUCKET__DBT_ARTIFACTS') }}"
2525
CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
26-
BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
2726
DBT_PROJECT_DIR: /app
2827
DBT_PROFILES_DIR: /app
2928
DBT_DATABASE: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
3029
DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
3130
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3231
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3332

34-
secrets:
35-
- deploy_type: volume
36-
deploy_target: /secrets/jobs-data/
37-
secret: jobs-data
38-
key: service-account.json
39-
4033
k8s_resources:
4134
request_memory: 2.0Gi
4235
request_cpu: 1

0 commit comments

Comments
 (0)