Skip to content

Commit b52c9e8

Browse files
ohriteohrite
committed
Use workflow identity federation in composer
1 parent 4f7ff72 commit b52c9e8

22 files changed

+27
-136
lines changed

‎airflow/dags/deploy_dbt_docs/deploy_dbt_docs_site.yml‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,18 @@ arguments:
1616
is_delete_operator_pod: true
1717
get_logs: true
1818
priority_class_name: dbt-high-priority
19+
service_account_name: composer-service-account
1920

2021
env_vars:
2122
AIRFLOW_ENV: "{{ env_var('AIRFLOW_ENV') }}"
2223
GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
23-
BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
2424
CALITP_BUCKET__DBT_ARTIFACTS: "{{ env_var('CALITP_BUCKET__DBT_ARTIFACTS') }}"
2525
CALITP_BUCKET__DBT_DOCS: "{{ env_var('CALITP_BUCKET__DBT_DOCS') }}"
2626
CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
2727
DBT_PROJECT_DIR: /app
2828
DBT_PROFILE_DIR: /app
2929
DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
3030

31-
secrets:
32-
- deploy_type: volume
33-
deploy_target: /secrets/jobs-data/
34-
secret: jobs-data
35-
key: service-account.json
36-
3731
tolerations:
3832
- key: pod-role
3933
operator: Equal

‎airflow/dags/parse_and_validate_rt_v2/parse_rt_service_alerts.yml‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,18 @@ arguments:
1616

1717
is_delete_operator_pod: true
1818
get_logs: true
19+
service_account_name: composer-service-account
1920

2021
env_vars:
2122
CALITP_BUCKET__AIRTABLE: "{{ env_var('CALITP_BUCKET__AIRTABLE') }}"
2223
CALITP_BUCKET__GTFS_RT_RAW: "{{ env_var('CALITP_BUCKET__GTFS_RT_RAW') }}"
2324
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2425
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2526
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
26-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2727
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2828
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
2929
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3030

31-
secrets:
32-
- deploy_type: volume
33-
deploy_target: /secrets/jobs-data/
34-
secret: jobs-data
35-
key: service_account.json
36-
3731
k8s_resources:
3832
request_memory: 2.0Gi
3933
request_cpu: 1

‎airflow/dags/parse_and_validate_rt_v2/parse_rt_trip_updates.yml‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,20 @@ arguments:
1414
- "{{ execution_date.replace(minute=0, second=0).format('YYYY-MM-DDTHH:mm:ss') }}"
1515
- "--verbose"
1616

17-
1817
is_delete_operator_pod: true
1918
get_logs: true
19+
service_account_name: composer-service-account
2020

2121
env_vars:
2222
CALITP_BUCKET__AIRTABLE: "{{ env_var('CALITP_BUCKET__AIRTABLE') }}"
2323
CALITP_BUCKET__GTFS_RT_RAW: "{{ env_var('CALITP_BUCKET__GTFS_RT_RAW') }}"
2424
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2525
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2626
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2827
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2928
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3029
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3130

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3831
k8s_resources:
3932
request_memory: 2.0Gi
4033
request_cpu: 1
@@ -44,6 +37,7 @@ tolerations:
4437
operator: Equal
4538
value: computetask
4639
effect: NoSchedule
40+
4741
affinity:
4842
nodeAffinity:
4943
requiredDuringSchedulingIgnoredDuringExecution:

‎airflow/dags/parse_and_validate_rt_v2/parse_rt_vehicle_positions.yml‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,18 @@ arguments:
1616

1717
is_delete_operator_pod: true
1818
get_logs: true
19+
service_account_name: composer-service-account
1920

2021
env_vars:
2122
CALITP_BUCKET__AIRTABLE: "{{ env_var('CALITP_BUCKET__AIRTABLE') }}"
2223
CALITP_BUCKET__GTFS_RT_RAW: "{{ env_var('CALITP_BUCKET__GTFS_RT_RAW') }}"
2324
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2425
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2526
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
26-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2727
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2828
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
2929
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3030

31-
secrets:
32-
- deploy_type: volume
33-
deploy_target: /secrets/jobs-data/
34-
secret: jobs-data
35-
key: service_account.json
36-
3731
k8s_resources:
3832
request_memory: 2.0Gi
3933
request_cpu: 1
@@ -43,6 +37,7 @@ tolerations:
4337
operator: Equal
4438
value: computetask
4539
effect: NoSchedule
40+
4641
affinity:
4742
nodeAffinity:
4843
requiredDuringSchedulingIgnoredDuringExecution:

‎airflow/dags/parse_and_validate_rt_v2/validate_rt_service_alerts.yml‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,18 @@ arguments:
1717

1818
is_delete_operator_pod: true
1919
get_logs: true
20+
service_account_name: composer-service-account
2021

2122
env_vars:
2223
CALITP_BUCKET__AIRTABLE: "{{ env_var('CALITP_BUCKET__AIRTABLE') }}"
2324
CALITP_BUCKET__GTFS_RT_RAW: "{{ env_var('CALITP_BUCKET__GTFS_RT_RAW') }}"
2425
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2526
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2627
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2828
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2929
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3030
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3131

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3832
k8s_resources:
3933
request_memory: 5.0Gi
4034
request_cpu: 2

‎airflow/dags/parse_and_validate_rt_v2/validate_rt_trip_updates.yml‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,18 @@ arguments:
1717

1818
is_delete_operator_pod: true
1919
get_logs: true
20+
service_account_name: composer-service-account
2021

2122
env_vars:
2223
CALITP_BUCKET__AIRTABLE: "{{ env_var('CALITP_BUCKET__AIRTABLE') }}"
2324
CALITP_BUCKET__GTFS_RT_RAW: "{{ env_var('CALITP_BUCKET__GTFS_RT_RAW') }}"
2425
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2526
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2627
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2828
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2929
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3030
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3131

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3832
k8s_resources:
3933
request_memory: 5.0Gi
4034
request_cpu: 2

‎airflow/dags/parse_and_validate_rt_v2/validate_rt_vehicle_positions.yml‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,18 @@ arguments:
1717

1818
is_delete_operator_pod: true
1919
get_logs: true
20+
service_account_name: composer-service-account
2021

2122
env_vars:
2223
CALITP_BUCKET__AIRTABLE: "{{ env_var('CALITP_BUCKET__AIRTABLE') }}"
2324
CALITP_BUCKET__GTFS_RT_RAW: "{{ env_var('CALITP_BUCKET__GTFS_RT_RAW') }}"
2425
CALITP_BUCKET__GTFS_RT_PARSED: "{{ env_var('CALITP_BUCKET__GTFS_RT_PARSED') }}"
2526
CALITP_BUCKET__GTFS_RT_VALIDATION: "{{ env_var('CALITP_BUCKET__GTFS_RT_VALIDATION') }}"
2627
CALITP_BUCKET__GTFS_SCHEDULE_RAW: "{{ env_var('CALITP_BUCKET__GTFS_SCHEDULE_RAW') }}"
27-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
2828
GRAAS_SERVER_URL: "{{ env_var('GRAAS_SERVER_URL') }}"
2929
SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
3030
SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"
3131

32-
secrets:
33-
- deploy_type: volume
34-
deploy_target: /secrets/jobs-data/
35-
secret: jobs-data
36-
key: service_account.json
37-
3832
k8s_resources:
3933
request_memory: 5.0Gi
4034
request_cpu: 2

‎airflow/dags/publish_open_data/publish_california_open_data.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,13 @@ arguments:
1212

1313
is_delete_operator_pod: true
1414
get_logs: true
15+
service_account_name: composer-service-account
1516

1617
env_vars:
17-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
1818
CALITP_BUCKET__DBT_ARTIFACTS: "{{ env_var('CALITP_BUCKET__DBT_ARTIFACTS') }}"
1919
CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
2020

2121
secrets:
22-
- deploy_type: volume
23-
deploy_target: /secrets/jobs-data/
24-
secret: jobs-data
25-
key: service-account.json
2622
- deploy_type: env
2723
deploy_target: CALITP_CKAN_GTFS_SCHEDULE_KEY
2824
secret: jobs-data

‎airflow/dags/scrape_feed_aggregators/scrape_mobility_database.yml‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,10 @@ arguments:
1111

1212
is_delete_operator_pod: true
1313
get_logs: true
14+
service_account_name: composer-service-account
1415

1516
env_vars:
1617
CALITP_BUCKET__AGGREGATOR_SCRAPER: "{{ env_var('CALITP_BUCKET__AGGREGATOR_SCRAPER') }}"
17-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
18-
19-
secrets:
20-
- deploy_type: volume
21-
deploy_target: /secrets/jobs-data/
22-
secret: jobs-data
23-
key: service_account.json
2418

2519
tolerations:
2620
- key: pod-role

‎airflow/dags/scrape_feed_aggregators/scrape_transitland.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,16 @@ arguments:
1111

1212
is_delete_operator_pod: true
1313
get_logs: true
14+
service_account_name: composer-service-account
1415

1516
env_vars:
1617
CALITP_BUCKET__AGGREGATOR_SCRAPER: "{{ env_var('CALITP_BUCKET__AGGREGATOR_SCRAPER') }}"
17-
GOOGLE_APPLICATION_CREDENTIALS: /secrets/jobs-data/service_account.json
1818

1919
secrets:
2020
- deploy_type: env
2121
deploy_target: TRANSITLAND_API_KEY
2222
secret: jobs-data
2323
key: transitland-api-key
24-
- deploy_type: volume
25-
deploy_target: /secrets/jobs-data/
26-
secret: jobs-data
27-
key: service_account.json
2824

2925
tolerations:
3026
- key: pod-role

0 commit comments

Comments
 (0)