Fix target name on Cosmos payment DAG (#4018)

* Use env var for target name on Cosmos Payment DAG
* Fix misspelled DBT_PROFILES_DIR var

[#3780]

Author: erikamov

airflow/dags/cosmos_payment_dag.py

@@ -1,8 +1,11 @@
+import os
 from datetime import datetime
 
 from cosmos import DbtDag, ProfileConfig, ProjectConfig, RenderConfig
 from cosmos.constants import TestBehavior
 
+DBT_TARGET = os.environ.get("DBT_TARGET")
+
 cosmos_payment = DbtDag(
     # dbt/cosmos-specific parameters
     project_config=ProjectConfig(
@@ -12,7 +15,7 @@
         seeds_relative_path="seeds/",
     ),
     profile_config=ProfileConfig(
-        target_name="staging",
+        target_name=DBT_TARGET,
         profile_name="calitp_warehouse",
         profiles_yml_filepath="/home/airflow/gcs/data/warehouse/profiles.yml",
     ),

airflow/dags/deploy_dbt_docs/deploy_dbt_docs_site.yml

@@ -25,7 +25,7 @@ env_vars:
   CALITP_BUCKET__DBT_DOCS: "{{ env_var('CALITP_BUCKET__DBT_DOCS') }}"
   CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
 
 secrets:

airflow/dags/transform_warehouse/dbt_run_and_upload_artifacts.yml

@@ -27,7 +27,7 @@ env_vars:
   CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_DATABASE: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"

airflow/dags/transform_warehouse/dbt_test.yml

@@ -27,7 +27,7 @@ env_vars:
   GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
   SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"

airflow/dags/transform_warehouse_full_refresh/dbt_run_and_upload_artifacts.yml

@@ -26,7 +26,7 @@ env_vars:
   CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_DATABASE: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"

airflow/dags/transform_warehouse_full_refresh/dbt_test.yml

@@ -28,7 +28,7 @@ env_vars:
   GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
   SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"

airflow/dags/transform_warehouse_full_refresh_sunday/dbt_run_and_upload_artifacts_full_refresh_exclude_rt.yml

@@ -26,7 +26,7 @@ env_vars:
   CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_DATABASE: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"

airflow/dags/transform_warehouse_full_refresh_sunday/dbt_run_and_upload_artifacts_select_rt.yml

@@ -29,7 +29,7 @@ env_vars:
   CALITP_BUCKET__PUBLISH: "{{ env_var('CALITP_BUCKET__PUBLISH') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_DATABASE: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"

airflow/dags/transform_warehouse_full_refresh_sunday/dbt_test_exclude_rt.yml

@@ -27,7 +27,7 @@ env_vars:
   GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
   SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"

airflow/dags/transform_warehouse_full_refresh_sunday/dbt_test_select_rt.yml

@@ -27,7 +27,7 @@ env_vars:
   GOOGLE_CLOUD_PROJECT: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
   BIGQUERY_KEYFILE_LOCATION: /secrets/jobs-data/service_account.json
   DBT_PROJECT_DIR: /app
-  DBT_PROFILE_DIR: /app
+  DBT_PROFILES_DIR: /app
   DBT_TARGET: "{{ env_var('DBT_TARGET') }}"
   SENTRY_DSN: "{{ env_var('SENTRY_DSN') }}"
   SENTRY_ENVIRONMENT: "{{ env_var('SENTRY_ENVIRONMENT') }}"