Merge pull request #248 from cityofaustin/ch-docker-con

Updating remaining DAGs docker_conn_id

dags/atd_bond_reporting.py

@@ -107,6 +107,7 @@
     t1 = DockerOperator(
         task_id="microstrategy_report_2020_bond_expenses",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command='python atd-bond-reporting/microstrategy_to_s3.py -r "2020 Bond Expenses Obligated"',
         environment=env_vars,
@@ -120,6 +121,7 @@
     t2 = DockerOperator(
         task_id="microstrategy_report_all_bonds_expenses",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command='python atd-bond-reporting/microstrategy_to_s3.py -r "All bonds Expenses Obligated"',
         environment=env_vars,
@@ -133,6 +135,7 @@
     t3 = DockerOperator(
         task_id="microstrategy_report_fdu_expenses_quarterly",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command='python atd-bond-reporting/microstrategy_to_s3.py -r "FDU Expenses by Quarter"',
         environment=env_vars,
@@ -146,6 +149,7 @@
     t4 = DockerOperator(
         task_id="microstrategy_report_2020_bond_metadata",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command='python atd-bond-reporting/microstrategy_to_s3.py -r "2020 Division Group and Unit"',
         environment=env_vars,
@@ -159,6 +163,7 @@
     t5 = DockerOperator(
         task_id="bond_data_to_postgres",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python atd-bond-reporting/bond_data.py",
         environment=env_vars,
@@ -172,6 +177,7 @@
     t6 = DockerOperator(
         task_id="bond_data_processing",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python atd-bond-reporting/bond_calculations.py",
         environment=env_vars,
@@ -185,6 +191,7 @@
     t7 = DockerOperator(
         task_id="bond_quarterly_data_processing",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python atd-bond-reporting/quarterly_reporting.py",
         environment=env_vars,

dags/atd_finance_data_fdus.py

@@ -132,6 +132,7 @@
     t1 = DockerOperator(
         task_id="fdus_to_s3",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 upload_to_s3.py fdus",
         environment=data_tracker_env,
@@ -143,6 +144,7 @@
     t2 = DockerOperator(
         task_id="fdus_to_socrata",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_socrata.py --dataset fdus",
         environment=finance_purchasing_env,

dags/atd_finance_data_master_agreements.py

@@ -132,6 +132,7 @@
     t1 = DockerOperator(
         task_id="master_agreements_to_s3",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 upload_to_s3.py master_agreements",
         environment=data_tracker_env,
@@ -143,6 +144,7 @@
     t2 = DockerOperator(
         task_id="master_agreements_to_finance_purchasing",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py master_agreements finance-purchasing",
         environment=finance_purchasing_env,

dags/atd_finance_data_objects.py

@@ -132,6 +132,7 @@
     t1 = DockerOperator(
         task_id="objects_to_s3",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 upload_to_s3.py objects",
         environment=data_tracker_env,
@@ -143,6 +144,7 @@
     t2 = DockerOperator(
         task_id="objects_to_finance_purchasing",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py objects finance-purchasing",
         environment=finance_purchasing_env,

dags/atd_finance_data_subprojects.py

@@ -132,6 +132,7 @@
     t1 = DockerOperator(
         task_id="subprojects_to_s3",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 upload_to_s3.py subprojects",
         environment=data_tracker_env,
@@ -143,6 +144,7 @@
     t2 = DockerOperator(
         task_id="subprojects_to_data_tracker",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py subprojects finance-purchasing",
         environment=finance_purchasing_env,
@@ -154,6 +156,7 @@
     t3 = DockerOperator(
         task_id="subprojects_to_socrata",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_socrata.py --dataset subprojects",
         environment=finance_purchasing_env,

dags/atd_finance_data_task_orders.py

@@ -132,6 +132,7 @@
     t1 = DockerOperator(
         task_id="task_orders_to_s3",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 upload_to_s3.py task_orders",
         environment=data_tracker_env,
@@ -143,6 +144,7 @@
     t2 = DockerOperator(
         task_id="task_orders_to_data_tracker",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py task_orders data-tracker",
         environment=data_tracker_env,
@@ -154,6 +156,7 @@
     t3 = DockerOperator(
         task_id="task_orders_to_finance_purchasing",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py task_orders finance-purchasing",
         environment=finance_purchasing_env,
@@ -165,6 +168,7 @@
     t4 = DockerOperator(
         task_id="task_orders_to_socrata",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_socrata.py --dataset task_orders",
         environment=finance_purchasing_env,

dags/atd_finance_data_units.py

@@ -132,6 +132,7 @@
     t1 = DockerOperator(
         task_id="units_orders_to_s3",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 upload_to_s3.py units",
         environment=data_tracker_env,
@@ -143,6 +144,7 @@
     t2 = DockerOperator(
         task_id="units_to_data_tracker",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py units data-tracker",
         environment=data_tracker_env,
@@ -154,6 +156,7 @@
     t3 = DockerOperator(
         task_id="units_to_socrata",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_socrata.py --dataset dept_units",
         environment=finance_purchasing_env,
@@ -165,6 +168,7 @@
     t4 = DockerOperator(
         task_id="units_to_finance_purchasing",
         image="atddocker/atd-finance-data:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="python3 s3_to_knack.py units finance-purchasing",
         environment=finance_purchasing_env,

dags/atd_kits_dms_message.py

@@ -65,6 +65,7 @@
 
     t1 = DockerOperator(
         task_id="update_knack_dms_message",
+        docker_conn_id="docker_default",
         image=docker_image,
         auto_remove=True,
         command="python ./atd-kits/atd-kits/dms_message_pub.py",

dags/atd_kits_sig_stat_pub.py

@@ -66,6 +66,7 @@
     t1 = DockerOperator(
         task_id="atd_kits_sig_status_to_socrata",
         image="atddocker/atd-kits:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="./atd-kits/atd-kits/signal_status_publisher.py",
         environment=env_vars,

dags/atd_knack_artbox_signals.py

@@ -95,6 +95,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_artbox_signals_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name_dest} -c {container_dest} {date_filter_arg}",
         environment=env_vars_t1,
@@ -106,6 +107,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_data_tracker_signals_to_smo",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_knack.py -a {app_name_src} -c {container_src} {date_filter_arg} -dest {app_name_dest}",
         environment=env_vars_t2,

dags/atd_knack_arterial_managment_locations.py

@@ -68,6 +68,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_arterial_managment_locations_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -79,6 +80,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_arterial_managment_locations_to_agol",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,

dags/atd_knack_banner.py

@@ -58,6 +58,7 @@
     update_employees = DockerOperator(
         task_id="update_employees",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-banner/update_employees.py",
         environment=env_vars,

dags/atd_knack_cctv_cameras.py

@@ -80,6 +80,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_cctv_cameras_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -91,6 +92,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_cctv_cameras_to_socrata",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -101,6 +103,7 @@
     t3 = DockerOperator(
         task_id="atd_knack_cctv_cameras_to_agol",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,

dags/atd_knack_corridor_retiming.py

@@ -72,6 +72,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_corridor_retiming_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -83,6 +84,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_corridor_retiming_to_socrata",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,

dags/atd_knack_data_tracker_sr_asset_assign.py

@@ -60,6 +60,7 @@
     t1 = DockerOperator(
         task_id="service_request_asset_assign",
         image= "atddocker/atd-knack-services:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/sr_asset_assign.py -a data-tracker -c view_2362 -s signals",
         environment=env_vars,

dags/atd_knack_data_tracker_street_segment_updater.py

@@ -55,6 +55,7 @@
     t1 = DockerOperator(
         task_id="update_street_segments",
         image="atddocker/atd-knack-services:production",
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/knack_street_seg_updater.py -a data-tracker -c view_1198 {date_filter_arg}",
         environment=env_vars,

dags/atd_knack_detectors.py

@@ -80,6 +80,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_detectors_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -91,6 +92,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_detectors_to_socrata",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -101,6 +103,7 @@
     t3 = DockerOperator(
         task_id="atd_knack_detectors_to_agol",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,

dags/atd_knack_esb_311.py

@@ -76,6 +76,7 @@
     t1 = DockerOperator(
         task_id="knack_amd_data_tracker_activities_to_311",
         image=DOCKER_IMAGE,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="./atd-knack-311/send_knack_messages_to_esb.py data-tracker",
         environment=env_vars_data_tracker,
@@ -89,6 +90,7 @@
     t2 = DockerOperator(
         task_id="knack_amd_signs_markings_activities_to_311",
         image=DOCKER_IMAGE,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command="./atd-knack-311/send_knack_messages_to_esb.py signs-markings",
         environment=env_vars_signs_markings,

dags/atd_knack_inventory_items_finance_to_data_tracker.py

@@ -108,6 +108,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_finance_inventory_items_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name_src} -c {container_src} {date_filter_arg}",
         environment=env_vars_t1,
@@ -119,6 +120,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_data_tracker_inventory_items_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name_dest} -c {container_dest} {date_filter_arg}",
         environment=env_vars_t2,
@@ -129,6 +131,7 @@
     t3 = DockerOperator(
         task_id="atd_knack_update_data_tracker_inventory_items_from_finance_inventory",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_knack.py -a {app_name_src} -c {container_src} {date_filter_arg} -dest {app_name_dest}",
         environment=env_vars_t3,

dags/atd_knack_inventory_items_nightly_snapshot.py

@@ -84,6 +84,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_inventory_items_nightly_snapshot_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -95,6 +96,7 @@
     t2 = DockerOperator(
         task_id="atd_knack_inventory_items_nightly_snapshot_to_socrata",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -105,6 +107,7 @@
     t3 = DockerOperator(
         task_id="atd_knack_inventory_items_nightly_snapshot_socrata_backup",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/backup_socrata.py -a {app_name} -c {container}",
         environment=env_vars,