From bd4e0d145b5ee7bde1d22509b4954a586962ae78 Mon Sep 17 00:00:00 2001 From: Charlie Henry Date: Tue, 27 Aug 2024 10:36:51 -0500 Subject: [PATCH] updating docker conn id --- dags/atd_bond_reporting.py | 7 +++++++ dags/atd_finance_data_fdus.py | 2 ++ dags/atd_finance_data_master_agreements.py | 2 ++ dags/atd_finance_data_objects.py | 2 ++ dags/atd_finance_data_subprojects.py | 3 +++ dags/atd_finance_data_task_orders.py | 4 ++++ dags/atd_finance_data_units.py | 4 ++++ dags/atd_kits_dms_message.py | 1 + dags/atd_kits_sig_stat_pub.py | 1 + dags/atd_knack_artbox_signals.py | 2 ++ dags/atd_knack_arterial_managment_locations.py | 2 ++ dags/atd_knack_banner.py | 1 + dags/atd_knack_cctv_cameras.py | 3 +++ dags/atd_knack_corridor_retiming.py | 2 ++ dags/atd_knack_data_tracker_sr_asset_assign.py | 1 + dags/atd_knack_data_tracker_street_segment_updater.py | 1 + dags/atd_knack_detectors.py | 3 +++ dags/atd_knack_esb_311.py | 2 ++ ...td_knack_inventory_items_finance_to_data_tracker.py | 3 +++ dags/atd_knack_inventory_items_nightly_snapshot.py | 3 +++ dags/atd_knack_inventory_transactions.py | 2 ++ dags/atd_knack_markings_materials.py | 2 ++ dags/atd_knack_markings_specifications.py | 2 ++ dags/atd_knack_markings_work_orders_jobs.py | 4 ++++ dags/atd_knack_metrobike_kiosks.py | 3 +++ dags/atd_knack_mmc_activities_to_socrata.py | 2 ++ dags/atd_knack_mmc_issues.py | 2 ++ dags/atd_knack_secondary_signals.py | 1 + dags/atd_knack_signal_cabinets.py | 3 +++ dags/atd_knack_signal_detection_status_log.py | 2 ++ dags/atd_knack_signal_studies.py | 2 ++ dags/atd_knack_signal_work_orders.py | 2 ++ dags/atd_knack_signs_markings_time_logs.py | 5 ++++- dags/atd_knack_tcp_submissions.py | 2 ++ dags/atd_knack_work_orders_markings.py | 4 ++++ dags/atd_metrobike_trips.py | 1 + dags/atd_moped_components_to_agol.py | 1 + dags/atd_road_conditions_socrata.py | 1 + dags/atd_service_bot_intake.py | 1 + dags/atd_service_bot_issues_to_socrata.py | 1 + dags/atd_signal_comms.py | 10 ++++++++++ dags/atd_trail_counters.py | 1 + dags/dts_finances_report_publishing.py | 2 ++ dags/vz_afd_ems_import.py | 1 + 44 files changed, 105 insertions(+), 1 deletion(-) diff --git a/dags/atd_bond_reporting.py b/dags/atd_bond_reporting.py index f3591001..90060aeb 100644 --- a/dags/atd_bond_reporting.py +++ b/dags/atd_bond_reporting.py @@ -107,6 +107,7 @@ t1 = DockerOperator( task_id="microstrategy_report_2020_bond_expenses", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command='python atd-bond-reporting/microstrategy_to_s3.py -r "2020 Bond Expenses Obligated"', environment=env_vars, @@ -120,6 +121,7 @@ t2 = DockerOperator( task_id="microstrategy_report_all_bonds_expenses", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command='python atd-bond-reporting/microstrategy_to_s3.py -r "All bonds Expenses Obligated"', environment=env_vars, @@ -133,6 +135,7 @@ t3 = DockerOperator( task_id="microstrategy_report_fdu_expenses_quarterly", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command='python atd-bond-reporting/microstrategy_to_s3.py -r "FDU Expenses by Quarter"', environment=env_vars, @@ -146,6 +149,7 @@ t4 = DockerOperator( task_id="microstrategy_report_2020_bond_metadata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command='python atd-bond-reporting/microstrategy_to_s3.py -r "2020 Division Group and Unit"', environment=env_vars, @@ -159,6 +163,7 @@ t5 = DockerOperator( task_id="bond_data_to_postgres", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command="python atd-bond-reporting/bond_data.py", environment=env_vars, @@ -172,6 +177,7 @@ t6 = DockerOperator( task_id="bond_data_processing", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command="python atd-bond-reporting/bond_calculations.py", environment=env_vars, @@ -185,6 +191,7 @@ t7 = DockerOperator( task_id="bond_quarterly_data_processing", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command="python atd-bond-reporting/quarterly_reporting.py", environment=env_vars, diff --git a/dags/atd_finance_data_fdus.py b/dags/atd_finance_data_fdus.py index 9f3564fd..d31f4757 100644 --- a/dags/atd_finance_data_fdus.py +++ b/dags/atd_finance_data_fdus.py @@ -132,6 +132,7 @@ t1 = DockerOperator( task_id="fdus_to_s3", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 upload_to_s3.py fdus", environment=data_tracker_env, @@ -143,6 +144,7 @@ t2 = DockerOperator( task_id="fdus_to_socrata", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_socrata.py --dataset fdus", environment=finance_purchasing_env, diff --git a/dags/atd_finance_data_master_agreements.py b/dags/atd_finance_data_master_agreements.py index e9ec13a4..01b92c18 100644 --- a/dags/atd_finance_data_master_agreements.py +++ b/dags/atd_finance_data_master_agreements.py @@ -132,6 +132,7 @@ t1 = DockerOperator( task_id="master_agreements_to_s3", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 upload_to_s3.py master_agreements", environment=data_tracker_env, @@ -143,6 +144,7 @@ t2 = DockerOperator( task_id="master_agreements_to_finance_purchasing", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py master_agreements finance-purchasing", environment=finance_purchasing_env, diff --git a/dags/atd_finance_data_objects.py b/dags/atd_finance_data_objects.py index 45ee656f..165d1ba8 100644 --- a/dags/atd_finance_data_objects.py +++ b/dags/atd_finance_data_objects.py @@ -132,6 +132,7 @@ t1 = DockerOperator( task_id="objects_to_s3", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 upload_to_s3.py objects", environment=data_tracker_env, @@ -143,6 +144,7 @@ t2 = DockerOperator( task_id="objects_to_finance_purchasing", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py objects finance-purchasing", environment=finance_purchasing_env, diff --git a/dags/atd_finance_data_subprojects.py b/dags/atd_finance_data_subprojects.py index 2cac21a3..e8c5056d 100644 --- a/dags/atd_finance_data_subprojects.py +++ b/dags/atd_finance_data_subprojects.py @@ -132,6 +132,7 @@ t1 = DockerOperator( task_id="subprojects_to_s3", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 upload_to_s3.py subprojects", environment=data_tracker_env, @@ -143,6 +144,7 @@ t2 = DockerOperator( task_id="subprojects_to_data_tracker", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py subprojects finance-purchasing", environment=finance_purchasing_env, @@ -154,6 +156,7 @@ t3 = DockerOperator( task_id="subprojects_to_socrata", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_socrata.py --dataset subprojects", environment=finance_purchasing_env, diff --git a/dags/atd_finance_data_task_orders.py b/dags/atd_finance_data_task_orders.py index ba436e8e..4b482f45 100644 --- a/dags/atd_finance_data_task_orders.py +++ b/dags/atd_finance_data_task_orders.py @@ -132,6 +132,7 @@ t1 = DockerOperator( task_id="task_orders_to_s3", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 upload_to_s3.py task_orders", environment=data_tracker_env, @@ -143,6 +144,7 @@ t2 = DockerOperator( task_id="task_orders_to_data_tracker", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py task_orders data-tracker", environment=data_tracker_env, @@ -154,6 +156,7 @@ t3 = DockerOperator( task_id="task_orders_to_finance_purchasing", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py task_orders finance-purchasing", environment=finance_purchasing_env, @@ -165,6 +168,7 @@ t4 = DockerOperator( task_id="task_orders_to_socrata", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_socrata.py --dataset task_orders", environment=finance_purchasing_env, diff --git a/dags/atd_finance_data_units.py b/dags/atd_finance_data_units.py index a1d58554..76efe025 100644 --- a/dags/atd_finance_data_units.py +++ b/dags/atd_finance_data_units.py @@ -132,6 +132,7 @@ t1 = DockerOperator( task_id="units_orders_to_s3", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 upload_to_s3.py units", environment=data_tracker_env, @@ -143,6 +144,7 @@ t2 = DockerOperator( task_id="units_to_data_tracker", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py units data-tracker", environment=data_tracker_env, @@ -154,6 +156,7 @@ t3 = DockerOperator( task_id="units_to_socrata", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_socrata.py --dataset dept_units", environment=finance_purchasing_env, @@ -165,6 +168,7 @@ t4 = DockerOperator( task_id="units_to_finance_purchasing", image="atddocker/atd-finance-data:production", + docker_conn_id="docker_default", auto_remove=True, command="python3 s3_to_knack.py units finance-purchasing", environment=finance_purchasing_env, diff --git a/dags/atd_kits_dms_message.py b/dags/atd_kits_dms_message.py index 586ee53f..a5d74b4c 100644 --- a/dags/atd_kits_dms_message.py +++ b/dags/atd_kits_dms_message.py @@ -65,6 +65,7 @@ t1 = DockerOperator( task_id="update_knack_dms_message", + docker_conn_id="docker_default", image=docker_image, auto_remove=True, command="python ./atd-kits/atd-kits/dms_message_pub.py", diff --git a/dags/atd_kits_sig_stat_pub.py b/dags/atd_kits_sig_stat_pub.py index cf9806d6..64d52efc 100644 --- a/dags/atd_kits_sig_stat_pub.py +++ b/dags/atd_kits_sig_stat_pub.py @@ -66,6 +66,7 @@ t1 = DockerOperator( task_id="atd_kits_sig_status_to_socrata", image="atddocker/atd-kits:production", + docker_conn_id="docker_default", auto_remove=True, command="./atd-kits/atd-kits/signal_status_publisher.py", environment=env_vars, diff --git a/dags/atd_knack_artbox_signals.py b/dags/atd_knack_artbox_signals.py index e7b01326..27a314b3 100644 --- a/dags/atd_knack_artbox_signals.py +++ b/dags/atd_knack_artbox_signals.py @@ -95,6 +95,7 @@ t1 = DockerOperator( task_id="atd_knack_artbox_signals_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name_dest} -c {container_dest} {date_filter_arg}", environment=env_vars_t1, @@ -106,6 +107,7 @@ t2 = DockerOperator( task_id="atd_knack_data_tracker_signals_to_smo", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_knack.py -a {app_name_src} -c {container_src} {date_filter_arg} -dest {app_name_dest}", environment=env_vars_t2, diff --git a/dags/atd_knack_arterial_managment_locations.py b/dags/atd_knack_arterial_managment_locations.py index 503a084b..5c0ba7be 100644 --- a/dags/atd_knack_arterial_managment_locations.py +++ b/dags/atd_knack_arterial_managment_locations.py @@ -68,6 +68,7 @@ t1 = DockerOperator( task_id="atd_knack_arterial_managment_locations_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -79,6 +80,7 @@ t2 = DockerOperator( task_id="atd_knack_arterial_managment_locations_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_banner.py b/dags/atd_knack_banner.py index d87887c5..fc63c708 100644 --- a/dags/atd_knack_banner.py +++ b/dags/atd_knack_banner.py @@ -58,6 +58,7 @@ update_employees = DockerOperator( task_id="update_employees", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-banner/update_employees.py", environment=env_vars, diff --git a/dags/atd_knack_cctv_cameras.py b/dags/atd_knack_cctv_cameras.py index 29488afa..d34266cc 100644 --- a/dags/atd_knack_cctv_cameras.py +++ b/dags/atd_knack_cctv_cameras.py @@ -80,6 +80,7 @@ t1 = DockerOperator( task_id="atd_knack_cctv_cameras_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -91,6 +92,7 @@ t2 = DockerOperator( task_id="atd_knack_cctv_cameras_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -101,6 +103,7 @@ t3 = DockerOperator( task_id="atd_knack_cctv_cameras_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_corridor_retiming.py b/dags/atd_knack_corridor_retiming.py index 83a28d42..d807d95b 100644 --- a/dags/atd_knack_corridor_retiming.py +++ b/dags/atd_knack_corridor_retiming.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_corridor_retiming_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_corridor_retiming_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_data_tracker_sr_asset_assign.py b/dags/atd_knack_data_tracker_sr_asset_assign.py index 4550b00c..cb6e6f2a 100644 --- a/dags/atd_knack_data_tracker_sr_asset_assign.py +++ b/dags/atd_knack_data_tracker_sr_asset_assign.py @@ -60,6 +60,7 @@ t1 = DockerOperator( task_id="service_request_asset_assign", image= "atddocker/atd-knack-services:production", + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/sr_asset_assign.py -a data-tracker -c view_2362 -s signals", environment=env_vars, diff --git a/dags/atd_knack_data_tracker_street_segment_updater.py b/dags/atd_knack_data_tracker_street_segment_updater.py index 43ff5714..494c7f72 100644 --- a/dags/atd_knack_data_tracker_street_segment_updater.py +++ b/dags/atd_knack_data_tracker_street_segment_updater.py @@ -55,6 +55,7 @@ t1 = DockerOperator( task_id="update_street_segments", image="atddocker/atd-knack-services:production", + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/knack_street_seg_updater.py -a data-tracker -c view_1198 {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_detectors.py b/dags/atd_knack_detectors.py index 4ed38476..aa4be2d1 100644 --- a/dags/atd_knack_detectors.py +++ b/dags/atd_knack_detectors.py @@ -80,6 +80,7 @@ t1 = DockerOperator( task_id="atd_knack_detectors_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -91,6 +92,7 @@ t2 = DockerOperator( task_id="atd_knack_detectors_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -101,6 +103,7 @@ t3 = DockerOperator( task_id="atd_knack_detectors_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_esb_311.py b/dags/atd_knack_esb_311.py index fa501f90..1b656c1a 100644 --- a/dags/atd_knack_esb_311.py +++ b/dags/atd_knack_esb_311.py @@ -76,6 +76,7 @@ t1 = DockerOperator( task_id="knack_amd_data_tracker_activities_to_311", image=DOCKER_IMAGE, + docker_conn_id="docker_default", auto_remove=True, command="./atd-knack-311/send_knack_messages_to_esb.py data-tracker", environment=env_vars_data_tracker, @@ -89,6 +90,7 @@ t2 = DockerOperator( task_id="knack_amd_signs_markings_activities_to_311", image=DOCKER_IMAGE, + docker_conn_id="docker_default", auto_remove=True, command="./atd-knack-311/send_knack_messages_to_esb.py signs-markings", environment=env_vars_signs_markings, diff --git a/dags/atd_knack_inventory_items_finance_to_data_tracker.py b/dags/atd_knack_inventory_items_finance_to_data_tracker.py index 8de56d6f..f034f210 100644 --- a/dags/atd_knack_inventory_items_finance_to_data_tracker.py +++ b/dags/atd_knack_inventory_items_finance_to_data_tracker.py @@ -108,6 +108,7 @@ t1 = DockerOperator( task_id="atd_knack_finance_inventory_items_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name_src} -c {container_src} {date_filter_arg}", environment=env_vars_t1, @@ -119,6 +120,7 @@ t2 = DockerOperator( task_id="atd_knack_data_tracker_inventory_items_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name_dest} -c {container_dest} {date_filter_arg}", environment=env_vars_t2, @@ -129,6 +131,7 @@ t3 = DockerOperator( task_id="atd_knack_update_data_tracker_inventory_items_from_finance_inventory", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_knack.py -a {app_name_src} -c {container_src} {date_filter_arg} -dest {app_name_dest}", environment=env_vars_t3, diff --git a/dags/atd_knack_inventory_items_nightly_snapshot.py b/dags/atd_knack_inventory_items_nightly_snapshot.py index d43527f6..f037a6f2 100644 --- a/dags/atd_knack_inventory_items_nightly_snapshot.py +++ b/dags/atd_knack_inventory_items_nightly_snapshot.py @@ -84,6 +84,7 @@ t1 = DockerOperator( task_id="atd_knack_inventory_items_nightly_snapshot_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -95,6 +96,7 @@ t2 = DockerOperator( task_id="atd_knack_inventory_items_nightly_snapshot_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -105,6 +107,7 @@ t3 = DockerOperator( task_id="atd_knack_inventory_items_nightly_snapshot_socrata_backup", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/backup_socrata.py -a {app_name} -c {container}", environment=env_vars, diff --git a/dags/atd_knack_inventory_transactions.py b/dags/atd_knack_inventory_transactions.py index c865180d..4b756120 100644 --- a/dags/atd_knack_inventory_transactions.py +++ b/dags/atd_knack_inventory_transactions.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_inventory_transactions_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_inventory_transactions_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_markings_materials.py b/dags/atd_knack_markings_materials.py index dba42a53..60cf29a6 100644 --- a/dags/atd_knack_markings_materials.py +++ b/dags/atd_knack_markings_materials.py @@ -68,6 +68,7 @@ t1 = DockerOperator( task_id="atd_knack_markings_materials_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -80,6 +81,7 @@ t2 = DockerOperator( task_id="atd_knack_markings_materials_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_markings_specifications.py b/dags/atd_knack_markings_specifications.py index bb67ea41..8d7ce9fe 100644 --- a/dags/atd_knack_markings_specifications.py +++ b/dags/atd_knack_markings_specifications.py @@ -69,6 +69,7 @@ t1 = DockerOperator( task_id="atd_knack_markings_specifications_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -81,6 +82,7 @@ t2 = DockerOperator( task_id="atd_knack_markings_specifications_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_markings_work_orders_jobs.py b/dags/atd_knack_markings_work_orders_jobs.py index 151bd881..37202a25 100644 --- a/dags/atd_knack_markings_work_orders_jobs.py +++ b/dags/atd_knack_markings_work_orders_jobs.py @@ -81,6 +81,7 @@ t1 = DockerOperator( task_id="atd_knack_markings_work_orders_jobs_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -93,6 +94,7 @@ t2 = DockerOperator( task_id="atd_knack_markings_work_orders_jobs_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -104,6 +106,7 @@ t3 = DockerOperator( task_id="atd_knack_markings_jobs_agol_build_markings_segment_geometries", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f'./atd-knack-services/services/agol_build_markings_segment_geometries.py -l markings_jobs {date_filter_arg}', environment=env_vars, @@ -114,6 +117,7 @@ t4 = DockerOperator( task_id="atd_knack_markings_work_orders_jobs_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f'./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}', environment=env_vars, diff --git a/dags/atd_knack_metrobike_kiosks.py b/dags/atd_knack_metrobike_kiosks.py index 4d0c8785..01bce37e 100644 --- a/dags/atd_knack_metrobike_kiosks.py +++ b/dags/atd_knack_metrobike_kiosks.py @@ -80,6 +80,7 @@ t1 = DockerOperator( task_id="atd_knack_metrobike_kiosks_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -91,6 +92,7 @@ t2 = DockerOperator( task_id="atd_knack_metrobike_kiosks_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -101,6 +103,7 @@ t3 = DockerOperator( task_id="atd_knack_metrobike_kiosks_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_mmc_activities_to_socrata.py b/dags/atd_knack_mmc_activities_to_socrata.py index a3154898..c9c5496a 100644 --- a/dags/atd_knack_mmc_activities_to_socrata.py +++ b/dags/atd_knack_mmc_activities_to_socrata.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_mmc_activities_to_socrata_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_mmc_activities_to_socrata_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_mmc_issues.py b/dags/atd_knack_mmc_issues.py index efecf107..748a5cf5 100644 --- a/dags/atd_knack_mmc_issues.py +++ b/dags/atd_knack_mmc_issues.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_mmc_issues_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_mmc_issues_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_secondary_signals.py b/dags/atd_knack_secondary_signals.py index 12e2f03c..a1fce9a5 100644 --- a/dags/atd_knack_secondary_signals.py +++ b/dags/atd_knack_secondary_signals.py @@ -46,6 +46,7 @@ t1 = DockerOperator( task_id="update_secondary_signals", image="atddocker/atd-knack-services:production", + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/secondary_signals_updater.py -a data-tracker -c view_197", environment=env_vars, diff --git a/dags/atd_knack_signal_cabinets.py b/dags/atd_knack_signal_cabinets.py index 1bc44592..c594fcd1 100644 --- a/dags/atd_knack_signal_cabinets.py +++ b/dags/atd_knack_signal_cabinets.py @@ -80,6 +80,7 @@ t1 = DockerOperator( task_id="atd_knack_signal_cabinets_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -91,6 +92,7 @@ t2 = DockerOperator( task_id="atd_knack_signal_cabinets_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -101,6 +103,7 @@ t3 = DockerOperator( task_id="atd_knack_signal_cabinets_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_signal_detection_status_log.py b/dags/atd_knack_signal_detection_status_log.py index 6ccfcbfe..6024cd1a 100644 --- a/dags/atd_knack_signal_detection_status_log.py +++ b/dags/atd_knack_signal_detection_status_log.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_traffic_signal_detection_status_log_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_traffic_signal_detection_status_log_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_signal_studies.py b/dags/atd_knack_signal_studies.py index 16baf316..510364cc 100644 --- a/dags/atd_knack_signal_studies.py +++ b/dags/atd_knack_signal_studies.py @@ -77,6 +77,7 @@ def get_env_vars(): t1 = DockerOperator( task_id="atd_knack_signal_studies_to_postgrest", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container}", @@ -89,6 +90,7 @@ def get_env_vars(): t2 = DockerOperator( task_id="atd_knack_signal_studies_to_socrata", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container}", diff --git a/dags/atd_knack_signal_work_orders.py b/dags/atd_knack_signal_work_orders.py index 33a344ee..d2ee0678 100644 --- a/dags/atd_knack_signal_work_orders.py +++ b/dags/atd_knack_signal_work_orders.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_signal_work_orders_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_signal_work_orders_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_signs_markings_time_logs.py b/dags/atd_knack_signs_markings_time_logs.py index f9bbf809..6dd9642c 100644 --- a/dags/atd_knack_signs_markings_time_logs.py +++ b/dags/atd_knack_signs_markings_time_logs.py @@ -74,6 +74,7 @@ t1 = DockerOperator( task_id="atd_knack_signs_time_logs_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container_signs} {date_filter_arg}", environment=env_vars, @@ -85,11 +86,11 @@ t2 = DockerOperator( task_id="atd_knack_markings_time_logs_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container_markings} {date_filter_arg}", environment=env_vars, tty=True, - force_pull=True, mount_tmp_dir=False, ) @@ -97,6 +98,7 @@ t3 = DockerOperator( task_id="atd_knack_signs_time_logs_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f'./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container_signs} {date_filter_arg}', environment=env_vars, @@ -107,6 +109,7 @@ t4 = DockerOperator( task_id="atd_knack_markings_time_logs_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f'./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container_markings} {date_filter_arg}', environment=env_vars, diff --git a/dags/atd_knack_tcp_submissions.py b/dags/atd_knack_tcp_submissions.py index 24c86628..a5448c10 100644 --- a/dags/atd_knack_tcp_submissions.py +++ b/dags/atd_knack_tcp_submissions.py @@ -72,6 +72,7 @@ t1 = DockerOperator( task_id="atd_knack_tcp_submissions_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -83,6 +84,7 @@ t2 = DockerOperator( task_id="atd_knack_tcp_submissions_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_knack_work_orders_markings.py b/dags/atd_knack_work_orders_markings.py index 46c3af1d..aebcb9f7 100644 --- a/dags/atd_knack_work_orders_markings.py +++ b/dags/atd_knack_work_orders_markings.py @@ -81,6 +81,7 @@ t1 = DockerOperator( task_id="atd_knack_markings_work_orders_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -93,6 +94,7 @@ t2 = DockerOperator( task_id="atd_knack_markings_work_orders_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -104,6 +106,7 @@ t3 = DockerOperator( task_id="atd_knack_markings_work_orders_agol_build_markings_segment_geometries", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f'./atd-knack-services/services/agol_build_markings_segment_geometries.py -l markings_jobs {date_filter_arg}', environment=env_vars, @@ -114,6 +117,7 @@ t4 = DockerOperator( task_id="atd_knack_markings_work_orders_to_socrata", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f'./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}', environment=env_vars, diff --git a/dags/atd_metrobike_trips.py b/dags/atd_metrobike_trips.py index cb183add..7a78dcb7 100644 --- a/dags/atd_metrobike_trips.py +++ b/dags/atd_metrobike_trips.py @@ -61,6 +61,7 @@ t1 = DockerOperator( task_id="atd_metrobike_trips_socrata", image="atddocker/atd-metrobike:production", + docker_conn_id="docker_default", auto_remove=True, command="python publish_trips.py", environment=env_vars, diff --git a/dags/atd_moped_components_to_agol.py b/dags/atd_moped_components_to_agol.py index c9b022e3..704c8327 100644 --- a/dags/atd_moped_components_to_agol.py +++ b/dags/atd_moped_components_to_agol.py @@ -61,6 +61,7 @@ t1 = DockerOperator( task_id="moped_components_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python components_to_agol.py {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_road_conditions_socrata.py b/dags/atd_road_conditions_socrata.py index 88f2ca1f..ee942064 100644 --- a/dags/atd_road_conditions_socrata.py +++ b/dags/atd_road_conditions_socrata.py @@ -61,6 +61,7 @@ t1 = DockerOperator( task_id="road_conditions_socrata", image="atddocker/atd-road-conditions:production", + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-road-conditions/socrata.py {date_filter_arg}", environment=env_vars, diff --git a/dags/atd_service_bot_intake.py b/dags/atd_service_bot_intake.py index 39655e7d..baffce40 100644 --- a/dags/atd_service_bot_intake.py +++ b/dags/atd_service_bot_intake.py @@ -66,6 +66,7 @@ def get_env_vars(): DockerOperator( task_id="dts_sr_to_github", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command="./atd-service-bot/intake.py", diff --git a/dags/atd_service_bot_issues_to_socrata.py b/dags/atd_service_bot_issues_to_socrata.py index c5cab6db..4919bb9d 100644 --- a/dags/atd_service_bot_issues_to_socrata.py +++ b/dags/atd_service_bot_issues_to_socrata.py @@ -78,6 +78,7 @@ def get_env_vars(): DockerOperator( task_id="dts_github_to_socrata", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command="./atd-service-bot/issues_to_socrata.py", diff --git a/dags/atd_signal_comms.py b/dags/atd_signal_comms.py index d729c313..de35aa96 100644 --- a/dags/atd_signal_comms.py +++ b/dags/atd_signal_comms.py @@ -91,6 +91,7 @@ def get_start_date(**context): cameras_s3 = DockerOperator( task_id="run_comm_check_cameras", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/run_comm_check.py camera --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -103,6 +104,7 @@ def get_start_date(**context): detectors_s3 = DockerOperator( task_id="run_comm_check_detectors", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/run_comm_check.py detector --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -115,6 +117,7 @@ def get_start_date(**context): dms_s3 = DockerOperator( task_id="run_comm_check_dms", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/run_comm_check.py digital_message_sign --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -127,6 +130,7 @@ def get_start_date(**context): battery_backup_s3 = DockerOperator( task_id="run_comm_check_battery_backup", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/run_comm_check.py cabinet_battery_backup --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -139,6 +143,7 @@ def get_start_date(**context): signal_monitors_s3 = DockerOperator( task_id="run_comm_check_signal_monitors", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/run_comm_check.py signal_monitor --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -151,6 +156,7 @@ def get_start_date(**context): cameras_socrata = DockerOperator( task_id="socrata_pub_cameras", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/socrata_pub.py camera --start {start_date} -v --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -163,6 +169,7 @@ def get_start_date(**context): detectors_socrata = DockerOperator( task_id="socrata_pub_detectors", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/socrata_pub.py detector --start {start_date} -v --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -175,6 +182,7 @@ def get_start_date(**context): dms_socrata = DockerOperator( task_id="socrata_pub_dms", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/socrata_pub.py digital_message_sign --start {start_date} -v --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -187,6 +195,7 @@ def get_start_date(**context): battery_backup_socrata = DockerOperator( task_id="socrata_pub_battery_backup", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/socrata_pub.py cabinet_battery_backup --start {start_date} -v --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, @@ -199,6 +208,7 @@ def get_start_date(**context): signal_monitors_socrata = DockerOperator( task_id="socrata_pub_signal_monitors", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"python atd-signal-comms/socrata_pub.py signal_monitor --start {start_date} -v --env {DEPLOYMENT_ENVIRONMENT}", environment=env_vars, diff --git a/dags/atd_trail_counters.py b/dags/atd_trail_counters.py index 3ff74dc4..ef59256f 100644 --- a/dags/atd_trail_counters.py +++ b/dags/atd_trail_counters.py @@ -64,6 +64,7 @@ t1 = DockerOperator( task_id="trail_counter_data_publish", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command=f"python counter_data.py --start {start}", diff --git a/dags/dts_finances_report_publishing.py b/dags/dts_finances_report_publishing.py index 3517f8b2..a4e68719 100644 --- a/dags/dts_finances_report_publishing.py +++ b/dags/dts_finances_report_publishing.py @@ -98,6 +98,7 @@ t1 = DockerOperator( task_id="download_microstrategy_reports", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command=f"python etl/rev_exp_report_to_s3.py", @@ -109,6 +110,7 @@ t2 = DockerOperator( task_id="update_socrata", image=docker_image, + docker_conn_id="docker_default", api_version="auto", auto_remove=True, command=f"python etl/mstro_reports_to_socrata.py", diff --git a/dags/vz_afd_ems_import.py b/dags/vz_afd_ems_import.py index ab9f96c5..daa08ef0 100644 --- a/dags/vz_afd_ems_import.py +++ b/dags/vz_afd_ems_import.py @@ -46,6 +46,7 @@ def etl_afd_import(): task_id="run_ems_import", environment=dict(os.environ), image="atddocker/vz-afd-ems-import:production", + docker_conn_id="docker_default", auto_remove=True, entrypoint=["/entrypoint.sh"], command=["afd"],