Add reference files to targets pipeline

Also add cip_program_data_summary target
city-of-baltimore · Aug 14, 2024 · e28fcec · e28fcec
1 parent dd3f79f
commit e28fcec
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 35 deletions.
diff --git a/R/summarise_cip_data_fy.R b/R/summarise_cip_data_fy.R
@@ -0,0 +1,25 @@
+#' Summarise CIP data by fiscal year
+#'
+#' [summarise_cip_data_fy()] wraps [dplyr::summarise()] to combine fiscal year
+#' amount columns grouped by some other variables.
+#'
+#' @param data Input data frame with numeric fiscal year columns.
+#' @param fy_col_prefix String that the fiscal year columns start with. Passed
+#'   to [tidyselect::starts_with()] and is not case sensitive.
+#' @returns A data frame with FY columns summed by the column names passed to
+#'   .by
+summarise_cip_data_fy <- function(data,
+                                  .by = NULL,
+                                  fy_col_prefix = "FY",
+                                  na.rm = TRUE) {
+  data |>
+    summarise(
+      across(
+        starts_with(fy_col_prefix),
+        \(x) {
+          sum(x, na.rm = na.rm)
+        }
+      ),
+      .by = all_of(.by)
+    )
+}
diff --git a/_targets.R b/_targets.R
@@ -44,6 +44,7 @@ tar_source()
 
 if (FALSE) {
   download_fy2530_report_references()
+  targets::tar_delete("report_reference_files_src")
 }
 
 report_reference_files <- rlang::set_names(
@@ -63,10 +64,18 @@ report_stages <- read_curr_fy_report_stages(
 
 ## Load setup data ----
 setup_cip_report <- tar_plan(
+  report_reference_files_src = rlang::set_names(
+    fs::dir_ls(path_tar_user(), glob = "*.csv"),
+    fs::path_ext_remove(
+      fs::path_file(
+        fs::dir_ls(path_tar_user(), glob = "*.csv")
+      )
+    )
+  ),
 
   # Read reference CSV files downloaded w/ download_fy2530_report_references
   report_references = purrr::map(
-    report_reference_files,
+    report_reference_files_src,
     readr::read_csv
   ),
   report_stage_reference = report_references[["fy2530_report_stages"]],
@@ -94,7 +103,7 @@ load_cip_program <- tar_plan(
     purrr::map(
       purrr::set_names(
         cip_program_files,
-        c("Requests", "PC", "BOE", "BOF", "Ordinance")
+        report_stages$stage
       ),
       \(x) {
         if (!fs::file_exists(x)) {
@@ -110,6 +119,17 @@ load_cip_program <- tar_plan(
       }
     )
   ),
+
+  # Get summary of program data by fiscal year and project
+  cip_program_data_summary = cip_program_data |>
+    map(
+      \(x) {
+        x |>
+          summarise_cip_data_fy(
+            .by = "project_code"
+          )
+      }
+    ),
   cip_program_report_stage = cip_program_data[[getOption("current_report_stage")]]
 )
 
@@ -282,6 +302,7 @@ render_cip_report <- tar_plan(
         title_short = getOption("current_report_title_short")
       )
     ),
+    cue = tar_cue(mode = "always"),
     error = "continue"
   ),
   tar_target(

diff --git a/_targets/meta/meta b/_targets/meta/meta
@@ -1,37 +1,39 @@
 name|type|data|command|depend|seed|path|time|size|bytes|format|repository|iteration|parent|children|seconds|warnings|error
-.Random.seed|object|1afe6cfc94b33e52|||||||||||||||
-additional_asset_locations|stem|6165c48acd85f298|8f8d3538b84bdbee|ca7ed4e4bcca0fc7|-253565380||t19943.1484880092s|f4602a179ca0a0cd|31577|qs|local|vector|||0.009||
+.Random.seed|object|b7bff5cc9199b919|||||||||||||||
+additional_asset_locations|stem|6165c48acd85f298|8f8d3538b84bdbee|ca7ed4e4bcca0fc7|-253565380||t19949.5839231125s|f4602a179ca0a0cd|31577|qs|local|vector|||0.047||
 additional_asset_locations_file|stem|e6884637290299af|d5f19b892c51afde|2aa4d62b3139a50f|-826618815|_targets/user/data/spatial/Baltimore-City_Additional-Locations.gpkg|t19937.1881431875s|b97d805b29404ef1|163840|file|local|vector|||0||
-asset_list|stem|19866128c0d7d145|5bfaea842ac224d1|c7833b6de6cd0d6d|-2077660488||t19943.148506584s|732c5a66348cf4ef|307015|qs|local|vector|||0.04||
-baltimore_asset_list|stem|5a3878585d722d76|93707876b0fd6bca|5403417bc473f3f5|-1198557630||t19943.1484888911s|edaef61f942c67e6|245126|qs|local|vector|||0.066||
+asset_list|stem|19866128c0d7d145|5bfaea842ac224d1|c7833b6de6cd0d6d|-2077660488||t19949.5839553717s|732c5a66348cf4ef|307015|qs|local|vector|||0.027||
+baltimore_asset_list|stem|5a3878585d722d76|93707876b0fd6bca|5403417bc473f3f5|-1198557630||t19949.5839240629s|edaef61f942c67e6|245126|qs|local|vector|||0.07||
 baltimore_asset_list_file|stem|363980bdff3f285a|222b8cd2910a4d65|2aa4d62b3139a50f|-1602440123|_targets/user/data/spatial/Baltimore-City_Asset-List.gpkg|t19940.8474880601s|f1433d62d40ca3fe|1622016|file|local|vector|||0||
 bind_cip_comparison_cols|function|26c293d18210c9d6|||||||||||||||
 bind_coords|function|b91c88436b8ae82b|||||||||||||||
-cip_locations|stem|60fcc68ea7ea9b97|b81e53db1fb40d56|b74f116bf1c40fb3|937396978||t19943.1485095918s|abf1b58e6538e254|93666|qs|local|vector|||0.247||
-cip_program_data|stem|2494ef41e66f5ced|66bf16fdc4ba9507|341a47e9cd25868c|-2073228994||t19943.1485059806s|cefa89c6455d4627|50533|qs|local|vector|||0.275||
+cip_locations|stem|50a1bf550cc97993|b81e53db1fb40d56|d42d8e29ae3b2680|937396978||t19949.5839575794s|dba05d9d773379cd|93685|qs|local|vector|||0.18||
+cip_program_data|stem|d06a7fffd8730dea|f744849a2f823526|913c04d2536a1077|-2073228994||t19949.5839315004s|cefa89c6455d4627|50533|qs|local|vector|||0.392||
+cip_program_data_summary|stem|cc201b8ac9e468b0|a533e87e4b1e0265|e3ee83cdbd36247e|1695324782||t19949.5850379201s|1bbf6ba209177a4c|11337|qs|local|vector|||0.038||
 cip_program_files|pattern|7ed5f1f6ca6a900e|dbe7c840af462c1c||986144214||||437689|file|local|vector||cip_program_files_4dc671ce2c846df2*cip_program_files_a557ad6ba14339da*cip_program_files_d16c830ca6061a9b*cip_program_files_2ff78b65aa0b9754*cip_program_files_5c5313b7fd997772|0||
 cip_program_files_2ff78b65aa0b9754|branch|49e049374c8bc029|dbe7c840af462c1c|2c530c1562a7fbd1|66194966|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Six-Year_CIP_BOE-Recommendations.xlsx|t19832.9580888433s|2f85fb7ded300987|89700|file|local|vector|cip_program_files||0||
 cip_program_files_4dc671ce2c846df2|branch|fb70276da899bd41|dbe7c840af462c1c|2c530c1562a7fbd1|-277278387|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Six-Year_CIP_Requests.xlsx|t19759.8953547585s|8f90e8147a3cdb1b|83757|file|local|vector|cip_program_files||0||
 cip_program_files_5c5313b7fd997772|branch|871a19e98bf6f23f|dbe7c840af462c1c|2c530c1562a7fbd1|-1373071283|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Six-Year_CIP_Ordinance.xlsx|t19928.027692697s|963911bc88990069|89694|file|local|vector|cip_program_files||0||
 cip_program_files_a557ad6ba14339da|branch|bbf2de596b0d481c|dbe7c840af462c1c|2c530c1562a7fbd1|-1912704343|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Six-Year_CIP_PC-Recommendations.xlsx|t19788.601528825s|779076d8e7babe5a|86800|file|local|vector|cip_program_files||0||
 cip_program_files_d16c830ca6061a9b|branch|05510c963895d7e4|dbe7c840af462c1c|2c530c1562a7fbd1|-1661903930|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Six-Year_CIP_BOF-Recommendations.xlsx|t19823.1067308027s|5b7c091897e29fef|87738|file|local|vector|cip_program_files||0||
-cip_program_files_files|stem|37c886fbe741a791|cb5c9aeda4e55ab6|91a5524b523f839f|-1721406497||t19943.1488507679s|2b142edbc17bd98c|214|rds|local|vector||cip_program_files_files_0ba79ff1fe9de0f4*cip_program_files_files_3d5e8e820bc73124*cip_program_files_files_3e10ac3bd9cd87ff*cip_program_files_files_4fe77910116edd69*cip_program_files_files_926305c27864228b|0.001||
-cip_program_report_stage|stem|0ec461cc0f25b64c|2a1bf582c8b598c2|bff60de530a3ea7f|985604013||t19943.1485066427s|5b789c05012296ef|16150|qs|local|vector|||0||
-cip_project_details|stem|9ba42afea8eaac74|4e54b3813274e72c|060c9cdf36c08a03|-571135643||t19943.148502681s|287e970335af41ca|505925|qs|local|vector|||1.163||
-cip_project_details_src|stem|7a617a45f004936e|e79c9262188300af|e621bc1a3e0b38e6|-894791588||t19943.1484876758s|c63cfad0e615c3a1|476399|qs|local|vector|||0.313||
-cip_project_details_src_file|stem|bf9a3ae046951fa0|d01ee6e6cd96667b|2aa4d62b3139a50f|-175129281|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Project_Details.xlsx|t19832.76471042s|2def68712d8a896b|1549883|file|local|vector|||0||
-cip_projects|stem|b7923f8986664937|09bdf2b4209f2bc8|761a61b52206bacf|820655482||t19943.1485132982s|7130ee4932262338|602702|qs|local|vector|||0.3||
-cip_report_data|stem|c7795053aa488f63|36ebb8248875bd9a|46b9791eac128bf5|1052679394||t19943.1485149568s|42242902e24840d0|391928|qs|local|vector|||0.125||
+cip_program_files_files|stem|37c886fbe741a791|cb5c9aeda4e55ab6|34e85949619e01ff|-1721406497||t19949.6269432344s|2b142edbc17bd98c|214|rds|local|vector||cip_program_files_files_0ba79ff1fe9de0f4*cip_program_files_files_3d5e8e820bc73124*cip_program_files_files_3e10ac3bd9cd87ff*cip_program_files_files_4fe77910116edd69*cip_program_files_files_926305c27864228b|0.043||
+cip_program_report_stage|stem|0ec461cc0f25b64c|2a1bf582c8b598c2|fccecf921efbd73c|985604013||t19949.5839498243s|5b789c05012296ef|16150|qs|local|vector|||0||
+cip_project_details|stem|e8500f168481f776|4e54b3813274e72c|f9d92339a131bbea|-571135643||t19949.5839497322s|d39d2bd950784566|506386|qs|local|vector|||1.274||
+cip_project_details_review|stem|313962686591a2b2|833ecc675f32bb06|2911dc9da9c6470f|1818972574||t19949.5839549125s|9f29ced6cdfa5f1b|157874|qs|local|vector|||0.059||
+cip_project_details_src|stem|450031b6bf52c9b0|e79c9262188300af|7aabf75d0ff1dc04|-894791588||t19949.5839264283s|1ba7934a4d782c22|468929|qs|local|vector|||0.188||
+cip_project_details_src_file|stem|dd0676be457a3fb3|d01ee6e6cd96667b|2aa4d62b3139a50f|-175129281|_targets/user/data/Adaptive-Planning/Capital_Projects_-_Project_Details.xlsx|t19948.6487085423s|5c3c8287b3bd7784|1558360|file|local|vector|||0||
+cip_projects|stem|0283bc3a41d8fda6|09bdf2b4209f2bc8|53d98736d2644e8a|820655482||t19949.5839614128s|ab31dfcf5dd8af58|603687|qs|local|vector|||0.309||
+cip_report_data|stem|ecbcce369c81536f|36ebb8248875bd9a|3fb898c3bd5f777f|1052679394||t19949.5839630769s|e218d8707959dbd2|392831|qs|local|vector|||0.122||
 cli_alert_new_col|function|729c6c101a962f03|||||||||||||||
 cols_label_ext|function|db08fe2f1025beb1|||||||||||||||
 cols_width_ext|function|5cc7ed248af7edcf|||||||||||||||
 combine_cip_project_data|function|5be1584a332f30a4|||||||||||||||
 compare_cip_stages|object|4fa7c077df415266|||||||||||||||
 convert_to_coords|function|65f5ade082c4f377|||||||||||||||
-curr_fy_additional_locations|stem|a6deb8cd29674a5a|0ec0fb3d7d67b9c5|3848e9264c4fb110|-620554702||t19943.1484890372s|569b2d599f465119|1249|qs|local|vector|||0.004||
-curr_fy_additional_locations_file|stem|ab94eca05fc280a9|9f81e6bb5d448c80|2aa4d62b3139a50f|316771958|_targets/user/data/spatial/FY2025-CIP-Additions.geojson|t19845.8852644063s|1044a2ddf7dfce7d|2106|file|local|vector|||0.001||
-curr_fy_dot_locations|stem|65544b7193a2850b|1cfd4b140a0771a1|d1a22ba930bf24a7|-1569617743||t19943.1484838671s|2dc3993eed8b78d7|32678|qs|local|vector|||0.329||
-curr_report_stage_reference|stem|a113b31fa6ade888|180502a1f117572d|62f0af408239bd67|1743703070||t19943.1484889542s|f9f676f264d3eac5|464|qs|local|vector|||0.001||
+curr_fy_additional_locations|stem|a6deb8cd29674a5a|0ec0fb3d7d67b9c5|3848e9264c4fb110|-620554702||t19949.5839541197s|569b2d599f465119|1249|qs|local|vector|||0.004||
+curr_fy_additional_locations_file|stem|ab94eca05fc280a9|9f81e6bb5d448c80|2aa4d62b3139a50f|316771958|_targets/user/data/spatial/FY2025-CIP-Additions.geojson|t19845.8852644063s|1044a2ddf7dfce7d|2106|file|local|vector|||0||
+curr_fy_dot_locations|stem|65544b7193a2850b|1cfd4b140a0771a1|cd59b8f560892f35|-1569617743||t19949.5839347434s|2dc3993eed8b78d7|32678|qs|local|vector|||0.277||
+curr_report_stage_reference|stem|6ee1d7a72c7e20d2|180502a1f117572d|b6d48f3d94215e3e|1743703070||t19949.5839541548s|0b511ff9f117fb66|620|qs|local|vector|||0.001||
 derive_account_type_col|function|abc9589aa1b7f014|||||||||||||||
 derive_fiscal_year_col|function|f0cd61f2360b027f|||||||||||||||
 derive_location_asset_id_col|function|9b6ee3e22b35dbbb|||||||||||||||
@@ -59,25 +61,25 @@ format_prj_data|function|2c4febc9c0d236f9|||||||||||||||
 format_project_detail_updates|function|fbe52f34e8fabdb0|||||||||||||||
 format_project_locations|function|b28006f849cd2e44|||||||||||||||
 format_project_name|function|e03748267a8e8cbd|||||||||||||||
-format_request_program|function|621050df701de622|||||||||||||||
+format_request_program|function|216dcff30a0de292|||||||||||||||
 format_revenue_category_name_xwalk|function|be4ef9077358eed9|||||||||||||||
 format_workday_names|function|f1c3a36bbbd53756|||||||||||||||
 fy_from_gt|function|63aeac0a3a433b21|||||||||||||||
 get_agency_contract_xwalk|function|acb5446dbca5651e|||||||||||||||
 gt_full_summary|function|616c3f3590e6550b|||||||||||||||
-gt_group_summary_tables|function|8dbe13bbf3a3a707|||||||||||||||
+gt_group_summary_tables|function|65a859032e1a3fe9|||||||||||||||
 gt_prj_est_cost|function|7ad469f2b63920ff|||||||||||||||
 gt_prj_fy_requests|function|9d3e057acfbc564f|||||||||||||||
 gt_revenue_category_fy|function|d931f0e096812a3b|||||||||||||||
 gt_revenue_category_reference|function|b2820f5bb2ad6f5b|||||||||||||||
 gt_simple|function|01b1b4e3060d1965|||||||||||||||
-gt_summary_table_agency|function|eb4ae673d752d7eb|||||||||||||||
-gt_summary_table_source|function|c2b900cb0547ab02|||||||||||||||
+gt_summary_table_agency|function|7f7054fab437d6bf|||||||||||||||
+gt_summary_table_source|function|bcac4c341ad8f8b4|||||||||||||||
 has_county_value|function|9f265468e670a33d|||||||||||||||
 join_agency_reference|function|f4463e009053e0b3|||||||||||||||
 join_project_locations|function|8fd1ef8cef3639d5|||||||||||||||
 join_recommendation_data|function|510c10d6209a21b4|||||||||||||||
-join_request_program_type|function|2053eb3bc58e3de1|||||||||||||||
+join_request_program_type|function|2fed392e8e058f8d|||||||||||||||
 kbl_agency_projects|function|5021ad13b071fe73|||||||||||||||
 kbl_comparison|function|3dbaca193e614d14|||||||||||||||
 kbl_comparison_list|function|663a3b0c233c832e|||||||||||||||
@@ -89,8 +91,8 @@ left_join_asset_data|function|00b474eb11d0660c|||||||||||||||
 left_join_coalesce|function|6d053544762c49ef|||||||||||||||
 load_agency_reference|function|2c0539610d9e0d7f|||||||||||||||
 load_cip_locations|object|a99adf80a99e7a8a|||||||||||||||
-load_cip_program|object|447bf9f39bb5bb54|||||||||||||||
-load_cip_project_details|object|adea5c5516957d46|||||||||||||||
+load_cip_program|object|b3dc61c9dd70f257|||||||||||||||
+load_cip_project_details|object|830b7053aab20ad0|||||||||||||||
 load_cip_project_locations|function|a554562e5ff6103a|||||||||||||||
 load_cip_report_data|function|201cf13b62b57cb3|||||||||||||||
 load_dot_intersections|function|bb83b261697409ee|||||||||||||||
@@ -99,7 +101,7 @@ load_p_hierarchy_xwalks|function|b008ab17907e2591|||||||||||||||
 load_packages|function|5395f5ff8f389730|||||||||||||||
 load_sheets_list|function|7884813e96a53b83|||||||||||||||
 locator_basemap_theme|function|f6d9e6ae34265c57|||||||||||||||
-match_request_program|function|e1979b99d6b98573|||||||||||||||
+match_request_program|function|a7814159662525c9|||||||||||||||
 n_what|function|8925472159697b3e|||||||||||||||
 nest_by_project_code|function|bee8499baf8f14b0|||||||||||||||
 path_tar_user|function|2151d081b1d9a2e8|||||||||||||||
@@ -126,14 +128,17 @@ read_curr_fy_report_stages|function|090e358fa6f40ba5|||||||||||||||
 read_fy25_dot_locations|function|6eee8ecf3d780f1c|||||||||||||||
 read_sf_data|function|85a88b9ad0cbd341|||||||||||||||
 rename_with_dictionary|function|e9d4125cfaf565ca|||||||||||||||
-render_cip_report|object|d431c26a8b71acfc|||||||||||||||
-report_qmd|stem|3b56b8ffa0e62a6f|c651d125314ebc67|2c530c1562a7fbd1|-1421695845||t19943.1548100958s|28e76c0917097715|38|qs|local|vector|||514.866||
+render_cip_report|object|f1afee7da12f1817|||||||||||||||
+replace_pdf_page|function|daa017de6c9d1463|||||||||||||||
+report_qmd|stem|3b56b8ffa0e62a6f|937061efd8794ea4|2c530c1562a7fbd1|-1421695845||t19949.6318864885s|28e76c0917097715|38|qs|local|vector|||426.999||
+report_qmd_cover|stem|b15420a7088b4278|34d1f101b11947ce|0c42b47540843759|-194422161||t19949.5915881463s|4a02b7493001b355|136|qs|local|vector|||1.537||
 report_reference_files|object|cf70ad4abf6cde4a|||||||||||||||
-report_references|stem|6361f4203463ec44|a85d2bfca963ef69|5f4fe7af8f2608db|-1957102286||t19943.1484209293s|b50c90ce1d006cb0|88794|qs|local|vector|||0.065||
-report_site_qmd|stem|3b56b8ffa0e62a6f|e81935b58d237421|2c530c1562a7fbd1|-1724555645||t19943.1488506433s|28e76c0917097715|38|qs|local|vector|||6.544||
-report_stage_reference|stem|84eb931668dfe176|752b35273016d7b5|b83b03b970293867|-1362881503||t19943.1484800256s|683c1f19d664410c|592|qs|local|vector|||0||
-report_stages|object|422678c383b1460b|||||||||||||||
-setup_cip_report|object|e34bc503c285f8d1|||||||||||||||
+report_reference_files_src|stem|2d8a19432e97f0e1|dcae1acb4fb07358|2aa4d62b3139a50f|-488301870||t19949.5838911002s|922598d76622d54c|287|qs|local|vector|||0||
+report_references|stem|90a8265e1664555c|002cfcfd5b986f76|dd72be52305f5e1b|-1957102286||t19949.5839268841s|a6fd26165f828d56|88993|qs|local|vector|||0.033||
+report_site_qmd|stem|3b56b8ffa0e62a6f|e81935b58d237421|2c530c1562a7fbd1|-1724555645||t19949.5850370756s|28e76c0917097715|38|qs|local|vector|||6.475||
+report_stage_reference|stem|98042abae7ab7c50|752b35273016d7b5|2c6944ff3598e03c|-1362881503||t19949.5839347885s|8f61f408405af29b|843|qs|local|vector|||0||
+report_stages|object|1e64810077d19a58|||||||||||||||
+setup_cip_report|object|8ebf5bfd6e534613|||||||||||||||
 st_join_mapbaltimore|function|490a2296df671618|||||||||||||||
 str_extract_agency_contract_id|function|92c32958c24aa152|||||||||||||||
 str_extract_all_project_codes|function|0b9cc2b55fbf6f51|||||||||||||||
@@ -147,6 +152,7 @@ str_extract_spend_category_code|function|04b3614e9ad707e4|||||||||||||||
 str_remove_trim|function|34ac6a5a85a8c18a|||||||||||||||
 str_squish_across|function|66bb992a1207ac41|||||||||||||||
 summarise_cip_comparison|function|f651d301f190b4c7|||||||||||||||
+summarise_cip_data_fy|function|e83a56d92e71ca9c|||||||||||||||
 summarise_cip_fy_comparison|function|9d6541d41ae01e91|||||||||||||||
-summary_tables|stem|411cb673f6b0e08f|ed2f82b623109a93|8d2271d080f6517b|-69483146||t19943.1485157628s|8e93b26b39802778|10148|qs|local|vector|||0.066||
+summary_tables|stem|8de06fea442e14db|ed2f82b623109a93|e398cdc55aada87e|-69483146||t19949.631891054s|b21237146d388330|10185|qs|local|vector|||0.288||
 vec_fmt_currency_plain|function|37098b64d24dfbf1|||||||||||||||