From 4559689d04cd9c713f45a50095e401faef773f7e Mon Sep 17 00:00:00 2001 From: Margaux Sleckman Date: Thu, 3 Nov 2022 14:50:43 -0700 Subject: [PATCH 1/4] adding first abstract text updates to text_00_parent --- in_text/text_00_parent.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/in_text/text_00_parent.yml b/in_text/text_00_parent.yml index 8c76cb9..2cde882 100644 --- a/in_text/text_00_parent.yml +++ b/in_text/text_00_parent.yml @@ -2,12 +2,15 @@ title: >- Estimating Stream Salinity Impairment Across the Delaware River Basin Using Space- and Time-Aware Machine Learning abstract: >- - [CHANGE] This data release contains information to support water quality modeling in the Delaware River Basin (DRB). - These data support both process-based and machine learning approaches to water quality modeling, including - the prediction of stream temperature. Reservoirs in the DRB serve an important role as a source of drinking - water, but also affect downstream water quality. Therefore, this data release includes data that - characterize both rivers and a subset of reservoirs in the basin. This release provides an update - to many of the files provided in a previous data release (Oliver et al., 2021). + + Stream salinity has been increasing for the last four decades across the Delaware River Basin (DRB), + partially due to groundwater storage and release of road salt that is applied for deicing. If left unmanaged, + this can have proximate consequences for infrastructure and ecosystems. This data release contains data used for + by the Machine learning models aimed at integrating finer-scale spacio-temporal dynamics to estimate stream salinity across + the Delaware River Basin and provide insight into salinity-impaired reaches and the processes that drive changes in stream salinity across the DRB, + which is critical to informed salinity management. This data release includes data that characterize both + rivers and a subset of reservoirs in the basin. + The data are stored in 4 child folders: 1) spatial information, 2) observations, and 3) model driver data, 4) predictions From 0acf4942729cb8dcaea227cfbce22ac3054ef390 Mon Sep 17 00:00:00 2001 From: Margaux Sleckman Date: Thu, 3 Nov 2022 14:57:28 -0700 Subject: [PATCH 2/4] parent metadata edits --- in_text/text_00_parent.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/in_text/text_00_parent.yml b/in_text/text_00_parent.yml index 2cde882..f2b0bd8 100644 --- a/in_text/text_00_parent.yml +++ b/in_text/text_00_parent.yml @@ -14,14 +14,15 @@ abstract: >- The data are stored in 4 child folders: 1) spatial information, 2) observations, and 3) model driver data, 4) predictions - [CHANGE]
  • 1) Spatial Information - Spatial data used for modeling efforts in the Delaware River Basin
    • - +
  • 1) Spatial Information - [CHANGE] Spatial data used for modeling efforts in the Delaware River Basin
    • - a shapefile of polylines for the river segments, point data for observation locations, and polygons for the three (Pepacton, Cannonsville, and Neversink) reservoirs in this dataset. -
  • 2) Observations - Reservoir (surface levels, releases, diversions, water temperature) and river (water temperature and flow) +
  • 2) Observations - [CHANGE] Reservoir (surface levels, releases, diversions, water temperature) and river (water temperature and flow) observations that can be used to train and test water quality models.
    • -
  • 3) Model driver data - Driver data used to force water quality models, including +
  • 3) Model driver data - [CHANGE] Driver data used to force water quality models, including stream reach distance matrices and daily meteorology data from NOAA GEFS and gridMET. This child item also includes the inputs and outputs of an uncalibrated run of PRMS-SNTemp which predicts mean water temperature at all reaches in the DRB. +
  • 3) Predictions - This data compilation was funded by the USGS. From 964cc7742432dc0c0bc57896e6a8fec42510d664 Mon Sep 17 00:00:00 2001 From: Margaux Sleckman Date: Thu, 3 Nov 2022 15:49:49 -0700 Subject: [PATCH 3/4] adding in_data and out_data folders --- in_data/.empty | 0 out_data/.empty | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 in_data/.empty create mode 100644 out_data/.empty diff --git a/in_data/.empty b/in_data/.empty new file mode 100644 index 0000000..e69de29 diff --git a/out_data/.empty b/out_data/.empty new file mode 100644 index 0000000..e69de29 From b5757b5420e3f926600231b089211e0046f383f7 Mon Sep 17 00:00:00 2001 From: Margaux Sleckman Date: Mon, 7 Nov 2022 09:35:32 -0800 Subject: [PATCH 4/4] tst --- 1_spatial.yml | 91 +++------------ 2_observations.yml | 95 +-------------- 3_driver.yml | 45 ++++--- 4_predictions.yml | 36 +++--- remake.yml | 73 ++++++------ src/1_spatial.yml | 96 --------------- src/2_observations.yml | 105 ----------------- src/3_config.yml | 37 ------ src/4_inputs.yml | 87 -------------- src/5_predictions.yml | 35 ------ src/README.md | 2 - src/drb-temp-data-release.Rproj | 13 --- src/fetch_filter_functions.R | 39 ++++--- src/file_functions.R | 26 +---- src/remake.yml | 200 -------------------------------- src/sb_functions.R | 5 +- src/spatial_functions.R | 67 ++++++----- 17 files changed, 160 insertions(+), 892 deletions(-) delete mode 100644 src/1_spatial.yml delete mode 100644 src/2_observations.yml delete mode 100644 src/3_config.yml delete mode 100644 src/4_inputs.yml delete mode 100644 src/5_predictions.yml delete mode 100644 src/README.md delete mode 100644 src/drb-temp-data-release.Rproj delete mode 100644 src/remake.yml diff --git a/1_spatial.yml b/1_spatial.yml index c85464d..287bc26 100644 --- a/1_spatial.yml +++ b/1_spatial.yml @@ -6,7 +6,7 @@ packages: - rgdal - sf - zip - + sources: - src/spatial_functions.R - src/fetch_filter_functions.R @@ -15,82 +15,19 @@ sources: targets: 1_spatial: depends: - - river_metadata - - out_data/study_stream_reaches.zip - - reservoir_metadata - - out_data/study_reservoirs.zip - - out_data/study_monitoring_sites.zip - - # not sure what this metadata should be, if anything - # one option is to include a reach to HRU crosswalk - #out_data/river_reach_metadata.csv: - # command: create_metadata_file(target_name, - # streams_sf = modeled_streams_sf, - # stream_to_hru = 'XX') - - modeled_network_sf: - command: retrieve_network(network_sf_fl = '../delaware-model-prep/1_network/out/network.rds') - - network_vertices_sf: - command: retrieve_vertices(network_sf_fl = '../delaware-model-prep/1_network/out/network.rds') - - # should include shapefile of HRUs - #hrus_sf: - #command: retrieve_hrus(hrus_sf_fl = 'XX') + - out_data/XX_geospatial_area_WG84.zip + - geospatial_area_metadata - # include map of network, maybe with HRUs? - #out_data/modeling_domain_map.png: - # command: plot_domain_map(target_name, - # network_sf = modeled_network_sf, - #plot_crs = I("+init=epsg:2811")) - - river_metadata: - command: extract_feature(network_vertices_sf) - - river_metadata2: - command: extract_feature(modeled_network_sf) - - out_data/study_stream_reaches.zip: - command: sf_to_zip(target_name, - sf_object = modeled_network_sf, - layer_name = I('study_stream_reaches')) - - #out_data/01_spatial_hru.zip: - # command: sf_to_zip(target_name, - # sf_object = hrus_sf, - # layer_name = I('study_hrus')) - - network_lordville: - command: readRDS(file = '../delaware-model-prep/9_collaborator_data/umn/network_subset_lordville.rds') - - lordville_sites: - command: get_sites(network_lordville) + # create target of spatial file + geospatial_area_WG84: + command: read_spatial_file(path = 'in_data/example_data/drb_shp/physiographic_regions_DRB.shp', selected_crs = 4326) - # Define scope of reservoir modeling data in this repo - reservoir_modeling_site_ids: - command: c(I(c( - Pepacton = 'nhdhr_151957878', - Cannonsville = 'nhdhr_120022743'))) - - reservoir_polygons: - command: fetch_filter_res_polygons( - out_rds = target_name, - in_ind = "../lake-temperature-model-prep/1_crosswalk_fetch/out/canonical_lakes_sf.rds.ind", - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - reservoir_metadata: - command: extract_feature(reservoir_polygons) - - out_data/study_reservoirs.zip: - command: sf_to_zip(zip_filename = target_name, - sf_object = reservoir_polygons, layer_name = I('study_reservoirs')) - - monitoring_sites: - command: readRDS(file = '../delaware-model-prep/2_observations/out/drb_filtered_sites.rds') + # grab metadata + geospatial_area_metadata: + command: extract_feature(geospatial_area_WG84) - out_data/study_monitoring_sites.zip: - command: reduce_and_zip(zip_filename = target_name, - in_dat = monitoring_sites, - layer_name = I('study_monitoring_sites')) - \ No newline at end of file + # Output geospatial area shp + out_data/XX_geospatial_area_WG84.zip: + command: sf_to_zip(target_name, + sf_object = geospatial_area_WG84, + layer_name = I('geospatial_area_WG84')) diff --git a/2_observations.yml b/2_observations.yml index 9adf3c1..6d25c32 100644 --- a/2_observations.yml +++ b/2_observations.yml @@ -7,99 +7,14 @@ packages: sources: - src/file_functions.R + - src/fetch_filter_functions.R targets: 2_observations: depends: - - out_data/temperature_observations_drb.zip - - out_data/temperature_observations_lordville.zip - - out_data/temperature_observations_forecast_sites.zip - - out_data/flow_observations_drb.zip - - out_data/flow_observations_lordville.zip - - out_data/flow_observations_forecast_sites.zip - - out_data/reservoir_releases_total.csv - - out_data/reservoir_releases_by_type_drb.csv - - out_data/reservoir_releases_by_type_lordville.csv - - out_data/reservoir_realsat_monthly_surface_area.csv - - out_data/reservoir_io_obs.csv - - out_data/reservoir_temp_obs.csv - - out_data/reservoir_level_obs.csv + - out_data/XX_observations.zip - ##### Transfer of files from delaware-model-prep ##### # daily flow and temperature data - out_data/temperature_observations_drb.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_temp_full.csv') - - out_data/temperature_observations_lordville.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_temp_subset_lordville.csv') - - out_data/temperature_observations_forecast_sites.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/2_observations/out/obs_temp_priority_sites.csv') - - out_data/flow_observations_drb.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_flow_full.csv') - - out_data/flow_observations_lordville.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_flow_subset_lordville.csv') - - out_data/flow_observations_forecast_sites.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/2_observations/out/obs_flow_priority_sites.csv') - - out_data/reservoir_releases_total.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/complete_reservoir_releases.csv', to = target_name) - - out_data/reservoir_releases_by_type_drb.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/reservoir_releases.csv', to = target_name) - - out_data/reservoir_releases_by_type_lordville.csv: - command: filter_reservoirs(out_file = target_name, in_dat = 'out_data/reservoir_releases_by_type.csv', keep = I(c('Pepacton', 'Cannonsville'))) - - # get GRAND IDs of reservoirs above Lordville - reservoir_lordville: - command: c(I(c('1550', '2192'))) - - out_data/reservoir_realsat_monthly_surface_area.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/realsat_monthly_surface_area.csv', to = target_name) - - out_data/reservoir_io_obs.csv: - command: copy_filter_feather( - out_csv = target_name, - in_feather = '../delaware-model-prep/9_collaborator_data/res/res_io_obs.feather', - site_ids = reservoir_modeling_site_ids) - - ##### Transfer and filtering of files of file from lake-temperature-model-prep ##### - out_data/reservoir_temp_obs.csv: - command: fetch_filter_tibble( - out_csv = target_name, - in_ind = '../lake-temperature-model-prep/7b_temp_merge/out/drb_daily_reservoir_temps.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_level_nwis.csv: - command: fetch_filter_tibble( - out_csv = target_name, - in_ind = '../lake-temperature-model-prep/7a_nwis_munge/out/drb_reservoirs_waterlevels_munged.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_level_nycdep.rds: - command: fetch_filter_nycdep( - out_rds = target_name, - in_ind = '../lake-temperature-model-prep/7a_nwis_munge/out/NYC_DEP_reservoir_waterlevels.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_level_usgs_historical.rds: - command: fetch_filter_historical( - out_rds = target_name, - in_ind = '../delaware-model-prep/2_observations/out/interpolated_daily_reservoir_water_budget_components.csv.ind', - in_repo = I('../delaware-model-prep/'), - xwalk = I(c('Cannonsville' = 'nhdhr_120022743', 'Pepacton' = 'nhdhr_151957878'))) - - out_data/reservoir_level_obs.csv: - command: combine_level_sources( - out_csv = target_name, - nwis_levels = 'out_data/reservoir_level_nwis.csv', - nyc_levels = 'out_data/reservoir_level_nycdep.rds', - hist_levels = 'out_data/reservoir_level_usgs_historical.rds') - + out_data/XX_observations.zip: + command: zip_obs(out_file = target_name, in_file = 'in_data/example_data/example_temp_drb_220101.csv') + \ No newline at end of file diff --git a/3_driver.yml b/3_driver.yml index 271b964..97fcac4 100644 --- a/3_driver.yml +++ b/3_driver.yml @@ -1,37 +1,32 @@ -target_default: 3_config +target_default: 3_driver include: packages: - dplyr - - RJSONIO - zip + - readr + - sf sources: - - src/fetch_filter_functions.R - src/file_functions.R + - src/fetch_filter_functions.R targets: - 3_config: + 3_drivers: depends: - - out_data/reservoir_nml_values.json - - ##### Transfer and filtering of files from lake-temperature-model-prep ##### - - # nml values in a nested list with one top-level element per site, saved as JSON - # read in with RJSONIO::fromJSON() - out_data/reservoir_nml_values.json: - command: fetch_filter_nml( - out_json = target_name, - in_ind = '../lake-temperature-model-prep/7_config_merge/out/nml_list.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - ##### Archive of files from res-temperature-process-models ##### - - # .nml files manually copied and renamed from res-temperature-process-models - out_data/reservoir_nml_files.zip: - command: zip_files( - out_file = target_name, - 'in_data/glm3_cal_nhdhr_120022743.nml', - 'in_data/glm3_cal_nhdhr_151957878.nml') + - out_data/XX_driver_data_1.zip + - out_data/XX_driver_data_2.zip + + ## Gridmet + out_data/driver_data_1.csv: + command: file.copy(from = 'in_data/example_data/gridmet_sample1.csv', to = target_name) + + out_data/XX_driver_data_1.zip: + command: zip_this(out_file = target_name, .object = 'out_data/driver_data_1.csv') + + out_data/driver_data_2.csv: + command: file.copy(from = 'in_data/example_data/gridmet_sample2.csv', to = target_name) + + out_data/XX_driver_data_2.zip: + command: zip_this(out_file = target_name, .object = 'out_data/driver_data_2.csv') diff --git a/4_predictions.yml b/4_predictions.yml index c43301c..0ef9c74 100644 --- a/4_predictions.yml +++ b/4_predictions.yml @@ -4,32 +4,24 @@ include: packages: - dplyr + - zip + - readr + - sf sources: + - src/file_functions.R - src/fetch_filter_functions.R targets: - 5_predictions: + 3_driver: depends: - - out_data/reservoir_io_sntemp.csv - - out_data/reservoir_downstream_preds.csv - - out_data/reservoir_outlet_depth_preds.csv - - out_data/dwallin_stream_preds.csv + - out_data/XX_forecasts1.zip + - out_data/XX_forecasts2.zip - out_data/reservoir_io_sntemp.csv: - command: copy_filter_feather( - out_csv = target_name, - in_feather = '../delaware-model-prep/9_collaborator_data/res/res_io_sntemp.feather', - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_downstream_preds.csv: - command: file.copy(from = '../res-temperature-process-models/5_extract/out/downstream_preds.csv', to = target_name) - - out_data/reservoir_outlet_depth_preds.csv: - command: file.copy(from = '../res-temperature-process-models/5_extract/out/outlet_depth_preds.csv', to = target_name) - - out_data/dwallin_stream_preds.csv: - command: file.copy(from = '../delaware-model-prep/3_predictions/out/dwallin_stream_preds.csv', to = target_name) - - out_data/forecast[2021-04-16_2021-07-16]_files.zip: - command: file.copy(from = 'in_data/forecast[2021-04-16_2021-07-16]_files.zip', to = target_name) \ No newline at end of file + # Forecast dataset 1 + out_data/XX_forecasts1.zip: + command: zip_this(out_file = target_name, .object = 'in_data/example_data/predictions_data_1.csv') + + # Forecast dataset 2 + out_data/XX_forecasts2.zip: + command: zip_this(out_file = target_name, .object = 'in_data/example_data/predictions_data_2.csv') \ No newline at end of file diff --git a/remake.yml b/remake.yml index eb6e035..896fe78 100644 --- a/remake.yml +++ b/remake.yml @@ -1,8 +1,8 @@ include: - 1_spatial.yml - 2_observations.yml - - 3_drivers.yml - - 4_forecasts.yml + - 3_driver.yml + - 4_predictions.yml packages: - yaml @@ -29,20 +29,20 @@ targets: depends: - 00_parent_sb_xml - - 01_spatial_sb_xml - - 01_spatial_sb_data -# - log/01_spatial_sb_data.csv - - - 02_observations_sb_data - - 02_observations_sb_xml -# - log/02_observations_sb_data.csv - - - 03_drivers_sb_xml - - 03_drivers_sb_data -# - log/03_drivers_sb_data.csv - - - 04_forecasts_sb_xml - - 04_forecasts_sb_data +# - 01_spatial_sb_xml +# - 01_spatial_sb_data +# # - log/01_spatial_sb_data.csv +# +# - 02_observations_sb_data +# - 02_observations_sb_xml +# # - log/02_observations_sb_data.csv +# +# - 03_drivers_sb_xml +# - 03_drivers_sb_data +# # - log/03_drivers_sb_data.csv +# +# - 04_predictions_sb_xml +# - 04_predictions_sb_data # - log/04_forecasts_sb_data.csv # - 04_inputs_sb_meteo @@ -99,7 +99,8 @@ targets: 01_spatial_sb_data: command: sb_replace_files( sb_id = sbid_01_spatial, - "out_data/XX_geospatial_area_WG84.zip" + "out_data/XX_geospatial_area_WG84.zip", + geospatial_area_metadata ) ### Push all spatial data + metadata xml to sb @@ -135,7 +136,8 @@ targets: 02_observations_sb_data: command: sb_replace_files( sbid_02_observations, - "out_data/XX_observations.zip" + "out_data/XX_observations.zip", + geospatial_area_metadata ) ## Push all obs data + metadata xml to sb via log file @@ -154,7 +156,7 @@ targets: command: render( filename = target_name, "in_text/text_SHARED.yml", - "in_text/text_03_drivers.yml", + "in_text/text_03_driver.yml", geospatial_area_metadata ) @@ -164,7 +166,7 @@ targets: sbid_03_drivers, xml_file = I('03_drivers.xml'), "in_text/text_SHARED.yml", - "in_text/text_03_drivers.yml", + "in_text/text_03_driver.yml", geospatial_area_metadata ) @@ -173,7 +175,8 @@ targets: command: sb_replace_files( sbid_03_drivers, 'out_data/XX_driver_data_1.zip', - 'out_data/XX_driver_data_2.zip' + 'out_data/XX_driver_data_2.zip', + geospatial_area_metadata ) ## Push all driver data + metadata xml to sb via log file @@ -190,28 +193,28 @@ targets: ## FORECASTS ## ## Build metadata xml for forecasts and save locally to out_xml - out_xml/04_forecasts.xml: + out_xml/04_predictions.xml: command: render( filename = target_name, "in_text/text_SHARED.yml", - "in_text/text_04_forecasts.yml", + "in_text/text_04_predictions.yml", geospatial_area_metadata ) ## Push forecasts metadata to sciencebase - 04_forecasts_sb_xml: + 04_predictions_sb_xml: command: sb_render_post_xml( - sbid_04_forecasts, - xml_file = I('04_forecasts.xml'), + sbid_04_predictions, + xml_file = I('04_predictions.xml'), "in_text/text_SHARED.yml", - "in_text/text_04_forecasts.yml", + "in_text/text_04_predictions.yml", geospatial_area_metadata ) ## Push forecasts data to sciencebase - 04_forecasts_sb_data: + 04_predictions_sb_data: command: sb_replace_files( - sbid_04_forecasts, + sbid_04_predictions, 'out_data/XX_forecasts1.zip', 'out_data/XX_forecasts2.zip' ) @@ -231,16 +234,16 @@ targets: # ------ SB IDs -------- sbid_00_parent: - command: c(I('XX_sb_id_parent')) # + command: c(I('6365866cd34ebe442507d0c7')) # 6365866cd34ebe442507d0c7 sbid_01_spatial: - command: c(I('XX_sb_id_spatial')) # + command: c(I('63658810d34ebe442507d0d0')) # sbid_02_observations: - command: c(I('XX_sb_id_observations')) # + command: c(I('6365882fd34ebe442507d0d3')) # sbid_03_drivers: - command: c(I('XX_sb_id_drivers')) # + command: c(I('6365886bd34ebe442507d0d5')) # - sbid_04_forecasts: - command: c(I('XX_sb_id_forecasts')) # \ No newline at end of file + sbid_04_predictions: + command: c(I('6365887ad34ebe442507d0d7')) # \ No newline at end of file diff --git a/src/1_spatial.yml b/src/1_spatial.yml deleted file mode 100644 index c85464d..0000000 --- a/src/1_spatial.yml +++ /dev/null @@ -1,96 +0,0 @@ -target_default: 1_spatial - -packages: - - dplyr - - meddle - - rgdal - - sf - - zip - -sources: - - src/spatial_functions.R - - src/fetch_filter_functions.R - - src/file_functions.R - -targets: - 1_spatial: - depends: - - river_metadata - - out_data/study_stream_reaches.zip - - reservoir_metadata - - out_data/study_reservoirs.zip - - out_data/study_monitoring_sites.zip - - # not sure what this metadata should be, if anything - # one option is to include a reach to HRU crosswalk - #out_data/river_reach_metadata.csv: - # command: create_metadata_file(target_name, - # streams_sf = modeled_streams_sf, - # stream_to_hru = 'XX') - - modeled_network_sf: - command: retrieve_network(network_sf_fl = '../delaware-model-prep/1_network/out/network.rds') - - network_vertices_sf: - command: retrieve_vertices(network_sf_fl = '../delaware-model-prep/1_network/out/network.rds') - - # should include shapefile of HRUs - #hrus_sf: - #command: retrieve_hrus(hrus_sf_fl = 'XX') - - # include map of network, maybe with HRUs? - #out_data/modeling_domain_map.png: - # command: plot_domain_map(target_name, - # network_sf = modeled_network_sf, - #plot_crs = I("+init=epsg:2811")) - - river_metadata: - command: extract_feature(network_vertices_sf) - - river_metadata2: - command: extract_feature(modeled_network_sf) - - out_data/study_stream_reaches.zip: - command: sf_to_zip(target_name, - sf_object = modeled_network_sf, - layer_name = I('study_stream_reaches')) - - #out_data/01_spatial_hru.zip: - # command: sf_to_zip(target_name, - # sf_object = hrus_sf, - # layer_name = I('study_hrus')) - - network_lordville: - command: readRDS(file = '../delaware-model-prep/9_collaborator_data/umn/network_subset_lordville.rds') - - lordville_sites: - command: get_sites(network_lordville) - - # Define scope of reservoir modeling data in this repo - reservoir_modeling_site_ids: - command: c(I(c( - Pepacton = 'nhdhr_151957878', - Cannonsville = 'nhdhr_120022743'))) - - reservoir_polygons: - command: fetch_filter_res_polygons( - out_rds = target_name, - in_ind = "../lake-temperature-model-prep/1_crosswalk_fetch/out/canonical_lakes_sf.rds.ind", - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - reservoir_metadata: - command: extract_feature(reservoir_polygons) - - out_data/study_reservoirs.zip: - command: sf_to_zip(zip_filename = target_name, - sf_object = reservoir_polygons, layer_name = I('study_reservoirs')) - - monitoring_sites: - command: readRDS(file = '../delaware-model-prep/2_observations/out/drb_filtered_sites.rds') - - out_data/study_monitoring_sites.zip: - command: reduce_and_zip(zip_filename = target_name, - in_dat = monitoring_sites, - layer_name = I('study_monitoring_sites')) - \ No newline at end of file diff --git a/src/2_observations.yml b/src/2_observations.yml deleted file mode 100644 index 9adf3c1..0000000 --- a/src/2_observations.yml +++ /dev/null @@ -1,105 +0,0 @@ -target_default: 2_observations - -packages: - - readr - - zip - - zoo - -sources: - - src/file_functions.R - -targets: - 2_observations: - depends: - - out_data/temperature_observations_drb.zip - - out_data/temperature_observations_lordville.zip - - out_data/temperature_observations_forecast_sites.zip - - out_data/flow_observations_drb.zip - - out_data/flow_observations_lordville.zip - - out_data/flow_observations_forecast_sites.zip - - out_data/reservoir_releases_total.csv - - out_data/reservoir_releases_by_type_drb.csv - - out_data/reservoir_releases_by_type_lordville.csv - - out_data/reservoir_realsat_monthly_surface_area.csv - - out_data/reservoir_io_obs.csv - - out_data/reservoir_temp_obs.csv - - out_data/reservoir_level_obs.csv - - ##### Transfer of files from delaware-model-prep ##### - # daily flow and temperature data - out_data/temperature_observations_drb.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_temp_full.csv') - - out_data/temperature_observations_lordville.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_temp_subset_lordville.csv') - - out_data/temperature_observations_forecast_sites.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/2_observations/out/obs_temp_priority_sites.csv') - - out_data/flow_observations_drb.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_flow_full.csv') - - out_data/flow_observations_lordville.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/obs_flow_subset_lordville.csv') - - out_data/flow_observations_forecast_sites.zip: - command: zip_obs(out_file = target_name, in_file = '../delaware-model-prep/2_observations/out/obs_flow_priority_sites.csv') - - out_data/reservoir_releases_total.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/complete_reservoir_releases.csv', to = target_name) - - out_data/reservoir_releases_by_type_drb.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/reservoir_releases.csv', to = target_name) - - out_data/reservoir_releases_by_type_lordville.csv: - command: filter_reservoirs(out_file = target_name, in_dat = 'out_data/reservoir_releases_by_type.csv', keep = I(c('Pepacton', 'Cannonsville'))) - - # get GRAND IDs of reservoirs above Lordville - reservoir_lordville: - command: c(I(c('1550', '2192'))) - - out_data/reservoir_realsat_monthly_surface_area.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/realsat_monthly_surface_area.csv', to = target_name) - - out_data/reservoir_io_obs.csv: - command: copy_filter_feather( - out_csv = target_name, - in_feather = '../delaware-model-prep/9_collaborator_data/res/res_io_obs.feather', - site_ids = reservoir_modeling_site_ids) - - ##### Transfer and filtering of files of file from lake-temperature-model-prep ##### - out_data/reservoir_temp_obs.csv: - command: fetch_filter_tibble( - out_csv = target_name, - in_ind = '../lake-temperature-model-prep/7b_temp_merge/out/drb_daily_reservoir_temps.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_level_nwis.csv: - command: fetch_filter_tibble( - out_csv = target_name, - in_ind = '../lake-temperature-model-prep/7a_nwis_munge/out/drb_reservoirs_waterlevels_munged.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_level_nycdep.rds: - command: fetch_filter_nycdep( - out_rds = target_name, - in_ind = '../lake-temperature-model-prep/7a_nwis_munge/out/NYC_DEP_reservoir_waterlevels.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_level_usgs_historical.rds: - command: fetch_filter_historical( - out_rds = target_name, - in_ind = '../delaware-model-prep/2_observations/out/interpolated_daily_reservoir_water_budget_components.csv.ind', - in_repo = I('../delaware-model-prep/'), - xwalk = I(c('Cannonsville' = 'nhdhr_120022743', 'Pepacton' = 'nhdhr_151957878'))) - - out_data/reservoir_level_obs.csv: - command: combine_level_sources( - out_csv = target_name, - nwis_levels = 'out_data/reservoir_level_nwis.csv', - nyc_levels = 'out_data/reservoir_level_nycdep.rds', - hist_levels = 'out_data/reservoir_level_usgs_historical.rds') - diff --git a/src/3_config.yml b/src/3_config.yml deleted file mode 100644 index 271b964..0000000 --- a/src/3_config.yml +++ /dev/null @@ -1,37 +0,0 @@ -target_default: 3_config - -include: - -packages: - - dplyr - - RJSONIO - - zip - -sources: - - src/fetch_filter_functions.R - - src/file_functions.R - -targets: - 3_config: - depends: - - out_data/reservoir_nml_values.json - - ##### Transfer and filtering of files from lake-temperature-model-prep ##### - - # nml values in a nested list with one top-level element per site, saved as JSON - # read in with RJSONIO::fromJSON() - out_data/reservoir_nml_values.json: - command: fetch_filter_nml( - out_json = target_name, - in_ind = '../lake-temperature-model-prep/7_config_merge/out/nml_list.rds.ind', - in_repo = I('../lake-temperature-model-prep/'), - site_ids = reservoir_modeling_site_ids) - - ##### Archive of files from res-temperature-process-models ##### - - # .nml files manually copied and renamed from res-temperature-process-models - out_data/reservoir_nml_files.zip: - command: zip_files( - out_file = target_name, - 'in_data/glm3_cal_nhdhr_120022743.nml', - 'in_data/glm3_cal_nhdhr_151957878.nml') diff --git a/src/4_inputs.yml b/src/4_inputs.yml deleted file mode 100644 index 697e624..0000000 --- a/src/4_inputs.yml +++ /dev/null @@ -1,87 +0,0 @@ -target_default: 4_inputs - -include: - -packages: - - dplyr - - zip - - readr - - sf - -sources: - - src/file_functions.R - - src/fetch_filter_functions.R - -targets: - 4_inputs: - depends: - - out_data/distance_matrix.csv - - out_data/distance_matrix_lordville.csv - - out_data/reach_attributes.csv - - out_data/reach_attributes_lordville.csv - - out_data/reservoir_features.csv - - out_data/reservoir_features_lordville.csv - - out_data/sntemp_inputs_outputs.zip - - out_data/sntemp_inputs_outputs_lordville.zip - - out_data/reservoir_interpolated_daily_water_budget_components.csv - - out_data/noaagefs_meteo_files.zip - - log/reservoir_meteo_files.yml - - #weather_drivers: - # command: extract_weather_drivers() - - #out_data/weather_drivers.zip: - # command: zip_this(out_file = target_name, in_file = weather_drivers) - - out_data/distance_matrix_drb.csv: - command: get_distance_matrix(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/distance_matrix.rds') - - ##### Transfer of files from delaware-model-prep ##### - out_data/distance_matrix_lordville.csv: - command: get_distance_matrix(out_file = target_name, in_file = '../delaware-model-prep/9_collaborator_data/umn/distance_matrix_lordville.rds') - - out_data/reach_attributes_drb.csv: - command: extract_reach_attributes( - res_file = '../delaware-model-prep/1_network/out/segments_relative_to_reservoirs.rds', - attr_file = '../delaware-model-prep/1_network/out/network.rds', - out_file = target_name) - - # filter to the segments upstream of Lordville - out_data/reach_attributes_lordville.csv: - command: filter_to_subset(out_file = target_name, in_file = 'out_data/reach_attributes.csv', segments = lordville_sites) - - out_data/reservoir_features_drb.csv: - command: file.copy(from = '../delaware-model-prep/9_collaborator_data/umn/reservoir_features.csv', to = target_name) - - out_data/reservoir_features_lordville.csv: - command: filter_res_ids(in_file = 'out_data/reservoir_features.csv', out_file = target_name, res_keep = reservoir_lordville) - - out_data/sntemp_inputs_outputs_drb.csv: - command: get_sntemp_output(out_file = target_name, in_file = '../delaware-model-prep/3_predictions/out/uncal_sntemp_input_output.feather') - - out_data/sntemp_inputs_outputs_lordville.csv: - command: filter_to_subset(out_file = target_name, in_file = 'out_data/sntemp_inputs_outputs.csv', segments = lordville_sites) - - out_data/sntemp_inputs_outputs_drb.zip: - command: zip_obs(out_file = target_name, in_file = 'out_data/sntemp_inputs_outputs.csv') - - out_data/sntemp_inputs_outputs_lordville.zip: - command: zip_obs(out_file = target_name, in_file = 'out_data/sntemp_inputs_outputs_lordville.csv') - - out_data/reservoir_interpolated_daily_water_budget_components.csv: - command: file.copy(from = '../delaware-model-prep/2_observations/out/interpolated_daily_reservoir_water_budget_components.csv', to = target_name) - - # NOAA GEFS meteo files for forecasts - out_data/noaagefs_meteo_files.zip: - command: file.copy(from = 'in_data/met_drivers.zip', to = target_name) - - # For now, we're putting meteo files manually into in_data, so all we - # do here is confirm that the requisite files exist. - # Not quite robust pipelining - if the files in in_data change - # but the filenames in nml_list.rds don't, - # this target will need to be force-rebuilt - log/reservoir_meteo_files.yml: - command: fetch_meteo_files( - out_yml = target_name, - nml_rds = 'out_data/reservoir_nml_list.rds') - \ No newline at end of file diff --git a/src/5_predictions.yml b/src/5_predictions.yml deleted file mode 100644 index e9caefd..0000000 --- a/src/5_predictions.yml +++ /dev/null @@ -1,35 +0,0 @@ -target_default: 5_predictions - -include: - -packages: - - dplyr - -sources: - - src/fetch_filter_functions.R - -targets: - 5_predictions: - depends: - - out_data/reservoir_io_sntemp.csv - - out_data/reservoir_downstream_preds.csv - - out_data/reservoir_outlet_depth_preds.csv - - out_data/dwallin_stream_preds.csv - - out_data/reservoir_io_sntemp.csv: - command: copy_filter_feather( - out_csv = target_name, - in_feather = '../delaware-model-prep/9_collaborator_data/res/res_io_sntemp.feather', - site_ids = reservoir_modeling_site_ids) - - out_data/reservoir_downstream_preds.csv: - command: file.copy(from = '../res-temperature-process-models/5_extract/out/downstream_preds.csv', to = target_name) - - out_data/reservoir_outlet_depth_preds.csv: - command: file.copy(from = '../res-temperature-process-models/5_extract/out/outlet_depth_preds.csv', to = target_name) - - out_data/dwallin_stream_preds.csv: - command: file.copy(from = '../delaware-model-prep/3_predictions/out/dwallin_stream_preds.csv', to = target_name) - - out_data/forecast[2021-04-16_2021-07-16]_files.zip: - command: file.copy(from = 'in_data/forecast[2021-04-16_2021-07-16]_files.zip', to = target_name) \ No newline at end of file diff --git a/src/README.md b/src/README.md deleted file mode 100644 index ab43df9..0000000 --- a/src/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# drb-temp-data-release -Data release in support of water temperature modeling in the Delaware River Basin diff --git a/src/drb-temp-data-release.Rproj b/src/drb-temp-data-release.Rproj deleted file mode 100644 index 8e3c2eb..0000000 --- a/src/drb-temp-data-release.Rproj +++ /dev/null @@ -1,13 +0,0 @@ -Version: 1.0 - -RestoreWorkspace: Default -SaveWorkspace: Default -AlwaysSaveHistory: Default - -EnableCodeIndexing: Yes -UseSpacesForTab: Yes -NumSpacesForTab: 2 -Encoding: UTF-8 - -RnwWeave: Sweave -LaTeX: pdfLaTeX diff --git a/src/fetch_filter_functions.R b/src/fetch_filter_functions.R index 6776274..dae2e9e 100644 --- a/src/fetch_filter_functions.R +++ b/src/fetch_filter_functions.R @@ -6,36 +6,39 @@ # navigates you into another repo and the part that should be used to call # gd_get within that repo. -fetch_filter_res_polygons <- function(out_rds, in_ind, in_repo, site_ids) { +fetch_filter_res_polygons <- function(out_rds, in_dat, site_ids, in_repo = NULL) { # pull the data file down to that other repo - gd_get_elsewhere(gsub(in_repo, '', in_ind, fixed=TRUE), in_repo) - + if(!is.null(in_repo)){ + gd_get_elsewhere(gsub(in_repo, '', in_dat, fixed=TRUE), in_repo) + } + + if(endsWith(in_dat, 'rds.ind')){ + data <- as_data_file(in_dat) } + if(endsWith(in_dat, '.rds')){ + data <- in_dat} + else{stop('in_dat should be rds file of indicator rds file (rds.ind)')} + # read and filter to just the specified sites - as_data_file(in_ind) %>% + data %>% readRDS() %>% filter(site_id %in% !!site_ids) %>% st_zm(drop = TRUE, what = "ZM") # the canonical lakes file has 3D multipolygons but the Z range is 0 to 0, so let's drop it down to 2D # return object rather than writing file as other functions in this .R file do } -fetch_filter_tibble <- function(out_csv, in_ind, in_repo, site_ids) { - # pull the data file down to that other repo - gd_get_elsewhere(gsub(in_repo, '', in_ind, fixed=TRUE), in_repo) +fetch_filter_tibble <- function(out_csv, in_dat, site_ids) { # read and filter to just the specified sites - as_data_file(in_ind) %>% - readRDS() %>% + readRDS(in_dat) %>% filter(site_id %in% !!site_ids) %>% readr::write_csv(out_csv) } -fetch_filter_nycdep <- function(out_rds, in_ind, in_repo, site_ids) { +fetch_filter_nycdep <- function(out_rds, in_dat, site_ids) { # pull the data file down to that other repo - gd_get_elsewhere(gsub(in_repo, '', in_ind, fixed=TRUE), in_repo) # read and filter to just the specified sites - nycdep_data <- as_data_file(in_ind) %>% - readRDS() %>% + nycdep_data <- readRDS(in_dat) %>% filter(site_id %in% !!site_ids) # filter out erroneous Pepacton observation and save as rds @@ -43,12 +46,11 @@ fetch_filter_nycdep <- function(out_rds, in_ind, in_repo, site_ids) { saveRDS(out_rds) } -fetch_filter_historical <- function(out_rds, in_ind, in_repo, xwalk) { - # pull the data file down to that other repo - gd_get_elsewhere(gsub(in_repo, '', in_ind, fixed=TRUE), in_repo) +fetch_filter_historical <- function(out_rds, in_dat, xwalk) { + # read and filter to just the specified sites - as_data_file(in_ind) %>% + in_dat %>% readr::read_csv(col_types = 'ccnnncnnnnnnnnnc') %>% mutate(site_id = xwalk[reservoir]) %>% filter(site_id %in% as.character(xwalk)) %>% @@ -117,4 +119,5 @@ copy_filter_feather <- function(out_csv, in_feather, site_ids) { arrow::read_feather(in_feather) %>% filter(res_id %in% site_ids) %>% readr::write_csv(out_csv) -} + +} \ No newline at end of file diff --git a/src/file_functions.R b/src/file_functions.R index 22b37e5..8c7ab62 100644 --- a/src/file_functions.R +++ b/src/file_functions.R @@ -1,7 +1,7 @@ zip_this <- function(out_file, .object){ if ('data.frame' %in% class(.object)){ filepath <- basename(out_file) %>% tools::file_path_sans_ext() %>% paste0('.csv') %>% file.path(tempdir(), .) - write_csv(.object, path = filepath) + write_csv(.object, file = filepath) zip_this(out_file = out_file, .object = filepath) } else if (class(.object) == 'character' & all(file.exists(.object))){ # works for single or multiple files @@ -37,7 +37,7 @@ get_distance_matrix <- function(out_file, in_file) { mutate(from = from) %>% select(from, everything()) - readr::write_csv(out, path = out_file) + readr::write_csv(out, file = out_file) } get_sntemp_output <- function(out_file, in_file){ @@ -52,7 +52,6 @@ extract_reach_attributes <- function(res_file, attr_file, out_file) { out <- left_join(res, select(attr, -subseg_updown, -start_pt, -end_pt, -to_subseg)) %>% mutate(subseg_length = round(subseg_length, 2)) - readr::write_csv(out, out_file) @@ -91,23 +90,6 @@ filter_res_ids <- function(in_file, out_file, res_keep) { readr::write_csv(dat, out_file) } -#' Call gd_get (without any scipiper target builds) on a file in another repo on -#' this file system, downloading that file from a shared Drive cache into that -#' other repo -#' -#' @param ind_file the indicator file of the data file in the other repo, to be -#' downloaded to the corresponding location in that other repo -#' @param repo the relative file path of the other repo. Usually it'll be right -#' alongside the current repo, e.g., '../lake-temperature-model-prep' -gd_get_elsewhere <- function(ind_file, repo, ...) { - # switch to the elsewhere repo, with plans to always switch back to this one before exiting the function - this_repo <- getwd() - on.exit(setwd(this_repo)) - setwd(repo) - - # fetch the file down to that elsewhere repo - gd_get(ind_file, ...) -} # Write a layer of an sf object as a zipped-up shapefile sf_to_zip <- function(zip_filename, sf_object, layer_name){ @@ -162,7 +144,9 @@ combine_level_sources <- function(out_csv, nwis_levels, nyc_levels, hist_levels) arrange(site_id, date) %>% group_by(site_id) %>% mutate(surface_elevation_m = na.approx(surface_elevation_m)) %>% - mutate(data_type = ifelse(is.na(data_type), 'daily interpolated', data_type)) + mutate(data_type = ifelse(is.na(data_type), 'daily interpolated', data_type), + source = ifelse(is.na(source), 'interpolated', source)) %>% + select(-source_id) out_dat$surface_elevation_m <- round(out_dat$surface_elevation_m, 2) readr::write_csv(out_dat, out_csv) diff --git a/src/remake.yml b/src/remake.yml deleted file mode 100644 index a2a6a5d..0000000 --- a/src/remake.yml +++ /dev/null @@ -1,200 +0,0 @@ - -include: - - 1_spatial.yml - - 2_observations.yml - - 3_config.yml - - 4_inputs.yml - - 5_predictions.yml - - -packages: - - yaml - - dplyr - - tidyr - - meddle # at least v0.0.12 - - readr - - feather - - rgdal - - stringr - - sf - - sbtools - - dssecrets - - scipiper - - tidyverse - -sources: - #- src/spatial_utils.R - - src/sb_functions.R - -targets: - all: - depends: - - 00_parent_sb_xml - - 01_spatial_sb_xml - - 01_spatial_sb_shp - - 02_observations_sb_data - - 02_observations_sb_xml - - 03_config_sb_xml - - 03_config_sb_data - - 04_inputs_sb_xml - - 04_inputs_sb_data - - 04_inputs_sb_meteo - - 05_predictions_sb_xml - - 05_predictions_sb_data - -# in case you want to mess w/ the xml alone: - out_xml/00_parent.xml: - command: render(filename = target_name, - "in_text/text_SHARED.yml", - "in_text/text_00_parent.yml", - river_metadata2) - - 00_parent_sb_xml: - command: sb_render_post_xml(sbid_00_parent, - xml_file = I('00_parent.xml'), - "in_text/text_SHARED.yml", - "in_text/text_00_parent.yml", - river_metadata2) - - 01_spatial_sb_xml: - command: sb_render_post_xml(sbid_01_spatial, - xml_file = I('01_spatial.xml'), - "in_text/text_SHARED.yml", - "in_text/text_01_spatial.yml", - river_metadata2) - - out_xml/01_spatial_fgdc_metadata.xml: - command: render(filename = target_name, - "in_text/text_SHARED.yml", - "in_text/text_01_spatial.yml", - river_metadata2) - - 01_spatial_sb_shp: - command: sb_replace_files(sbid_01_spatial, - "out_data/study_stream_reaches.zip", - "out_data/study_reservoirs.zip", - "out_data/study_monitoring_sites.zip") - - 02_observations_sb_xml: - command: sb_render_post_xml(sbid_02_observations, - xml_file = I('02_observations.xml'), - "in_text/text_SHARED.yml", - "in_text/text_02_observations.yml", - river_metadata2) - - out_xml/02_observations_fgdc_metadata.xml: - command: render(filename = target_name, - "in_text/text_SHARED.yml", - "in_text/text_02_observations.yml", - river_metadata2) - - 02_observations_sb_data: - command: sb_replace_files(sbid_02_observations, - "out_data/temperature_observations_drb.zip", - "out_data/temperature_observations_lordville.zip", - "out_data/temperature_observations_forecast_sites.zip", - "out_data/flow_observations_drb.zip", - "out_data/flow_observations_lordville.zip", - "out_data/flow_observations_forecast_sites.zip", - "out_data/reservoir_releases_total.csv", - "out_data/reservoir_releases_by_type_drb.csv", - "out_data/reservoir_releases_by_type_lordville.csv", - "out_data/reservoir_realsat_monthly_surface_area.csv", - "out_data/reservoir_io_obs.csv", - "out_data/reservoir_temp_obs.csv", - "out_data/reservoir_level_obs.csv") - - 03_config_sb_xml: - command: sb_render_post_xml(sbid_03_config, - xml_file = I('03_config.xml'), - "in_text/text_SHARED.yml", - "in_text/text_03_config.yml", - reservoir_metadata) - - out_xml/03_config_fgdc_metadata.xml: - command: render(filename = target_name, - "in_text/text_SHARED.yml", - "in_text/text_03_config.yml", - reservoir_metadata) - - 03_config_sb_data: - command: sb_replace_files(sbid_03_config, - "out_data/reservoir_nml_values.json", - "out_data/reservoir_nml_files.zip") - - 04_inputs_sb_xml: - command: sb_render_post_xml(sbid_04_inputs, - xml_file = I('04_inputs.xml'), - "in_text/text_SHARED.yml", - "in_text/text_04_inputs.yml", - river_metadata2) - - 04_inputs_sb_data: - command: sb_replace_files(sbid_04_inputs, - "out_data/distance_matrix_drb.csv", - "out_data/distance_matrix_lordville.csv", - "out_data/reservoir_features_drb.csv", - "out_data/reservoir_features_lordville.csv", - "out_data/sntemp_inputs_outputs_drb.zip", - "out_data/sntemp_inputs_outputs_lordville.zip", - "out_data/reach_attributes_drb.csv", - "out_data/reach_attributes_lordville.csv", - "out_data/reservoir_interpolated_daily_water_budget_components.csv", - "out_data/noaagefs_meteo_files.zip") - - out_xml/04_inputs_fgdc_metadata.xml: - command: render(filename = target_name, - "in_text/text_SHARED.yml", - "in_text/text_04_inputs.yml", - river_metadata2) - - 04_inputs_sb_meteo: - command: sb_replace_files(sbid_04_inputs, - "in_data/NLDAS_time[0.367700]_x[397]_y[136].csv", - "in_data/NLDAS_time[0.367700]_x[401]_y[136].csv", - file_hash = 'log/reservoir_meteo_files.yml') - - 05_predictions_sb_xml: - command: sb_render_post_xml(sbid_05_predictions, - xml_file = I('05_predictions.xml'), - "in_text/text_SHARED.yml", - "in_text/text_05_predictions.yml", - river_metadata2) - - out_xml/05_predictions_fgdc_metadata.xml: - command: render(filename = target_name, - "in_text/text_SHARED.yml", - "in_text/text_05_predictions.yml", - river_metadata2) - - 05_predictions_sb_data: - command: sb_replace_files(sbid_05_predictions, - "out_data/reservoir_io_sntemp.csv", - "out_data/reservoir_downstream_preds.csv", - "out_data/reservoir_outlet_depth_preds.csv", - "out_data/dwallin_stream_preds.csv", - "out_data/forecast[2021-04-16_2021-07-16]_files.zip", - "in_data/dwallin_functions.R") - - # ------ SB IDs -------- - - sbid_00_parent: - command: c(I('5f6a26af82ce38aaa2449100')) - - sbid_01_spatial: - command: c(I('5f6a285d82ce38aaa244912e')) - - sbid_02_observations: - command: c(I('5f6a287382ce38aaa2449131')) - - sbid_03_config: - command: c(I('5f6a288982ce38aaa2449133')) - - sbid_04_inputs: - command: c(I('5f6a289982ce38aaa2449135')) - - sbid_05_predictions: - command: c(I('5f6a28a782ce38aaa2449137')) - - sb_id_06_evaluation: - command: c(I('5f6a28ba82ce38aaa2449139')) diff --git a/src/sb_functions.R b/src/sb_functions.R index 14ecfe0..c443b7d 100644 --- a/src/sb_functions.R +++ b/src/sb_functions.R @@ -1,4 +1,4 @@ -sb_replace_files <- function(sb_id, ..., file_hash){ +sb_replace_file_meta <- function(sb_id, ..., file_hash){ if (!sbtools::is_logged_in()){ @@ -18,7 +18,6 @@ sb_replace_files <- function(sb_id, ..., file_hash){ if (length(files) > 0){ item_replace_files(sb_id, files = files) } - } sb_render_post_xml <- function(sb_id, ..., xml_file = NULL){ @@ -29,6 +28,6 @@ sb_render_post_xml <- function(sb_id, ..., xml_file = NULL){ render(filename = xml_file, ...) - sb_replace_files(sb_id = sb_id, xml_file) + sb_replace_file_meta(sb_id = sb_id, xml_file) } \ No newline at end of file diff --git a/src/spatial_functions.R b/src/spatial_functions.R index ec050f9..581989d 100644 --- a/src/spatial_functions.R +++ b/src/spatial_functions.R @@ -4,7 +4,8 @@ retrieve_network <- function(network_sf_fl) { select(-start_pt, -end_pt, -subseg_updown) %>% mutate(subseg_length = round(subseg_length, 2)) %>% mutate(from_segs = ifelse(from_segs == '', NA, from_segs)) %>% - rename(subsegid = subseg_id, subsegseg = subseg_seg, subseglen = subseg_length, fromsegs = from_segs, toseg = to_seg, tosubseg = to_subseg, segidnat = seg_id_nat) + rename(subsegid = subseg_id, subsegseg = subseg_seg, subseglen = subseg_length, + fromsegs = from_segs, toseg = to_seg, tosubseg = to_subseg, segidnat = seg_id_nat) out <- sf::st_transform(out, crs = 4326) return(out) } @@ -20,48 +21,62 @@ sf_to_zip <- function(zip_filename, sf_object, layer_name){ cdir <- getwd() on.exit(setwd(cdir)) dsn <- tempdir() - - sf_out <- dplyr::select(sf_object, seg_id_nat, geometry) - + + sf_out <- dplyr::select(sf_object, segidnat, geometry) + sf::st_write(sf_out, dsn = dsn, layer = layer_name, driver="ESRI Shapefile", delete_dsn=TRUE) # overwrites - - files_to_zip <- data.frame(filepath = dir(dsn, full.names = TRUE), stringsAsFactors = FALSE) %>% - mutate(filename = basename(filepath)) %>% - filter(str_detect(string = filename, pattern = layer_name)) %>% pull(filename) - setwd(dsn) - # zip::zip works across platforms - zip::zip(file.path(cdir, zip_filename), files = files_to_zip) - setwd(cdir) -} -sf_to_zip2 <- function(zip_filename, sf_object, layer_name){ - cdir <- getwd() - on.exit(setwd(cdir)) - dsn <- tempdir() - - sf::st_write(sf_object, dsn = dsn, layer = layer_name, driver="ESRI Shapefile", delete_dsn=TRUE) # overwrites - files_to_zip <- data.frame(filepath = dir(dsn, full.names = TRUE), stringsAsFactors = FALSE) %>% mutate(filename = basename(filepath)) %>% - filter(str_detect(string = filename, pattern = layer_name)) %>% pull(filename) - + filter(grepl(pattern = layer_name, x = filename)) %>% + pull(filename) + setwd(dsn) # zip::zip works across platforms zip::zip(file.path(cdir, zip_filename), files = files_to_zip) setwd(cdir) } +# +# sf_to_zip2 <- function(zip_filename, sf_object, layer_name){ +# cdir <- getwd() +# on.exit(setwd(cdir)) +# dsn <- tempdir() +# +# sf::st_write(sf_object, dsn = dsn, layer = layer_name, driver="ESRI Shapefile", delete_dsn=TRUE) # overwrites +# +# files_to_zip <- data.frame(filepath = dir(dsn, full.names = TRUE), stringsAsFactors = FALSE) %>% +# mutate(filename = basename(filepath)) %>% +# filter(grepl(pattern = layer_name, x = filename)) %>% +# pull(filename) +# +# setwd(dsn) +# # zip::zip works across platforms +# zip::zip(file.path(cdir, zip_filename), files = files_to_zip) +# setwd(cdir) +# } reduce_and_zip <- function(zip_filename, in_dat, layer_name) { out <- in_dat %>% select(-ID, -bird_filtered, -fish_filtered) %>% - mutate(site_type = ifelse(site_type %in% c('Stream', 'stream'), 'ST', site_type)) %>% + mutate(site_type = case_when( + site_type %in% c('Stream', 'stream', 'ST') ~ 'Stream', + site_type %in% 'ST-TS' ~ 'Tidal Stream', + TRUE ~ site_type)) %>% distinct() %>% - rename(siteid = site_id, sitetype = site_type, origsource = original_source, - subsegid = subseg_id, fishdist = fish_dist_to_outlet_m, + rename(siteid = site_id, sitetype = site_type, origsource = original_source, + subsegid = subseg_id, fishdist = fish_dist_to_outlet_m, birddist = bird_dist_to_subseg_m, segidnat = seg_id_nat) %>% mutate(birddist = round(birddist, 2), fishdist = round(fishdist, 2)) %>% st_as_sf(coords = c('longitude', 'latitude'), crs = 4326) - + sf_to_zip2(zip_filename = zip_filename, sf_object = out, layer_name = layer_name) +} + + +read_spatial_file <- function(path, selected_crs){ + out <- sf::read_sf(path) %>% + sf::st_transform(read_file, crs = selected_crs) + + return(out) } \ No newline at end of file