push trained model weights for 0_baseline_LSTM and 2_multitask_dense …

2a_model.R

@@ -189,14 +189,14 @@ p2a_targets_list <- list(
         list(model_id = "0_baseline_LSTM",
              snakefile_dir = "0_baseline_LSTM",
              config_path = stringr::str_remove(p2a_config_baseline_LSTM_yml, "2a_model/src/models/")),
-        # the 1_ models use the same model and therefore the same Snakefile
-        # as the 0_baseline_LSTM run
-        list(model_id = "1_metab_multitask",
-             snakefile_dir = "0_baseline_LSTM",
-             config_path = stringr::str_remove(p2a_config_metab_multitask_yml, "2a_model/src/models/")),
-        list(model_id = "1a_multitask_do_gpp_er",
-             snakefile_dir = "0_baseline_LSTM",
-             config_path = stringr::str_remove(p2a_config_1a_metab_multitask_yml, "2a_model/src/models/")),
+         #the 1_ models use the same model and therefore the same Snakefile
+         #as the 0_baseline_LSTM run
+        #list(model_id = "1_metab_multitask",
+             #snakefile_dir = "0_baseline_LSTM",
+             #config_path = stringr::str_remove(p2a_config_metab_multitask_yml, "2a_model/src/models/")),
+        #list(model_id = "1a_multitask_do_gpp_er",
+             #snakefile_dir = "0_baseline_LSTM",
+             #config_path = stringr::str_remove(p2a_config_1a_metab_multitask_yml, "2a_model/src/models/")),
         list(model_id = "2_multitask_dense",
              snakefile_dir = "2_multitask_dense",
              config_path = stringr::str_remove(p2a_config_multitask_dense_yml, "2a_model/src/models/"))
@@ -219,16 +219,18 @@ p2a_targets_list <- list(
     # this path is relative to the Snakefile
     prepped_data_file <- file.path("../../../out/models",p2a_model_ids$model_id, "prepped.npz")
 
+    # make sure the directory is unlocked (this has been a hangup for me) 
+    system(stringr::str_glue("snakemake  -s {snakefile_path} --configfile {config_path} --unlock"))
     # First create the prepped data files if they are not already.
     # These are needed to make the predictions.
     system(stringr::str_glue("snakemake {prepped_data_file} -s {snakefile_path} --configfile {config_path} -j"))
 
     # Then touch all of the existing files. This makes the weights "up-to-date"
     # so snakemake doesn't train the models again
-    system(stringr::str_glue("snakemake -s {snakefile_path} --configfile {config_path} -j --touch"))
+    system(stringr::str_glue("snakemake -s {snakefile_path} --configfile {config_path} -j --touch --rerun-incomplete"))
 
     # then run the snakemake pipeline to produce the predictions and metric files
-    system(stringr::str_glue("snakemake -s {snakefile_path} --configfile {config_path} -j --rerun-incomplete"))
+    system(stringr::str_glue("snakemake -s {snakefile_path} --configfile {config_path} -j --rerun-incomplete "))
 
     # print out the metrics file name for the target
     file.path("2a_model/out/models", p2a_model_ids$model_id, "exp_overall_metrics.csv")

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_1/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_2/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_3/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_4/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_5/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_6/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_7/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_8/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/0_baseline_LSTM/nstates_10/nep_100/rep_9/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_1/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_2/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_3/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_4/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_5/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_6/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_7/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_8/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/2_multitask_dense/nstates_10/nep_100/rep_9/train_weights/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "."
+all_model_checkpoint_paths: "."

2a_model/out/models/add_weights.sh

@@ -1,6 +1,6 @@
 files=(checkpoint .data-00000-of-00001 .index .snakemake_timestamp)
 
-models=(0_baseline_LSTM 1_metab_multitask 1a_multitask_do_gpp_er)
+models=(0_baseline_LSTM 2_multitask_dense)
 
 for f in ${files[@]}; do
     for d in ${models[@]}; do

2a_model/src/models/0_baseline_LSTM/config.yml

@@ -1,8 +1,8 @@
-exp_name: "0_baseline_LSTM"
+exp_name: 0_baseline_LSTM
 y_vars:
-  - "do_min"
-  - "do_mean"
-  - "do_max"
+  - do_min
+  - do_mean
+  - do_max
 lambdas:
   - 1
   - 1

2a_model/src/models/1_metab_multitask/1a_multitask_do_gpp_er.yml

@@ -1,13 +1,13 @@
-exp_name: "1a_multitask_do_gpp_er"
+exp_name: 1a_multitask_do_gpp_er
 y_vars:
-  - "do_min"
-  - "do_mean"
-  - "do_max"
-  - "GPP"
-  - "ER"
-  - "K600"
-  - "depth"
-  - "temp.water"
+  - do_min
+  - do_mean
+  - do_max
+  - GPP
+  - ER
+  - K600
+  - depth
+  - temp.water
 lambdas:
   - 1
   - 1

2a_model/src/models/1_metab_multitask/config.yml

@@ -1,13 +1,13 @@
-exp_name: "1_metab_multitask"
+exp_name: 1_metab_multitask
 y_vars:
-  - "do_min"
-  - "do_mean"
-  - "do_max"
-  - "GPP"
-  - "ER"
-  - "K600"
-  - "depth"
-  - "temp.water"
+  - do_min
+  - do_mean
+  - do_max
+  - GPP
+  - ER
+  - K600
+  - depth
+  - temp.water
 lambdas:
   - 1
   - 1

2a_model/src/models/2_multitask_dense/config.yml

@@ -1,13 +1,13 @@
-exp_name: "2_multitask_dense"
+exp_name: 2_multitask_dense
 y_vars:
-  - "do_min"
-  - "do_mean"
-  - "do_max"
-  - "GPP"
-  - "ER"
-  - "K600"
-  - "depth"
-  - "temp.water"
+  - do_min
+  - do_mean
+  - do_max
+  - GPP
+  - ER
+  - K600
+  - depth
+  - temp.water
 lambdas:
   - 1
   - 1

2a_model/src/models/config_base.yml

@@ -1,6 +1,6 @@
-out_dir: "../../../out/models"
+out_dir: ../../../out/models
 seed: False
-num_replicates: 1
+num_replicates: 10
 trn_offset: 1.0
 tst_val_offset: 1.0
 epochs: 100
@@ -10,27 +10,47 @@ recurrent_dropout: 0.2
 finetune_learning_rate: 0.01
 early_stopping: False
 validation_sites:
-  - "01472104"
-  - "01473500"
-  - "01481500"
+  - '01472104'
+  - '01473500'
+  - 01481500
 test_sites:
-  - "01475530"
-  - "01475548"
+  - '01475530'
+  - 01475548
 train_start_date: '1980-01-01'
 train_end_date: '2014-10-01'
 val_start_date: '2014-10-01'
 val_end_date: '2015-10-01'
 test_start_date: '2015-10-01'
 test_end_date: '2022-10-01'
 x_vars:
-  - "pr"
-  - "SLOPE"
-  - "tmmx"
-  - "tmmn"
-  - "srad"
-  - "CAT_BASIN_SLOPE"
-  - "CAT_ELEV_MEAN"
-  - "CAT_IMPV11"
-  - "CAT_CNPY11_BUFF100"
-  - "CAT_TWI"
+  - tmmn
+  - tmmx
+  - pr
+  - srad
+  - SLOPE
+  - TOTDASQKM
+  - CAT_BASIN_SLOPE
+  - TOT_BASIN_SLOPE
+  - CAT_ELEV_MEAN
+  - CAT_RDX
+  - CAT_BFI
+  - CAT_EWT
+  - CAT_TWI
+  - CAT_PPT7100_ANN
+  - TOT_PPT7100_ANN
+  - CAT_RUN7100
+  - CAT_CNPY11_BUFF100
+  - CAT_IMPV11
+  - TOT_IMPV11
+  - CAT_NLCD11_wetland
+  - TOT_NLCD11_wetland
+  - CAT_SANDAVE
+  - CAT_PERMAVE
+  - TOT_PERMAVE
+  - CAT_RFACT
+  - CAT_WTDEP
+  - TOT_WTDEP
+  - CAT_NPDES_MAJ
+  - CAT_NDAMS2010
+  - CAT_NORM_STORAGE2010
 

_targets.R

@@ -110,7 +110,7 @@ base_config_options <- list(
   out_dir = "../../../out/models",
   # random seed for training; If FALSE, no seed. Otherwise, specify the seed:
   seed = FALSE,
-  num_replicates = 1,
+  num_replicates = 10,
   trn_offset = 1,
   tst_val_offset = 1,
   epochs = 100,