Merge pull request #190 from COMPASS-DOE/error-handling

* Don't trap mclapply errors; convert warnings to errors * Only strip off SINGLE-DIGIT _{0-9)min suffixes
COMPASS-DOE · Jul 18, 2024 · d114ed8 · d114ed8
2 parents ff56caf + acdf920
commit d114ed8
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 72 deletions.
diff --git a/synoptic/L0.qmd b/synoptic/L0.qmd
@@ -45,6 +45,11 @@ if(packageVersion("compasstools") < "0.2") {
          "devtools::install_github('COMPASS-DOE/compasstools')")
 }
 
+# Warnings are not allowed here, as this usually means a file
+# problem that we want to fix immediately
+oldwarn <- options()$warn
+options(warn = 2)
+
 files_to_process <- list.files(RAW, pattern = "*.dat$", 
                                full.names = TRUE, recursive = TRUE)
 
@@ -81,7 +86,7 @@ f <- function(fn, new_dir) {
     # datalogger table. If this ends with "_1min" or "_5min" then strip that
     # out; this refers to the temporary tables enabled during the TEMPEST floods.
     # We want these processed as normal data, just on a finer time resolution.
-    PATTERN <- "_[0-9]+min"
+    PATTERN <- "_[0-9]{1}min"
     if(grepl(PATTERN, dat$Table[1])) {
         message("\tRemoving _{num}min from table name: ", dat$Table[1])
         dat$Table <- gsub(PATTERN, "", dat$Table)
@@ -158,14 +163,7 @@ if(params$run_parallel) {
 }
 
 log_info("About to L0", logfile = params$logfile)
-tryCatch({
-    out <- apply_func(files_to_process, f, new_dir = L0)
-},
-error = function(e) {
-    log_warning("L0: an error occurred!", logfile = params$logfile)
-    log_info(as.character(e), logfile = params$logfile)
-    stop(e)
-})
+out <- apply_func(files_to_process, f, new_dir = L0)
 ```
 
 ## Summary
@@ -196,4 +194,7 @@ Git commit `r GIT_COMMIT`.
 
 ```{r reproducibility}
 sessionInfo()
+
+# Restore old warning setting
+options(warn = oldwarn)
 ```
diff --git a/synoptic/L1.qmd b/synoptic/L1.qmd
@@ -44,6 +44,10 @@ if(packageVersion("compasstools") < "0.2") {
          "devtools::install_github('COMPASS-DOE/compasstools')")
 }
 
+# Warnings are not allowed here
+oldwarn <- options()$warn
+options(warn = 2)
+
 source("helpers.R")
 
 # Get the column metadata file
@@ -142,15 +146,10 @@ if(params$run_parallel) {
 }
 
 log_info("About to L1", logfile = params$logfile)
-tryCatch({
-    out <- apply_func(names(dirs_to_process), f, 
-                      dirs_to_process = dirs_to_process, out_dir = L1)
-},
-error = function(e) {
-    log_warning("L1: an error occurred!", logfile = params$logfile)
-    log_info(as.character(e), logfile = params$logfile)
-    stop(e)
-})
+out <- apply_func(names(dirs_to_process),
+                  f, 
+                  dirs_to_process = dirs_to_process, 
+                  out_dir = L1)
 
 out_df <- do.call("rbind", out)
 
@@ -299,4 +298,7 @@ Git commit `r GIT_COMMIT`.
 
 ```{r reproducibility}
 sessionInfo()
+
+# Restore old warning setting
+options(warn = oldwarn)
 ```
diff --git a/synoptic/L1_normalize.qmd b/synoptic/L1_normalize.qmd
@@ -283,16 +283,10 @@ if(params$run_parallel) {
 }
 
 log_info("About to L1_normalize", logfile = params$logfile)
-tryCatch({
-    out <- apply_func(files_to_process, f,
-                      out_dir = L1_NORMALIZE,
-                      design_table = dt_ex)
-},
-error = function(e) {
-    log_warning("L1_normalize: an error occurred!", logfile = params$logfile)
-    log_info(as.character(e), logfile = params$logfile)
-    stop(e)
-})
+out <- apply_func(files_to_process, 
+                  f,
+                  out_dir = L1_NORMALIZE,
+                  design_table = dt_ex)
 ```
 
 ## Summary

diff --git a/synoptic/helpers.R b/synoptic/helpers.R
@@ -133,7 +133,7 @@ write_to_folders <- function(x, root_dir, data_level, site, plot,
                 na_string <- NA_STRING_L1
                 write_this_plot <- TRUE
                 p <- ggplot(x, aes(TIMESTAMP, Value, group = paste(Instrument_ID, Sensor_ID))) +
-                    geom_line() +
+                    geom_line(na.rm = TRUE) +
                     facet_wrap(~research_name, scales = "free") +
                     ggtitle(filename) +
                     theme(axis.text = element_text(size = 6),

diff --git a/synoptic/metadata/design_table.csv b/synoptic/metadata/design_table.csv
@@ -2447,46 +2447,46 @@ RedoxMU3_83,Table1,BattV,TMP,,,,,,,,
 RedoxMU3_83,Table1,RedoxRa({1:20}),TMP,,,,,,,,
 RedoxMU3_83,Table1,RedoxRb({1:20}),TMP,,,,,,,,
 ,,,,,,,,,,,
-GCREW_MET,GCREW_MET,Format,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,RECORD,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Statname,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Rain_in_Tot,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Rain_cm_Tot,GCW,W,CS700,,,,,wx_gcrew_rain15,NOTE IN CM
-GCREW_MET,GCREW_MET,AirTC_Avg,GCW,W,CS EE181,,,,,wx_tempavg15,
-GCREW_MET,GCREW_MET,RH,GCW,W,CS EE181,,,,,wx_rh15,
-GCREW_MET,GCREW_MET,CleanRH,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,VP,GCW,W,CS106,,,,,wx_vappress15,
-GCREW_MET,GCREW_MET,VPD,GCW,W,CS106,,,,,,Removed per SC email 2024-05-10
-GCREW_MET,GCREW_MET,SVP,GCW,W,CS106,,,,,wx_svp15,
-GCREW_MET,GCREW_MET,Barometric_Pressure_PB110B,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Barometric_Pressure_PB110B_Avg,GCW,W,CS106,,,,,wx_barpress15,
-GCREW_MET,GCREW_MET,SlrW_Avg,GCW,W,LI-200R,,,,,wx_slr_fd15,
-GCREW_MET,GCREW_MET,SlrMJ_Tot,GCW,W,LI-200R,,,,,wx_slr_tf15,
-GCREW_MET,GCREW_MET,PAR_Den_C_Avg,GCW,W,LI-190R,,,,,wx_par_den15,
-GCREW_MET,GCREW_MET,PAR_Tot_C_Tot,GCW,W,LI-190R,,,,,wx_par_tot15,
-GCREW_MET,GCREW_MET,Temp10cm_Avg,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Temp25cm_Avg,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Temp90cm_Avg,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Above75cm_plate_Avg,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Aquatroll_ID(2),GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Depth,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Temperature,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Actual_Conductivity,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Specific_Conductivity,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Salinity,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,TDS,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Water_Density,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Pressure,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,Resistivity,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WindDir_RM_Avg,GCW,W,RM Young 05103-10-L,,,,,wx_winddir15,
-GCREW_MET,GCREW_MET,WindDir_RM,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WS_ms_RM_Max,GCW,W,RM Young 05103-10-L,,,,,wx_maxws15,
-GCREW_MET,GCREW_MET,WS_ms_RM_Min,GCW,W,RM Young 05103-10-L,,,,,wx_minws15,
-GCREW_MET,GCREW_MET,WS_ms_RM_Avg,GCW,W,RM Young 05103-10-L,,,,,wx_windspeed15,
-GCREW_MET,GCREW_MET,WS_ms_RM,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WindDir_WS_Avg,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WindDir_WS,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WS_ms_WS_Max,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WS_ms_WS_Min,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WS_ms_WS_Avg,GCW,W,,,,,,,
-GCREW_MET,GCREW_MET,WS_ms_WS,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Format,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,RECORD,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Statname,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Rain_in_Tot,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Rain_cm_Tot,GCW,W,CS700,,,,,wx_gcrew_rain15,NOTE IN CM
+GCREW_MET,GCREW_MET_15min,AirTC_Avg,GCW,W,CS EE181,,,,,wx_tempavg15,
+GCREW_MET,GCREW_MET_15min,RH,GCW,W,CS EE181,,,,,wx_rh15,
+GCREW_MET,GCREW_MET_15min,CleanRH,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,VP,GCW,W,CS106,,,,,wx_vappress15,
+GCREW_MET,GCREW_MET_15min,VPD,GCW,W,CS106,,,,,,Removed per SC email 2024-05-10
+GCREW_MET,GCREW_MET_15min,SVP,GCW,W,CS106,,,,,wx_svp15,
+GCREW_MET,GCREW_MET_15min,Barometric_Pressure_PB110B,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Barometric_Pressure_PB110B_Avg,GCW,W,CS106,,,,,wx_barpress15,
+GCREW_MET,GCREW_MET_15min,SlrW_Avg,GCW,W,LI-200R,,,,,wx_slr_fd15,
+GCREW_MET,GCREW_MET_15min,SlrMJ_Tot,GCW,W,LI-200R,,,,,wx_slr_tf15,
+GCREW_MET,GCREW_MET_15min,PAR_Den_C_Avg,GCW,W,LI-190R,,,,,wx_par_den15,
+GCREW_MET,GCREW_MET_15min,PAR_Tot_C_Tot,GCW,W,LI-190R,,,,,wx_par_tot15,
+GCREW_MET,GCREW_MET_15min,Temp10cm_Avg,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Temp25cm_Avg,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Temp90cm_Avg,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Above75cm_plate_Avg,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Aquatroll_ID(2),GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Depth,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Temperature,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Actual_Conductivity,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Specific_Conductivity,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Salinity,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,TDS,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Water_Density,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Pressure,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,Resistivity,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WindDir_RM_Avg,GCW,W,RM Young 05103-10-L,,,,,wx_winddir15,
+GCREW_MET,GCREW_MET_15min,WindDir_RM,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WS_ms_RM_Max,GCW,W,RM Young 05103-10-L,,,,,wx_maxws15,
+GCREW_MET,GCREW_MET_15min,WS_ms_RM_Min,GCW,W,RM Young 05103-10-L,,,,,wx_minws15,
+GCREW_MET,GCREW_MET_15min,WS_ms_RM_Avg,GCW,W,RM Young 05103-10-L,,,,,wx_windspeed15,
+GCREW_MET,GCREW_MET_15min,WS_ms_RM,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WindDir_WS_Avg,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WindDir_WS,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WS_ms_WS_Max,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WS_ms_WS_Min,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WS_ms_WS_Avg,GCW,W,,,,,,,
+GCREW_MET,GCREW_MET_15min,WS_ms_WS,GCW,W,,,,,,,