Write basic data viz plot for each L1 data file

.github/workflows/check-workflow.yaml

@@ -25,6 +25,7 @@ jobs:
             any::readr
             any::lubridate
             any::RSQLite
+            any::ggplot2
       - name: Install other needed packages
         run: |
           pak::pkg_install("COMPASS-DOE/compasstools")

synoptic/L1.qmd

@@ -10,6 +10,7 @@ params:
   L1_METADATA: "L1_metadata/"
   debug: false
   remove_input_files: false
+  write_plots: true
   logfile: ""
 date: now
 date-format: "YYYY-MM-DD HH:mm:ssZ"
@@ -30,6 +31,8 @@ This script
 ```{r init}
 #| include: false
 
+library(ggplot2)
+theme_set(theme_bw())
 library(compasstools)
 if(packageVersion("compasstools") < "0.2") {
     stop("Please update to latest version of compasstools!\n",
@@ -95,7 +98,8 @@ f <- function(dir_name, dirs_to_process, out_dir) {
     write_to_folders(dat, 
                      root_dir = out_dir, 
                      data_level = "L1",
-                     site = site)
+                     site = site,
+                     write_plots = params$write_plots)
 
     # Add any OOB flags to the flag database
     flg <- dat[!is.na(dat$OOB) & dat$OOB == 1, "ID"]

synoptic/data_TEST/L1/.gitignore

@@ -1,3 +1,5 @@
 *.csv
 */README.md
 *.txt
+*.pdf
+*.png

synoptic/data_TEST/L1/README.md

@@ -2,3 +2,5 @@
 
 This folder holds COMPASS-FME Level 1 data: long form, bounds QA/QC flags applied,
 unit conversion done, and sorted into year-month-logger folders.
+
+A basic data visualization file is also written for each data file.

synoptic/helpers.R

@@ -78,7 +78,7 @@ read_csv_group <- function(files, col_types = NULL,
 # This is used to split the data for sorting into <yyyy>_<mm> folders
 # Returns a list of filenames written (names) and number of data lines (values)
 write_to_folders <- function(x, root_dir, data_level, site,
-                             logger, table, quiet = FALSE) {
+                             logger, table, quiet = FALSE, write_plots = TRUE) {
     years <- year(x$TIMESTAMP)
     months <- sprintf("%02i", month(x$TIMESTAMP)) # add leading zero if needed
 
@@ -89,6 +89,8 @@ write_to_folders <- function(x, root_dir, data_level, site,
         }
 
         for(m in unique(months)) {
+            write_this_plot <- FALSE
+
             if(is.na(m)) {
                 stop(data_level, " invalid month ", m)
             }
@@ -118,6 +120,12 @@ write_to_folders <- function(x, root_dir, data_level, site,
                 folder <- file.path(root_dir, paste(site, y, sep = "_"))
                 filename <- paste0(paste(site, time_period, data_level, sep = "_"), ".csv")
                 na_string <- NA_STRING_L1
+                write_this_plot <- TRUE
+                p <- ggplot(x, aes(TIMESTAMP, value, group = design_link)) +
+                    geom_line() +
+                    facet_wrap(~research_name, scales = "free") +
+                    ggtitle(filename) +
+                    theme(axis.text = element_text(size = 6))
             } else if(data_level == "L2") {
                 folder <- file.path(root_dir, paste(site, y, sep = "_"))
                 filename <- paste0(paste(site, time_period, table, data_level, sep = "_"), ".csv")
@@ -153,6 +161,12 @@ write_to_folders <- function(x, root_dir, data_level, site,
                 stop("File ", fn, "was not written")
             }
 
+            # Write basic QA/QC plot
+            if(write_plots && write_this_plot) {
+                fn_p <- gsub("csv$", "pdf", fn)
+                ggsave(fn_p, plot = p, width = 10, height = 8)
+            }
+
             lines_written[[fn]] <- nrow(dat)
         } # for m
     } # for y