From 7b104e0874ebfdc2eb278368e84e4f8c87da7700 Mon Sep 17 00:00:00 2001
From: Ben Bond-Lamberty <bondlamberty@pnnl.gov>
Date: Mon, 17 Jun 2024 13:43:20 -0400
Subject: [PATCH] Add cumulative-data plot code

---
 .../code_examples/cumulative-observations.R   | 45 +++++++++++++++++++
 synoptic/docs/making-a-new-release.md         |  4 +-
 2 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 synoptic/data/L1/code_examples/cumulative-observations.R

diff --git a/synoptic/data/L1/code_examples/cumulative-observations.R b/synoptic/data/L1/code_examples/cumulative-observations.R
new file mode 100644
index 0000000..9d2983e
--- /dev/null
+++ b/synoptic/data/L1/code_examples/cumulative-observations.R
@@ -0,0 +1,45 @@
+# Calculate and plot cumulative observations over time
+
+
+# Find and count lines for all files in the L1 folder
+fls <- list.files("./data/L1/", pattern = "*.csv$", full.names = TRUE, recursive = TRUE)
+
+results <- data.frame(file = basename(fls), rows = NA_real_)
+
+for(i in seq_along(fls)) {
+    message(basename(fls[i]))
+    results$rows[i] <- length(readLines(fls[i])) - 1
+}
+
+# An example of how to parse the filenames into useful information:
+# site, plot, time range, data level, and version number
+library(tidyr)
+results <- separate(results, file, sep = "_", into = c("Site", "Plot", "Timerange","Level","version"))
+results <- separate(results, Timerange, sep = "-", into = c("Begin", "End"))
+results$Date <- as.Date(results$Begin, format = "%Y%m%d")
+
+# Compute cumulative observations by site and date
+library(dplyr)
+results %>%
+    complete(Site, Date, fill = list(rows = 0)) %>%
+    group_by(Site, Date) %>%
+    summarise(n = sum(rows, na.rm = TRUE), .groups = "drop") %>%
+    arrange(Date) %>%
+    group_by(Site) %>%
+    mutate(cum_n = cumsum(n)) ->
+    smry
+
+# Make a nice graph
+library(ggplot2)
+theme_set(theme_bw())
+library(scales)
+library(viridis)
+
+p2 <- ggplot(smry, aes(Date, cum_n, fill = Site)) +
+    geom_area(alpha = 0.8 , linewidth = 0.5, colour = "white") +
+    xlab("Year") + ylab("COMPASS-FME environmental sensor observations") +
+    scale_fill_viridis(discrete = TRUE) +
+    theme(axis.title = element_text(size = 14)) +
+    scale_y_continuous(labels = unit_format(unit = "M", scale = 1e-6))
+print(p2)
+ggsave("~/Desktop/sensors.png", height = 6, width = 8)
diff --git a/synoptic/docs/making-a-new-release.md b/synoptic/docs/making-a-new-release.md
index 725e7ac..f236f86 100644
--- a/synoptic/docs/making-a-new-release.md
+++ b/synoptic/docs/making-a-new-release.md
@@ -47,8 +47,8 @@ script). Of course, this is much slower.
 11. Double-check the final release README file.
 
 12. You may want to clean up the resulting L1 folder; for example,
-remove (`find ./ -name ".DS_Store" | xargs rm`) hidden files created by
-MacOS.
+remove unwanted hidden files (`find ./ -name ".DS_Store" | xargs rm`)
+or 'stub' data (`find ./ -name "*20240501-20240501*" | xargs rm`).
 
 13. Push the data (including `L1`, `Raw`, `L0`, and `Logs`) to the
 COMPASS HPC. For example: