From 0992167f9e4ba01120869a9238a73e48ec568f86 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Fri, 1 Dec 2023 19:57:48 -0500 Subject: [PATCH 01/10] Create design_link_shift_example.R --- synoptic/design_link_shift_example.R | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 synoptic/design_link_shift_example.R diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R new file mode 100644 index 0000000..ad0e5bc --- /dev/null +++ b/synoptic/design_link_shift_example.R @@ -0,0 +1,36 @@ +# Preparing for the design_link table to have a "valid_until" column +# that we can handle EFFICIENTLY + +# Two objects (sensors) at time points 1:3 +test_data <- data.frame(obj = c(1,1,1,2,2,2), time = c(1,2,3,1,2,3)) +# Object 2 changes its design link after time 2 +test_dt <- data.frame(obj = c(1,2,2), + dl = c("A", "B", "C"), + valid_until = c(NA, 2, NA)) + +shifting_objects <- test_dt[!is.na(test_dt$valid_until),]$obj + +x <- merge(test_data, test_dt) + +# Identify entries where the time is after valid_until +x$past_valid_time <- !is.na(x$valid_until) & x$time > x$valid_until +# Identify objects and times where a shift occurs +pastvalids <- x[x$past_valid_time, c("obj", "time")] +pastvalids$shift_happened <- TRUE +x <- merge(x, pastvalids, all.x = TRUE) +x$shift_happened[is.na(x$shift_happened)] <- FALSE +# Identify shift objects +x$shift_object <- x$obj %in% shifting_objects + +# Whew! Now we can +# 1. Delete past-valid-time entries +x <- x[!x$past_valid_time,] +# 2. Retain rows that are not shift objects... +x <- x[!x$shift_object | + # or are shift objects and a shift happened + # (i.e. something got deleted in step 1 above) + (x$shift_object & x$shift_happened) | + # or are shift objects, no shift happened, with a non-NA valid_until + (x$shift_object & !x$shift_happened & !is.na(x$valid_until)), + ] + From e4b105e368930aebdea8a05cf9e02d9f63e2faa3 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Fri, 1 Dec 2023 20:00:10 -0500 Subject: [PATCH 02/10] Update design_link_shift_example.R --- synoptic/design_link_shift_example.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R index ad0e5bc..3c403df 100644 --- a/synoptic/design_link_shift_example.R +++ b/synoptic/design_link_shift_example.R @@ -8,18 +8,18 @@ test_dt <- data.frame(obj = c(1,2,2), dl = c("A", "B", "C"), valid_until = c(NA, 2, NA)) -shifting_objects <- test_dt[!is.na(test_dt$valid_until),]$obj - x <- merge(test_data, test_dt) # Identify entries where the time is after valid_until x$past_valid_time <- !is.na(x$valid_until) & x$time > x$valid_until # Identify objects and times where a shift occurs +# A "shift" is an observation (object + time) where we change design links pastvalids <- x[x$past_valid_time, c("obj", "time")] pastvalids$shift_happened <- TRUE x <- merge(x, pastvalids, all.x = TRUE) x$shift_happened[is.na(x$shift_happened)] <- FALSE # Identify shift objects +shifting_objects <- test_dt[!is.na(test_dt$valid_until),]$obj x$shift_object <- x$obj %in% shifting_objects # Whew! Now we can From 115caa1fc47e995d9560296c3d71bb4e138659c6 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 06:13:50 -0500 Subject: [PATCH 03/10] Works except for two-shift cases --- synoptic/design_link_shift_example.R | 108 +++++++++++++++++++++------ 1 file changed, 85 insertions(+), 23 deletions(-) diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R index 3c403df..22f61ae 100644 --- a/synoptic/design_link_shift_example.R +++ b/synoptic/design_link_shift_example.R @@ -10,27 +10,89 @@ test_dt <- data.frame(obj = c(1,2,2), x <- merge(test_data, test_dt) -# Identify entries where the time is after valid_until -x$past_valid_time <- !is.na(x$valid_until) & x$time > x$valid_until -# Identify objects and times where a shift occurs -# A "shift" is an observation (object + time) where we change design links -pastvalids <- x[x$past_valid_time, c("obj", "time")] -pastvalids$shift_happened <- TRUE -x <- merge(x, pastvalids, all.x = TRUE) -x$shift_happened[is.na(x$shift_happened)] <- FALSE -# Identify shift objects -shifting_objects <- test_dt[!is.na(test_dt$valid_until),]$obj -x$shift_object <- x$obj %in% shifting_objects - -# Whew! Now we can -# 1. Delete past-valid-time entries -x <- x[!x$past_valid_time,] -# 2. Retain rows that are not shift objects... -x <- x[!x$shift_object | - # or are shift objects and a shift happened - # (i.e. something got deleted in step 1 above) - (x$shift_object & x$shift_happened) | - # or are shift objects, no shift happened, with a non-NA valid_until - (x$shift_object & !x$shift_happened & !is.na(x$valid_until)), - ] +# The design links might not be stable over time; for example, if a tree +# dies, its sensor might get reassigned to a new tree. In this case the +# design_link table will have two entries, one for the old assignment and +# one for the new. We know which one to use by the "valid_until" column, +# which give a end date for a design link. Most entries will be NA. +# +# So, when the design_link table is merged with a data table, if a reassignment +# has occurred, some data rows will get repeated with the different possible +# design links. +# +# This function uses the object name (group identifier -- typically, Logger+ +# Table+Loggernet_variable), timestamp, and valid_until timestamps to identify +# which rows to keep (correct design_link assignment) and which to drop. +valid_entries <- function(objects, times, valid_until) { + # If no non-NA valid_until entries, nothing will change + if(all(is.na(valid_until))) { + return(rep(TRUE, length(objects))) + } + + # Identify entries where the time is after valid_until + # Most valid_until entries will likely be NA + past_valid_time <- !is.na(valid_until) & times > valid_until + # If nothing is past valid time, nothing changed, just drop NA rows + if(all(!past_valid_time)) { + return(!is.na(valid_until)) + } + + # Identify 'shift objects': they passed a time_valid + shift_objects <- unique(objects[past_valid_time]) + # Identify 'shift happened' object/times + x <- data.frame(obj = objects, time = times, valid_until = valid_until, + past_valid_time = past_valid_time) + y <- x[past_valid_time,] + y <- y[!duplicated(y),] + y$sh <- TRUE # mark 'shift happened' + z <- merge(x, y, all.x = TRUE) + z$sh[is.na(z$sh)] <- FALSE + shift_happened <- z$sh + + # con <- aggregate(valid_until ~ obj + time, data = z[!past_valid_time,], + # FUN = min, na.rm = TRUE) + # names(con)[3] <- "controlling" + # z <- merge(z, con, all.x = TRUE) + # controlling_vu <- z$controlling + + # Identify which entries to retain + retain <- + # objects that have no shift possibility + !objects %in% shift_objects | + # shift objects in which a shift happened + (objects %in% shift_objects & shift_happened) | + # shift objects but no shift happened and non-NA valid_until + (shift_objects & !shift_happened & !is.na(valid_until)) + # Definitely want to drop entries that have exceeded valid_until, however + retain[past_valid_time] <- FALSE + retain +} +x <- merge(test_data, test_dt) +debug(valid_entries) +valid_entries(x$obj, x$time, x$valid_until) + +# No shifting objects +ret <- valid_entries(c(1, 1, 1), c(1, 2, 3), c(NA, NA, NA)) +stopifnot(all(ret)) +# One object, shift time is never reached +ret <- valid_entries(c(1, 1, 1, 1), c(1, 1, 2, 2), c(4, NA, 4, NA)) +stopifnot(ret == c(TRUE, FALSE, TRUE, FALSE)) +# One object, shift time is in the past +ret <- valid_entries(c(1, 1, 1, 1), c(3, 3, 4, 4), c(2, NA, 2, NA)) +stopifnot(ret == c(FALSE, TRUE, FALSE, TRUE)) +# One object, shifts +ret <- valid_entries(c(1, 1, 1, 1), c(2, 2, 3, 3), c(2, NA, 2, NA)) +stopifnot(ret == c(TRUE, FALSE, FALSE, TRUE)) +# One objects, shifts twice (valid_untils at 1 and 2) +ret <- valid_entries(objects = rep(1, 9), + times = c(1, 1, 1, 2, 2, 2, 3, 3, 3), + valid_until = c(1, 2, NA, 1, 2, NA, 1, 2, NA)) + +# Two objects, one shifts +ret <- valid_entries(objects = c(1, 1, 1, 2, 2, 2, 2, 2, 2), + times = c(1, 2, 3, 1, 1, 2, 2, 3, 3), + valid_until = c(NA, NA, NA, 2, NA, 2, NA, 2, NA)) +stopifnot(ret == c(TRUE, TRUE, TRUE, # obj 1 + TRUE, FALSE, TRUE, FALSE, FALSE, TRUE)) # obj 2 + From 896d9e712a47a883c9e1b55b3abec67a04d00bf2 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 06:52:52 -0500 Subject: [PATCH 04/10] Much simpler and clearer! --- synoptic/design_link_shift_example.R | 63 +++++++++------------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R index 22f61ae..a52358a 100644 --- a/synoptic/design_link_shift_example.R +++ b/synoptic/design_link_shift_example.R @@ -24,51 +24,22 @@ x <- merge(test_data, test_dt) # Table+Loggernet_variable), timestamp, and valid_until timestamps to identify # which rows to keep (correct design_link assignment) and which to drop. valid_entries <- function(objects, times, valid_until) { - # If no non-NA valid_until entries, nothing will change - if(all(is.na(valid_until))) { - return(rep(TRUE, length(objects))) - } - - # Identify entries where the time is after valid_until - # Most valid_until entries will likely be NA - past_valid_time <- !is.na(valid_until) & times > valid_until - # If nothing is past valid time, nothing changed, just drop NA rows - if(all(!past_valid_time)) { - return(!is.na(valid_until)) - } - - # Identify 'shift objects': they passed a time_valid - shift_objects <- unique(objects[past_valid_time]) - # Identify 'shift happened' object/times - x <- data.frame(obj = objects, time = times, valid_until = valid_until, - past_valid_time = past_valid_time) - y <- x[past_valid_time,] - y <- y[!duplicated(y),] - y$sh <- TRUE # mark 'shift happened' + # Any NA valid_until entries apply into the far future + valid_until[is.na(valid_until)] <- 99999 # max int basically + past_valid_time <- times > valid_until + x <- data.frame(obj = objects, time = times, vu = valid_until) + # Compute the minimum valid_until entry for each object and time that is + # not past the valid_until point; this is the 'controlling' value + y <- aggregate(vu ~ obj + time, data = x[!past_valid_time,], FUN = min) + names(y)[3] <- "controlling" + # Figure out controlling valid_until for each object/time z <- merge(x, y, all.x = TRUE) - z$sh[is.na(z$sh)] <- FALSE - shift_happened <- z$sh + # An NA controlling entry means there is none + z$controlling[is.na(z$controlling)] <- FALSE - # con <- aggregate(valid_until ~ obj + time, data = z[!past_valid_time,], - # FUN = min, na.rm = TRUE) - # names(con)[3] <- "controlling" - # z <- merge(z, con, all.x = TRUE) - # controlling_vu <- z$controlling - - # Identify which entries to retain - retain <- - # objects that have no shift possibility - !objects %in% shift_objects | - # shift objects in which a shift happened - (objects %in% shift_objects & shift_happened) | - # shift objects but no shift happened and non-NA valid_until - (shift_objects & !shift_happened & !is.na(valid_until)) - # Definitely want to drop entries that have exceeded valid_until, however - retain[past_valid_time] <- FALSE - retain + return(z$vu == z$controlling) } -x <- merge(test_data, test_dt) -debug(valid_entries) + valid_entries(x$obj, x$time, x$valid_until) # No shifting objects @@ -87,12 +58,16 @@ stopifnot(ret == c(TRUE, FALSE, FALSE, TRUE)) ret <- valid_entries(objects = rep(1, 9), times = c(1, 1, 1, 2, 2, 2, 3, 3, 3), valid_until = c(1, 2, NA, 1, 2, NA, 1, 2, NA)) - +stopifnot(ret == c(TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, TRUE)) # Two objects, one shifts ret <- valid_entries(objects = c(1, 1, 1, 2, 2, 2, 2, 2, 2), times = c(1, 2, 3, 1, 1, 2, 2, 3, 3), valid_until = c(NA, NA, NA, 2, NA, 2, NA, 2, NA)) stopifnot(ret == c(TRUE, TRUE, TRUE, # obj 1 TRUE, FALSE, TRUE, FALSE, FALSE, TRUE)) # obj 2 - +# There's a valid_until but no new entry +ret <- valid_entries(objects = c(1, 1), + times = c(1, 2), + valid_until = c(1, 1)) +stopifnot(ret == c(TRUE, FALSE)) From d91a1df38951cc1daa7ee9263b584d9710b94152 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 06:57:53 -0500 Subject: [PATCH 05/10] Update design_link_shift_example.R --- synoptic/design_link_shift_example.R | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R index a52358a..2390a5b 100644 --- a/synoptic/design_link_shift_example.R +++ b/synoptic/design_link_shift_example.R @@ -1,14 +1,6 @@ # Preparing for the design_link table to have a "valid_until" column # that we can handle EFFICIENTLY -# Two objects (sensors) at time points 1:3 -test_data <- data.frame(obj = c(1,1,1,2,2,2), time = c(1,2,3,1,2,3)) -# Object 2 changes its design link after time 2 -test_dt <- data.frame(obj = c(1,2,2), - dl = c("A", "B", "C"), - valid_until = c(NA, 2, NA)) - -x <- merge(test_data, test_dt) # The design links might not be stable over time; for example, if a tree # dies, its sensor might get reassigned to a new tree. In this case the @@ -40,8 +32,22 @@ valid_entries <- function(objects, times, valid_until) { return(z$vu == z$controlling) } +# Sample data. We have two objects (sensors) at time points 1:3 +test_data <- data.frame(obj = c(1, 1, 1, 2, 2, 2), time = c(1, 2, 3, 1, 2, 3)) +# Object 2 changes its design link after time 2 +test_dt <- data.frame(obj = c(1,2,2), + dl = c("A", "B", "C"), + valid_until = c(NA, 2, NA)) +# Merge the 'data' with the 'design link table' +x <- merge(test_data, test_dt) +# Call valid_entries. It figures out that all the object 1 entries should be +# retained, but 1 of 2 entries in each timestep should be dropped for object 2. +# This is because there are two design_table entries for it (see above); the +# first ends at time point 2, and the second is indefinite after that. valid_entries(x$obj, x$time, x$valid_until) +# Test code + # No shifting objects ret <- valid_entries(c(1, 1, 1), c(1, 2, 3), c(NA, NA, NA)) stopifnot(all(ret)) @@ -59,7 +65,7 @@ ret <- valid_entries(objects = rep(1, 9), times = c(1, 1, 1, 2, 2, 2, 3, 3, 3), valid_until = c(1, 2, NA, 1, 2, NA, 1, 2, NA)) stopifnot(ret == c(TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, TRUE)) -# Two objects, one shifts +# Two objects, only one shifts ret <- valid_entries(objects = c(1, 1, 1, 2, 2, 2, 2, 2, 2), times = c(1, 2, 3, 1, 1, 2, 2, 3, 3), valid_until = c(NA, NA, NA, 2, NA, 2, NA, 2, NA)) From 8262e4ee906d56deeb6cfcc2045fd69d958354b7 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 07:01:29 -0500 Subject: [PATCH 06/10] Update design_link_shift_example.R --- synoptic/design_link_shift_example.R | 1 + 1 file changed, 1 insertion(+) diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R index 2390a5b..895a36b 100644 --- a/synoptic/design_link_shift_example.R +++ b/synoptic/design_link_shift_example.R @@ -19,6 +19,7 @@ valid_entries <- function(objects, times, valid_until) { # Any NA valid_until entries apply into the far future valid_until[is.na(valid_until)] <- 99999 # max int basically past_valid_time <- times > valid_until + # Create a data frame to aggregate and then merge, below x <- data.frame(obj = objects, time = times, vu = valid_until) # Compute the minimum valid_until entry for each object and time that is # not past the valid_until point; this is the 'controlling' value From 5320e9d6bdacdd9ebdd068de097ca57b464d752a Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 09:44:56 -0500 Subject: [PATCH 07/10] Update design_link_shift_example.R --- synoptic/design_link_shift_example.R | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R index 895a36b..7a4d6b6 100644 --- a/synoptic/design_link_shift_example.R +++ b/synoptic/design_link_shift_example.R @@ -6,18 +6,19 @@ # dies, its sensor might get reassigned to a new tree. In this case the # design_link table will have two entries, one for the old assignment and # one for the new. We know which one to use by the "valid_until" column, -# which give a end date for a design link. Most entries will be NA. +# which give a end date for a design link--or, most commonly, it will be +# empty (NA) indicating that there is no end date. # # So, when the design_link table is merged with a data table, if a reassignment # has occurred, some data rows will get repeated with the different possible # design links. # -# This function uses the object name (group identifier -- typically, Logger+ +# This function uses the object (i.e. group identifier; typically, Logger+ # Table+Loggernet_variable), timestamp, and valid_until timestamps to identify # which rows to keep (correct design_link assignment) and which to drop. valid_entries <- function(objects, times, valid_until) { # Any NA valid_until entries apply into the far future - valid_until[is.na(valid_until)] <- 99999 # max int basically + valid_until[is.na(valid_until)] <- 2932896 # this is 12/31/9999 past_valid_time <- times > valid_until # Create a data frame to aggregate and then merge, below x <- data.frame(obj = objects, time = times, vu = valid_until) From 21e6beb44161e77c82be045a42270e113a620814 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 09:47:04 -0500 Subject: [PATCH 08/10] Move code into helpers.R --- synoptic/design_link_shift_example.R | 81 ---------------------------- synoptic/helpers.R | 79 +++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 81 deletions(-) delete mode 100644 synoptic/design_link_shift_example.R diff --git a/synoptic/design_link_shift_example.R b/synoptic/design_link_shift_example.R deleted file mode 100644 index 7a4d6b6..0000000 --- a/synoptic/design_link_shift_example.R +++ /dev/null @@ -1,81 +0,0 @@ -# Preparing for the design_link table to have a "valid_until" column -# that we can handle EFFICIENTLY - - -# The design links might not be stable over time; for example, if a tree -# dies, its sensor might get reassigned to a new tree. In this case the -# design_link table will have two entries, one for the old assignment and -# one for the new. We know which one to use by the "valid_until" column, -# which give a end date for a design link--or, most commonly, it will be -# empty (NA) indicating that there is no end date. -# -# So, when the design_link table is merged with a data table, if a reassignment -# has occurred, some data rows will get repeated with the different possible -# design links. -# -# This function uses the object (i.e. group identifier; typically, Logger+ -# Table+Loggernet_variable), timestamp, and valid_until timestamps to identify -# which rows to keep (correct design_link assignment) and which to drop. -valid_entries <- function(objects, times, valid_until) { - # Any NA valid_until entries apply into the far future - valid_until[is.na(valid_until)] <- 2932896 # this is 12/31/9999 - past_valid_time <- times > valid_until - # Create a data frame to aggregate and then merge, below - x <- data.frame(obj = objects, time = times, vu = valid_until) - # Compute the minimum valid_until entry for each object and time that is - # not past the valid_until point; this is the 'controlling' value - y <- aggregate(vu ~ obj + time, data = x[!past_valid_time,], FUN = min) - names(y)[3] <- "controlling" - # Figure out controlling valid_until for each object/time - z <- merge(x, y, all.x = TRUE) - # An NA controlling entry means there is none - z$controlling[is.na(z$controlling)] <- FALSE - - return(z$vu == z$controlling) -} - -# Sample data. We have two objects (sensors) at time points 1:3 -test_data <- data.frame(obj = c(1, 1, 1, 2, 2, 2), time = c(1, 2, 3, 1, 2, 3)) -# Object 2 changes its design link after time 2 -test_dt <- data.frame(obj = c(1,2,2), - dl = c("A", "B", "C"), - valid_until = c(NA, 2, NA)) -# Merge the 'data' with the 'design link table' -x <- merge(test_data, test_dt) -# Call valid_entries. It figures out that all the object 1 entries should be -# retained, but 1 of 2 entries in each timestep should be dropped for object 2. -# This is because there are two design_table entries for it (see above); the -# first ends at time point 2, and the second is indefinite after that. -valid_entries(x$obj, x$time, x$valid_until) - -# Test code - -# No shifting objects -ret <- valid_entries(c(1, 1, 1), c(1, 2, 3), c(NA, NA, NA)) -stopifnot(all(ret)) -# One object, shift time is never reached -ret <- valid_entries(c(1, 1, 1, 1), c(1, 1, 2, 2), c(4, NA, 4, NA)) -stopifnot(ret == c(TRUE, FALSE, TRUE, FALSE)) -# One object, shift time is in the past -ret <- valid_entries(c(1, 1, 1, 1), c(3, 3, 4, 4), c(2, NA, 2, NA)) -stopifnot(ret == c(FALSE, TRUE, FALSE, TRUE)) -# One object, shifts -ret <- valid_entries(c(1, 1, 1, 1), c(2, 2, 3, 3), c(2, NA, 2, NA)) -stopifnot(ret == c(TRUE, FALSE, FALSE, TRUE)) -# One objects, shifts twice (valid_untils at 1 and 2) -ret <- valid_entries(objects = rep(1, 9), - times = c(1, 1, 1, 2, 2, 2, 3, 3, 3), - valid_until = c(1, 2, NA, 1, 2, NA, 1, 2, NA)) -stopifnot(ret == c(TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, TRUE)) -# Two objects, only one shifts -ret <- valid_entries(objects = c(1, 1, 1, 2, 2, 2, 2, 2, 2), - times = c(1, 2, 3, 1, 1, 2, 2, 3, 3), - valid_until = c(NA, NA, NA, 2, NA, 2, NA, 2, NA)) -stopifnot(ret == c(TRUE, TRUE, TRUE, # obj 1 - TRUE, FALSE, TRUE, FALSE, FALSE, TRUE)) # obj 2 -# There's a valid_until but no new entry -ret <- valid_entries(objects = c(1, 1), - times = c(1, 2), - valid_until = c(1, 1)) -stopifnot(ret == c(TRUE, FALSE)) - diff --git a/synoptic/helpers.R b/synoptic/helpers.R index ea87b0a..89e9e30 100644 --- a/synoptic/helpers.R +++ b/synoptic/helpers.R @@ -248,3 +248,82 @@ list_directories <- function(dir_list, outfile = "", prefix = "", list_directories(subdirs, outfile, prefix = newprefix, list_files = list_files) } } + + +# The design links might not be stable over time; for example, if a tree +# dies, its sensor might get reassigned to a new tree. In this case the +# design_link table will have two entries, one for the old assignment and +# one for the new. We know which one to use by the "valid_until" column, +# which give a end date for a design link--or, most commonly, it will be +# empty (NA) indicating that there is no end date. +# +# So, when the design_link table is merged with a data table, if a reassignment +# has occurred, some data rows will get repeated with the different possible +# design links. +# +# This function uses the object (i.e. group identifier; typically, Logger+ +# Table+Loggernet_variable), timestamp, and valid_until timestamps to identify +# which rows to keep (correct design_link assignment) and which to drop. +valid_entries <- function(objects, times, valid_until) { + # Any NA valid_until entries apply into the far future + valid_until[is.na(valid_until)] <- 2932896 # this is 12/31/9999 + past_valid_time <- times > valid_until + # Create a data frame to aggregate and then merge, below + x <- data.frame(obj = objects, time = times, vu = valid_until) + # Compute the minimum valid_until entry for each object and time that is + # not past the valid_until point; this is the 'controlling' value + y <- aggregate(vu ~ obj + time, data = x[!past_valid_time,], FUN = min) + names(y)[3] <- "controlling" + # Figure out controlling valid_until for each object/time + z <- merge(x, y, all.x = TRUE) + # An NA controlling entry means there is none + z$controlling[is.na(z$controlling)] <- FALSE + + return(z$vu == z$controlling) +} + +# Sample data. We have two objects (sensors) at time points 1:3 +test_data <- data.frame(obj = c(1, 1, 1, 2, 2, 2), time = c(1, 2, 3, 1, 2, 3)) +# Object 2 changes its design link after time 2 +test_dt <- data.frame(obj = c(1,2,2), + dl = c("A", "B", "C"), + valid_until = c(NA, 2, NA)) +# Merge the 'data' with the 'design link table' +x <- merge(test_data, test_dt) +# Call valid_entries. It figures out that all the object 1 entries should be +# retained, but 1 of 2 entries in each timestep should be dropped for object 2. +# This is because there are two design_table entries for it (see above); the +# first ends at time point 2, and the second is indefinite after that. +valid_entries(x$obj, x$time, x$valid_until) + +# Test code for valid_entries + +# No shifting objects +ret <- valid_entries(c(1, 1, 1), c(1, 2, 3), c(NA, NA, NA)) +stopifnot(all(ret)) +# One object, shift time is never reached +ret <- valid_entries(c(1, 1, 1, 1), c(1, 1, 2, 2), c(4, NA, 4, NA)) +stopifnot(ret == c(TRUE, FALSE, TRUE, FALSE)) +# One object, shift time is in the past +ret <- valid_entries(c(1, 1, 1, 1), c(3, 3, 4, 4), c(2, NA, 2, NA)) +stopifnot(ret == c(FALSE, TRUE, FALSE, TRUE)) +# One object, shifts +ret <- valid_entries(c(1, 1, 1, 1), c(2, 2, 3, 3), c(2, NA, 2, NA)) +stopifnot(ret == c(TRUE, FALSE, FALSE, TRUE)) +# One objects, shifts twice (valid_untils at 1 and 2) +ret <- valid_entries(objects = rep(1, 9), + times = c(1, 1, 1, 2, 2, 2, 3, 3, 3), + valid_until = c(1, 2, NA, 1, 2, NA, 1, 2, NA)) +stopifnot(ret == c(TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, TRUE)) +# Two objects, only one shifts +ret <- valid_entries(objects = c(1, 1, 1, 2, 2, 2, 2, 2, 2), + times = c(1, 2, 3, 1, 1, 2, 2, 3, 3), + valid_until = c(NA, NA, NA, 2, NA, 2, NA, 2, NA)) +stopifnot(ret == c(TRUE, TRUE, TRUE, # obj 1 + TRUE, FALSE, TRUE, FALSE, FALSE, TRUE)) # obj 2 +# There's a valid_until but no new entry +ret <- valid_entries(objects = c(1, 1), + times = c(1, 2), + valid_until = c(1, 1)) +stopifnot(ret == c(TRUE, FALSE)) + From c8ba0458225edc2c3cb9544fad7dd445a1b776a0 Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 14:44:02 -0500 Subject: [PATCH 09/10] Integrate the multiple-design link code into L1_normalize --- synoptic/L1_normalize.qmd | 18 +- synoptic/data_TEST/design_table.csv | 594 ++++++++++++++-------------- synoptic/helpers.R | 8 +- 3 files changed, 316 insertions(+), 304 deletions(-) diff --git a/synoptic/L1_normalize.qmd b/synoptic/L1_normalize.qmd index 70a8fd1..9c53b21 100644 --- a/synoptic/L1_normalize.qmd +++ b/synoptic/L1_normalize.qmd @@ -47,7 +47,7 @@ if(packageVersion("compasstools") < "0.2") { # Read the design table (everything must have an entry) DESIGN_TABLE <- file.path(params$DATA_ROOT, params$DESIGN_TABLE) -dt <- read_csv(DESIGN_TABLE, col_types = "ccccccc") +dt <- read_csv(DESIGN_TABLE, col_types = "cccccDcc") dt$note <- NULL # For compactness, the design table may have expansions. For example, # "DiffVoltA_Avg({1:8})" -> "DiffVoltA_Avg(1)", "DiffVoltA_Avg(2)", etc., with @@ -56,7 +56,7 @@ dt_ex <- compasstools::expand_df(dt) # Read the variable metadata table METADATA_VARS_TABLE <- file.path(params$DATA_ROOT, params$METADATA_VARS_TABLE) -mt <- read_csv(METADATA_VARS_TABLE, col_types = "ccccccccddc") +mt <- read_csv(METADATA_VARS_TABLE, col_types = "ccccccddc") # ...and create the units table (everything must have an entry) ut <- mt[!is.na(mt$research_name),] # filter ut$new_unit = ut$final_units # rename @@ -128,7 +128,19 @@ f <- function(fn, out_dir, design_table) { dat <- merge(dat, design_table, by = c("Logger", "Table", "loggernet_variable"), sort = FALSE) - # This is a left join, and should not have changed number of rows; + # This is a left join, and normally should not have changed the number of rows + # The exception would be if a sensor has been reassigned; in that case it will have + # >1 entry in the design table, with the "valid_until" column controlling when the + # old assignment becomes invalid and the new one takes over. Call valid_entries() + # (in helpers.R) to figure out which mappings should apply. + message("\tChecking for multiple-match design links...") + #browser() + dat_retain <- valid_entries(objects = dat$loggernet_variable, + times = ymd_hms(dat$TIMESTAMP, tz = "EST"), + valid_until = dat$valid_until) + message("\tDropping ", sum(!dat_retain), " out-of-date design links") + dat <- dat[dat_retain,] + # i.e., there must be exactly one match for every loggernet variable if(nrow(dat) > old_rows) { counts <- aggregate(design_link ~ loggernet_variable, data = test, diff --git a/synoptic/data_TEST/design_table.csv b/synoptic/data_TEST/design_table.csv index 50fa9fc..932f355 100644 --- a/synoptic/data_TEST/design_table.csv +++ b/synoptic/data_TEST/design_table.csv @@ -1,297 +1,297 @@ -Site,Logger,Table,loggernet_variable,design_link,research_name,note -PTR,Compass_PTR_UP_313,CheckTable,Format,,, -PTR,Compass_PTR_UP_313,CheckTable,RECORD,,, -PTR,Compass_PTR_UP_313,CheckTable,BattV,BattV-PTR-UP,voltage, -PTR,Compass_PTR_UP_313,CheckTable,SolarV,SolarV-PTR-UP,voltage, -PTR,Compass_PTR_UP_313,CheckTable,Batt_CHECK,,, -PTR,Compass_PTR_UP_313,CheckTable,Solar_CHECK,,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(1),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(2),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(3),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(4),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(5),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(6),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(7),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(8),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(9),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(10),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(11),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(12),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(13),,, -PTR,Compass_PTR_UP_313,CheckTable,Flag(14),,, -PTR,Compass_PTR_UP_313,CheckTable,Statname,,, -PTR,Compass_PTR_UP_313,CheckTable,PB,,, -,,,,,, -PTR,Compass_PTR_UP_313,SapflowA,TIMESTAMP,Timestamp,timestamp, -PTR,Compass_PTR_UP_313,SapflowA,RECORD,,, -PTR,Compass_PTR_UP_313,SapflowA,PB,,, -PTR,Compass_PTR_UP_313,SapflowA,Format,,, -PTR,Compass_PTR_UP_313,SapflowA,Statname,,, -PTR,Compass_PTR_UP_313,SapflowA,BattV_Avg,Logger-PTR-UP,voltage, -PTR,Compass_PTR_UP_313,SapflowA,DiffVoltA_Avg({1:8}),SF-PTR-UP-0{1:8},sapflow_2.5, -PTR,Compass_PTR_UP_313,SapflowA,DiffVoltA_Avg({9:12}),"SF-PTR-UP-0{1,2,3,8}D",sapflow_5, -,,,,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,RECORD,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,PB,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Format,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Statname,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,BattV_Avg,GW-BatVolt-PTR-UP,voltage, -PTR,Compass_PTR_UP_313,WaterLevel600A,Barometric_Pressure600A,GW-BPress-UP,gw_pressure, -PTR,Compass_PTR_UP_313,WaterLevel600A,Temperature600A,GW-Temp-PTR-UP,gw_temperature, -PTR,Compass_PTR_UP_313,WaterLevel600A,Aquatroll_IDA(1),,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Actual_Conductivity600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Specific_Conductivity600A,GW-Salinity-PTR-TR,gw_salinity, -PTR,Compass_PTR_UP_313,WaterLevel600A,Salinity600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,TDS600A,GW-Density-PTR-UP,gw_density, -PTR,Compass_PTR_UP_313,WaterLevel600A,Water_Density600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Resistivity600A,GW-ph-PTR-UP,gw_ph, -PTR,Compass_PTR_UP_313,WaterLevel600A,pH600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,pH_mV600A,GW-ORP-PTR-UP,gw_ph_orp, -PTR,Compass_PTR_UP_313,WaterLevel600A,pH_ORP600A,GW-RDOconc-PTR-UP,gw_rdo_concentration, -PTR,Compass_PTR_UP_313,WaterLevel600A,RDO_concen600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,RDO_perc_sat600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,RDO_part_Pressure600A,GW-Press-PTR-UP,gw_pressure, -PTR,Compass_PTR_UP_313,WaterLevel600A,Pressure600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Depth600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Voltage,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Voltage_Ext600A,,, -PTR,Compass_PTR_UP_313,WaterLevel600A,Battery_Int600A,,, -,,,,,, -PTR,Compass_PTR_UP_313,TerosTableA,RECORD,,, -PTR,Compass_PTR_UP_313,TerosTableA,PB,,, -PTR,Compass_PTR_UP_313,TerosTableA,Format,,, -PTR,Compass_PTR_UP_313,TerosTableA,Statname,,, -PTR,Compass_PTR_UP_313,TerosTableA,BattV_Avg,TEROS-BatVolt-PTR-UP,voltage, -PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({1:4},1)",SoilVWC-PTR-UP-0{1:4},soil_vwc_10cm, -PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({5:6},1)",SoilVWC-PTR-UP-0{5:6},soil_vwc_30cm, -PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({1:4},2)",SoilT-PTR-UP-0{1:4},soil_temp_10cm, -PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({5:6},2)",SoilT-PTR-UP-0{5:6},soil_temp_30cm, -PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({1:4},3)",SoilEC-PTR-UP-0{1:4},soil_EC_10cm, -PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({5:6},3)",SoilEC-PTR-UP-0{5:6},soil_EC_30cm, -,,,,,, -PTR,Compass_PTR_TR_312,CheckTable,Format,,, -PTR,Compass_PTR_TR_312,CheckTable,RECORD,,, -PTR,Compass_PTR_TR_312,CheckTable,BattV,BattV-PTR-TR,voltage, -PTR,Compass_PTR_TR_312,CheckTable,SolarV,SolarV-PTR-TR,voltage, -PTR,Compass_PTR_TR_312,CheckTable,Batt_CHECK,,, -PTR,Compass_PTR_TR_312,CheckTable,Solar_CHECK,,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(1),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(2),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(3),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(4),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(5),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(6),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(7),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(8),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(9),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(10),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(11),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(12),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(13),,, -PTR,Compass_PTR_TR_312,CheckTable,Flag(14),,, -PTR,Compass_PTR_TR_312,CheckTable,Statname,,, -PTR,Compass_PTR_TR_312,CheckTable,PB,,, -,,,,,, -PTR,Compass_PTR_TR_312,SapflowB,TIMESTAMP,Timestamp,timestamp, -PTR,Compass_PTR_TR_312,SapflowB,RECORD,,, -PTR,Compass_PTR_TR_312,SapflowB,PB,,, -PTR,Compass_PTR_TR_312,SapflowB,Format,,, -PTR,Compass_PTR_TR_312,SapflowB,Statname,,, -PTR,Compass_PTR_TR_312,SapflowB,BattV_Avg,Logger-PTR-TR,voltage, -PTR,Compass_PTR_TR_312,SapflowB,DiffVoltB_Avg({1:8}),SF-PTR-TR-0{1:8},sapflow_2.5, -PTR,Compass_PTR_TR_312,SapflowB,DiffVoltB_Avg({9:12}),"SF-PTR-TR-0{1,5,2,7}D",sapflow_5, -,,,,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,RECORD,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,PB,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Format,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Statname,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,BattV_Avg,GW-BatVolt-PTR-TR,voltage, -PTR,Compass_PTR_TR_312,WaterLevel600B,Barometric_Pressure600B,GW-BPress-TR,gw_pressure, -PTR,Compass_PTR_TR_312,WaterLevel600B,Temperature600B,GW-Temp-PTR-TR,gw_temperature, -PTR,Compass_PTR_TR_312,WaterLevel600B,Aquatroll_IDB(1),,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Actual_Conductivity600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Specific_Conductivity600B,GW-Salinity-PTR-TR,gw_salinity, -PTR,Compass_PTR_TR_312,WaterLevel600B,Salinity600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,TDS600B,GW-Density-PTR-TR,gw_density, -PTR,Compass_PTR_TR_312,WaterLevel600B,Water_Density600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Resistivity600B,GW-ph-PTR-TR,gw_ph, -PTR,Compass_PTR_TR_312,WaterLevel600B,pH600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,pH_mV600B,GW-ORP-PTR-TR,gw_ph_orp, -PTR,Compass_PTR_TR_312,WaterLevel600B,pH_ORP600B,GW-RDOconc-PTR-TR,gw_rdo_concentration, -PTR,Compass_PTR_TR_312,WaterLevel600B,RDO_concen600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,RDO_perc_sat600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,RDO_part_Pressure600B,GW-Press-PTR-TR,gw_pressure,check w PR: is this atm p or water p -PTR,Compass_PTR_TR_312,WaterLevel600B,Pressure600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Depth600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Voltage,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Voltage_Ext600B,,, -PTR,Compass_PTR_TR_312,WaterLevel600B,Battery_Int600B,,, -,,,,,, -PTR,Compass_PTR_TR_312,TerosTableB,RECORD,,, -PTR,Compass_PTR_TR_312,TerosTableB,PB,,, -PTR,Compass_PTR_TR_312,TerosTableB,Format,,, -PTR,Compass_PTR_TR_312,TerosTableB,Statname,,, -PTR,Compass_PTR_TR_312,TerosTableB,BattV_Avg,TEROS-BatVolt-PTR-TR,voltage, -PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({1:4},1)",SoilVWC-PTR-TR-0{1:4},soil_vwc_10cm, -PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({5:6},1)",SoilVWC-PTR-TR-0{5:6},soil_vwc_30cm, -PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({1:4},2)",SoilT-PTR-TR-0{1:4},soil_temp_10cm, -PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({5:6},2)",SoilT-PTR-TR-0{5:6},soil_temp_30cm, -PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({1:4},3)",SoilEC-PTR-TR-0{1:4},soil_EC_10cm, -PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({5:6},3)",SoilEC-PTR-TR-0{5:6},soil_EC_30cm, -,,,,,, -PTR,Compass_PTR_W_311,CheckTable,Format,,, -PTR,Compass_PTR_W_311,CheckTable,RECORD,,, -PTR,Compass_PTR_W_311,CheckTable,BattV,BattV-PTR-W,voltage, -PTR,Compass_PTR_W_311,CheckTable,SolarV,SolarV-PTR-W,voltage, -PTR,Compass_PTR_W_311,CheckTable,Batt_CHECK,,, -PTR,Compass_PTR_W_311,CheckTable,Solar_CHECK,,, -PTR,Compass_PTR_W_311,CheckTable,Flag(1),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(2),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(3),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(4),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(5),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(6),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(7),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(8),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(9),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(10),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(11),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(12),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(13),,, -PTR,Compass_PTR_W_311,CheckTable,Flag(14),,, -PTR,Compass_PTR_W_311,CheckTable,Statname,,, -PTR,Compass_PTR_W_311,CheckTable,PB,,, -,,,,,, -PTR,Compass_PTR_W_311,WaterLevel600C,RECORD,,, -PTR,Compass_PTR_W_311,WaterLevel600C,PB,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Format,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Statname,,, -PTR,Compass_PTR_W_311,WaterLevel600C,BattV_Avg,GL-BatVolt-PTR-W,voltage, -PTR,Compass_PTR_W_311,WaterLevel600C,Barometric_Pressure600C,GL-BPress-UP,gw_pressure, -PTR,Compass_PTR_W_311,WaterLevel600C,Temperature600C,GW-Temp-PTR-W,gw_temperature, -PTR,Compass_PTR_W_311,WaterLevel600C,Aquatroll_IDC(1),,, -PTR,Compass_PTR_W_311,WaterLevel600C,Actual_Conductivity600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Specific_Conductivity600C,GW-Salinity-PTR-W,gw_salinity, -PTR,Compass_PTR_W_311,WaterLevel600C,Salinity600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,TDS600C,GW-Density-PTR-W,gw_density, -PTR,Compass_PTR_W_311,WaterLevel600C,Water_Density600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Resistivity600C,GW-ph-PTR-W,gw_ph, -PTR,Compass_PTR_W_311,WaterLevel600C,pH600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,pH_mV600C,GW-ORP-PTR-W,gw_ph_orp, -PTR,Compass_PTR_W_311,WaterLevel600C,pH_ORP600C,GW-RDOconc-PTR-W,gw_rdo_concentration, -PTR,Compass_PTR_W_311,WaterLevel600C,RDO_concen600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,RDO_perc_sat600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,RDO_part_Pressure600C,GW-Press-PTR-W,gw_pressure, -PTR,Compass_PTR_W_311,WaterLevel600C,Pressure600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Depth600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Voltage,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Voltage_Ext600C,,, -PTR,Compass_PTR_W_311,WaterLevel600C,Battery_Int600C,,, -,,,,,, -PTR,Compass_PTR_W_311,TerosTableC,RECORD,,, -PTR,Compass_PTR_W_311,TerosTableC,PB,,, -PTR,Compass_PTR_W_311,TerosTableC,Format,,, -PTR,Compass_PTR_W_311,TerosTableC,Statname,,, -PTR,Compass_PTR_W_311,TerosTableC,BattV_Avg,TEROS-BatVolt-PTR-W,voltage, -PTR,Compass_PTR_W_311,TerosTableC,"TerosC({1:4},1)",SoilVWC-PTR-W-0{1:4},soil_vwc_10cm, -PTR,Compass_PTR_W_311,TerosTableC,"TerosC({5:6},1)",SoilVWC-PTR-W-0{5:6},soil_vwc_30cm, -PTR,Compass_PTR_W_311,TerosTableC,"TerosC({1:4},2)",SoilT-PTR-W-0{1:4},soil_temp_10cm, -PTR,Compass_PTR_W_311,TerosTableC,"TerosC({5:6},2)",SoilT-PTR-W-0{5:6},soil_temp_30cm, -PTR,Compass_PTR_W_311,TerosTableC,"TerosC({1:4},3)",SoilEC-PTR-W-0{1:4},soil_EC_10cm, -PTR,Compass_PTR_W_311,TerosTableC,"TerosC({5:6},3)",SoilEC-PTR-W-0{5:6},soil_EC_30cm, -,,,,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,Format,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,RECORD,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,SlrFD_W_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,SlrTF_MJ_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,Rain_mm_Tot,CLIM-Rain-PTR-W,wx_rain, -PTR,Compass_PTR_W_311,ClimaVue50_15min,WS_ms_S_WVT,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,WindDir_D1_WVT,CLIM-WindDirD1-PTR-W,wx_winddir, -PTR,Compass_PTR_W_311,ClimaVue50_15min,WindDir_SD1_WVT,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,MaxWS_ms_Max,CLIM-MaxWSMax-PTR-W,wx_maxws, -PTR,Compass_PTR_W_311,ClimaVue50_15min,MaxWS_ms_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,Invalid_Wind_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_Avg,CLIM-TempAvg-PTR-W,wx_tempavg, -PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_Max,CLIM-TempMax-PTR-W,wx_tempmax, -PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_Min,CLIM-TempTmin-PTR-W,wx_tempmin, -PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_TMn,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,VP_mbar_Avg,CLIM-VP-PTR-W,wx_vappress, -PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar,CLIM-BP-PTR-W,wx_barpress, -PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_TMn,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,RH,CLIM-RH-PTR-W,wx_rh, -PTR,Compass_PTR_W_311,ClimaVue50_15min,RHT_C_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,TiltNS_deg_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,TiltWE_deg_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,Strikes_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,Dist_km_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,Dist_km_TMn,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,PAR_Den_C_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,PAR_Tot_C_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_15min,CVMeta,,, -,,,,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,Format,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,RECORD,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,BattV_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,Rain_mm_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,SlrFD_W_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,SlrTF_MJ_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,WS_ms_WVc(1),,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,WS_ms_WVc(2),,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,WS_ms_S_WVT,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,MaxWS_ms_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,MaxWS_ms_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_TMn,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,VP_mbar_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_TMn,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,RH_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,RH_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,RHT_C_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,RHT_C_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_TMn,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltWE_deg_Max,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltWE_deg_TMx,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltWE_deg_Min,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,PAR_Den_C_Avg,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,PAR_Tot_C_Tot,,, -PTR,Compass_PTR_W_311,ClimaVue50_24hr,CVMeta,,, -,,,,,, -PTR,Compass_PTR_W_311,ExoTable,Format,,, -PTR,Compass_PTR_W_311,ExoTable,RECORD,,, -PTR,Compass_PTR_W_311,ExoTable,PB,,, -PTR,Compass_PTR_W_311,ExoTable,Statname,,, -PTR,Compass_PTR_W_311,ExoTable,Date,,, -PTR,Compass_PTR_W_311,ExoTable,Time,,, -PTR,Compass_PTR_W_311,ExoTable,Conductivity,EXO-Cond-PTR-W,sonde_conductivity, -PTR,Compass_PTR_W_311,ExoTable,FDOM_QSU,EXO-FDOM-PTR-W,sonde_fdom, -PTR,Compass_PTR_W_311,ExoTable,FDOM_RFU,,, -PTR,Compass_PTR_W_311,ExoTable,NLF_conductivity,,, -PTR,Compass_PTR_W_311,ExoTable,ODO_sat,,, -PTR,Compass_PTR_W_311,ExoTable,ODO_local,,, -PTR,Compass_PTR_W_311,ExoTable,ODO_MgL,,, -PTR,Compass_PTR_W_311,ExoTable,Pressure_psia,,, -PTR,Compass_PTR_W_311,ExoTable,Salinity_PPT,EXO-Salinity-PTR-W,sonde_salinity, -PTR,Compass_PTR_W_311,ExoTable,Specific_Conductivity_uScm,EXO-SpCond-PTR-W,sonde_spcond, -PTR,Compass_PTR_W_311,ExoTable,Wiper_Position_mv,,, -PTR,Compass_PTR_W_311,ExoTable,pH,EXO-pH-PTR-W,sonde_ph, -PTR,Compass_PTR_W_311,ExoTable,pH_mv,,, -PTR,Compass_PTR_W_311,ExoTable,Temp_C,EXO-Temp-PTR-W,sonde_temp, -PTR,Compass_PTR_W_311,ExoTable,Depth_m,EXO-Depth-PTR-W,sonde_depth, -PTR,Compass_PTR_W_311,ExoTable,Battery_v,EXO-BatteryV-PTR-W,sonde_battery, -PTR,Compass_PTR_W_311,ExoTable,Cable_v,EXO-CableV-PTR-W,sonde_cable, -PTR,Compass_PTR_W_311,ExoTable,Wiper_Current_ma,EXO-Wiper-PTR-W,sonde_wiper, -PTR,Compass_PTR_W_311,ExoTable,sn,,, -PTR,Compass_PTR_W_311,ExoTable,snn,,, +Site,Logger,Table,loggernet_variable,design_link,valid_until,research_name,note +PTR,Compass_PTR_UP_313,CheckTable,Format,,,, +PTR,Compass_PTR_UP_313,CheckTable,RECORD,,,, +PTR,Compass_PTR_UP_313,CheckTable,BattV,BattV-PTR-UP,,voltage, +PTR,Compass_PTR_UP_313,CheckTable,SolarV,SolarV-PTR-UP,,voltage, +PTR,Compass_PTR_UP_313,CheckTable,Batt_CHECK,,,, +PTR,Compass_PTR_UP_313,CheckTable,Solar_CHECK,,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(1),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(2),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(3),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(4),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(5),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(6),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(7),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(8),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(9),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(10),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(11),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(12),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(13),,,, +PTR,Compass_PTR_UP_313,CheckTable,Flag(14),,,, +PTR,Compass_PTR_UP_313,CheckTable,Statname,,,, +PTR,Compass_PTR_UP_313,CheckTable,PB,,,, +,,,,,,, +PTR,Compass_PTR_UP_313,SapflowA,TIMESTAMP,Timestamp,,timestamp, +PTR,Compass_PTR_UP_313,SapflowA,RECORD,,,, +PTR,Compass_PTR_UP_313,SapflowA,PB,,,, +PTR,Compass_PTR_UP_313,SapflowA,Format,,,, +PTR,Compass_PTR_UP_313,SapflowA,Statname,,,, +PTR,Compass_PTR_UP_313,SapflowA,BattV_Avg,Logger-PTR-UP,,voltage, +PTR,Compass_PTR_UP_313,SapflowA,DiffVoltA_Avg({1:8}),SF-PTR-UP-0{1:8},,sapflow_2.5, +PTR,Compass_PTR_UP_313,SapflowA,DiffVoltA_Avg({9:12}),"SF-PTR-UP-0{1,2,3,8}D",,sapflow_5, +,,,,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,RECORD,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,PB,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Format,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Statname,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,BattV_Avg,GW-BatVolt-PTR-UP,,voltage, +PTR,Compass_PTR_UP_313,WaterLevel600A,Barometric_Pressure600A,GW-BPress-UP,,gw_pressure, +PTR,Compass_PTR_UP_313,WaterLevel600A,Temperature600A,GW-Temp-PTR-UP,,gw_temperature, +PTR,Compass_PTR_UP_313,WaterLevel600A,Aquatroll_IDA(1),,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Actual_Conductivity600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Specific_Conductivity600A,GW-Salinity-PTR-TR,,gw_salinity, +PTR,Compass_PTR_UP_313,WaterLevel600A,Salinity600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,TDS600A,GW-Density-PTR-UP,,gw_density, +PTR,Compass_PTR_UP_313,WaterLevel600A,Water_Density600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Resistivity600A,GW-ph-PTR-UP,,gw_ph, +PTR,Compass_PTR_UP_313,WaterLevel600A,pH600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,pH_mV600A,GW-ORP-PTR-UP,,gw_ph_orp, +PTR,Compass_PTR_UP_313,WaterLevel600A,pH_ORP600A,GW-RDOconc-PTR-UP,,gw_rdo_concentration, +PTR,Compass_PTR_UP_313,WaterLevel600A,RDO_concen600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,RDO_perc_sat600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,RDO_part_Pressure600A,GW-Press-PTR-UP,,gw_pressure, +PTR,Compass_PTR_UP_313,WaterLevel600A,Pressure600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Depth600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Voltage,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Voltage_Ext600A,,,, +PTR,Compass_PTR_UP_313,WaterLevel600A,Battery_Int600A,,,, +,,,,,,, +PTR,Compass_PTR_UP_313,TerosTableA,RECORD,,,, +PTR,Compass_PTR_UP_313,TerosTableA,PB,,,, +PTR,Compass_PTR_UP_313,TerosTableA,Format,,,, +PTR,Compass_PTR_UP_313,TerosTableA,Statname,,,, +PTR,Compass_PTR_UP_313,TerosTableA,BattV_Avg,TEROS-BatVolt-PTR-UP,,voltage, +PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({1:4},1)",SoilVWC-PTR-UP-0{1:4},,soil_vwc_10cm, +PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({5:6},1)",SoilVWC-PTR-UP-0{5:6},,soil_vwc_30cm, +PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({1:4},2)",SoilT-PTR-UP-0{1:4},,soil_temp_10cm, +PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({5:6},2)",SoilT-PTR-UP-0{5:6},,soil_temp_30cm, +PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({1:4},3)",SoilEC-PTR-UP-0{1:4},,soil_EC_10cm, +PTR,Compass_PTR_UP_313,TerosTableA,"TerosA({5:6},3)",SoilEC-PTR-UP-0{5:6},,soil_EC_30cm, +,,,,,,, +PTR,Compass_PTR_TR_312,CheckTable,Format,,,, +PTR,Compass_PTR_TR_312,CheckTable,RECORD,,,, +PTR,Compass_PTR_TR_312,CheckTable,BattV,BattV-PTR-TR,,voltage, +PTR,Compass_PTR_TR_312,CheckTable,SolarV,SolarV-PTR-TR,,voltage, +PTR,Compass_PTR_TR_312,CheckTable,Batt_CHECK,,,, +PTR,Compass_PTR_TR_312,CheckTable,Solar_CHECK,,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(1),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(2),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(3),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(4),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(5),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(6),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(7),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(8),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(9),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(10),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(11),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(12),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(13),,,, +PTR,Compass_PTR_TR_312,CheckTable,Flag(14),,,, +PTR,Compass_PTR_TR_312,CheckTable,Statname,,,, +PTR,Compass_PTR_TR_312,CheckTable,PB,,,, +,,,,,,, +PTR,Compass_PTR_TR_312,SapflowB,TIMESTAMP,Timestamp,,timestamp, +PTR,Compass_PTR_TR_312,SapflowB,RECORD,,,, +PTR,Compass_PTR_TR_312,SapflowB,PB,,,, +PTR,Compass_PTR_TR_312,SapflowB,Format,,,, +PTR,Compass_PTR_TR_312,SapflowB,Statname,,,, +PTR,Compass_PTR_TR_312,SapflowB,BattV_Avg,Logger-PTR-TR,,voltage, +PTR,Compass_PTR_TR_312,SapflowB,DiffVoltB_Avg({1:8}),SF-PTR-TR-0{1:8},,sapflow_2.5, +PTR,Compass_PTR_TR_312,SapflowB,DiffVoltB_Avg({9:12}),"SF-PTR-TR-0{1,5,2,7}D",,sapflow_5, +,,,,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,RECORD,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,PB,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Format,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Statname,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,BattV_Avg,GW-BatVolt-PTR-TR,,voltage, +PTR,Compass_PTR_TR_312,WaterLevel600B,Barometric_Pressure600B,GW-BPress-TR,,gw_pressure, +PTR,Compass_PTR_TR_312,WaterLevel600B,Temperature600B,GW-Temp-PTR-TR,,gw_temperature, +PTR,Compass_PTR_TR_312,WaterLevel600B,Aquatroll_IDB(1),,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Actual_Conductivity600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Specific_Conductivity600B,GW-Salinity-PTR-TR,,gw_salinity, +PTR,Compass_PTR_TR_312,WaterLevel600B,Salinity600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,TDS600B,GW-Density-PTR-TR,,gw_density, +PTR,Compass_PTR_TR_312,WaterLevel600B,Water_Density600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Resistivity600B,GW-ph-PTR-TR,,gw_ph, +PTR,Compass_PTR_TR_312,WaterLevel600B,pH600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,pH_mV600B,GW-ORP-PTR-TR,,gw_ph_orp, +PTR,Compass_PTR_TR_312,WaterLevel600B,pH_ORP600B,GW-RDOconc-PTR-TR,,gw_rdo_concentration, +PTR,Compass_PTR_TR_312,WaterLevel600B,RDO_concen600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,RDO_perc_sat600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,RDO_part_Pressure600B,GW-Press-PTR-TR,,gw_pressure,check w PR: is this atm p or water p +PTR,Compass_PTR_TR_312,WaterLevel600B,Pressure600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Depth600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Voltage,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Voltage_Ext600B,,,, +PTR,Compass_PTR_TR_312,WaterLevel600B,Battery_Int600B,,,, +,,,,,,, +PTR,Compass_PTR_TR_312,TerosTableB,RECORD,,,, +PTR,Compass_PTR_TR_312,TerosTableB,PB,,,, +PTR,Compass_PTR_TR_312,TerosTableB,Format,,,, +PTR,Compass_PTR_TR_312,TerosTableB,Statname,,,, +PTR,Compass_PTR_TR_312,TerosTableB,BattV_Avg,TEROS-BatVolt-PTR-TR,,voltage, +PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({1:4},1)",SoilVWC-PTR-TR-0{1:4},,soil_vwc_10cm, +PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({5:6},1)",SoilVWC-PTR-TR-0{5:6},,soil_vwc_30cm, +PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({1:4},2)",SoilT-PTR-TR-0{1:4},,soil_temp_10cm, +PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({5:6},2)",SoilT-PTR-TR-0{5:6},,soil_temp_30cm, +PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({1:4},3)",SoilEC-PTR-TR-0{1:4},,soil_EC_10cm, +PTR,Compass_PTR_TR_312,TerosTableB,"TerosB({5:6},3)",SoilEC-PTR-TR-0{5:6},,soil_EC_30cm, +,,,,,,, +PTR,Compass_PTR_W_311,CheckTable,Format,,,, +PTR,Compass_PTR_W_311,CheckTable,RECORD,,,, +PTR,Compass_PTR_W_311,CheckTable,BattV,BattV-PTR-W,,voltage, +PTR,Compass_PTR_W_311,CheckTable,SolarV,SolarV-PTR-W,,voltage, +PTR,Compass_PTR_W_311,CheckTable,Batt_CHECK,,,, +PTR,Compass_PTR_W_311,CheckTable,Solar_CHECK,,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(1),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(2),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(3),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(4),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(5),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(6),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(7),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(8),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(9),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(10),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(11),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(12),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(13),,,, +PTR,Compass_PTR_W_311,CheckTable,Flag(14),,,, +PTR,Compass_PTR_W_311,CheckTable,Statname,,,, +PTR,Compass_PTR_W_311,CheckTable,PB,,,, +,,,,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,RECORD,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,PB,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Format,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Statname,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,BattV_Avg,GL-BatVolt-PTR-W,,voltage, +PTR,Compass_PTR_W_311,WaterLevel600C,Barometric_Pressure600C,GL-BPress-UP,,gw_pressure, +PTR,Compass_PTR_W_311,WaterLevel600C,Temperature600C,GW-Temp-PTR-W,,gw_temperature, +PTR,Compass_PTR_W_311,WaterLevel600C,Aquatroll_IDC(1),,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Actual_Conductivity600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Specific_Conductivity600C,GW-Salinity-PTR-W,,gw_salinity, +PTR,Compass_PTR_W_311,WaterLevel600C,Salinity600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,TDS600C,GW-Density-PTR-W,,gw_density, +PTR,Compass_PTR_W_311,WaterLevel600C,Water_Density600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Resistivity600C,GW-ph-PTR-W,,gw_ph, +PTR,Compass_PTR_W_311,WaterLevel600C,pH600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,pH_mV600C,GW-ORP-PTR-W,,gw_ph_orp, +PTR,Compass_PTR_W_311,WaterLevel600C,pH_ORP600C,GW-RDOconc-PTR-W,,gw_rdo_concentration, +PTR,Compass_PTR_W_311,WaterLevel600C,RDO_concen600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,RDO_perc_sat600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,RDO_part_Pressure600C,GW-Press-PTR-W,,gw_pressure, +PTR,Compass_PTR_W_311,WaterLevel600C,Pressure600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Depth600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Voltage,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Voltage_Ext600C,,,, +PTR,Compass_PTR_W_311,WaterLevel600C,Battery_Int600C,,,, +,,,,,,, +PTR,Compass_PTR_W_311,TerosTableC,RECORD,,,, +PTR,Compass_PTR_W_311,TerosTableC,PB,,,, +PTR,Compass_PTR_W_311,TerosTableC,Format,,,, +PTR,Compass_PTR_W_311,TerosTableC,Statname,,,, +PTR,Compass_PTR_W_311,TerosTableC,BattV_Avg,TEROS-BatVolt-PTR-W,,voltage, +PTR,Compass_PTR_W_311,TerosTableC,"TerosC({1:4},1)",SoilVWC-PTR-W-0{1:4},,soil_vwc_10cm, +PTR,Compass_PTR_W_311,TerosTableC,"TerosC({5:6},1)",SoilVWC-PTR-W-0{5:6},,soil_vwc_30cm, +PTR,Compass_PTR_W_311,TerosTableC,"TerosC({1:4},2)",SoilT-PTR-W-0{1:4},,soil_temp_10cm, +PTR,Compass_PTR_W_311,TerosTableC,"TerosC({5:6},2)",SoilT-PTR-W-0{5:6},,soil_temp_30cm, +PTR,Compass_PTR_W_311,TerosTableC,"TerosC({1:4},3)",SoilEC-PTR-W-0{1:4},,soil_EC_10cm, +PTR,Compass_PTR_W_311,TerosTableC,"TerosC({5:6},3)",SoilEC-PTR-W-0{5:6},,soil_EC_30cm, +,,,,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,Format,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,RECORD,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,SlrFD_W_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,SlrTF_MJ_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,Rain_mm_Tot,CLIM-Rain-PTR-W,,wx_rain, +PTR,Compass_PTR_W_311,ClimaVue50_15min,WS_ms_S_WVT,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,WindDir_D1_WVT,CLIM-WindDirD1-PTR-W,,wx_winddir, +PTR,Compass_PTR_W_311,ClimaVue50_15min,WindDir_SD1_WVT,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,MaxWS_ms_Max,CLIM-MaxWSMax-PTR-W,,wx_maxws, +PTR,Compass_PTR_W_311,ClimaVue50_15min,MaxWS_ms_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,Invalid_Wind_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_Avg,CLIM-TempAvg-PTR-W,,wx_tempavg, +PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_Max,CLIM-TempMax-PTR-W,,wx_tempmax, +PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_Min,CLIM-TempTmin-PTR-W,,wx_tempmin, +PTR,Compass_PTR_W_311,ClimaVue50_15min,AirT_C_TMn,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,VP_mbar_Avg,CLIM-VP-PTR-W,,wx_vappress, +PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar,CLIM-BP-PTR-W,,wx_barpress, +PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,BP_mbar_TMn,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,RH,CLIM-RH-PTR-W,,wx_rh, +PTR,Compass_PTR_W_311,ClimaVue50_15min,RHT_C_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,TiltNS_deg_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,TiltWE_deg_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,Strikes_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,Dist_km_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,Dist_km_TMn,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,PAR_Den_C_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,PAR_Tot_C_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_15min,CVMeta,,,, +,,,,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,Format,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,RECORD,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,BattV_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,Rain_mm_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,SlrFD_W_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,SlrTF_MJ_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,WS_ms_WVc(1),,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,WS_ms_WVc(2),,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,WS_ms_S_WVT,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,MaxWS_ms_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,MaxWS_ms_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,AirT_C_TMn,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,VP_mbar_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,BP_mbar_TMn,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,RH_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,RH_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,RHT_C_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,RHT_C_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltNS_deg_TMn,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltWE_deg_Max,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltWE_deg_TMx,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,TiltWE_deg_Min,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,PAR_Den_C_Avg,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,PAR_Tot_C_Tot,,,, +PTR,Compass_PTR_W_311,ClimaVue50_24hr,CVMeta,,,, +,,,,,,, +PTR,Compass_PTR_W_311,ExoTable,Format,,,, +PTR,Compass_PTR_W_311,ExoTable,RECORD,,,, +PTR,Compass_PTR_W_311,ExoTable,PB,,,, +PTR,Compass_PTR_W_311,ExoTable,Statname,,,, +PTR,Compass_PTR_W_311,ExoTable,Date,,,, +PTR,Compass_PTR_W_311,ExoTable,Time,,,, +PTR,Compass_PTR_W_311,ExoTable,Conductivity,EXO-Cond-PTR-W,,sonde_conductivity, +PTR,Compass_PTR_W_311,ExoTable,FDOM_QSU,EXO-FDOM-PTR-W,,sonde_fdom, +PTR,Compass_PTR_W_311,ExoTable,FDOM_RFU,,,, +PTR,Compass_PTR_W_311,ExoTable,NLF_conductivity,,,, +PTR,Compass_PTR_W_311,ExoTable,ODO_sat,,,, +PTR,Compass_PTR_W_311,ExoTable,ODO_local,,,, +PTR,Compass_PTR_W_311,ExoTable,ODO_MgL,,,, +PTR,Compass_PTR_W_311,ExoTable,Pressure_psia,,,, +PTR,Compass_PTR_W_311,ExoTable,Salinity_PPT,EXO-Salinity-PTR-W,,sonde_salinity, +PTR,Compass_PTR_W_311,ExoTable,Specific_Conductivity_uScm,EXO-SpCond-PTR-W,,sonde_spcond, +PTR,Compass_PTR_W_311,ExoTable,Wiper_Position_mv,,,, +PTR,Compass_PTR_W_311,ExoTable,pH,EXO-pH-PTR-W,,sonde_ph, +PTR,Compass_PTR_W_311,ExoTable,pH_mv,,,, +PTR,Compass_PTR_W_311,ExoTable,Temp_C,EXO-Temp-PTR-W,,sonde_temp, +PTR,Compass_PTR_W_311,ExoTable,Depth_m,EXO-Depth-PTR-W,,sonde_depth, +PTR,Compass_PTR_W_311,ExoTable,Battery_v,EXO-BatteryV-PTR-W,,sonde_battery, +PTR,Compass_PTR_W_311,ExoTable,Cable_v,EXO-CableV-PTR-W,,sonde_cable, +PTR,Compass_PTR_W_311,ExoTable,Wiper_Current_ma,EXO-Wiper-PTR-W,,sonde_wiper, +PTR,Compass_PTR_W_311,ExoTable,sn,,,, +PTR,Compass_PTR_W_311,ExoTable,snn,,,, diff --git a/synoptic/helpers.R b/synoptic/helpers.R index 89e9e30..add2c21 100644 --- a/synoptic/helpers.R +++ b/synoptic/helpers.R @@ -266,7 +266,7 @@ list_directories <- function(dir_list, outfile = "", prefix = "", # which rows to keep (correct design_link assignment) and which to drop. valid_entries <- function(objects, times, valid_until) { # Any NA valid_until entries apply into the far future - valid_until[is.na(valid_until)] <- 2932896 # this is 12/31/9999 + valid_until[is.na(valid_until)] <- ymd_hms("2999-12-31 11:59:00") past_valid_time <- times > valid_until # Create a data frame to aggregate and then merge, below x <- data.frame(obj = objects, time = times, vu = valid_until) @@ -277,9 +277,9 @@ valid_entries <- function(objects, times, valid_until) { # Figure out controlling valid_until for each object/time z <- merge(x, y, all.x = TRUE) # An NA controlling entry means there is none - z$controlling[is.na(z$controlling)] <- FALSE - - return(z$vu == z$controlling) + valids <- z$vu == z$controlling + valids[is.na(valids)] <- FALSE + valids } # Sample data. We have two objects (sensors) at time points 1:3 From 8525134b52343f788084f02e1868d6cd93227afc Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Sat, 2 Dec 2023 17:44:43 -0500 Subject: [PATCH 10/10] Remove unneeded valid_until column --- synoptic/L1_normalize.qmd | 4 ++-- synoptic/helpers.R | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/synoptic/L1_normalize.qmd b/synoptic/L1_normalize.qmd index 9c53b21..abf92be 100644 --- a/synoptic/L1_normalize.qmd +++ b/synoptic/L1_normalize.qmd @@ -134,14 +134,14 @@ f <- function(fn, out_dir, design_table) { # old assignment becomes invalid and the new one takes over. Call valid_entries() # (in helpers.R) to figure out which mappings should apply. message("\tChecking for multiple-match design links...") - #browser() dat_retain <- valid_entries(objects = dat$loggernet_variable, times = ymd_hms(dat$TIMESTAMP, tz = "EST"), valid_until = dat$valid_until) message("\tDropping ", sum(!dat_retain), " out-of-date design links") dat <- dat[dat_retain,] + dat$valid_until <- NULL - # i.e., there must be exactly one match for every loggernet variable + # At this point, there should be exactly one match for every loggernet variable if(nrow(dat) > old_rows) { counts <- aggregate(design_link ~ loggernet_variable, data = test, FUN = function(x) length(unique(x))) diff --git a/synoptic/helpers.R b/synoptic/helpers.R index add2c21..040684b 100644 --- a/synoptic/helpers.R +++ b/synoptic/helpers.R @@ -265,20 +265,26 @@ list_directories <- function(dir_list, outfile = "", prefix = "", # Table+Loggernet_variable), timestamp, and valid_until timestamps to identify # which rows to keep (correct design_link assignment) and which to drop. valid_entries <- function(objects, times, valid_until) { + # Nothing to do if there are no valid_until entries + if(all(is.na(valid_until))) return(rep(TRUE, length(objects()))) + # Any NA valid_until entries apply into the far future valid_until[is.na(valid_until)] <- ymd_hms("2999-12-31 11:59:00") past_valid_time <- times > valid_until + # Create a data frame to aggregate and then merge, below x <- data.frame(obj = objects, time = times, vu = valid_until) # Compute the minimum valid_until entry for each object and time that is # not past the valid_until point; this is the 'controlling' value y <- aggregate(vu ~ obj + time, data = x[!past_valid_time,], FUN = min) names(y)[3] <- "controlling" + # Figure out controlling valid_until for each object/time z <- merge(x, y, all.x = TRUE) # An NA controlling entry means there is none valids <- z$vu == z$controlling valids[is.na(valids)] <- FALSE + valids }