Merge branch 'master' into issue1198_addfirstwake

NEWS.md

@@ -1,8 +1,14 @@
-# CHANGES IN GGIR VERSION 3.1-5
+# CHANGES IN GGIR VERSION 3.1-?
 
 - Part 5: Fixed minor bug in g.part5.addfirstwake causing the first wake is not correctly added when
 no SIBs are detected from the beginning of the recording until the first detected night. #1198
 
+- Part 5: Add parameters require_complete_lastnight_part5 to control whether last window is included if last night is incomplete. #1196
+
+- General: GGIR version look-up in .onattach() no longer crashes when computer is offline, fixes #1203.
+
+- Reports: The calendar_date and filename columns in reports have been standardized, as %Y-%m-%d and the input accelerometer file name, respectively. #1197
+
 # CHANGES IN GGIR VERSION 3.1-4
 
 - Part 3: Update threshold used for HorAngle to 60 degree, and auto-setting HASPT.ignore.invalid to NA when NotWorn guider is used. #1186

R/GGIR.R

@@ -369,7 +369,7 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
       g.report.part2(metadatadir = metadatadir, f0 = f0, f1 = f1,
                      maxdur = params_cleaning[["maxdur"]],
                      store.long = store.long, params_output,
-                     verbose = verbose)
+                     verbose = verbose, desiredtz = params_general[["desiredtz"]])
     }
   }
   if (length(which(do.report == 4)) > 0) {

R/check_params.R

@@ -117,7 +117,7 @@ check_params = function(params_sleep = c(), params_metrics = c(),
     boolean_params = c("epochvalues2csv", "save_ms5rawlevels", "save_ms5raw_without_invalid",
                        "storefolderstructure", "dofirstpage", "visualreport", "week_weekend_aggregate.part5",
                        "do.part3.pdf", "outliers.only", "do.visual", "do.sibreport", "visualreport_without_invalid",
-                       "do.part2.pdf")
+                       "do.part2.pdf", "require_complete_lastnight_part5")
     character_params = c("save_ms5raw_format", "timewindow", "sep_reports", "sep_config",
                          "dec_reports", "dec_config")
     check_class("output", params = params_output, parnames = numeric_params, parclass = "numeric")

R/g.part5.R

@@ -170,6 +170,7 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
         # convert to character/numeric if stored as factor in metashort and metalong
         M$metashort = correctOlderMilestoneData(M$metashort)
         M$metalong = correctOlderMilestoneData(M$metalong)
+        filename = filename_dir
         # load output g.part3
         longitudinal_axis = NULL # initialise var that is part of ms3.out
         load(paste0(metadatadir, "/meta/ms3.out/", fnames.ms3[i]))
@@ -600,7 +601,9 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
                                            includedaycrit.part5 = params_cleaning[["includedaycrit.part5"]],
                                            ID = ID,
                                            params_output = params_output,
-                                           params_247 = params_247)
+                                           params_247 = params_247,
+                                           filename = filename,
+                                           timewindow = timewindowi)
                   }
                 }
               }
@@ -662,7 +665,9 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
                 if (length(GGIRversion) != 1) GGIRversion = sessionInfo()$otherPkgs$GGIR$Version
               }
               output$GGIRversion = GGIRversion
-              save(output, tail_expansion_log, GGIRversion,
+              # Capture final timestamp to ease filtering last window in g.report.part5
+              last_timestamp = time_POSIX[length(time_POSIX)] 
+              save(output, tail_expansion_log, GGIRversion, last_timestamp,
                    file = paste(metadatadir, ms5.out, "/", fnames.ms3[i], sep = ""))
             }
           }

R/g.part5.savetimeseries.R

@@ -2,8 +2,10 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
                                   DaCleanFile = NULL,
                                   includedaycrit.part5 = 2/3, ID = NULL,
                                   params_output,
-                                  params_247 = NULL) {
-
+                                  params_247 = NULL,
+                                  filename = "",
+                                  timewindow = NULL) {
+
   ms5rawlevels = data.frame(date_time = ts$time, class_id = LEVELS,
                             # class_name = rep("",Nts),
                             stringsAsFactors = FALSE)
@@ -23,6 +25,17 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
   names(mdat)[which(names(mdat) == "nonwear")] = "invalidepoch"
   names(mdat)[which(names(mdat) == "diur")] = "SleepPeriodTime"
   mdat = mdat[,-which(names(mdat) == "date_time")]
+  if ("require_complete_lastnight_part5" %in% names(params_output) &&
+      params_output[["require_complete_lastnight_part5"]] == TRUE) {
+    last_timestamp = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H"))
+    if ((timewindow == "MM" || timewindow == "OO") && last_timestamp < 9) {
+      mdat$window[which(mdat$window == max(mdat$window))] = 0 
+    }
+    if (timewindow == "WW" && last_timestamp < 15) {
+      mdat$window[which(mdat$window == max(mdat$window))] = 0 
+    }
+  }
+
   # Add invalid day indicator
   mdat$invalid_wakinghours = mdat$invalid_sleepperiod =  mdat$invalid_fullwindow = 100
   wakeup = which(diff(c(mdat$SleepPeriodTime,0)) == -1) + 1 # first epoch of each day
@@ -92,7 +105,7 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
       mdat$timestamp = as.POSIXct(mdat$timenum, origin = "1970-01-01",tz = desiredtz)
       rawlevels_fname = gsub(pattern = ".csv", replacement = ".RData", x = rawlevels_fname)
       fname = unique(rawlevels_fname[grep("*RData$", rawlevels_fname)])
-      save(mdat, file = fname)
+      save(mdat, filename, file = fname)
     }
     #===============================
     rm(mdat)

R/g.part6.R

@@ -116,6 +116,7 @@ g.part6 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(),
       } else {
         mdat = data.table::fread(file = paste0(metadatadir, "/meta/ms5.outraw/", 
                                                params_phyact[["part6_threshold_combi"]],  "/", fnames.ms5raw[i]), data.table = FALSE)
+        filename = fnames.ms5raw[i]
       }
       nfeatures = 50
       summary = matrix(NA, nfeatures, 1)
@@ -188,7 +189,7 @@ g.part6 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(),
       summary[fi] = format(starttime)
       s_names[fi] = "starttime"
       fi = fi + 1
-      summary[fi] = gsub(pattern = "[.]RData|[.]csv", replacement = "", x = fnames.ms5raw[i])
+      summary[fi] = gsub(pattern = "[.]RData$|[.]csv$", replacement = "", x = filename)
       s_names[fi] = "filename"
       fi = fi + 1
       summary[fi] = ifelse(test = nrow(ts) == 1,
@@ -281,9 +282,9 @@ g.part6 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(),
         }
         output_part6$GGIRversion = GGIRversion
         save(output_part6, GGIRversion, file = paste0(metadatadir, ms6.out, "/",
-                                         gsub(pattern = "[.]csv|[.]RData",
-                                              replacement = "",
-                                              x = fnames.ms5raw[i]), ".RData"))
+                                                      gsub(pattern = "[.]csv|[.]RData",
+                                                           replacement = "",
+                                                           x = fnames.ms5raw[i]), ".RData"))
       }
       rm(output_part6, summary)
     }

R/g.report.part2.R

@@ -1,5 +1,6 @@
 g.report.part2 = function(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0,
-                          store.long = FALSE, params_output, verbose = TRUE) {
+                          store.long = FALSE, params_output, verbose = TRUE,
+                          desiredtz = "") {
   ms2.out = "/meta/ms2.out"
   if (file.exists(paste0(metadatadir,ms2.out))) {
     if (length(dir(paste0(metadatadir,ms2.out))) == 0) {
@@ -239,7 +240,14 @@ g.report.part2 = function(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0,
       # tidy up data.frames
       SUMMARY_clean = tidyup_df(SUMMARY)
       daySUMMARY_clean = tidyup_df(daySUMMARY)
-
+      daySUMMARY_clean$start_time = daySUMMARY_clean$calendar_date
+      # reorder to have starttime next to calendar_date
+      old_vars = which(colnames(daySUMMARY_clean) != "start_time")
+      new_var = which(colnames(daySUMMARY_clean) == "start_time")
+      daySUMMARY_clean = daySUMMARY_clean[, c(old_vars[1:3], new_var, old_vars[4:length(old_vars)])]
+      # format calendar dates
+      dd = iso8601chartime2POSIX(daySUMMARY_clean$calendar_date, tz = desiredtz) 
+      daySUMMARY_clean$calendar_date = format(dd, format = "%Y-%m-%d")
       #===============================================================================
       # store final matrices again
       data.table::fwrite(x = SUMMARY_clean, file = paste0(metadatadir, "/results/part2_summary.csv"),

R/g.report.part4.R

@@ -76,6 +76,9 @@ g.report.part4 = function(datadir = c(), metadatadir = c(), loglocation = c(),
     }
     nightsummary2 = as.data.frame(do.call(rbind, lapply(fnames.ms4, myfun)), stringsAsFactors = FALSE)
     nightsummary2$night = as.numeric(gsub(" ", "", nightsummary2$night))
+    nightsummary2$calendar_date = as.Date(nightsummary2$calendar_date, format = "%d/%m/%Y")
+    nightsummary2$calendar_date = format(nightsummary2$calendar_date, format = "%Y-%m-%d")
+    nightsummary2$filename = gsub(".RData$", "", nightsummary2$filename)
     # ====================================== Add non-wearing during SPT from part 5, if it is availabe:
     ms5.out = "/meta/ms5.out"
     if (file.exists(paste(metadatadir, ms5.out, sep = ""))) {

R/g.report.part5.R

@@ -47,21 +47,32 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
 
       x$wear_min_day = (1 - (x$nonwear_perc_day / 100)) * x$dur_day_min #valid minute during waking hours
       x$wear_perc_day = 100 - x$nonwear_perc_day #wear percentage during waking hours
+      x$lasttimestamp = as.numeric(x$lasttimestamp)
 
       minimumValidMinutesMM = 0 # default
       if (length(params_cleaning[["includedaycrit"]]) == 2) {
         minimumValidMinutesMM = params_cleaning[["includedaycrit"]][2] * 60
       }
+      if (params_output[["require_complete_lastnight_part5"]] == FALSE) {
+        x$lastnight = FALSE
+      } else {
+        x$lastnight = x$window_number == max(x$window_number)
+      }
       if (window == "WW" | window == "OO") {
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
+                          ((x$lastnight == TRUE & x$lasttimestamp >= 15 & window == "WW") |
+                             (x$lastnight == TRUE & x$lasttimestamp >= 9 & window == "OO") |
+                             x$lastnight == FALSE) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
       } else if (window == "MM") {
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
+                          ((x$lastnight == TRUE & x$lasttimestamp > 9) |
+                             x$lastnight == FALSE) &
                           x$dur_day_spt_min >= (params_cleaning[["minimum_MM_length.part5"]] * 60) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
@@ -118,12 +129,17 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                  " take a few minutes\n"))
     }
     myfun = function(x, expectedCols = c()) {
-      tail_expansion_log = NULL
+      tail_expansion_log = last_timestamp = output = NULL
       load(file = x)
       cut = which(output[, 1] == "")
       if (length(cut) > 0 & length(cut) < nrow(output)) {
         output = output[-cut, which(colnames(output) != "")]
       }
+      if (exists("last_timestamp") == TRUE) {
+        output$lasttimestamp = as.numeric(format(last_timestamp, "%H"))
+      } else {
+        output$lasttimestamp = Inf # use dummy value
+      }
       out = as.matrix(output)
       if (length(expectedCols) > 0) {
         tmp = as.data.frame(matrix(0, 0, length(expectedCols)))
@@ -184,6 +200,9 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
       outputfinal = outputfinal[,-cut]
     }
 
+    # revise filename
+    outputfinal$filename = gsub(".RData$", "", outputfinal$filename)
+
     # order data.frame
     outputfinal$window_number = as.numeric(gsub(" ", "", outputfinal$window_number))
     outputfinal = outputfinal[order(outputfinal$filename, outputfinal$window_number, outputfinal$window), ]
@@ -269,8 +288,11 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                   sep = params_output[["sep_reports"]],
                   dec = params_output[["dec_reports"]])
                 # store all summaries in csv files with cleaning criteria
-                validdaysi = getValidDayIndices(x = OF3, window = uwi[j],
+                validdaysi = getValidDayIndices(x = OF3_clean, window = uwi[j],
                                                 params_cleaning = params_cleaning)
+                if ("lasttimestamp" %in% colnames(OF3_clean)) {
+                  OF3_clean = OF3_clean[, -which(colnames(OF3_clean) == "lasttimestamp")]
+                }
                 if (length(validdaysi) > 0) {
                   data.table::fwrite(
                     OF3_clean[validdaysi, ],

R/load_params.R

@@ -115,7 +115,8 @@ load_params = function(topic = c("sleep", "metrics", "rawdata",
                          do.sibreport = FALSE, do.part2.pdf = TRUE,
                          sep_reports = ",", sep_config = ",", 
                          dec_reports = ".", dec_config = ".", 
-                         visualreport_without_invalid = TRUE)
+                         visualreport_without_invalid = TRUE,
+                         require_complete_lastnight_part5 = FALSE)
 
   }
   if ("general" %in% topic) {

R/zzz.R

@@ -13,6 +13,7 @@
   }
   pkgs <- available.packages(repos = repos)
   cran_version <- package_version(pkgs[which(pkgs[,1] == "GGIR"),"Version"])
+  if (length(cran_version) == 0) return() # handle no internet connection
   local_version <- packageVersion("GGIR")
   behind_cran <- cran_version > local_version
   if (interactive()) {

man/GGIR.Rd

@@ -1539,6 +1539,13 @@ GGIR(mode = 1:5,
           If TRUE, then reports generated with \code{visualreport = TRUE} only show
           the windows with sufficiently valid data according to \code{includedaycrit}
           when viewingwindow = 1 or \code{includenightcrit} when viewingwindow = 2}
+      \item{require_complete_lastnight_part5}{    
+          Boolean (default = FALSE).
+          When set to TRUE: The last WW window is excluded if recording ends 
+          between midnight and 3pm; The last OO and MM window are excluded if 
+          recording ends between midnight and 9am. This to avoid risk that recording
+          end biases the sleep estimates for the last night.
+      }
     }
   }
 }

man/g.part5.savetimeseries.Rd

@@ -12,7 +12,9 @@
                               DaCleanFile = NULL,
                               includedaycrit.part5 = 2/3,
                               ID = NULL, params_output,
-                              params_247 = NULL)
+                              params_247 = NULL,
+                              filename = "",
+                              timewindow = NULL)
 }
 \arguments{
   \item{ts}{
@@ -25,7 +27,7 @@
      See \link{GGIR}.
   }
   \item{rawlevels_fname}{
-    Path to the file where the output should be stored
+    Path to the file where the output (time series) should be stored
   }
   \item{DaCleanFile}{
     Content of data_cleaning_file as documented in \link{g.report.part5}.
@@ -48,6 +50,13 @@
   \item{params_247}{
     See \link{GGIR}
   }
+  \item{filename}{
+    Character (default = "") indicating the name of the accelerometer data file 
+    that was used as input. This name will be stored inside the time series output file.
+  }
+  \item{timewindow}{
+    See \link{GGIR}
+  }
 }
 \value{
   Function does not provide output, it only prepare data for saving
@@ -56,4 +65,4 @@
 \keyword{internal}
 \author{
   Vincent T van Hees <v.vanhees@accelting.com>
-}
+}

man/g.report.part2.Rd

@@ -9,7 +9,7 @@
 }
 \usage{
   g.report.part2(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0,
-  store.long = FALSE, params_output, verbose = TRUE)
+  store.long = FALSE, params_output, verbose = TRUE, desiredtz = "")
 }
 \arguments{
   \item{metadatadir}{
@@ -39,6 +39,9 @@
   \item{verbose}{
     See details in \link{GGIR}.
   }
+  \item{desiredtz}{
+    See details in \link{GGIR}.
+  }
 }
 \value{
 Function does not produce data, but only writes reports

tests/testthat/test_load_check_params.R

@@ -17,7 +17,7 @@ test_that("load_params can load parameters", {
   expect_equal(length(params$params_247), 22)
   expect_equal(length(params$params_cleaning), 24)
   expect_equal(length(params$params_phyact), 14)
-  expect_equal(length(params$params_output), 21)
+  expect_equal(length(params$params_output), 22)
   expect_equal(length(params$params_general), 17)
 
   params_sleep = params$params_sleep

tests/testthat/test_part6.R

@@ -21,14 +21,17 @@ test_that("Part 6 with household co-analysis", {
   data(data.ts)
   mdat = data.ts
   mdat$timenum = mdat$timenum - (5 * 60) 
-  save(mdat, file = paste0(dn, "/800-900-001_left wrist.RData"))
+  filename = "800-900-001_left wrist.bin"
+  save(mdat, filename, file = paste0(dn, "/800-900-001_left wrist.RData"))
   mdat$timenum = mdat$timenum + (7 * 60)
-  save(mdat, file = paste0(dn, "/800-900-002_left wrist.RData"))
+  filename = "800-900-002_left wrist.bin"
+  save(mdat, filename, file = paste0(dn, "/800-900-002_left wrist.RData"))
   mdat$timenum = mdat$timenum + (14 * 60)
-  save(mdat, file = paste0(dn, "/800-900-003_left wrist.RData"))
+  filename = "800-900-003_left wrist.bin"
+  save(mdat, filename, file = paste0(dn, "/800-900-003_left wrist.RData"))
 
   # Run household co-analysis  
-  # Update parameters to align with datset
+  # Update parameters to align with dataset
   params_general = load_params(topic = "general")$params_general
   params_general[["desiredtz"]] = "America/Curacao"
   params_phyact = load_params(topic = "phyact")$params_phyact