Remove regex special character from dataset name variable and add in …

…open_dataset() call
Sage-Bionetworks · Jul 12, 2024 · 3df7f23 · 3df7f23
1 parent eac6b70
commit 3df7f23
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/scripts/process-data/fitbitecg.R b/scripts/process-data/fitbitecg.R
@@ -100,7 +100,7 @@ ecg_stat_summarize <- function(df) {
   return(result)
 }
 
-dataset <- "fitbitecg$"
+dataset <- "fitbitecg"
 
 cat(paste0("\n----", glue::glue("Transforming data for {dataset}"), "----\n"))
 
@@ -112,7 +112,7 @@ vars <-
 
 # Load the desired subset of this dataset in memory and do some feature engineering for derived variables
 df <- 
-  arrow::open_dataset(s3$path(str_subset(dataset_paths, dataset))) %>% 
+  arrow::open_dataset(s3$path(str_subset(dataset_paths, paste0(dataset, "$")))) %>% 
   select(all_of(c(vars))) %>% 
   filter(ResultClassification %in% c("Normal Sinus Rhythm", "Atrial Fibrillation")) %>% 
   rename(StartDate = StartTime) %>% 

diff --git a/scripts/process-data/fitbitsleeplogs.R b/scripts/process-data/fitbitsleeplogs.R
@@ -126,7 +126,7 @@ sleeplogs_stat_summarize <- function(df) {
   return(result)
 }
 
-dataset <- "fitbitsleeplogs$"
+dataset <- "fitbitsleeplogs"
 
 cat(paste0("\n----", glue::glue("Transforming data for {dataset}"), "----\n"))
 
@@ -138,7 +138,7 @@ vars <-
 
 # Load the desired subset of this dataset in memory and do some feature engineering for derived variables
 df <- 
-  arrow::open_dataset(s3$path(str_subset(dataset_paths, dataset))) %>% 
+  arrow::open_dataset(s3$path(str_subset(dataset_paths, paste0(dataset, "$")))) %>% 
   select(all_of(c(vars, "LogId"))) %>% 
   collect() %>% 
   distinct() %>%