From 3df7f235037b64c147360d3fedb9ba10fbc42ee2 Mon Sep 17 00:00:00 2001 From: Pranav Anbarasu Date: Fri, 12 Jul 2024 03:07:07 +0000 Subject: [PATCH] Remove regex special character from dataset name variable and add in open_dataset() call --- scripts/process-data/fitbitecg.R | 4 ++-- scripts/process-data/fitbitsleeplogs.R | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/process-data/fitbitecg.R b/scripts/process-data/fitbitecg.R index 9033457..5fee30b 100644 --- a/scripts/process-data/fitbitecg.R +++ b/scripts/process-data/fitbitecg.R @@ -100,7 +100,7 @@ ecg_stat_summarize <- function(df) { return(result) } -dataset <- "fitbitecg$" +dataset <- "fitbitecg" cat(paste0("\n----", glue::glue("Transforming data for {dataset}"), "----\n")) @@ -112,7 +112,7 @@ vars <- # Load the desired subset of this dataset in memory and do some feature engineering for derived variables df <- - arrow::open_dataset(s3$path(str_subset(dataset_paths, dataset))) %>% + arrow::open_dataset(s3$path(str_subset(dataset_paths, paste0(dataset, "$")))) %>% select(all_of(c(vars))) %>% filter(ResultClassification %in% c("Normal Sinus Rhythm", "Atrial Fibrillation")) %>% rename(StartDate = StartTime) %>% diff --git a/scripts/process-data/fitbitsleeplogs.R b/scripts/process-data/fitbitsleeplogs.R index 7a475f4..ea122b4 100644 --- a/scripts/process-data/fitbitsleeplogs.R +++ b/scripts/process-data/fitbitsleeplogs.R @@ -126,7 +126,7 @@ sleeplogs_stat_summarize <- function(df) { return(result) } -dataset <- "fitbitsleeplogs$" +dataset <- "fitbitsleeplogs" cat(paste0("\n----", glue::glue("Transforming data for {dataset}"), "----\n")) @@ -138,7 +138,7 @@ vars <- # Load the desired subset of this dataset in memory and do some feature engineering for derived variables df <- - arrow::open_dataset(s3$path(str_subset(dataset_paths, dataset))) %>% + arrow::open_dataset(s3$path(str_subset(dataset_paths, paste0(dataset, "$")))) %>% select(all_of(c(vars, "LogId"))) %>% collect() %>% distinct() %>%