From 1e60016530d1b57d7ed51b3b8ff3437ee4ec33f7 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Fri, 1 Mar 2024 14:27:10 -0500 Subject: [PATCH 01/20] Introduce enable_repeat_nonrepeat, update tests --- R/checks.R | 47 +++-------- R/clean_redcap_long.R | 103 +++++++++++++++++++++++- R/read_redcap.R | 10 ++- man/clean_redcap_long.Rd | 12 ++- man/convert_mixed_instrument.Rd | 26 ++++++ man/get_mixed_structure_fields.Rd | 20 +++++ man/read_redcap.Rd | 8 +- tests/testthat/test-clean_redcap_long.R | 40 +++++++++ 8 files changed, 223 insertions(+), 43 deletions(-) create mode 100644 man/convert_mixed_instrument.Rd create mode 100644 man/get_mixed_structure_fields.Rd diff --git a/R/checks.R b/R/checks.R index b2a8a28b..efeee8ce 100644 --- a/R/checks.R +++ b/R/checks.R @@ -101,39 +101,7 @@ check_user_rights <- function(db_data, check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { - # Identify columns to check for repeat/nonrepeat behavior - safe_cols <- c( - names(db_data)[1], "redcap_event_name", - "redcap_repeat_instrument", "redcap_repeat_instance", - "redcap_data_access_group" - ) - - check_cols <- setdiff(names(db_data), safe_cols) - - # Set up check_data function that looks for repeating and nonrepeating - # behavior in a given column and returns a boolean - check_data <- function(db_data, check_col) { - # Repeating Check - rep <- any(!is.na(db_data[{{ check_col }}]) & !is.na(db_data["redcap_repeat_instrument"])) - - # Nonrepeating Check - nonrep <- any(!is.na(db_data[{{ check_col }}]) & is.na(db_data["redcap_repeat_instrument"])) - - rep & nonrep - } - - # Create a simple dataframe, loop through check columns and append - # dataframe with column being checked and the output of check_data - out <- data.frame() - for (i in seq_along(check_cols)) { - rep_and_nonrep <- db_data %>% - check_data(check_col = check_cols[i]) - - field <- check_cols[i] - - out <- rbind(out, data.frame(field, rep_and_nonrep)) - out - } + out <- get_mixed_structure_fields(db_data = db_data) # Filter for violations out <- out %>% @@ -142,11 +110,16 @@ check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { # Produce error message if violations detected if (nrow(out) > 0) { cli_abort(c("x" = "Instrument{?s} detected that ha{?s/ve} both repeating and - nonrepeating instances defined in the project: {out$field}"), - class = c("repeat_nonrepeat_instrument", "REDCapTidieR_cond"), - call = call + nonrepeating instances defined in the project: {out$field}", + "i" = "Set {.code enable_repeat_nonrepeat} to {.code TRUE} to override. + See the + {.href [Diving Deeper vigentte](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#longitudinal-redcap-projects)} + for more information." + ), + class = c("repeat_nonrepeat_instrument", "REDCapTidieR_cond"), + call = call ) - } +} } #' @title diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 122a4a4d..426cfd44 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -12,6 +12,10 @@ #' \code{REDCapR::redcap_metadata_read()$data} #' @param linked_arms Output of \code{link_arms}, linking instruments to REDCap #' events/arms +#' @param enable_repeat_nonrepeat A logical to allow for support of mixed repeating/non-repeating +#' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions +#' as repeating instruments with a single instance. Applies to longitudinal projects +#' only Default `FALSE`. #' #' @return #' Returns a \code{tibble} with list elements containing tidy dataframes. Users @@ -22,7 +26,8 @@ clean_redcap_long <- function(db_data_long, db_metadata_long, - linked_arms) { + linked_arms, + enable_repeat_nonrepeat = FALSE) { # Repeating Instrument Check ---- # Check if database supplied contains any repeating instruments to map onto # `redcap_repeat_*` variables @@ -34,7 +39,11 @@ clean_redcap_long <- function(db_data_long, assert_data_frame(db_metadata_long) if (has_repeat_forms) { - check_repeat_and_nonrepeat(db_data_long) + if (enable_repeat_nonrepeat) { + db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) + } else { + check_repeat_and_nonrepeat(db_data_long) + } } ## Repeating Instruments Logic ---- @@ -337,3 +346,93 @@ distill_repeat_table_long <- function(form_name, out %>% tibble() } + +#' @title Convert Mixed Structure Instruments to Repeating Instruments +#' +#' @description +#' For longitudinal projects where users set `enable_repeat_nonrepeat` to `TRUE`, +#' this function will handle the process of setting the nonrepeating parts of the +#' instrument to repeating ones with a single instance. +#' +#' @param db_data_long The longitudinal REDCap database output defined by +#' \code{REDCapR::redcap_read_oneshot()$data} +#' @param db_metadata_long The longitudinal REDCap metadata output defined by +#' \code{REDCapR::redcap_metadata_read()$data} +#' +#' @return +#' Returns a \code{tibble} with list elements containing tidy dataframes. Users +#' can access dataframes under the \code{redcap_data} column with reference to +#' \code{form_name} and \code{structure} column details. +#' +#' @keywords internal + +convert_mixed_instrument <- function(db_data_long, db_metadata_long) { + + mixed_structure_fields <- get_mixed_structure_fields(db_data_long) %>% + filter(rep_and_nonrep & !stringr::str_ends(field_name, "_form_complete")) %>% + left_join(db_metadata_long %>% select(field_name, form_name), + by = "field_name") + + for (i in 1:nrow(mixed_structure_fields)) { + field <- mixed_structure_fields$field_name[i] + form <- mixed_structure_fields$form_name[i] + + # Find column index in db_data_long with the specified field_name + col_index <- which(colnames(db_data_long) == field) + + # Update redcap_repeat_instance and redcap_repeat_instrument + db_data_long[, "redcap_repeat_instance"][!is.na(db_data_long[, col_index])] <- 1 + db_data_long[, "redcap_repeat_instrument"][!is.na(db_data_long[, col_index])] <- form + } + + db_data_long +} + +#' @title Get Mixed Structure Instrument List +#' +#' @description +#' Define fields in a given project that are used in both a repeating and +#' nonrepeating manner. +#' +#' @param db_data The REDCap database output generated by +#' \code{REDCapR::redcap_read_oneshot()$data} +#' +#' @returns a dataframe +#' +#' @keywords internal + +get_mixed_structure_fields <- function(db_data) { + # Identify columns to check for repeat/nonrepeat behavior + safe_cols <- c( + names(db_data)[1], "redcap_event_name", + "redcap_repeat_instrument", "redcap_repeat_instance", + "redcap_data_access_group" + ) + + check_cols <- setdiff(names(db_data), safe_cols) + + # Set up check_data function that looks for repeating and nonrepeating + # behavior in a given column and returns a boolean + check_data <- function(db_data, check_col) { + # Repeating Check + rep <- any(!is.na(db_data[{{ check_col }}]) & !is.na(db_data["redcap_repeat_instrument"])) + + # Nonrepeating Check + nonrep <- any(!is.na(db_data[{{ check_col }}]) & is.na(db_data["redcap_repeat_instrument"])) + + rep & nonrep + } + + # Create a simple dataframe, loop through check columns and append + # dataframe with column being checked and the output of check_data + out <- data.frame() + for (i in seq_along(check_cols)) { + rep_and_nonrep <- db_data %>% + check_data(check_col = check_cols[i]) + + field_name <- check_cols[i] + + out <- rbind(out, data.frame(field_name, rep_and_nonrep)) + } + out +} diff --git a/R/read_redcap.R b/R/read_redcap.R index 74e5cf25..9c769662 100644 --- a/R/read_redcap.R +++ b/R/read_redcap.R @@ -53,6 +53,10 @@ #' @param guess_max A positive [base::numeric] value #' passed to [readr::read_csv()] that specifies the maximum number of records to #' use for guessing column types. Default `.Machine$integer.max`. +#' @param enable_repeat_nonrepeat A logical to allow for support of mixed repeating/non-repeating +#' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions +#' as repeating instruments with a single instance. Applies to longitudinal projects +#' only Default `FALSE`. #' #' @examples #' \dontrun{ @@ -75,7 +79,8 @@ read_redcap <- function(redcap_uri, export_survey_fields = NULL, export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, - guess_max = .Machine$integer.max) { + guess_max = .Machine$integer.max, + enable_repeat_nonrepeat = FALSE) { check_arg_is_character(redcap_uri, len = 1, any.missing = FALSE) check_arg_is_character(token, len = 1, any.missing = FALSE) check_arg_is_valid_token(token) @@ -267,7 +272,8 @@ read_redcap <- function(redcap_uri, out <- clean_redcap_long( db_data_long = db_data, db_metadata_long = db_metadata, - linked_arms = linked_arms + linked_arms = linked_arms, + enable_repeat_nonrepeat = enable_repeat_nonrepeat ) } else { out <- clean_redcap( diff --git a/man/clean_redcap_long.Rd b/man/clean_redcap_long.Rd index 9926f46f..2b4cd2f9 100644 --- a/man/clean_redcap_long.Rd +++ b/man/clean_redcap_long.Rd @@ -4,7 +4,12 @@ \alias{clean_redcap_long} \title{Extract longitudinal REDCap databases into tidy tibbles} \usage{ -clean_redcap_long(db_data_long, db_metadata_long, linked_arms) +clean_redcap_long( + db_data_long, + db_metadata_long, + linked_arms, + enable_repeat_nonrepeat = FALSE +) } \arguments{ \item{db_data_long}{The longitudinal REDCap database output defined by @@ -15,6 +20,11 @@ clean_redcap_long(db_data_long, db_metadata_long, linked_arms) \item{linked_arms}{Output of \code{link_arms}, linking instruments to REDCap events/arms} + +\item{enable_repeat_nonrepeat}{A logical to allow for support of mixed repeating/non-repeating +instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions +as repeating instruments with a single instance. Applies to longitudinal projects +only Default \code{FALSE}.} } \value{ Returns a \code{tibble} with list elements containing tidy dataframes. Users diff --git a/man/convert_mixed_instrument.Rd b/man/convert_mixed_instrument.Rd new file mode 100644 index 00000000..96eed99c --- /dev/null +++ b/man/convert_mixed_instrument.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/clean_redcap_long.R +\name{convert_mixed_instrument} +\alias{convert_mixed_instrument} +\title{Convert Mixed Structure Instruments to Repeating Instruments} +\usage{ +convert_mixed_instrument(db_data_long, db_metadata_long) +} +\arguments{ +\item{db_data_long}{The longitudinal REDCap database output defined by +\code{REDCapR::redcap_read_oneshot()$data}} + +\item{db_metadata_long}{The longitudinal REDCap metadata output defined by +\code{REDCapR::redcap_metadata_read()$data}} +} +\value{ +Returns a \code{tibble} with list elements containing tidy dataframes. Users +can access dataframes under the \code{redcap_data} column with reference to +\code{form_name} and \code{structure} column details. +} +\description{ +For longitudinal projects where users set \code{enable_repeat_nonrepeat} to \code{TRUE}, +this function will handle the process of setting the nonrepeating parts of the +instrument to repeating ones with a single instance. +} +\keyword{internal} diff --git a/man/get_mixed_structure_fields.Rd b/man/get_mixed_structure_fields.Rd new file mode 100644 index 00000000..13340521 --- /dev/null +++ b/man/get_mixed_structure_fields.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/clean_redcap_long.R +\name{get_mixed_structure_fields} +\alias{get_mixed_structure_fields} +\title{Get Mixed Structure Instrument List} +\usage{ +get_mixed_structure_fields(db_data) +} +\arguments{ +\item{db_data}{The REDCap database output generated by +\code{REDCapR::redcap_read_oneshot()$data}} +} +\value{ +a dataframe +} +\description{ +Define fields in a given project that are used in both a repeating and +nonrepeating manner. +} +\keyword{internal} diff --git a/man/read_redcap.Rd b/man/read_redcap.Rd index a57f4880..c8cc3b67 100644 --- a/man/read_redcap.Rd +++ b/man/read_redcap.Rd @@ -12,7 +12,8 @@ read_redcap( export_survey_fields = NULL, export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, - guess_max = .Machine$integer.max + guess_max = .Machine$integer.max, + enable_repeat_nonrepeat = FALSE ) } \arguments{ @@ -45,6 +46,11 @@ from REDCapR API calls. Default \code{TRUE}.} \item{guess_max}{A positive \link[base:numeric]{base::numeric} value passed to \code{\link[readr:read_delim]{readr::read_csv()}} that specifies the maximum number of records to use for guessing column types. Default \code{.Machine$integer.max}.} + +\item{enable_repeat_nonrepeat}{A logical to allow for support of mixed repeating/non-repeating +instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions +as repeating instruments with a single instance. Applies to longitudinal projects +only Default \code{FALSE}.} } \value{ A \code{tibble} in which each row represents a REDCap instrument. It diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 97d87b14..1c5f7f49 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -208,3 +208,43 @@ test_that("distill_repeat_table_long no arms returns tables for REDCap dbs with any(c("redcap_repeat_instrument", "redcap_arm") %in% names(out)) ) }) + +test_that("get_mixed_structure_fields works", { + mixed_structure_db <- tibble::tribble( + ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, + 1, NA, NA, "A", + 2, "mixed_structure_form", 1, "B" + ) + + expected_out <- data.frame( + field_name = "mixed_structure_variable", + rep_and_nonrep = TRUE + ) + + out <- get_mixed_structure_fields(mixed_structure_db) + + expect_equal(out, expected_out) +}) + +test_that("convert_mixed_instrument works", { + mixed_structure_db <- tibble::tribble( + ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, + 1, NA, NA, "A", + 2, "mixed_structure_form", 1, "B" + ) + + mixed_structure_meta <- tibble::tribble( + ~field_name, ~form_name, + "mixed_structure_variable", "mixed_structure_form" + ) + + expected_out <-tibble::tribble( + ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, + 1, "mixed_structure_form", 1, "A", + 2, "mixed_structure_form", 1, "B" + ) + + out <- convert_mixed_instrument(mixed_structure_db, mixed_structure_meta) + + expect_equal(out, expected_out) +}) From 9976484451b6a207b71f456605a88c5e8403dcbf Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Fri, 1 Mar 2024 14:47:10 -0500 Subject: [PATCH 02/20] Address Styler & Linter --- R/checks.R | 24 ++++++++++++++---------- R/clean_redcap_long.R | 10 +++++----- tests/testthat/test-clean_redcap_long.R | 2 +- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/R/checks.R b/R/checks.R index efeee8ce..3467d4fe 100644 --- a/R/checks.R +++ b/R/checks.R @@ -105,21 +105,25 @@ check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { # Filter for violations out <- out %>% - filter(rep_and_nonrep) + filter(rep_and_nonrep) # nolint object_usage_linter # Produce error message if violations detected if (nrow(out) > 0) { - cli_abort(c("x" = "Instrument{?s} detected that ha{?s/ve} both repeating and + cli_abort( + c( + "x" = "Instrument{?s} detected that ha{?s/ve} both repeating and nonrepeating instances defined in the project: {out$field}", - "i" = "Set {.code enable_repeat_nonrepeat} to {.code TRUE} to override. - See the - {.href [Diving Deeper vigentte](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#longitudinal-redcap-projects)} - for more information." - ), - class = c("repeat_nonrepeat_instrument", "REDCapTidieR_cond"), - call = call + "i" = paste0( + "Set {.code enable_repeat_nonrepeat} to {.code TRUE} to override. ", + "See the ", + "{.href [Diving Deeper vigentte](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#longitudinal-redcap-projects)} ", # nolint line_length_linter + "for more information." + ) + ), + class = c("repeat_nonrepeat_instrument", "REDCapTidieR_cond"), + call = call ) -} + } } #' @title diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 426cfd44..4cbf9b26 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -367,13 +367,13 @@ distill_repeat_table_long <- function(form_name, #' @keywords internal convert_mixed_instrument <- function(db_data_long, db_metadata_long) { - mixed_structure_fields <- get_mixed_structure_fields(db_data_long) %>% - filter(rep_and_nonrep & !stringr::str_ends(field_name, "_form_complete")) %>% - left_join(db_metadata_long %>% select(field_name, form_name), - by = "field_name") + filter(rep_and_nonrep & !stringr::str_ends(field_name, "_form_complete")) %>% # nolint object_usage_linter + left_join(db_metadata_long %>% select(field_name, form_name), # nolint object_usage_linter + by = "field_name" + ) - for (i in 1:nrow(mixed_structure_fields)) { + for (i in seq_len(nrow(mixed_structure_fields))) { field <- mixed_structure_fields$field_name[i] form <- mixed_structure_fields$form_name[i] diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 1c5f7f49..af641a3f 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -238,7 +238,7 @@ test_that("convert_mixed_instrument works", { "mixed_structure_variable", "mixed_structure_form" ) - expected_out <-tibble::tribble( + expected_out <- tibble::tribble( ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, 1, "mixed_structure_form", 1, "A", 2, "mixed_structure_form", 1, "B" From 16986114d45645937dd3f76d47e12c46fc7ceb9e Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Fri, 1 Mar 2024 15:20:51 -0500 Subject: [PATCH 03/20] Check edits, microbenchmark rerun --- NAMESPACE | 1 + R/REDCapTidieR-package.R | 2 +- R/checks.R | 2 +- R/clean_redcap_long.R | 4 +- utility/microbenchmark_results.csv | 82 +++++++++++++++--------------- utility/test_creds.R | 3 +- 6 files changed, 49 insertions(+), 45 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index b3f04e42..137cf7a0 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -92,6 +92,7 @@ importFrom(rlang,try_fetch) importFrom(rlang,zap) importFrom(stringi,stri_split_fixed) importFrom(stringr,str_detect) +importFrom(stringr,str_ends) importFrom(stringr,str_replace) importFrom(stringr,str_replace_all) importFrom(stringr,str_squish) diff --git a/R/REDCapTidieR-package.R b/R/REDCapTidieR-package.R index 1f932d5f..c277dc2b 100644 --- a/R/REDCapTidieR-package.R +++ b/R/REDCapTidieR-package.R @@ -18,7 +18,7 @@ #' is_installed new_environment quo_get_expr try_fetch zap as_label #' @importFrom stringi stri_split_fixed #' @importFrom stringr str_detect str_replace str_replace_all str_squish str_trunc -#' str_trim +#' str_trim str_ends #' @importFrom tibble as_tibble is_tibble tibble #' @importFrom tidyr complete fill pivot_wider nest unnest unnest_wider #' @importFrom tidyselect all_of any_of ends_with eval_select everything diff --git a/R/checks.R b/R/checks.R index 3467d4fe..095bef6b 100644 --- a/R/checks.R +++ b/R/checks.R @@ -105,7 +105,7 @@ check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { # Filter for violations out <- out %>% - filter(rep_and_nonrep) # nolint object_usage_linter + filter(.data$rep_and_nonrep) # Produce error message if violations detected if (nrow(out) > 0) { diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 4cbf9b26..38d3c9a8 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -368,8 +368,8 @@ distill_repeat_table_long <- function(form_name, convert_mixed_instrument <- function(db_data_long, db_metadata_long) { mixed_structure_fields <- get_mixed_structure_fields(db_data_long) %>% - filter(rep_and_nonrep & !stringr::str_ends(field_name, "_form_complete")) %>% # nolint object_usage_linter - left_join(db_metadata_long %>% select(field_name, form_name), # nolint object_usage_linter + filter(.data$rep_and_nonrep & !str_ends(.data$field_name, "_form_complete")) %>% + left_join(db_metadata_long %>% select(.data$field_name, .data$form_name), by = "field_name" ) diff --git a/utility/microbenchmark_results.csv b/utility/microbenchmark_results.csv index ef042006..7bf59441 100644 --- a/utility/microbenchmark_results.csv +++ b/utility/microbenchmark_results.csv @@ -1,41 +1,43 @@ min,lq,mean,median,uq,max,neval -891.35,891.35,891.35,891.35,891.35,891.35,1 -1.42,1.42,1.42,1.42,1.42,1.42,1 -661.09,661.09,661.09,661.09,661.09,661.09,1 -2.23,2.23,2.23,2.23,2.23,2.23,1 -3.37,3.37,3.37,3.37,3.37,3.37,1 -699.25,699.25,699.25,699.25,699.25,699.25,1 -641.48,641.48,641.48,641.48,641.48,641.48,1 -596.4,596.4,596.4,596.4,596.4,596.4,1 -600.89,600.89,600.89,600.89,600.89,600.89,1 -747.66,747.66,747.66,747.66,747.66,747.66,1 -710.54,710.54,710.54,710.54,710.54,710.54,1 -632.86,632.86,632.86,632.86,632.86,632.86,1 -623.4,623.4,623.4,623.4,623.4,623.4,1 -588.44,588.44,588.44,588.44,588.44,588.44,1 -150.11,150.11,150.11,150.11,150.11,150.11,1 -654.46,654.46,654.46,654.46,654.46,654.46,1 -581.29,581.29,581.29,581.29,581.29,581.29,1 -1.06,1.06,1.06,1.06,1.06,1.06,1 -1.17,1.17,1.17,1.17,1.17,1.17,1 -601.02,601.02,601.02,601.02,601.02,601.02,1 -658.33,658.33,658.33,658.33,658.33,658.33,1 -635.63,635.63,635.63,635.63,635.63,635.63,1 -779.47,779.47,779.47,779.47,779.47,779.47,1 -610.22,610.22,610.22,610.22,610.22,610.22,1 -668.01,668.01,668.01,668.01,668.01,668.01,1 -720.67,720.67,720.67,720.67,720.67,720.67,1 -1.08,1.08,1.08,1.08,1.08,1.08,1 -1.07,1.07,1.07,1.07,1.07,1.07,1 -1.96,1.96,1.96,1.96,1.96,1.96,1 -1.48,1.48,1.48,1.48,1.48,1.48,1 -1.64,1.64,1.64,1.64,1.64,1.64,1 -2.16,2.16,2.16,2.16,2.16,2.16,1 -878.95,878.95,878.95,878.95,878.95,878.95,1 -1.73,1.73,1.73,1.73,1.73,1.73,1 -961.31,961.31,961.31,961.31,961.31,961.31,1 -1.14,1.14,1.14,1.14,1.14,1.14,1 -975.07,975.07,975.07,975.07,975.07,975.07,1 -1.55,1.55,1.55,1.55,1.55,1.55,1 -5.59,5.59,5.59,5.59,5.59,5.59,1 -8.51,8.51,8.51,8.51,8.51,8.51,1 +1.16,1.16,1.16,1.16,1.16,1.16,1 +1.83,1.83,1.83,1.83,1.83,1.83,1 +693.01,693.01,693.01,693.01,693.01,693.01,1 +3.69,3.69,3.69,3.69,3.69,3.69,1 +5.19,5.19,5.19,5.19,5.19,5.19,1 +805.32,805.32,805.32,805.32,805.32,805.32,1 +689.2,689.2,689.2,689.2,689.2,689.2,1 +573.82,573.82,573.82,573.82,573.82,573.82,1 +700.61,700.61,700.61,700.61,700.61,700.61,1 +615.47,615.47,615.47,615.47,615.47,615.47,1 +685.91,685.91,685.91,685.91,685.91,685.91,1 +702.75,702.75,702.75,702.75,702.75,702.75,1 +656.78,656.78,656.78,656.78,656.78,656.78,1 +664.89,664.89,664.89,664.89,664.89,664.89,1 +372.23,372.23,372.23,372.23,372.23,372.23,1 +848.39,848.39,848.39,848.39,848.39,848.39,1 +671.34,671.34,671.34,671.34,671.34,671.34,1 +1.05,1.05,1.05,1.05,1.05,1.05,1 +981.49,981.49,981.49,981.49,981.49,981.49,1 +626.57,626.57,626.57,626.57,626.57,626.57,1 +656.61,656.61,656.61,656.61,656.61,656.61,1 +642.03,642.03,642.03,642.03,642.03,642.03,1 +696.71,696.71,696.71,696.71,696.71,696.71,1 +694.78,694.78,694.78,694.78,694.78,694.78,1 +641.12,641.12,641.12,641.12,641.12,641.12,1 +752.85,752.85,752.85,752.85,752.85,752.85,1 +2.84,2.84,2.84,2.84,2.84,2.84,1 +2.33,2.33,2.33,2.33,2.33,2.33,1 +3.83,3.83,3.83,3.83,3.83,3.83,1 +3.5,3.5,3.5,3.5,3.5,3.5,1 +3.87,3.87,3.87,3.87,3.87,3.87,1 +4.96,4.96,4.96,4.96,4.96,4.96,1 +2.18,2.18,2.18,2.18,2.18,2.18,1 +3.69,3.69,3.69,3.69,3.69,3.69,1 +2.29,2.29,2.29,2.29,2.29,2.29,1 +2.3,2.3,2.3,2.3,2.3,2.3,1 +2.11,2.11,2.11,2.11,2.11,2.11,1 +3.3,3.3,3.3,3.3,3.3,3.3,1 +3.91,3.91,3.91,3.91,3.91,3.91,1 +13.27,13.27,13.27,13.27,13.27,13.27,1 +14.4,14.4,14.4,14.4,14.4,14.4,1 +41.54,41.54,41.54,41.54,41.54,41.54,1 diff --git a/utility/test_creds.R b/utility/test_creds.R index d42883aa..d794265c 100644 --- a/utility/test_creds.R +++ b/utility/test_creds.R @@ -31,6 +31,7 @@ redcaptidier_creds <- tibble::tribble( Sys.getenv("REDCAP_URI"), Sys.getenv("REDCAPTIDIER_LARGE_SPARSE_API"), "large sparse db", Sys.getenv("REDCAP_URI"), Sys.getenv("REDCAPTIDIER_DAG_API"), "data access groups", Sys.getenv("REDCAP_URI"), Sys.getenv("REDCAPTIDIER_LONGITUDINAL_DAG_API"), "longitudinal data access groups", + Sys.getenv("REDCAP_URI"), Sys.getenv("REDCAPTIDIER_MIXED_STRUCTURE_API"), "mixed structure repeat no repeat", Sys.getenv("REDCAP_URI"), Sys.getenv("PRODIGY_REDCAP_API"), "prodigy db", Sys.getenv("REDCAP_URI"), Sys.getenv("CART_COMP_REDCAP_API"), "cart comprehensive db", Sys.getenv("REDCAP_URI"), Sys.getenv("BMT_OUTCOMES_REDCAP_API"), "bmt outcomes db" @@ -41,7 +42,7 @@ creds <- rbind(ouhsc_creds, redcaptidier_creds) microbenchmark_fx <- function(redcap_uri, token, name, times = 1){ microbenchmark( - name = read_redcap(redcap_uri = redcap_uri, token = token), + name = read_redcap(redcap_uri = redcap_uri, token = token, enable_repeat_nonrepeat = TRUE), times = times ) } From 4f1b5e88e864b58b200eb7e9806f23f09b9a04c1 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Fri, 1 Mar 2024 15:32:36 -0500 Subject: [PATCH 04/20] Update version and NEWS --- DESCRIPTION | 2 +- NEWS.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3fd75276..a55b0122 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: REDCapTidieR Type: Package Title: Extract 'REDCap' Databases into Tidy 'Tibble's -Version: 1.0.0 +Version: 1.1.0 Authors@R: c( person("Richard", "Hanna", , "richardshanna91@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0009-0005-6496-8154")), diff --git a/NEWS.md b/NEWS.md index 41de6306..bb059add 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +# REDCapTidieR 1.1.0 + +- `read_redcap()` now supports instruments that follow a mixed repeating/non-repeating structure with the `enable_repeat_nonrepeat` parameter +- When enabled, instruments with mixed repeating/nonrepeating structure will be treated as single-instance repeating instruments + # REDCapTidieR 1.0.0 Version 1.0.0 From 1054e2c50cf7d89587d08125990a50205ddf2036 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 4 Mar 2024 09:36:58 -0500 Subject: [PATCH 05/20] Add test for clean_redcap_long and test data Addresses codecov warning --- inst/testdata/db_metadata_mixed_structure.RDS | Bin 0 -> 408 bytes inst/testdata/db_mixed_structure.RDS | Bin 0 -> 313 bytes .../db_mixed_structure_linked_arms.RDS | Bin 0 -> 359 bytes tests/testthat/test-clean_redcap_long.R | 35 ++++++++++++++++++ 4 files changed, 35 insertions(+) create mode 100644 inst/testdata/db_metadata_mixed_structure.RDS create mode 100644 inst/testdata/db_mixed_structure.RDS create mode 100644 inst/testdata/db_mixed_structure_linked_arms.RDS diff --git a/inst/testdata/db_metadata_mixed_structure.RDS b/inst/testdata/db_metadata_mixed_structure.RDS new file mode 100644 index 0000000000000000000000000000000000000000..46c6f835d8faab0af8fcb777a2ce3a421b1b9d43 GIT binary patch literal 408 zcmV;J0cZXniwFP!000001MO2iPs1<}HBF1uR+N4)BW5HjvGD^C5-S3U&2pSeV&T}W z?a;9C$N9LVHjs$I$U-E?=XbvM&Ug0xh!7Hye$*q;08KKuyS+NQKx~1FIgaUe0~z8x zG{ChZA=FM6tAa8TDJM`r@Epd5DV__JQK>V*Lk$e9OJVTc zHMx3p%1i5XL6JIB1u_MabqThSYUMa9m@}fHnbGB{RblTy=?7E}?Ax+1g@tJn@APwM z^kx})dVr(%j6dxAOSkb{G8-Bnp7p+qB~4Plif|Qbf>mP6)YRz_^7;GXxqd$~SQd^cXZWK%V!R3EJq5+7%(YC%RK34gQ%vfd3M%ay!Y*iRV(q3yR z`|srckT$t3w2`IccAoBrmYqK*UP;f0eZV@qamwah?!NS-DY$F1J^BuuS;dv# literal 0 HcmV?d00001 diff --git a/inst/testdata/db_mixed_structure_linked_arms.RDS b/inst/testdata/db_mixed_structure_linked_arms.RDS new file mode 100644 index 0000000000000000000000000000000000000000..6a57433271695f69b1a80a6aae79760b7b4f45cc GIT binary patch literal 359 zcmV-t0hsyC}33h7JCSoZ_9K6ho-2E2a({Pwd2M}DYhb$${5j~YLBFaFXTTwb!v?Yx(OWqngfg)&4kCzXPKTP}e&I F008Wes?q=e literal 0 HcmV?d00001 diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index af641a3f..56649292 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -28,6 +28,15 @@ db_metadata_long_noarms <- readRDS( linked_arms_long_noarms <- readRDS( system.file("testdata/linked_arms_long_noarms.RDS", package = "REDCapTidieR") ) +db_mixed_structure <- readRDS( + system.file("testdata/db_mixed_structure.RDS", package = "REDCapTidieR") +) +db_metadata_mixed_structure <- readRDS( + system.file("testdata/db_metadata_mixed_structure.RDS", package = "REDCapTidieR") +) +db_mixed_structure_linked_arms <- readRDS( + system.file("testdata/db_mixed_structure_linked_arms.RDS", package = "REDCapTidieR") +) # Run Tests ---- test_that("clean_redcap_long with arms works", { @@ -72,6 +81,32 @@ test_that("clean_redcap_long without arms works", { expect_true(!is.null(out$redcap_data)) }) +test_that("clean_redcap_long with mixed structure works", { + # Required since amendments take place before clean_redcap_long call in read_redcap + db_metadata_mixed_structure <- update_field_names(db_metadata_mixed_structure) + + expect_error( + clean_redcap_long( + db_data_long = db_mixed_structure, + db_metadata_long = db_metadata_mixed_structure + ), + class = "repeat_nonrepeat_instrument" + ) + + out <- clean_redcap_long( + db_data_long = db_mixed_structure, + db_metadata_long = db_metadata_mixed_structure, + linked_arms = db_mixed_structure_linked_arms, + enable_repeat_nonrepeat = TRUE + ) + + # Check general structure + expect_true(is_tibble(out)) + expect_true("repeating" %in% out$structure) + expect_true("nonrepeating" %in% out$structure) + expect_true(!is.null(out$redcap_data)) +}) + test_that("distill_nonrepeat_table_long tibble contains expected columns for longitudinal REDCap databases with arms", { ## Check longitudinal structure with arms ---- out <- distill_nonrepeat_table_long( From 5de704f5bd46f435cf4e32683b1318de929a875d Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 4 Mar 2024 10:21:37 -0500 Subject: [PATCH 06/20] Fix overwrite bug, simplify code fixed a bug where existing >1 repeating instance were getting overwritten as 1 --- R/clean_redcap_long.R | 12 +++++++----- inst/testdata/db_mixed_structure.RDS | Bin 313 -> 327 bytes tests/testthat/test-clean_redcap_long.R | 8 ++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 38d3c9a8..31c28897 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -377,12 +377,14 @@ convert_mixed_instrument <- function(db_data_long, db_metadata_long) { field <- mixed_structure_fields$field_name[i] form <- mixed_structure_fields$form_name[i] - # Find column index in db_data_long with the specified field_name - col_index <- which(colnames(db_data_long) == field) + # Create a logical mask for rows needing update + update_mask <- is.na(db_data_long$redcap_repeat_instance) & !is.na(db_data_long[[field]]) - # Update redcap_repeat_instance and redcap_repeat_instrument - db_data_long[, "redcap_repeat_instance"][!is.na(db_data_long[, col_index])] <- 1 - db_data_long[, "redcap_repeat_instrument"][!is.na(db_data_long[, col_index])] <- form + # Update redcap_repeat_instance + db_data_long$redcap_repeat_instance <- if_else(update_mask, 1, db_data_long$redcap_repeat_instance) + + # Update redcap_repeat_instrument + db_data_long$redcap_repeat_instrument <- if_else(update_mask, form, db_data_long$redcap_repeat_instrument) } db_data_long diff --git a/inst/testdata/db_mixed_structure.RDS b/inst/testdata/db_mixed_structure.RDS index 800058cd47db30ad0c3325f2edd51faf5178f80e..970217505c38a5a205cd743f3d49f73f65c373a8 100644 GIT binary patch literal 327 zcmV-N0l5AjiwFP!000001C>(EPQx$^HvJLnCMG7(;DWRlU{ZVH1}AQCKp^cFMQJ8g zTGA@bfEzpuPsjrxz*&=SO#um4iQCV1K7VxP7+vV!Y9XAPg-A=k_ z=9BM`c@_n@?b%ZjRX)un6}&b>z!kjWhTI=sV(gMk>w~tMvr^@x$ek9&b4DxDOw*Lu zz^M&15=kM=fCMJl__QCVlDX=UbU&xPFeh8cOqj4zR4cAs#l*h`uI}uo`?0O|M6(sG ZooELRbE@flsf;*)&u@zRcSTPF002K!oS*;z literal 313 zcmV-90ml9xiwFP!000001AS4=PQx$^Htj0tCaO&!;DWRlU{ZJD#*G^s5J=pjD9xlw zOIoEFaD!*z33&m2oHky623Sg*+Wx+Nwjaj;5W*-7AdK({(Zl`q#U;j$@Z$ha;tH-l zpthk$n>GXxqd$~SQd^cXZWK%V!R3EJq5+7%(YC%RK34gQ%vfd3M%ay!Y*iRV(q3yR z`|srckT$t3w2`IccAoBrmYqK*UP;f0eZV@qamwah?!NS-DY$F1J^BuuS;dv# diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 56649292..67083a5d 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -265,7 +265,9 @@ test_that("convert_mixed_instrument works", { mixed_structure_db <- tibble::tribble( ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, 1, NA, NA, "A", - 2, "mixed_structure_form", 1, "B" + 2, "mixed_structure_form", 1, "B", + 3, "repeat_form", 1, "C", + 4, "repeat_form", 2, "D" ) mixed_structure_meta <- tibble::tribble( @@ -276,7 +278,9 @@ test_that("convert_mixed_instrument works", { expected_out <- tibble::tribble( ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, 1, "mixed_structure_form", 1, "A", - 2, "mixed_structure_form", 1, "B" + 2, "mixed_structure_form", 1, "B", + 3, "repeat_form", 1, "C", + 4, "repeat_form", 2, "D" ) out <- convert_mixed_instrument(mixed_structure_db, mixed_structure_meta) From b22ddfcf919e7849f63a0c6401fdec978d7af49a Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 4 Mar 2024 13:40:27 -0500 Subject: [PATCH 07/20] Update structure for mixed instruments --- R/clean_redcap_long.R | 5 ++++- tests/testthat/test-clean_redcap_long.R | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 31c28897..f1948c6a 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -33,6 +33,7 @@ clean_redcap_long <- function(db_data_long, # `redcap_repeat_*` variables has_repeat_forms <- "redcap_repeat_instance" %in% names(db_data_long) + has_mixed_structure_forms <- FALSE # Apply checkmate checks assert_data_frame(db_data_long) @@ -41,6 +42,8 @@ clean_redcap_long <- function(db_data_long, if (has_repeat_forms) { if (enable_repeat_nonrepeat) { db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) + mixed_structure_forms <- get_mixed_structure_fields(db_data_long) + has_mixed_structure_forms <- ifelse(any(mixed_structure_forms$rep_and_nonrep), TRUE, has_mixed_structure_forms) } else { check_repeat_and_nonrepeat(db_data_long) } @@ -64,7 +67,7 @@ clean_redcap_long <- function(db_data_long, linked_arms ) ), - structure = "repeating" + structure = ifelse(has_mixed_structure_forms, "mixed", "repeating") ) } diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 67083a5d..4fc7f65b 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -102,7 +102,7 @@ test_that("clean_redcap_long with mixed structure works", { # Check general structure expect_true(is_tibble(out)) - expect_true("repeating" %in% out$structure) + expect_true("mixed" %in% out$structure) expect_true("nonrepeating" %in% out$structure) expect_true(!is.null(out$redcap_data)) }) From cc7bc731550cb3f524fa79f7b62668120e07752b Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 4 Mar 2024 14:19:36 -0500 Subject: [PATCH 08/20] Update API, update vignettes/articles --- NEWS.md | 2 +- R/checks.R | 4 ++-- R/clean_redcap_long.R | 10 ++++----- R/read_redcap.R | 8 +++---- man/clean_redcap_long.Rd | 6 ++--- man/convert_mixed_instrument.Rd | 2 +- man/read_redcap.Rd | 6 ++--- tests/testthat/test-clean_redcap_long.R | 2 +- utility/test_creds.R | 2 +- vignettes/articles/diving_deeper.Rmd | 30 +++++++++++++++++++++---- vignettes/glossary.Rmd | 2 +- 11 files changed, 48 insertions(+), 26 deletions(-) diff --git a/NEWS.md b/NEWS.md index bb059add..a040a909 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # REDCapTidieR 1.1.0 -- `read_redcap()` now supports instruments that follow a mixed repeating/non-repeating structure with the `enable_repeat_nonrepeat` parameter +- `read_redcap()` now supports instruments that follow a mixed repeating/non-repeating structure with the `enable_mixed_structure` parameter - When enabled, instruments with mixed repeating/nonrepeating structure will be treated as single-instance repeating instruments # REDCapTidieR 1.0.0 diff --git a/R/checks.R b/R/checks.R index 095bef6b..a564ae0f 100644 --- a/R/checks.R +++ b/R/checks.R @@ -114,9 +114,9 @@ check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { "x" = "Instrument{?s} detected that ha{?s/ve} both repeating and nonrepeating instances defined in the project: {out$field}", "i" = paste0( - "Set {.code enable_repeat_nonrepeat} to {.code TRUE} to override. ", + "Set {.code enable_mixed_structure} to {.code TRUE} to override. ", "See the ", - "{.href [Diving Deeper vigentte](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#longitudinal-redcap-projects)} ", # nolint line_length_linter + "{.href [Mixed Structure Instruments](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#mixed-structure-instruments)} ", # nolint line_length_linter "for more information." ) ), diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index f1948c6a..2208791f 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -12,10 +12,10 @@ #' \code{REDCapR::redcap_metadata_read()$data} #' @param linked_arms Output of \code{link_arms}, linking instruments to REDCap #' events/arms -#' @param enable_repeat_nonrepeat A logical to allow for support of mixed repeating/non-repeating +#' @param enable_mixed_structure A logical to allow for support of mixed repeating/non-repeating #' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions #' as repeating instruments with a single instance. Applies to longitudinal projects -#' only Default `FALSE`. +#' only. Default `FALSE`. #' #' @return #' Returns a \code{tibble} with list elements containing tidy dataframes. Users @@ -27,7 +27,7 @@ clean_redcap_long <- function(db_data_long, db_metadata_long, linked_arms, - enable_repeat_nonrepeat = FALSE) { + enable_mixed_structure = FALSE) { # Repeating Instrument Check ---- # Check if database supplied contains any repeating instruments to map onto # `redcap_repeat_*` variables @@ -40,7 +40,7 @@ clean_redcap_long <- function(db_data_long, assert_data_frame(db_metadata_long) if (has_repeat_forms) { - if (enable_repeat_nonrepeat) { + if (enable_mixed_structure) { db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) mixed_structure_forms <- get_mixed_structure_fields(db_data_long) has_mixed_structure_forms <- ifelse(any(mixed_structure_forms$rep_and_nonrep), TRUE, has_mixed_structure_forms) @@ -353,7 +353,7 @@ distill_repeat_table_long <- function(form_name, #' @title Convert Mixed Structure Instruments to Repeating Instruments #' #' @description -#' For longitudinal projects where users set `enable_repeat_nonrepeat` to `TRUE`, +#' For longitudinal projects where users set `enable_mixed_structure` to `TRUE`, #' this function will handle the process of setting the nonrepeating parts of the #' instrument to repeating ones with a single instance. #' diff --git a/R/read_redcap.R b/R/read_redcap.R index 9c769662..ac29ad58 100644 --- a/R/read_redcap.R +++ b/R/read_redcap.R @@ -53,10 +53,10 @@ #' @param guess_max A positive [base::numeric] value #' passed to [readr::read_csv()] that specifies the maximum number of records to #' use for guessing column types. Default `.Machine$integer.max`. -#' @param enable_repeat_nonrepeat A logical to allow for support of mixed repeating/non-repeating +#' @param enable_mixed_structure A logical to allow for support of mixed repeating/non-repeating #' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions #' as repeating instruments with a single instance. Applies to longitudinal projects -#' only Default `FALSE`. +#' only. Default `FALSE`. #' #' @examples #' \dontrun{ @@ -80,7 +80,7 @@ read_redcap <- function(redcap_uri, export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, guess_max = .Machine$integer.max, - enable_repeat_nonrepeat = FALSE) { + enable_mixed_structure = FALSE) { check_arg_is_character(redcap_uri, len = 1, any.missing = FALSE) check_arg_is_character(token, len = 1, any.missing = FALSE) check_arg_is_valid_token(token) @@ -273,7 +273,7 @@ read_redcap <- function(redcap_uri, db_data_long = db_data, db_metadata_long = db_metadata, linked_arms = linked_arms, - enable_repeat_nonrepeat = enable_repeat_nonrepeat + enable_mixed_structure = enable_mixed_structure ) } else { out <- clean_redcap( diff --git a/man/clean_redcap_long.Rd b/man/clean_redcap_long.Rd index 2b4cd2f9..d5c1c990 100644 --- a/man/clean_redcap_long.Rd +++ b/man/clean_redcap_long.Rd @@ -8,7 +8,7 @@ clean_redcap_long( db_data_long, db_metadata_long, linked_arms, - enable_repeat_nonrepeat = FALSE + enable_mixed_structure = FALSE ) } \arguments{ @@ -21,10 +21,10 @@ clean_redcap_long( \item{linked_arms}{Output of \code{link_arms}, linking instruments to REDCap events/arms} -\item{enable_repeat_nonrepeat}{A logical to allow for support of mixed repeating/non-repeating +\item{enable_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions as repeating instruments with a single instance. Applies to longitudinal projects -only Default \code{FALSE}.} +only. Default \code{FALSE}.} } \value{ Returns a \code{tibble} with list elements containing tidy dataframes. Users diff --git a/man/convert_mixed_instrument.Rd b/man/convert_mixed_instrument.Rd index 96eed99c..45913cd0 100644 --- a/man/convert_mixed_instrument.Rd +++ b/man/convert_mixed_instrument.Rd @@ -19,7 +19,7 @@ can access dataframes under the \code{redcap_data} column with reference to \code{form_name} and \code{structure} column details. } \description{ -For longitudinal projects where users set \code{enable_repeat_nonrepeat} to \code{TRUE}, +For longitudinal projects where users set \code{enable_mixed_structure} to \code{TRUE}, this function will handle the process of setting the nonrepeating parts of the instrument to repeating ones with a single instance. } diff --git a/man/read_redcap.Rd b/man/read_redcap.Rd index c8cc3b67..efb99940 100644 --- a/man/read_redcap.Rd +++ b/man/read_redcap.Rd @@ -13,7 +13,7 @@ read_redcap( export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, guess_max = .Machine$integer.max, - enable_repeat_nonrepeat = FALSE + enable_mixed_structure = FALSE ) } \arguments{ @@ -47,10 +47,10 @@ from REDCapR API calls. Default \code{TRUE}.} passed to \code{\link[readr:read_delim]{readr::read_csv()}} that specifies the maximum number of records to use for guessing column types. Default \code{.Machine$integer.max}.} -\item{enable_repeat_nonrepeat}{A logical to allow for support of mixed repeating/non-repeating +\item{enable_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions as repeating instruments with a single instance. Applies to longitudinal projects -only Default \code{FALSE}.} +only. Default \code{FALSE}.} } \value{ A \code{tibble} in which each row represents a REDCap instrument. It diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 4fc7f65b..65e32df4 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -97,7 +97,7 @@ test_that("clean_redcap_long with mixed structure works", { db_data_long = db_mixed_structure, db_metadata_long = db_metadata_mixed_structure, linked_arms = db_mixed_structure_linked_arms, - enable_repeat_nonrepeat = TRUE + enable_mixed_structure = TRUE ) # Check general structure diff --git a/utility/test_creds.R b/utility/test_creds.R index d794265c..439aa3fd 100644 --- a/utility/test_creds.R +++ b/utility/test_creds.R @@ -42,7 +42,7 @@ creds <- rbind(ouhsc_creds, redcaptidier_creds) microbenchmark_fx <- function(redcap_uri, token, name, times = 1){ microbenchmark( - name = read_redcap(redcap_uri = redcap_uri, token = token, enable_repeat_nonrepeat = TRUE), + name = read_redcap(redcap_uri = redcap_uri, token = token, enable_mixed_structure = TRUE), times = times ) } diff --git a/vignettes/articles/diving_deeper.Rmd b/vignettes/articles/diving_deeper.Rmd index 127c690f..5b25b11e 100644 --- a/vignettes/articles/diving_deeper.Rmd +++ b/vignettes/articles/diving_deeper.Rmd @@ -19,6 +19,7 @@ knitr::knit_exit() redcap_uri <- Sys.getenv("REDCAP_URI") superheroes_token <- Sys.getenv("SUPERHEROES_REDCAP_API") longitudinal_token <- Sys.getenv("REDCAPTIDIER_DEEP_DIVE_VIGNETTE_API") +mixed_token <- Sys.getenv("REDCAPTIDIER_MIXED_STRUCTURE_API") survey_token <- Sys.getenv("REDCAPTIDIER_CLASSIC_API") dag_token <- Sys.getenv("REDCAPTIDIER_DAG_API") ``` @@ -80,7 +81,7 @@ super_hero_powers |> rmarkdown::paged_table() ``` -## Longitudinal REDCap projects +## Longitudinal REDCap Projects REDCap supports two main mechanisms to allow collecting the same data multiple times: [**repeating instruments**](glossary.html#repeating-instrument) and [**longitudinal projects**](glossary.html#longitudinal-project). In addition, a longitudinal project may have [**arms**](glossary.html#arm) and/or [**repeating events**](glossary.html#repeating-event). @@ -156,8 +157,6 @@ adverse_events |> It is possible to have a repeating instrument designated to multiple events, however this is an uncommon pattern. REDCapTidieR supports this scenario as well. -REDCapTidieR does *not* allow you to have the same instrument designated both as a repeating and as a nonrepeating instrument in different events. It will throw an error if it detects this. - The `unscheduled` **event** is a **repeating event**. Like adverse events, unscheduled visits aren't tied to a pre-determined study visit, and a patient could have zero, one, or multiple unscheduled visits. On the other hand, you might want to record the same kinds of data for an unscheduled visit as for a pre-determined regular visit and collect data in the same instruments, for example `physical_exam` and `hematology`. The granularity of these tables is one row per study subject per event per *event instance*. The subject had two unscheduled visits. `redcap_event_instance` allows us to match `physical_exam` and `hematology` responses which occurred on the same unscheduled visit. ```{r, results='hold'} @@ -174,7 +173,30 @@ Note that REDCapTidieR allows for an instrument to be associated with both repea REDCapTidieR supports projects with multiple arms. If you have a project with multiple arms, there will be an additional column `redcap_arm` to identify the arm that the row is associated with. -## Categorical variables +### Mixed Structure Instruments + +By default, REDCapTidieR does not allow you to have the same instrument designated both as a repeating and as a nonrepeating instrument in different events (i.e. a "mixed structure instrument"), and will throw an error if this is detected: + +``` +Error in `clean_redcap_long()` at REDCapTidieR/R/read_redcap.R:272:5: +✖ Instruments detected that have both repeating and nonrepeating instances defined in the project: mixed_structure_1 and mixed_structure_form_complete +ℹ Set `enable_mixed_structure` to `TRUE` to override. See Mixed Structure Instruments for more information. +``` + +This is because such a design inherently goes against [tidy](glossary.html#tidy) data principles. + +However, as of REDCapTidieR v1.1.0 it is now possible to override this behavior by setting `enable_mixed_structure` in `read_redcap()` to `TRUE`. When enabled, nonrepeating variants of mixed structure instruments will be treated as repeating instruments with a single repeating instance. + +Users are cautioned when enabling this feature, since it changes definitions in the original data output. To visually assist with this, you will see that `structure` in the [supertibble](glossary.html#supertibble) will say "mixed": + +```{r} +read_redcap(redcap_uri, + mixed_token, + enable_mixed_structure = TRUE) |> + rmarkdown::paged_table() +``` + +## Categorical Variables REDCapTidieR performs a number of opinionated transformations on categorical [fields](glossary.html#field) to streamline exploring them and working with them. diff --git a/vignettes/glossary.Rmd b/vignettes/glossary.Rmd index 0d44116a..7382eaa1 100644 --- a/vignettes/glossary.Rmd +++ b/vignettes/glossary.Rmd @@ -227,7 +227,7 @@ The [skimr](https://docs.ropensci.org/skimr/) R package provides summary statist ### Structure {#structure} -The structure of an [instrument](#instrument) can be [repeating](#repeating-instrument) or [nonrepeating](#nonrepeating-instrument). The [supertibble](#supertibble) shows the instrument's structure in the `structure` column. The structure of a [project](#project) can be [classic](#classic-project), [longitudinal](#longitudinal-project), or longitudinal with [arms](#arm). The structure of an [event](#event) can be [repeating](#repeating-event) or [nonrepeating](#nonrepeating-event). The [granularity](#granularity) of a [data tibble](#data-tibble) depends on the structure of all three: the instrument, the project, and the events associated with the instrument. Note: REDCap does not allow repeating instruments inside a repeating event. See also: the section [Longitudinal REDCap projects](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#longitudinal-redcap-projects) in the [Diving Deeper vignette](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html). ↩︎ +The structure of an [instrument](#instrument) can be [repeating](#repeating-instrument), [nonrepeating](#nonrepeating-instrument), or mixed. The [supertibble](#supertibble) shows the instrument's structure in the `structure` column. The structure of a [project](#project) can be [classic](#classic-project), [longitudinal](#longitudinal-project), or longitudinal with [arms](#arm). The structure of an [event](#event) can be [repeating](#repeating-event) or [nonrepeating](#nonrepeating-event). As of REDCapTidier v1.1.0, [mixed structure instruments](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#mixed-structure-instruments) are supported. The [granularity](#granularity) of a [data tibble](#data-tibble) depends on the structure of all three: the instrument, the project, and the events associated with the instrument. Note: REDCap does not allow repeating instruments inside a repeating event. See also: the section [Longitudinal REDCap projects](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#longitudinal-redcap-projects) in the [Diving Deeper vignette](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html). ↩︎ ### Supertibble {#supertibble} From dba13eddb5553df6918c23e40761d3a114e5f990 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 4 Mar 2024 14:31:46 -0500 Subject: [PATCH 09/20] Update workflow secrets --- .github/workflows/R-CMD-check.yaml | 1 + .github/workflows/pkgdown.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index c9b86ec0..062e53d4 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -29,6 +29,7 @@ jobs: REDCAPTIDIER_LARGE_SPARSE_API: ${{ secrets.REDCAPTIDIER_LARGE_SPARSE_API }} REDCAPTIDIER_DAG_API: ${{ secrets.REDCAPTIDIER_DAG_API }} REDCAPTIDIER_LONGITUDINAL_DAG_API: ${{ secrets.REDCAPTIDIER_LONGITUDINAL_DAG_API }} + REDCAPTIDIER_MIXED_STRUCTURE_API: ${{ secrets.REDCAPTIDIER_MIXED_STRUCTURE_API }} steps: - name: Update Ubuntu, Install cURL Headers, add Libraries run: | diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 1dd23590..0b6da316 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -25,6 +25,7 @@ jobs: SUPERHEROES_REDCAP_API: ${{ secrets.SUPERHEROES_REDCAP_API }} REDCAPTIDIER_DEEP_DIVE_VIGNETTE_API: ${{ secrets.REDCAPTIDIER_DEEP_DIVE_VIGNETTE_API }} REDCAPTIDIER_DAG_API: ${{ secrets.REDCAPTIDIER_DAG_API }} + REDCAPTIDIER_MIXED_STRUCTURE_API: ${{ secrets.REDCAPTIDIER_MIXED_STRUCTURE_API }} steps: - uses: actions/checkout@v3 From 77e67f8b6792d78a5336f3696357ab9889cfa7ed Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 4 Mar 2024 14:46:18 -0500 Subject: [PATCH 10/20] Typo --- R/checks.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/checks.R b/R/checks.R index a564ae0f..2c876b22 100644 --- a/R/checks.R +++ b/R/checks.R @@ -115,7 +115,7 @@ check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { nonrepeating instances defined in the project: {out$field}", "i" = paste0( "Set {.code enable_mixed_structure} to {.code TRUE} to override. ", - "See the ", + "See ", "{.href [Mixed Structure Instruments](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#mixed-structure-instruments)} ", # nolint line_length_linter "for more information." ) From 357dc171e204636807e35ce13fb36d3e0e4fd2f4 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Tue, 5 Mar 2024 13:01:38 -0500 Subject: [PATCH 11/20] Update API enable -> allow, add logical check --- NEWS.md | 2 +- R/checks.R | 2 +- R/clean_redcap_long.R | 8 ++++---- R/read_redcap.R | 7 ++++--- man/clean_redcap_long.Rd | 4 ++-- man/convert_mixed_instrument.Rd | 2 +- man/read_redcap.Rd | 4 ++-- tests/testthat/test-clean_redcap_long.R | 2 +- utility/test_creds.R | 2 +- vignettes/articles/diving_deeper.Rmd | 6 +++--- 10 files changed, 20 insertions(+), 19 deletions(-) diff --git a/NEWS.md b/NEWS.md index a040a909..8753513c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # REDCapTidieR 1.1.0 -- `read_redcap()` now supports instruments that follow a mixed repeating/non-repeating structure with the `enable_mixed_structure` parameter +- `read_redcap()` now supports instruments that follow a mixed repeating/non-repeating structure with the `allow_mixed_structure` parameter - When enabled, instruments with mixed repeating/nonrepeating structure will be treated as single-instance repeating instruments # REDCapTidieR 1.0.0 diff --git a/R/checks.R b/R/checks.R index 2c876b22..06eb6e45 100644 --- a/R/checks.R +++ b/R/checks.R @@ -114,7 +114,7 @@ check_repeat_and_nonrepeat <- function(db_data, call = caller_env()) { "x" = "Instrument{?s} detected that ha{?s/ve} both repeating and nonrepeating instances defined in the project: {out$field}", "i" = paste0( - "Set {.code enable_mixed_structure} to {.code TRUE} to override. ", + "Set {.code allow_mixed_structure} to {.code TRUE} to override. ", "See ", "{.href [Mixed Structure Instruments](https://chop-cgtinformatics.github.io/REDCapTidieR/articles/diving_deeper.html#mixed-structure-instruments)} ", # nolint line_length_linter "for more information." diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 2208791f..2e3c82ba 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -12,7 +12,7 @@ #' \code{REDCapR::redcap_metadata_read()$data} #' @param linked_arms Output of \code{link_arms}, linking instruments to REDCap #' events/arms -#' @param enable_mixed_structure A logical to allow for support of mixed repeating/non-repeating +#' @param allow_mixed_structure A logical to allow for support of mixed repeating/non-repeating #' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions #' as repeating instruments with a single instance. Applies to longitudinal projects #' only. Default `FALSE`. @@ -27,7 +27,7 @@ clean_redcap_long <- function(db_data_long, db_metadata_long, linked_arms, - enable_mixed_structure = FALSE) { + allow_mixed_structure = FALSE) { # Repeating Instrument Check ---- # Check if database supplied contains any repeating instruments to map onto # `redcap_repeat_*` variables @@ -40,7 +40,7 @@ clean_redcap_long <- function(db_data_long, assert_data_frame(db_metadata_long) if (has_repeat_forms) { - if (enable_mixed_structure) { + if (allow_mixed_structure) { db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) mixed_structure_forms <- get_mixed_structure_fields(db_data_long) has_mixed_structure_forms <- ifelse(any(mixed_structure_forms$rep_and_nonrep), TRUE, has_mixed_structure_forms) @@ -353,7 +353,7 @@ distill_repeat_table_long <- function(form_name, #' @title Convert Mixed Structure Instruments to Repeating Instruments #' #' @description -#' For longitudinal projects where users set `enable_mixed_structure` to `TRUE`, +#' For longitudinal projects where users set `allow_mixed_structure` to `TRUE`, #' this function will handle the process of setting the nonrepeating parts of the #' instrument to repeating ones with a single instance. #' diff --git a/R/read_redcap.R b/R/read_redcap.R index ac29ad58..25334093 100644 --- a/R/read_redcap.R +++ b/R/read_redcap.R @@ -53,7 +53,7 @@ #' @param guess_max A positive [base::numeric] value #' passed to [readr::read_csv()] that specifies the maximum number of records to #' use for guessing column types. Default `.Machine$integer.max`. -#' @param enable_mixed_structure A logical to allow for support of mixed repeating/non-repeating +#' @param allow_mixed_structure A logical to allow for support of mixed repeating/non-repeating #' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions #' as repeating instruments with a single instance. Applies to longitudinal projects #' only. Default `FALSE`. @@ -80,7 +80,7 @@ read_redcap <- function(redcap_uri, export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, guess_max = .Machine$integer.max, - enable_mixed_structure = FALSE) { + allow_mixed_structure = FALSE) { check_arg_is_character(redcap_uri, len = 1, any.missing = FALSE) check_arg_is_character(token, len = 1, any.missing = FALSE) check_arg_is_valid_token(token) @@ -89,6 +89,7 @@ read_redcap <- function(redcap_uri, check_arg_is_logical(export_survey_fields, len = 1, any.missing = FALSE, null.ok = TRUE) check_arg_is_logical(export_data_access_groups, len = 1, any.missing = FALSE, null.ok = TRUE) check_arg_is_logical(suppress_redcapr_messages, len = 1, any.missing = FALSE) + check_arg_is_logical(allow_mixed_structure, len = 1, any.missing = FALSE) # Load REDCap Metadata ---- # Capture unexpected metadata API call errors @@ -273,7 +274,7 @@ read_redcap <- function(redcap_uri, db_data_long = db_data, db_metadata_long = db_metadata, linked_arms = linked_arms, - enable_mixed_structure = enable_mixed_structure + allow_mixed_structure = allow_mixed_structure ) } else { out <- clean_redcap( diff --git a/man/clean_redcap_long.Rd b/man/clean_redcap_long.Rd index d5c1c990..46e5fcf0 100644 --- a/man/clean_redcap_long.Rd +++ b/man/clean_redcap_long.Rd @@ -8,7 +8,7 @@ clean_redcap_long( db_data_long, db_metadata_long, linked_arms, - enable_mixed_structure = FALSE + allow_mixed_structure = FALSE ) } \arguments{ @@ -21,7 +21,7 @@ clean_redcap_long( \item{linked_arms}{Output of \code{link_arms}, linking instruments to REDCap events/arms} -\item{enable_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating +\item{allow_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions as repeating instruments with a single instance. Applies to longitudinal projects only. Default \code{FALSE}.} diff --git a/man/convert_mixed_instrument.Rd b/man/convert_mixed_instrument.Rd index 45913cd0..30b20104 100644 --- a/man/convert_mixed_instrument.Rd +++ b/man/convert_mixed_instrument.Rd @@ -19,7 +19,7 @@ can access dataframes under the \code{redcap_data} column with reference to \code{form_name} and \code{structure} column details. } \description{ -For longitudinal projects where users set \code{enable_mixed_structure} to \code{TRUE}, +For longitudinal projects where users set \code{allow_mixed_structure} to \code{TRUE}, this function will handle the process of setting the nonrepeating parts of the instrument to repeating ones with a single instance. } diff --git a/man/read_redcap.Rd b/man/read_redcap.Rd index efb99940..ca77ccaf 100644 --- a/man/read_redcap.Rd +++ b/man/read_redcap.Rd @@ -13,7 +13,7 @@ read_redcap( export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, guess_max = .Machine$integer.max, - enable_mixed_structure = FALSE + allow_mixed_structure = FALSE ) } \arguments{ @@ -47,7 +47,7 @@ from REDCapR API calls. Default \code{TRUE}.} passed to \code{\link[readr:read_delim]{readr::read_csv()}} that specifies the maximum number of records to use for guessing column types. Default \code{.Machine$integer.max}.} -\item{enable_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating +\item{allow_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions as repeating instruments with a single instance. Applies to longitudinal projects only. Default \code{FALSE}.} diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 65e32df4..3a2e00a5 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -97,7 +97,7 @@ test_that("clean_redcap_long with mixed structure works", { db_data_long = db_mixed_structure, db_metadata_long = db_metadata_mixed_structure, linked_arms = db_mixed_structure_linked_arms, - enable_mixed_structure = TRUE + allow_mixed_structure = TRUE ) # Check general structure diff --git a/utility/test_creds.R b/utility/test_creds.R index 439aa3fd..98e66109 100644 --- a/utility/test_creds.R +++ b/utility/test_creds.R @@ -42,7 +42,7 @@ creds <- rbind(ouhsc_creds, redcaptidier_creds) microbenchmark_fx <- function(redcap_uri, token, name, times = 1){ microbenchmark( - name = read_redcap(redcap_uri = redcap_uri, token = token, enable_mixed_structure = TRUE), + name = read_redcap(redcap_uri = redcap_uri, token = token, allow_mixed_structure = TRUE), times = times ) } diff --git a/vignettes/articles/diving_deeper.Rmd b/vignettes/articles/diving_deeper.Rmd index 5b25b11e..70812382 100644 --- a/vignettes/articles/diving_deeper.Rmd +++ b/vignettes/articles/diving_deeper.Rmd @@ -180,19 +180,19 @@ By default, REDCapTidieR does not allow you to have the same instrument designat ``` Error in `clean_redcap_long()` at REDCapTidieR/R/read_redcap.R:272:5: ✖ Instruments detected that have both repeating and nonrepeating instances defined in the project: mixed_structure_1 and mixed_structure_form_complete -ℹ Set `enable_mixed_structure` to `TRUE` to override. See Mixed Structure Instruments for more information. +ℹ Set `allow_mixed_structure` to `TRUE` to override. See Mixed Structure Instruments for more information. ``` This is because such a design inherently goes against [tidy](glossary.html#tidy) data principles. -However, as of REDCapTidieR v1.1.0 it is now possible to override this behavior by setting `enable_mixed_structure` in `read_redcap()` to `TRUE`. When enabled, nonrepeating variants of mixed structure instruments will be treated as repeating instruments with a single repeating instance. +However, as of REDCapTidieR v1.1.0 it is now possible to override this behavior by setting `allow_mixed_structure` in `read_redcap()` to `TRUE`. When enabled, nonrepeating variants of mixed structure instruments will be treated as repeating instruments with a single repeating instance. Users are cautioned when enabling this feature, since it changes definitions in the original data output. To visually assist with this, you will see that `structure` in the [supertibble](glossary.html#supertibble) will say "mixed": ```{r} read_redcap(redcap_uri, mixed_token, - enable_mixed_structure = TRUE) |> + allow_mixed_structure = TRUE) |> rmarkdown::paged_table() ``` From 9e373961f81a911adee413dd95cdafcab9003d19 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Tue, 5 Mar 2024 17:29:53 -0500 Subject: [PATCH 12/20] Fix order so nonrepeat mixed vals appear TODO Fix structure label assignment in supertibble --- R/clean_redcap_long.R | 60 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index 2e3c82ba..d219aacb 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -39,36 +39,13 @@ clean_redcap_long <- function(db_data_long, assert_data_frame(db_data_long) assert_data_frame(db_metadata_long) - if (has_repeat_forms) { - if (allow_mixed_structure) { - db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) - mixed_structure_forms <- get_mixed_structure_fields(db_data_long) - has_mixed_structure_forms <- ifelse(any(mixed_structure_forms$rep_and_nonrep), TRUE, has_mixed_structure_forms) - } else { - check_repeat_and_nonrepeat(db_data_long) - } - } - - ## Repeating Instruments Logic ---- + ## Repeating Forms Assignment ---- + # Needed first to inform nonrepeating forms logic if (has_repeat_forms) { repeated_forms <- db_data_long %>% filter(!is.na(.data$redcap_repeat_instrument)) %>% pull(.data$redcap_repeat_instrument) %>% unique() - - repeated_forms_tibble <- tibble( - redcap_form_name = repeated_forms, - redcap_data = map( - .data$redcap_form_name, - ~ distill_repeat_table_long( - .x, - db_data_long, - db_metadata_long, - linked_arms - ) - ), - structure = ifelse(has_mixed_structure_forms, "mixed", "repeating") - ) } ## Nonrepeating Instruments Logic ---- @@ -98,6 +75,35 @@ clean_redcap_long <- function(db_data_long, structure = "nonrepeating" ) + ## Repeating Instruments Logic ---- + if (has_repeat_forms) { + # If mixed structure allowed, retrieve mixed structure forms + if (allow_mixed_structure) { + mixed_structure_fields <- get_mixed_structure_fields(db_data_long) + has_mixed_structure_forms <- ifelse(any(mixed_structure_fields$rep_and_nonrep), TRUE, has_mixed_structure_forms) # Update + } else { + check_repeat_and_nonrepeat(db_data_long) + } + + if (has_mixed_structure_forms) { + db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) + } + + repeated_forms_tibble <- tibble( + redcap_form_name = repeated_forms, + redcap_data = map( + .data$redcap_form_name, + ~ distill_repeat_table_long( + .x, + db_data_long, + db_metadata_long, + linked_arms + ) + ), + structure = "repeating" # TODO: Fix this so can be "repeating" / "mixed" + ) + } + if (has_repeat_forms) { rbind(repeated_forms_tibble, nonrepeated_forms_tibble) } else { @@ -254,6 +260,7 @@ distill_repeat_table_long <- function(form_name, db_data_long, db_metadata_long, linked_arms) { + has_repeat_forms <- "redcap_repeat_instance" %in% names(db_data_long) my_record_id <- names(db_data_long)[1] @@ -370,10 +377,11 @@ distill_repeat_table_long <- function(form_name, #' @keywords internal convert_mixed_instrument <- function(db_data_long, db_metadata_long) { + mixed_structure_fields <- get_mixed_structure_fields(db_data_long) %>% filter(.data$rep_and_nonrep & !str_ends(.data$field_name, "_form_complete")) %>% left_join(db_metadata_long %>% select(.data$field_name, .data$form_name), - by = "field_name" + by = "field_name" ) for (i in seq_len(nrow(mixed_structure_fields))) { From 88fe92020294352def4834691882feb7ad12eab2 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Wed, 6 Mar 2024 09:49:31 -0500 Subject: [PATCH 13/20] Fix structure field in supertibble Minor cleaning, updating of convert_mixed_instrument function --- R/clean_redcap_long.R | 44 +++++++++++++++++++-------------- man/convert_mixed_instrument.Rd | 6 ++--- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index d219aacb..e671e578 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -78,17 +78,28 @@ clean_redcap_long <- function(db_data_long, ## Repeating Instruments Logic ---- if (has_repeat_forms) { # If mixed structure allowed, retrieve mixed structure forms + if (allow_mixed_structure) { - mixed_structure_fields <- get_mixed_structure_fields(db_data_long) - has_mixed_structure_forms <- ifelse(any(mixed_structure_fields$rep_and_nonrep), TRUE, has_mixed_structure_forms) # Update + # Retrieve mixed structure fields and forms + mixed_structure_ref <- get_mixed_structure_fields(db_data_long) %>% + filter(.data$rep_and_nonrep & !str_ends(.data$field_name, "_form_complete")) %>% + left_join(db_metadata_long %>% select(.data$field_name, .data$form_name), + by = "field_name" + ) + + # Update if project actually has mixed structure + has_mixed_structure_forms <- nrow(mixed_structure_ref) > 0 + + # Convert mixed instruments to mixed structure format + if (has_mixed_structure_forms) { + db_data_long <- convert_mixed_instrument(db_data_long, mixed_structure_ref) + } + } else { + # Throw error if mixed structure detected and not allowed check_repeat_and_nonrepeat(db_data_long) } - if (has_mixed_structure_forms) { - db_data_long <- convert_mixed_instrument(db_data_long, db_metadata_long) - } - repeated_forms_tibble <- tibble( redcap_form_name = repeated_forms, redcap_data = map( @@ -100,7 +111,8 @@ clean_redcap_long <- function(db_data_long, linked_arms ) ), - structure = "repeating" # TODO: Fix this so can be "repeating" / "mixed" + structure = case_when(has_mixed_structure_forms & redcap_form_name %in% mixed_structure_ref$form_name ~ "mixed", + TRUE ~ "repeating") ) } @@ -366,8 +378,8 @@ distill_repeat_table_long <- function(form_name, #' #' @param db_data_long The longitudinal REDCap database output defined by #' \code{REDCapR::redcap_read_oneshot()$data} -#' @param db_metadata_long The longitudinal REDCap metadata output defined by -#' \code{REDCapR::redcap_metadata_read()$data} +#' @param mixed_structure_ref Reference dataframe containing mixed structure +#' fields and forms. #' #' @return #' Returns a \code{tibble} with list elements containing tidy dataframes. Users @@ -376,17 +388,11 @@ distill_repeat_table_long <- function(form_name, #' #' @keywords internal -convert_mixed_instrument <- function(db_data_long, db_metadata_long) { - - mixed_structure_fields <- get_mixed_structure_fields(db_data_long) %>% - filter(.data$rep_and_nonrep & !str_ends(.data$field_name, "_form_complete")) %>% - left_join(db_metadata_long %>% select(.data$field_name, .data$form_name), - by = "field_name" - ) +convert_mixed_instrument <- function(db_data_long, mixed_structure_ref) { - for (i in seq_len(nrow(mixed_structure_fields))) { - field <- mixed_structure_fields$field_name[i] - form <- mixed_structure_fields$form_name[i] + for (i in seq_len(nrow(mixed_structure_ref))) { + field <- mixed_structure_ref$field_name[i] + form <- mixed_structure_ref$form_name[i] # Create a logical mask for rows needing update update_mask <- is.na(db_data_long$redcap_repeat_instance) & !is.na(db_data_long[[field]]) diff --git a/man/convert_mixed_instrument.Rd b/man/convert_mixed_instrument.Rd index 30b20104..ed3a2816 100644 --- a/man/convert_mixed_instrument.Rd +++ b/man/convert_mixed_instrument.Rd @@ -4,14 +4,14 @@ \alias{convert_mixed_instrument} \title{Convert Mixed Structure Instruments to Repeating Instruments} \usage{ -convert_mixed_instrument(db_data_long, db_metadata_long) +convert_mixed_instrument(db_data_long, mixed_structure_ref) } \arguments{ \item{db_data_long}{The longitudinal REDCap database output defined by \code{REDCapR::redcap_read_oneshot()$data}} -\item{db_metadata_long}{The longitudinal REDCap metadata output defined by -\code{REDCapR::redcap_metadata_read()$data}} +\item{mixed_structure_ref}{Reference dataframe containing mixed structure +fields and forms.} } \value{ Returns a \code{tibble} with list elements containing tidy dataframes. Users From bd1367eebd1629346d37fb4ebb722389ab310b70 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Wed, 6 Mar 2024 10:25:38 -0500 Subject: [PATCH 14/20] Update tests Fix case_when issue --- R/clean_redcap_long.R | 2 + inst/testdata/db_metadata_mixed_structure.RDS | Bin 408 -> 416 bytes inst/testdata/db_mixed_structure.RDS | Bin 327 -> 366 bytes .../db_mixed_structure_linked_arms.RDS | Bin 359 -> 367 bytes tests/testthat/test-clean_redcap_long.R | 55 +++++++++++++----- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index e671e578..c4858761 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -79,6 +79,8 @@ clean_redcap_long <- function(db_data_long, if (has_repeat_forms) { # If mixed structure allowed, retrieve mixed structure forms + mixed_structure_ref <- data.frame() + if (allow_mixed_structure) { # Retrieve mixed structure fields and forms mixed_structure_ref <- get_mixed_structure_fields(db_data_long) %>% diff --git a/inst/testdata/db_metadata_mixed_structure.RDS b/inst/testdata/db_metadata_mixed_structure.RDS index 46c6f835d8faab0af8fcb777a2ce3a421b1b9d43..490ebbdc615866fb2722c32942ee53c48d7329ae 100644 GIT binary patch literal 416 zcmV;R0bl+fiwFP!000001MO2yPs1<_Eo+C^CdNKEBhE-n;>Hg^NL&#}+@dt~+K4pS zl9ZJjKO6yWm%v)Y*cs6(P4a9%KiTd>KL~;_h{8?~cJUZ=@9(b9FAV3kMyv?&F-7kPu^a!EB)c4NcPF7sp)bd@DNm4js4oYnxAj zn&0u+vTsU>ER=SqbE_`5bzM$xu6|9ltDt|sKk65C@Vw^+rz>d=#syCt*q?I{38_qe zzN(qt1cnBf*66?O(p{fL=8`?@St zVqt3Gt$y~jTFrb<4{>!~@PmDSZ8x3?YJB5E2kzTgkYw&|A>MpVuu60`*Oj~=c>4fF KPrGfk1pol&e8B+# literal 408 zcmV;J0cZXniwFP!000001MO2iPs1<}HBF1uR+N4)BW5HjvGD^C5-S3U&2pSeV&T}W z?a;9C$N9LVHjs$I$U-E?=XbvM&Ug0xh!7Hye$*q;08KKuyS+NQKx~1FIgaUe0~z8x zG{ChZA=FM6tAa8TDJM`r@Epd5DV__JQK>V*Lk$e9OJVTc zHMx3p%1i5XL6JIB1u_MabqThSYUMa9m@}fHnbGB{RblTy=?7E}?Ax+1g@tJn@APwM z^kx})dVr(%j6dxAOSkb{G8-Bnp7p+qB~4Plif|Qbf>mP6)YRz_^7;!uSYvpJaE86KM*e>Em+97OjuY4r)^+s<^FU6sS5z_7p?MTu`k6^@Pf@UnQVtf z$^1>kEHC6LDppc-=d@<;Yb}mG$&)rWP}hoLLo@$AOUrJ?VH27*W@`FC&t5aLE7Nyq z)_bV8L~&a`ttxc&W2i8^Qwop&Yq4$Mk`R;xqs=-xHO1K>hGC9{*Ma`4C$rZ{$`g@e zFu-Gd;Dx<>8-*?vsnPlsTSS?R**vCMrHCWGWaSPkicXyCgkH{hsunK6BAq{N=6MY1EvAhoe9%977wmzY}kfC(SqkB5P|)e7ko04N(g|T MKYaiy`>O;106}c75C8xG literal 327 zcmV-N0l5AjiwFP!000001C>(EPQx$^HvJLnCMG7(;DWRlU{ZVH1}AQCKp^cFMQJ8g zTGA@bfEzpuPsjrxz*&=SO#um4iQCV1K7VxP7+vV!Y9XAPg-A=k_ z=9BM`c@_n@?b%ZjRX)un6}&b>z!kjWhTI=sV(gMk>w~tMvr^@x$ek9&b4DxDOw*Lu zz^M&15=kM=fCMJl__QCVlDX=UbU&xPFeh8cOqj4zR4cAs#l*h`uI}uo`?0O|M6(sG ZooELRbE@flsf;*)&u@zRcSTPF002K!oS*;z diff --git a/inst/testdata/db_mixed_structure_linked_arms.RDS b/inst/testdata/db_mixed_structure_linked_arms.RDS index 6a57433271695f69b1a80a6aae79760b7b4f45cc..5892f5b2b5fa913c2238fe21c6d4c053cfd9b292 100644 GIT binary patch literal 367 zcmV-#0g(P5iwFP!000001GQ33Ps1<_&DwOI0)~b-^8;YgIC15|F93;Kl%?*dl(ZvF zfFr+~!%CVabS+GXO&r|D&-Tl+oyRdkC`336P#BpbiXQIou5JuAGY&R(WUOzjx5K73 zj?t*fg%`*aSr94(S?mj5{~2tnIL`7+!WyU|NiKy$C*Pqnk-kAp3MDmDTEcehK$5ni z=vIo0#o-RZft`8{j%5Bi&!~XHttognpRz1JI*E}_n%bS#;Gh~^rhOZyI!m85kb1CH zfeYZRUWc}(inTAm#TuA5= zc35T&3wxrHGWG8~Rej*=!@WlPyPlIq&q{+C-p^WWjURLGa)&rOK3$o7h_BOAex1>$wxN>G1e>H>o&?T5=L64(hkO(0J}Rj;W&aMA{dJ NHlJVFIH|1!007omu<8H+ literal 359 zcmV-t0hsyC}33h7JCSoZ_9K6ho-2E2a({Pwd2M}DYhb$${5j~YLBFaFXTTwb!v?Yx(OWqngfg)&4kCzXPKTP}e&I F008Wes?q=e diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 3a2e00a5..93de218f 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -85,10 +85,12 @@ test_that("clean_redcap_long with mixed structure works", { # Required since amendments take place before clean_redcap_long call in read_redcap db_metadata_mixed_structure <- update_field_names(db_metadata_mixed_structure) + # Expect error when allow_mixed_structure not specified expect_error( clean_redcap_long( db_data_long = db_mixed_structure, - db_metadata_long = db_metadata_mixed_structure + db_metadata_long = db_metadata_mixed_structure, + linked_arms = db_mixed_structure_linked_arms ), class = "repeat_nonrepeat_instrument" ) @@ -100,11 +102,32 @@ test_that("clean_redcap_long with mixed structure works", { allow_mixed_structure = TRUE ) - # Check general structure + # Check general structure, check all three structure types present expect_true(is_tibble(out)) expect_true("mixed" %in% out$structure) expect_true("nonrepeating" %in% out$structure) + expect_true("repeating" %in% out$structure) expect_true(!is.null(out$redcap_data)) + + # Check redcap_data contents for mixed and nonrepeating structure + expected_mixed_data <- tibble::tribble( + ~record_id, ~redcap_event, ~redcap_form_instance, ~mixed_structure_1, ~form_status_complete, + 1, "event_1", 1, "Mixed Nonrepeat 1", 0, + 1, "event_2", 1, "Mixed Repeat 1", 0, + 1, "event_2", 2, "Mixed Repeat 2", 0 + ) + + expected_nonrepeat_data <- tibble::tribble( + ~record_id, ~redcap_event, ~nonrepeat_1, ~form_status_complete, + 1, "event_1", "Nonrepeat 1", 0, + 1, "event_2", "Nonrepeat 2", 0 + ) + + expect_equal(out$redcap_data[out$redcap_form_name == "mixed_structure_form"][[1]], + expected_mixed_data) + + expect_equal(out$redcap_data[out$redcap_form_name == "nonrepeat_form"][[1]], + expected_nonrepeat_data) }) test_that("distill_nonrepeat_table_long tibble contains expected columns for longitudinal REDCap databases with arms", { @@ -263,27 +286,27 @@ test_that("get_mixed_structure_fields works", { test_that("convert_mixed_instrument works", { mixed_structure_db <- tibble::tribble( - ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, - 1, NA, NA, "A", - 2, "mixed_structure_form", 1, "B", - 3, "repeat_form", 1, "C", - 4, "repeat_form", 2, "D" + ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, ~repeat_form_variable, + 1, NA, NA, "A", NA, + 2, "mixed_structure_form", 1, "B", NA, + 3, "repeat_form", 1, NA, "C", + 4, "repeat_form", 2, NA, "D" ) - mixed_structure_meta <- tibble::tribble( - ~field_name, ~form_name, - "mixed_structure_variable", "mixed_structure_form" + mixed_structure_ref <- tibble::tribble( + ~field_name, ~rep_and_nonrep, ~form_name, + "mixed_structure_variable", TRUE, "mixed_structure_form" ) expected_out <- tibble::tribble( - ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, - 1, "mixed_structure_form", 1, "A", - 2, "mixed_structure_form", 1, "B", - 3, "repeat_form", 1, "C", - 4, "repeat_form", 2, "D" + ~record_id, ~redcap_repeat_instrument, ~redcap_repeat_instance, ~mixed_structure_variable, ~repeat_form_variable, + 1, "mixed_structure_form", 1, "A", NA, + 2, "mixed_structure_form", 1, "B", NA, + 3, "repeat_form", 1, NA, "C", + 4, "repeat_form", 2, NA, "D" ) - out <- convert_mixed_instrument(mixed_structure_db, mixed_structure_meta) + out <- convert_mixed_instrument(mixed_structure_db, mixed_structure_ref) expect_equal(out, expected_out) }) From e47693688906443d9bd6c924fdec5631bb31ab86 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Wed, 6 Mar 2024 10:30:00 -0500 Subject: [PATCH 15/20] Styler and lintr --- R/clean_redcap_long.R | 11 +++++------ tests/testthat/test-clean_redcap_long.R | 12 ++++++++---- vignettes/articles/diving_deeper.Rmd | 5 +++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index c4858761..a2b55f7f 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -86,7 +86,7 @@ clean_redcap_long <- function(db_data_long, mixed_structure_ref <- get_mixed_structure_fields(db_data_long) %>% filter(.data$rep_and_nonrep & !str_ends(.data$field_name, "_form_complete")) %>% left_join(db_metadata_long %>% select(.data$field_name, .data$form_name), - by = "field_name" + by = "field_name" ) # Update if project actually has mixed structure @@ -96,7 +96,6 @@ clean_redcap_long <- function(db_data_long, if (has_mixed_structure_forms) { db_data_long <- convert_mixed_instrument(db_data_long, mixed_structure_ref) } - } else { # Throw error if mixed structure detected and not allowed check_repeat_and_nonrepeat(db_data_long) @@ -113,8 +112,10 @@ clean_redcap_long <- function(db_data_long, linked_arms ) ), - structure = case_when(has_mixed_structure_forms & redcap_form_name %in% mixed_structure_ref$form_name ~ "mixed", - TRUE ~ "repeating") + structure = case_when( + has_mixed_structure_forms & redcap_form_name %in% mixed_structure_ref$form_name ~ "mixed", + TRUE ~ "repeating" + ) ) } @@ -274,7 +275,6 @@ distill_repeat_table_long <- function(form_name, db_data_long, db_metadata_long, linked_arms) { - has_repeat_forms <- "redcap_repeat_instance" %in% names(db_data_long) my_record_id <- names(db_data_long)[1] @@ -391,7 +391,6 @@ distill_repeat_table_long <- function(form_name, #' @keywords internal convert_mixed_instrument <- function(db_data_long, mixed_structure_ref) { - for (i in seq_len(nrow(mixed_structure_ref))) { field <- mixed_structure_ref$field_name[i] form <- mixed_structure_ref$form_name[i] diff --git a/tests/testthat/test-clean_redcap_long.R b/tests/testthat/test-clean_redcap_long.R index 93de218f..04f479f0 100644 --- a/tests/testthat/test-clean_redcap_long.R +++ b/tests/testthat/test-clean_redcap_long.R @@ -123,11 +123,15 @@ test_that("clean_redcap_long with mixed structure works", { 1, "event_2", "Nonrepeat 2", 0 ) - expect_equal(out$redcap_data[out$redcap_form_name == "mixed_structure_form"][[1]], - expected_mixed_data) + expect_equal( + out$redcap_data[out$redcap_form_name == "mixed_structure_form"][[1]], + expected_mixed_data + ) - expect_equal(out$redcap_data[out$redcap_form_name == "nonrepeat_form"][[1]], - expected_nonrepeat_data) + expect_equal( + out$redcap_data[out$redcap_form_name == "nonrepeat_form"][[1]], + expected_nonrepeat_data + ) }) test_that("distill_nonrepeat_table_long tibble contains expected columns for longitudinal REDCap databases with arms", { diff --git a/vignettes/articles/diving_deeper.Rmd b/vignettes/articles/diving_deeper.Rmd index 70812382..8d4d5e48 100644 --- a/vignettes/articles/diving_deeper.Rmd +++ b/vignettes/articles/diving_deeper.Rmd @@ -191,8 +191,9 @@ Users are cautioned when enabling this feature, since it changes definitions in ```{r} read_redcap(redcap_uri, - mixed_token, - allow_mixed_structure = TRUE) |> + mixed_token, + allow_mixed_structure = TRUE +) |> rmarkdown::paged_table() ``` From 71f34adbeef3ac0597b0959f4179bafd07d47d4a Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Wed, 6 Mar 2024 16:54:51 -0500 Subject: [PATCH 16/20] Add in options for allow_mixed_structure --- R/read_redcap.R | 4 ++-- man/read_redcap.Rd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/read_redcap.R b/R/read_redcap.R index 25334093..e60b26d4 100644 --- a/R/read_redcap.R +++ b/R/read_redcap.R @@ -56,7 +56,7 @@ #' @param allow_mixed_structure A logical to allow for support of mixed repeating/non-repeating #' instruments. Setting to `TRUE` will treat the mixed instrument's non-repeating versions #' as repeating instruments with a single instance. Applies to longitudinal projects -#' only. Default `FALSE`. +#' only. Default `FALSE`. Can be set globally with `options(redcaptidier.allow.mixed.structure = FALSE)`. #' #' @examples #' \dontrun{ @@ -80,7 +80,7 @@ read_redcap <- function(redcap_uri, export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, guess_max = .Machine$integer.max, - allow_mixed_structure = FALSE) { + allow_mixed_structure = getOption("redcaptidier.allow.mixed.structure", FALSE)) { check_arg_is_character(redcap_uri, len = 1, any.missing = FALSE) check_arg_is_character(token, len = 1, any.missing = FALSE) check_arg_is_valid_token(token) diff --git a/man/read_redcap.Rd b/man/read_redcap.Rd index ca77ccaf..7cd5d2bb 100644 --- a/man/read_redcap.Rd +++ b/man/read_redcap.Rd @@ -13,7 +13,7 @@ read_redcap( export_data_access_groups = NULL, suppress_redcapr_messages = TRUE, guess_max = .Machine$integer.max, - allow_mixed_structure = FALSE + allow_mixed_structure = getOption("redcaptidier.allow.mixed.structure", FALSE) ) } \arguments{ @@ -50,7 +50,7 @@ use for guessing column types. Default \code{.Machine$integer.max}.} \item{allow_mixed_structure}{A logical to allow for support of mixed repeating/non-repeating instruments. Setting to \code{TRUE} will treat the mixed instrument's non-repeating versions as repeating instruments with a single instance. Applies to longitudinal projects -only. Default \code{FALSE}.} +only. Default \code{FALSE}. Can be set globally with \code{options(redcaptidier.allow.mixed.structure = FALSE)}.} } \value{ A \code{tibble} in which each row represents a REDCap instrument. It From 64c62809b39590845189b64c7b8ca4bf04f9e1a4 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Fri, 8 Mar 2024 13:58:17 -0500 Subject: [PATCH 17/20] Move convert_mixed_instrument to distill fx --- R/clean_redcap_long.R | 28 +++++++++++++++++++--------- man/distill_repeat_table_long.Rd | 10 +++++++++- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index a2b55f7f..f2fb1b5f 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -82,7 +82,7 @@ clean_redcap_long <- function(db_data_long, mixed_structure_ref <- data.frame() if (allow_mixed_structure) { - # Retrieve mixed structure fields and forms + # Retrieve mixed structure fields and forms in reference df mixed_structure_ref <- get_mixed_structure_fields(db_data_long) %>% filter(.data$rep_and_nonrep & !str_ends(.data$field_name, "_form_complete")) %>% left_join(db_metadata_long %>% select(.data$field_name, .data$form_name), @@ -91,14 +91,11 @@ clean_redcap_long <- function(db_data_long, # Update if project actually has mixed structure has_mixed_structure_forms <- nrow(mixed_structure_ref) > 0 - # Convert mixed instruments to mixed structure format - if (has_mixed_structure_forms) { - db_data_long <- convert_mixed_instrument(db_data_long, mixed_structure_ref) - } } else { - # Throw error if mixed structure detected and not allowed - check_repeat_and_nonrepeat(db_data_long) + if (!has_mixed_structure_forms) { + check_repeat_and_nonrepeat(db_data_long) + } } repeated_forms_tibble <- tibble( @@ -109,7 +106,9 @@ clean_redcap_long <- function(db_data_long, .x, db_data_long, db_metadata_long, - linked_arms + linked_arms, + has_mixed_structure_forms = has_mixed_structure_forms, + mixed_structure_ref = mixed_structure_ref ) ), structure = case_when( @@ -268,13 +267,19 @@ distill_nonrepeat_table_long <- function(form_name, #' \code{REDCapR::redcap_metadata_read()$data} #' @param linked_arms Output of \code{link_arms}, linking instruments to REDCap #' events/arms +#' @param has_mixed_structure Whether the instrument under evaluation has a mixed +#' structure. Default `FALSE`. +#' @param name mixed_structure_ref A mixed structure reference dataframe supplied +#' by `get_mixed_structure_fields()`. #' #' @keywords internal distill_repeat_table_long <- function(form_name, db_data_long, db_metadata_long, - linked_arms) { + linked_arms, + has_mixed_structure_forms = FALSE, + mixed_structure_ref = NULL) { has_repeat_forms <- "redcap_repeat_instance" %in% names(db_data_long) my_record_id <- names(db_data_long)[1] @@ -308,6 +313,11 @@ distill_repeat_table_long <- function(form_name, my_fields <- c(my_fields, "redcap_data_access_group") } + # If has mixed structure, convert form + if (has_mixed_structure_forms) { + db_data_long <- convert_mixed_instrument(db_data_long, mixed_structure_ref %>% filter(form_name == my_form)) + } + # Setup data for loop redcap_arm linking db_data_long <- db_data_long %>% add_partial_keys(var = .data$redcap_event_name) %>% diff --git a/man/distill_repeat_table_long.Rd b/man/distill_repeat_table_long.Rd index 814abbcb..6cecc0fc 100644 --- a/man/distill_repeat_table_long.Rd +++ b/man/distill_repeat_table_long.Rd @@ -8,7 +8,9 @@ distill_repeat_table_long( form_name, db_data_long, db_metadata_long, - linked_arms + linked_arms, + has_mixed_structure_forms = FALSE, + mixed_structure_ref = NULL ) } \arguments{ @@ -23,6 +25,12 @@ REDCap metadata.} \item{linked_arms}{Output of \code{link_arms}, linking instruments to REDCap events/arms} + +\item{has_mixed_structure}{Whether the instrument under evaluation has a mixed +structure. Default \code{FALSE}.} + +\item{name}{mixed_structure_ref A mixed structure reference dataframe supplied +by \code{get_mixed_structure_fields()}.} } \value{ A \code{tibble} of all data related to a specified \code{form_name} From 65d5baf88a5d916e22a385957b7dd5e304409607 Mon Sep 17 00:00:00 2001 From: Rich Hanna <38384694+rsh52@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:14:36 -0400 Subject: [PATCH 18/20] Update R/clean_redcap_long.R Co-authored-by: Ezra Porter <60618324+ezraporter@users.noreply.github.com> --- R/clean_redcap_long.R | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index f2fb1b5f..c875f1fe 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -93,9 +93,7 @@ clean_redcap_long <- function(db_data_long, has_mixed_structure_forms <- nrow(mixed_structure_ref) > 0 # Convert mixed instruments to mixed structure format } else { - if (!has_mixed_structure_forms) { - check_repeat_and_nonrepeat(db_data_long) - } + check_repeat_and_nonrepeat(db_data_long) } repeated_forms_tibble <- tibble( From e53815b129cc33390aa636b119d59a62b773113b Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 11 Mar 2024 09:32:01 -0400 Subject: [PATCH 19/20] Fix linter --- R/clean_redcap_long.R | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/R/clean_redcap_long.R b/R/clean_redcap_long.R index c875f1fe..56402796 100644 --- a/R/clean_redcap_long.R +++ b/R/clean_redcap_long.R @@ -33,7 +33,6 @@ clean_redcap_long <- function(db_data_long, # `redcap_repeat_*` variables has_repeat_forms <- "redcap_repeat_instance" %in% names(db_data_long) - has_mixed_structure_forms <- FALSE # Apply checkmate checks assert_data_frame(db_data_long) @@ -78,6 +77,7 @@ clean_redcap_long <- function(db_data_long, ## Repeating Instruments Logic ---- if (has_repeat_forms) { # If mixed structure allowed, retrieve mixed structure forms + has_mixed_structure_forms <- FALSE # nolint: object_usage_linter mixed_structure_ref <- data.frame() @@ -91,9 +91,8 @@ clean_redcap_long <- function(db_data_long, # Update if project actually has mixed structure has_mixed_structure_forms <- nrow(mixed_structure_ref) > 0 - # Convert mixed instruments to mixed structure format } else { - check_repeat_and_nonrepeat(db_data_long) + check_repeat_and_nonrepeat(db_data_long) } repeated_forms_tibble <- tibble( From 3b0c0f8e967a3c788393e2728e4c68358dc0c436 Mon Sep 17 00:00:00 2001 From: Richard Hanna Date: Mon, 11 Mar 2024 10:05:27 -0400 Subject: [PATCH 20/20] Update microbenchmark --- utility/microbenchmark_results.csv | 86 +++++++++++++++--------------- utility/test_creds.R | 3 +- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/utility/microbenchmark_results.csv b/utility/microbenchmark_results.csv index 7bf59441..3a08cbcd 100644 --- a/utility/microbenchmark_results.csv +++ b/utility/microbenchmark_results.csv @@ -1,43 +1,43 @@ -min,lq,mean,median,uq,max,neval -1.16,1.16,1.16,1.16,1.16,1.16,1 -1.83,1.83,1.83,1.83,1.83,1.83,1 -693.01,693.01,693.01,693.01,693.01,693.01,1 -3.69,3.69,3.69,3.69,3.69,3.69,1 -5.19,5.19,5.19,5.19,5.19,5.19,1 -805.32,805.32,805.32,805.32,805.32,805.32,1 -689.2,689.2,689.2,689.2,689.2,689.2,1 -573.82,573.82,573.82,573.82,573.82,573.82,1 -700.61,700.61,700.61,700.61,700.61,700.61,1 -615.47,615.47,615.47,615.47,615.47,615.47,1 -685.91,685.91,685.91,685.91,685.91,685.91,1 -702.75,702.75,702.75,702.75,702.75,702.75,1 -656.78,656.78,656.78,656.78,656.78,656.78,1 -664.89,664.89,664.89,664.89,664.89,664.89,1 -372.23,372.23,372.23,372.23,372.23,372.23,1 -848.39,848.39,848.39,848.39,848.39,848.39,1 -671.34,671.34,671.34,671.34,671.34,671.34,1 -1.05,1.05,1.05,1.05,1.05,1.05,1 -981.49,981.49,981.49,981.49,981.49,981.49,1 -626.57,626.57,626.57,626.57,626.57,626.57,1 -656.61,656.61,656.61,656.61,656.61,656.61,1 -642.03,642.03,642.03,642.03,642.03,642.03,1 -696.71,696.71,696.71,696.71,696.71,696.71,1 -694.78,694.78,694.78,694.78,694.78,694.78,1 -641.12,641.12,641.12,641.12,641.12,641.12,1 -752.85,752.85,752.85,752.85,752.85,752.85,1 -2.84,2.84,2.84,2.84,2.84,2.84,1 -2.33,2.33,2.33,2.33,2.33,2.33,1 -3.83,3.83,3.83,3.83,3.83,3.83,1 -3.5,3.5,3.5,3.5,3.5,3.5,1 -3.87,3.87,3.87,3.87,3.87,3.87,1 -4.96,4.96,4.96,4.96,4.96,4.96,1 -2.18,2.18,2.18,2.18,2.18,2.18,1 -3.69,3.69,3.69,3.69,3.69,3.69,1 -2.29,2.29,2.29,2.29,2.29,2.29,1 -2.3,2.3,2.3,2.3,2.3,2.3,1 -2.11,2.11,2.11,2.11,2.11,2.11,1 -3.3,3.3,3.3,3.3,3.3,3.3,1 -3.91,3.91,3.91,3.91,3.91,3.91,1 -13.27,13.27,13.27,13.27,13.27,13.27,1 -14.4,14.4,14.4,14.4,14.4,14.4,1 -41.54,41.54,41.54,41.54,41.54,41.54,1 +min,lq,mean,median,uq,max,neval,description,source +1.56,1.56,1.56,1.56,1.56,1.56,1,simple static (read-only) test project,ouhsc +2.08,2.08,2.08,2.08,2.08,2.08,1,longitudinal (read-only) ARM test project,ouhsc +841.23,841.23,841.23,841.23,841.23,841.23,1,simple write data,ouhsc +4.34,4.34,4.34,4.34,4.34,4.34,1,Russian Characters,ouhsc +6.67,6.67,6.67,6.67,6.67,6.67,1,"super-wide --3,000 columns",ouhsc +912.5,912.5,912.5,912.5,912.5,912.5,1,static (not longitudinal) survey test project,ouhsc +769.03,769.03,769.03,769.03,769.03,769.03,1,"Clinical Trial (Fake) --Read-only, contributed by @higgi13425",ouhsc +1.08,1.08,1.08,1.08,1.08,1.08,1,nonnumeric record_id,ouhsc +729.67,729.67,729.67,729.67,729.67,729.67,1,DAG Read,ouhsc +936.55,936.55,936.55,936.55,936.55,936.55,1,potentially problematic values,ouhsc +778.25,778.25,778.25,778.25,778.25,778.25,1,Repeating Instruments,ouhsc +1.62,1.62,1.62,1.62,1.62,1.62,1,simple write metadata,ouhsc +974.01,974.01,974.01,974.01,974.01,974.01,1,DAG Write -admin,ouhsc +791.47,791.47,791.47,791.47,791.47,791.47,1,DAG Write -group A,ouhsc +389.54,389.54,389.54,389.54,389.54,389.54,1,"super-wide #3--35,000 columns",ouhsc +827.92,827.92,827.92,827.92,827.92,827.92,1,Repeating Instruments --Sparse,ouhsc +636.88,636.88,636.88,636.88,636.88,636.88,1,Delete Single Arm,ouhsc +1.41,1.41,1.41,1.41,1.41,1.41,1,Delete Multiple Arm,ouhsc +1.14,1.14,1.14,1.14,1.14,1.14,1,longitudinal single arm,ouhsc +1.01,1.01,1.01,1.01,1.01,1.01,1,decimal comma and dot,ouhsc +997.98,997.98,997.98,997.98,997.98,997.98,1,decimal comma,ouhsc +900.25,900.25,900.25,900.25,900.25,900.25,1,decimal dot,ouhsc +1.04,1.04,1.04,1.04,1.04,1.04,1,Validation Types,ouhsc +1.03,1.03,1.03,1.03,1.03,1.03,1,Blank for Gray Status,ouhsc +903.09,903.09,903.09,903.09,903.09,903.09,1,Checkboxes 1,ouhsc +1.32,1.32,1.32,1.32,1.32,1.32,1,Vignette: Longitudinal & Repeating Measures,ouhsc +2.49,2.49,2.49,2.49,2.49,2.49,1,classic,redcaptidier +3.02,3.02,3.02,3.02,3.02,3.02,1,classic no repeat,redcaptidier +3.36,3.36,3.36,3.36,3.36,3.36,1,longitudinal,redcaptidier +3.67,3.67,3.67,3.67,3.67,3.67,1,longitudinal no arms,redcaptidier +5.06,5.06,5.06,5.06,5.06,5.06,1,longitudinal no repeat,redcaptidier +5.01,5.01,5.01,5.01,5.01,5.01,1,deep dive vignette,redcaptidier +2.11,2.11,2.11,2.11,2.11,2.11,1,repeat first instrument,redcaptidier +3.35,3.35,3.35,3.35,3.35,3.35,1,repeat event,redcaptidier +2.46,2.46,2.46,2.46,2.46,2.46,1,restricted access,redcaptidier +2,2,2,2,2,2,1,large sparse db,redcaptidier +1.94,1.94,1.94,1.94,1.94,1.94,1,data access groups,redcaptidier +3.28,3.28,3.28,3.28,3.28,3.28,1,longitudinal data access groups,redcaptidier +3.15,3.15,3.15,3.15,3.15,3.15,1,mixed structure repeat no repeat,redcaptidier +10.44,10.44,10.44,10.44,10.44,10.44,1,prodigy db,redcaptidier +15.1,15.1,15.1,15.1,15.1,15.1,1,cart comprehensive db,redcaptidier +48.32,48.32,48.32,48.32,48.32,48.32,1,bmt outcomes db,redcaptidier diff --git a/utility/test_creds.R b/utility/test_creds.R index 98e66109..97c51353 100644 --- a/utility/test_creds.R +++ b/utility/test_creds.R @@ -38,7 +38,7 @@ redcaptidier_creds <- tibble::tribble( ) # Combine Credentials -creds <- rbind(ouhsc_creds, redcaptidier_creds) +creds <- rbind(ouhsc_creds %>% mutate(source = "ouhsc"), redcaptidier_creds %>% mutate(source = "redcaptidier")) microbenchmark_fx <- function(redcap_uri, token, name, times = 1){ microbenchmark( @@ -58,4 +58,5 @@ for (i in seq_along(microbenchmark_results)) { out %>% select(-expr) %>% mutate(across(tidyselect::everything(), ~round(., digits = 2))) %>% + cbind(., "description" = creds$comment, "source" = creds$source) %>% readr::write_csv("utility/microbenchmark_results.csv")