diff --git a/DESCRIPTION b/DESCRIPTION index ebd6ab0..e12ffac 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: MethodEvaluation Type: Package Title: Package for Evaluation of Estimation Methods Version: 2.3.0 -Date: 2032-03-31 +Date: 2023-04-13 Authors@R: c( person("Martijn", "Schuemie", , "schuemie@ohdsi.org", role = c("aut", "cre")) ) @@ -21,7 +21,7 @@ URL: https://github.com/OHDSI/MethodEvaluation BugReports: https://github.com/OHDSI/MethodEvaluation/issues Depends: R (>= 3.5.0), - DatabaseConnector (>= 4.0.2), + DatabaseConnector (>= 6.0.0), FeatureExtraction (>= 3.0.0), Cyclops (>= 3.0.0) Imports: diff --git a/NEWS.md b/NEWS.md index a70e2b6..8ddeeab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,13 +7,7 @@ Changes: 2. Adding `packageCustomBenchmarkResults()` to support custom methods benchmarks. - -MethodEvaluation 2.2.1 -====================== - -Changes: - -1. Fixing seeds and setting `resetCoefficients = TRUE` to ensure reproducibility of positive control synthesis. +3. Fixing seeds and setting `resetCoefficients = TRUE` to ensure reproducibility of positive control synthesis. MethodEvaluation 2.2.0 diff --git a/R/CreateReferenceSetCohorts.R b/R/CreateReferenceSetCohorts.R index 3bee9bf..1e178dc 100644 --- a/R/CreateReferenceSetCohorts.R +++ b/R/CreateReferenceSetCohorts.R @@ -1,13 +1,13 @@ # Copyright 2023 Observational Health Data Sciences and Informatics # # This file is part of MethodEvaluation -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,9 +20,9 @@ #' This function will create the outcomes of interest and nesting cohorts referenced in the various #' reference sets. The outcomes of interest are derives using information like diagnoses, procedures, #' and drug prescriptions. The outcomes are stored in a table on the database server. -#' +#' #' For the 'ohdsiMethodsBenchmark' reference set, the exposures are taken from the drug_era table, and -#' are therefore not generated as separate cohorts, and an exposure cohort table therefore needn't be supplied. +#' are therefore not generated as separate cohorts, and an exposure cohort table therefore needn't be supplied. #' For the 'ohdsiDevelopment' reference set, exposure cohorts will be generated. #' #' @param connectionDetails An R object of type \code{ConnectionDetails} created using the @@ -35,24 +35,24 @@ #' @param cdmDatabaseSchema A database schema containing health care data in the OMOP Commond #' Data Model. Note that for SQL Server, botth the database and schema #' should be specified, e.g. 'cdm_schema.dbo'. -#' @param exposureDatabaseSchema The name of the database schema where the exposure cohorts will be +#' @param exposureDatabaseSchema The name of the database schema where the exposure cohorts will be #' created. Only needed if \code{referenceSet = 'ohdsiDevelopment'}. Note -#' that for SQL Server, both the database and schema should be specified, +#' that for SQL Server, both the database and schema should be specified, #' e.g. 'cdm_schema.dbo'. -#' @param exposureTable The name of the table that will be created to store the exposure +#' @param exposureTable The name of the table that will be created to store the exposure #' cohorts. Only needed if \code{referenceSet = 'ohdsiDevelopment'}. #' @param outcomeDatabaseSchema The database schema where the target outcome table is located. Note #' that for SQL Server, both the database and schema should be #' specified, e.g. 'cdm_schema.dbo' #' @param outcomeTable The name of the table where the outcomes will be stored. #' @param nestingDatabaseSchema (For the OHDSI Methods Benchmark and OHDSI Development Set only) The -#' database schema where the nesting outcome table is located. Note that +#' database schema where the nesting outcome table is located. Note that #' for SQL Server, both the database and schema should be specified, e.g. #' 'cdm_schema.dbo'. -#' @param nestingTable (For the OHDSI Methods Benchmark and OHDSI Development Set only) The +#' @param nestingTable (For the OHDSI Methods Benchmark and OHDSI Development Set only) The #' name of the table where the nesting cohorts will be stored. #' @param referenceSet The name of the reference set for which outcomes need to be created. -#' Currently supported are "omopReferenceSet", "euadrReferenceSet", +#' Currently supported are "omopReferenceSet", "euadrReferenceSet", #' "ohdsiMethodsBenchmark", and "ohdsiDevelopment". #' @param workFolder Name of local folder to place intermediary results; make sure to use #' forward slashes (/). Do not use a folder on a network drive since @@ -76,14 +76,29 @@ createReferenceSetCohorts <- function(connectionDetails, warning("The 'oracleTempSchema' argument is deprecated. Use 'tempEmulationSchema' instead.") tempEmulationSchema <- oracleTempSchema } + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertClass(connectionDetails, "ConnectionDetails", add = errorMessages) + checkmate::assertCharacter(tempEmulationSchema, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::assertCharacter(cdmDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(exposureDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(exposureTable, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeTable, len = 1, add = errorMessages) + checkmate::assertCharacter(nestingDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(nestingTable, len = 1, add = errorMessages) + checkmate::assertChoice(referenceSet, c("omopReferenceSet", "euadrReferenceSet", "ohdsiMethodsBenchmark", "ohdsiDevelopment"), add = errorMessages) + checkmate::assertCharacter(workFolder, len = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) + if (referenceSet == "omopReferenceSet") { ParallelLogger::logInfo("Generating HOIs for the OMOP reference set") renderedSql <- SqlRender::loadRenderTranslateSql("CreateOmopHois.sql", - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - cdm_database_schema = cdmDatabaseSchema, - outcome_database_schema = outcomeDatabaseSchema, - outcome_table = outcomeTable) + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + cdm_database_schema = cdmDatabaseSchema, + outcome_database_schema = outcomeDatabaseSchema, + outcome_table = outcomeTable + ) conn <- DatabaseConnector::connect(connectionDetails) DatabaseConnector::executeSql(conn, renderedSql) ParallelLogger::logInfo("Done") @@ -93,28 +108,33 @@ createReferenceSetCohorts <- function(connectionDetails, # TODO: add code for creating the EU-ADR HOIs } else if (referenceSet == "ohdsiMethodsBenchmark") { ParallelLogger::logInfo("Generating HOIs and nesting cohorts for the OHDSI Methods Benchmark") - if (outcomeDatabaseSchema == nestingDatabaseSchema && outcomeTable == nestingTable) + if (outcomeDatabaseSchema == nestingDatabaseSchema && outcomeTable == nestingTable) { stop("Outcome and nesting cohorts cannot be created in the same table") - createOhdsiNegativeControlCohorts(connectionDetails = connectionDetails, - cdmDatabaseSchema = cdmDatabaseSchema, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - nestingDatabaseSchema = nestingDatabaseSchema, - nestingTable = nestingTable, - tempEmulationSchema = tempEmulationSchema, - workFolder = workFolder) + } + createOhdsiNegativeControlCohorts( + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + nestingDatabaseSchema = nestingDatabaseSchema, + nestingTable = nestingTable, + tempEmulationSchema = tempEmulationSchema, + workFolder = workFolder + ) } else if (referenceSet == "ohdsiDevelopment") { ParallelLogger::logInfo("Generating HOIs and nesting cohorts for the OHDSI Development set") - createOhdsiDevelopmentNegativeControlCohorts(connectionDetails = connectionDetails, - cdmDatabaseSchema = cdmDatabaseSchema, - exposureDatabaseSchema = exposureDatabaseSchema, - exposureTable = exposureTable, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - nestingDatabaseSchema = nestingDatabaseSchema, - nestingTable = nestingTable, - tempEmulationSchema = tempEmulationSchema, - workFolder = workFolder) + createOhdsiDevelopmentNegativeControlCohorts( + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + exposureDatabaseSchema = exposureDatabaseSchema, + exposureTable = exposureTable, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + nestingDatabaseSchema = nestingDatabaseSchema, + nestingTable = nestingTable, + tempEmulationSchema = tempEmulationSchema, + workFolder = workFolder + ) } else { stop(paste("Unknow reference set:", referenceSet)) } @@ -134,104 +154,129 @@ createOhdsiDevelopmentNegativeControlCohorts <- function(connectionDetails, dir.create(workFolder, recursive = TRUE) } ohdsiDevelopmentNegativeControls <- readRDS(system.file("ohdsiDevelopmentNegativeControls.rds", - package = "MethodEvaluation")) - + package = "MethodEvaluation" + )) + connection <- DatabaseConnector::connect(connectionDetails) on.exit(DatabaseConnector::disconnect(connection)) - - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = "CreateCohortTable.sql", - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cohort_database_schema = outcomeDatabaseSchema, - cohort_table = outcomeTable) + + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "CreateCohortTable.sql", + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cohort_database_schema = outcomeDatabaseSchema, + cohort_table = outcomeTable + ) DatabaseConnector::executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE) - + if (outcomeDatabaseSchema != nestingDatabaseSchema | outcomeTable != nestingTable) { - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = "CreateCohortTable.sql", - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cohort_database_schema = nestingDatabaseSchema, - cohort_table = nestingTable) + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "CreateCohortTable.sql", + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cohort_database_schema = nestingDatabaseSchema, + cohort_table = nestingTable + ) DatabaseConnector::executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE) } if (outcomeDatabaseSchema != exposureDatabaseSchema | outcomeTable != exposureTable) { - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = "CreateCohortTable.sql", - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cohort_database_schema = exposureDatabaseSchema, - cohort_table = exposureTable) + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "CreateCohortTable.sql", + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cohort_database_schema = exposureDatabaseSchema, + cohort_table = exposureTable + ) DatabaseConnector::executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE) } - exposureCohorts <- data.frame(cohortName = c("ace_inhibitors", "thiazides_diuretics"), - cohortId = c(1, 2)) + exposureCohorts <- data.frame( + cohortName = c("ace_inhibitors", "thiazides_diuretics"), + cohortId = c(1, 2) + ) for (i in 1:nrow(exposureCohorts)) { ParallelLogger::logInfo(paste("Creating exposure cohort:", exposureCohorts$cohortName[i])) - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = paste0(exposureCohorts$cohortName[i], ".sql"), - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cdm_database_schema = cdmDatabaseSchema, - vocabulary_database_schema = cdmDatabaseSchema, - target_database_schema = exposureDatabaseSchema, - target_cohort_table = exposureTable, - target_cohort_id = exposureCohorts$cohortId[i]) + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = paste0(exposureCohorts$cohortName[i], ".sql"), + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cdm_database_schema = cdmDatabaseSchema, + vocabulary_database_schema = cdmDatabaseSchema, + target_database_schema = exposureDatabaseSchema, + target_cohort_table = exposureTable, + target_cohort_id = exposureCohorts$cohortId[i] + ) DatabaseConnector::executeSql(connection, sql) } ParallelLogger::logInfo("Creating negative control outcomes") outcomeCohorts <- ohdsiDevelopmentNegativeControls %>% distinct(cohortId = .data$outcomeId, cohortName = .data$outcomeName) sql <- SqlRender::loadRenderTranslateSql("NegativeControls.sql", - "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cdm_database_schema = cdmDatabaseSchema, - target_database_schema = outcomeDatabaseSchema, - target_cohort_table = outcomeTable, - outcome_ids = outcomeCohorts$cohortId) + "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cdm_database_schema = cdmDatabaseSchema, + target_database_schema = outcomeDatabaseSchema, + target_cohort_table = outcomeTable, + outcome_ids = outcomeCohorts$cohortId + ) DatabaseConnector::executeSql(connection, sql) - + ParallelLogger::logInfo("Creating nesting cohorts") nestingCohorts <- ohdsiDevelopmentNegativeControls %>% distinct(cohortId = .data$nestingId, cohortName = .data$nestingName) sql <- SqlRender::loadRenderTranslateSql("NestingCohorts.sql", - "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cdm_database_schema = cdmDatabaseSchema, - target_database_schema = nestingDatabaseSchema, - target_cohort_table = nestingTable, - nesting_ids = nestingCohorts$cohortId) + "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cdm_database_schema = cdmDatabaseSchema, + target_database_schema = nestingDatabaseSchema, + target_cohort_table = nestingTable, + nesting_ids = nestingCohorts$cohortId + ) DatabaseConnector::executeSql(connection, sql) - + ParallelLogger::logInfo("Counting cohorts") - exposureCohortCounts <- countCohorts(connection = connection, - cohortDatabaseSchema = exposureDatabaseSchema, - cohortTable = exposureTable, - cohortIds = exposureCohorts$cohortId) %>% + exposureCohortCounts <- countCohorts( + connection = connection, + cohortDatabaseSchema = exposureDatabaseSchema, + cohortTable = exposureTable, + cohortIds = exposureCohorts$cohortId + ) %>% right_join(exposureCohorts, by = "cohortId") %>% mutate(type = "Exposure") - outcomeCohortCounts <- countCohorts(connection = connection, - cohortDatabaseSchema = outcomeDatabaseSchema, - cohortTable = outcomeTable, - cohortIds = outcomeCohorts$cohortId) %>% + outcomeCohortCounts <- countCohorts( + connection = connection, + cohortDatabaseSchema = outcomeDatabaseSchema, + cohortTable = outcomeTable, + cohortIds = outcomeCohorts$cohortId + ) %>% right_join(outcomeCohorts, by = "cohortId") %>% mutate(type = "Outcome") - nestingCohortCounts <- countCohorts(connection = connection, - cohortDatabaseSchema = nestingDatabaseSchema, - cohortTable = nestingTable, - cohortIds = nestingCohorts$cohortId) %>% + nestingCohortCounts <- countCohorts( + connection = connection, + cohortDatabaseSchema = nestingDatabaseSchema, + cohortTable = nestingTable, + cohortIds = nestingCohorts$cohortId + ) %>% right_join(nestingCohorts, by = "cohortId") %>% mutate(type = "Nesting") cohortCounts <- bind_rows(exposureCohortCounts, outcomeCohortCounts, nestingCohortCounts) %>% - mutate(cohortEntries = case_when(is.na(.data$cohortEntries) ~ as.integer(0), - TRUE ~ as.integer(.data$cohortEntries)), - cohortSubjects = case_when(is.na(.data$cohortSubjects) ~ as.integer(0), - TRUE ~ as.integer(.data$cohortSubjects))) - readr::write_csv(cohortCounts, file.path(workFolder, "cohortCounts.csv")) - ParallelLogger::logInfo("Cohort counts written to ", file.path(workFolder, "cohortCounts.csv")) + mutate( + cohortEntries = case_when( + is.na(.data$cohortEntries) ~ as.integer(0), + TRUE ~ as.integer(.data$cohortEntries) + ), + cohortSubjects = case_when( + is.na(.data$cohortSubjects) ~ as.integer(0), + TRUE ~ as.integer(.data$cohortSubjects) + ) + ) + readr::write_csv(cohortCounts, file.path(workFolder, "cohortCounts.csv")) + ParallelLogger::logInfo("Cohort counts written to ", file.path(workFolder, "cohortCounts.csv")) } createOhdsiNegativeControlCohorts <- function(connectionDetails, @@ -243,40 +288,49 @@ createOhdsiNegativeControlCohorts <- function(connectionDetails, tempEmulationSchema, workFolder) { ohdsiNegativeControls <- readRDS(system.file("ohdsiNegativeControls.rds", - package = "MethodEvaluation")) - + package = "MethodEvaluation" + )) + connection <- DatabaseConnector::connect(connectionDetails) on.exit(DatabaseConnector::disconnect(connection)) - - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = "CreateCohortTable.sql", - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cohort_database_schema = outcomeDatabaseSchema, - cohort_table = outcomeTable) + + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "CreateCohortTable.sql", + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cohort_database_schema = outcomeDatabaseSchema, + cohort_table = outcomeTable + ) DatabaseConnector::executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE) - - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = "CreateCohortTable.sql", - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cohort_database_schema = nestingDatabaseSchema, - cohort_table = nestingTable) + + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "CreateCohortTable.sql", + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cohort_database_schema = nestingDatabaseSchema, + cohort_table = nestingTable + ) DatabaseConnector::executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE) - - complexOutcomeCohorts <- data.frame(sqlName = c("acute_pancreatitis", "gi_bleed", "stroke", "ibd"), - cohortId = c(1, 2, 3, 4)) + + complexOutcomeCohorts <- data.frame( + sqlName = c("acute_pancreatitis", "gi_bleed", "stroke", "ibd"), + cohortId = c(1, 2, 3, 4) + ) for (i in 1:nrow(complexOutcomeCohorts)) { ParallelLogger::logInfo(paste("Creating outcome:", complexOutcomeCohorts$sqlName[i])) - sql <- SqlRender::loadRenderTranslateSql(sqlFilename = paste0(complexOutcomeCohorts$sqlName[i], ".sql"), - packageName = "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cdm_database_schema = cdmDatabaseSchema, - vocabulary_database_schema = cdmDatabaseSchema, - target_database_schema = outcomeDatabaseSchema, - target_cohort_table = outcomeTable, - target_cohort_id = complexOutcomeCohorts$cohortId[i]) + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = paste0(complexOutcomeCohorts$sqlName[i], ".sql"), + packageName = "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cdm_database_schema = cdmDatabaseSchema, + vocabulary_database_schema = cdmDatabaseSchema, + target_database_schema = outcomeDatabaseSchema, + target_cohort_table = outcomeTable, + target_cohort_id = complexOutcomeCohorts$cohortId[i] + ) DatabaseConnector::executeSql(connection, sql) } ParallelLogger::logInfo("Creating other negative control outcomes") @@ -285,76 +339,94 @@ createOhdsiNegativeControlCohorts <- function(connectionDetails, distinct(cohortId = .data$outcomeId) %>% pull() sql <- SqlRender::loadRenderTranslateSql("NegativeControls.sql", - "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cdm_database_schema = cdmDatabaseSchema, - target_database_schema = outcomeDatabaseSchema, - target_cohort_table = outcomeTable, - outcome_ids = otherOutcomeCohortIds) + "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cdm_database_schema = cdmDatabaseSchema, + target_database_schema = outcomeDatabaseSchema, + target_cohort_table = outcomeTable, + outcome_ids = otherOutcomeCohortIds + ) DatabaseConnector::executeSql(connection, sql) outcomeCohorts <- ohdsiNegativeControls %>% - distinct(cohortId = .data$outcomeId, cohortName = .data$outcomeName) - + distinct(cohortId = .data$outcomeId, cohortName = .data$outcomeName) + ParallelLogger::logInfo("Creating nesting cohorts") nestingCohorts <- ohdsiNegativeControls %>% distinct(cohortId = .data$nestingId, cohortName = .data$nestingName) sql <- SqlRender::loadRenderTranslateSql("NestingCohorts.sql", - "MethodEvaluation", - dbms = connectionDetails$dbms, - tempEmulationSchema = tempEmulationSchema, - cdm_database_schema = cdmDatabaseSchema, - target_database_schema = nestingDatabaseSchema, - target_cohort_table = nestingTable, - nesting_ids = nestingCohorts$cohortId) + "MethodEvaluation", + dbms = connectionDetails$dbms, + tempEmulationSchema = tempEmulationSchema, + cdm_database_schema = cdmDatabaseSchema, + target_database_schema = nestingDatabaseSchema, + target_cohort_table = nestingTable, + nesting_ids = nestingCohorts$cohortId + ) DatabaseConnector::executeSql(connection, sql) - - exposureCohorts <- bind_rows(ohdsiNegativeControls %>% - distinct(cohortId = .data$targetId, cohortName = .data$targetName) , - ohdsiNegativeControls %>% - distinct(cohortId = .data$comparatorId, cohortName = .data$comparatorName)) %>% + + exposureCohorts <- bind_rows( + ohdsiNegativeControls %>% + distinct(cohortId = .data$targetId, cohortName = .data$targetName), + ohdsiNegativeControls %>% + distinct(cohortId = .data$comparatorId, cohortName = .data$comparatorName) + ) %>% distinct() - + ParallelLogger::logInfo("Counting cohorts") - exposureCohortCounts <- countDrugEras(connection = connection, - cdmDatabaseSchema = cdmDatabaseSchema, - cohortIds = exposureCohorts$cohortId) %>% + exposureCohortCounts <- countDrugEras( + connection = connection, + cdmDatabaseSchema = cdmDatabaseSchema, + cohortIds = exposureCohorts$cohortId + ) %>% right_join(exposureCohorts, by = "cohortId") %>% mutate(type = "Exposure") - outcomeCohortCounts <- countCohorts(connection = connection, - cohortDatabaseSchema = outcomeDatabaseSchema, - cohortTable = outcomeTable, - cohortIds = outcomeCohorts$cohortId) %>% + outcomeCohortCounts <- countCohorts( + connection = connection, + cohortDatabaseSchema = outcomeDatabaseSchema, + cohortTable = outcomeTable, + cohortIds = outcomeCohorts$cohortId + ) %>% right_join(outcomeCohorts, by = "cohortId") %>% mutate(type = "Outcome") - nestingCohortCounts <- countCohorts(connection = connection, - cohortDatabaseSchema = nestingDatabaseSchema, - cohortTable = nestingTable, - cohortIds = nestingCohorts$cohortId) %>% + nestingCohortCounts <- countCohorts( + connection = connection, + cohortDatabaseSchema = nestingDatabaseSchema, + cohortTable = nestingTable, + cohortIds = nestingCohorts$cohortId + ) %>% right_join(nestingCohorts, by = "cohortId") %>% mutate(type = "Nesting") cohortCounts <- bind_rows(exposureCohortCounts, outcomeCohortCounts, nestingCohortCounts) %>% - mutate(cohortEntries = case_when(is.na(.data$cohortEntries) ~ as.integer(0), - TRUE ~ as.integer(.data$cohortEntries)), - cohortSubjects = case_when(is.na(.data$cohortSubjects) ~ as.integer(0), - TRUE ~ as.integer(.data$cohortSubjects))) - readr::write_csv(cohortCounts, file.path(workFolder, "cohortCounts.csv")) - ParallelLogger::logInfo("Cohort counts written to ", file.path(workFolder, "cohortCounts.csv")) + mutate( + cohortEntries = case_when( + is.na(.data$cohortEntries) ~ as.integer(0), + TRUE ~ as.integer(.data$cohortEntries) + ), + cohortSubjects = case_when( + is.na(.data$cohortSubjects) ~ as.integer(0), + TRUE ~ as.integer(.data$cohortSubjects) + ) + ) + readr::write_csv(cohortCounts, file.path(workFolder, "cohortCounts.csv")) + ParallelLogger::logInfo("Cohort counts written to ", file.path(workFolder, "cohortCounts.csv")) } countCohorts <- function(connection, cohortDatabaseSchema, cohortTable, cohortIds) { sql <- "SELECT cohort_definition_id AS cohort_id, COUNT(*) AS cohort_entries, COUNT(DISTINCT subject_id) AS cohort_subjects - FROM @cohort_database_schema.@cohort_table + FROM @cohort_database_schema.@cohort_table WHERE cohort_definition_id IN (@cohort_ids) GROUP BY cohort_definition_id;" - cohortCounts <- DatabaseConnector::renderTranslateQuerySql(connection = connection, - sql = sql, - cohort_database_schema = cohortDatabaseSchema, - cohort_table = cohortTable, - cohort_ids = cohortIds, - snakeCaseToCamelCase = TRUE) + cohortCounts <- DatabaseConnector::renderTranslateQuerySql( + connection = connection, + sql = sql, + cohort_database_schema = cohortDatabaseSchema, + cohort_table = cohortTable, + cohort_ids = cohortIds, + snakeCaseToCamelCase = TRUE + ) return(cohortCounts) } @@ -365,11 +437,13 @@ countDrugEras <- function(connection, cdmDatabaseSchema, cohortIds) { FROM @cdm_database_schema.drug_era WHERE drug_concept_id IN (@cohort_ids) GROUP BY drug_concept_id;" - cohortCounts <- DatabaseConnector::renderTranslateQuerySql(connection = connection, - sql = sql, - cdm_database_schema = cdmDatabaseSchema, - cohort_ids = cohortIds, - snakeCaseToCamelCase = TRUE) + cohortCounts <- DatabaseConnector::renderTranslateQuerySql( + connection = connection, + sql = sql, + cdm_database_schema = cdmDatabaseSchema, + cohort_ids = cohortIds, + snakeCaseToCamelCase = TRUE + ) return(cohortCounts) } @@ -420,13 +494,13 @@ countDrugEras <- function(connection, cdmDatabaseSchema, cohortIds) { #' #' @export synthesizeReferenceSetPositiveControls <- function(connectionDetails, - cdmDatabaseSchema, oracleTempSchema = NULL, tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), - outcomeDatabaseSchema = cdmDatabaseSchema, - outcomeTable = "cohort", + cdmDatabaseSchema, exposureDatabaseSchema = cdmDatabaseSchema, exposureTable = "drug_era", + outcomeDatabaseSchema = cdmDatabaseSchema, + outcomeTable = "cohort", referenceSet = "ohdsiMethodsBenchmark", maxCores = 1, workFolder, @@ -435,89 +509,111 @@ synthesizeReferenceSetPositiveControls <- function(connectionDetails, warning("The 'oracleTempSchema' argument is deprecated. Use 'tempEmulationSchema' instead.") tempEmulationSchema <- oracleTempSchema } - if (!referenceSet %in% c("ohdsiMethodsBenchmark", "ohdsiDevelopment")) { - stop("Currently only supporting positive control synthesis for the ohdsiMethodsBenchmark and ohdsiDevelopment reference sets") - } + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertClass(connectionDetails, "ConnectionDetails", add = errorMessages) + checkmate::assertCharacter(tempEmulationSchema, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::assertCharacter(cdmDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(exposureDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(exposureTable, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeTable, len = 1, add = errorMessages) + checkmate::assertChoice(referenceSet, c("ohdsiMethodsBenchmark", "ohdsiDevelopment"), add = errorMessages) + checkmate::assertInt(maxCores, lower = 1, add = errorMessages) + checkmate::assertCharacter(workFolder, len = 1, add = errorMessages) + checkmate::assertCharacter(summaryFileName, len = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) injectionFolder <- file.path(workFolder, "SignalInjection") - if (!file.exists(injectionFolder)) + if (!file.exists(injectionFolder)) { dir.create(injectionFolder) - + } + injectionSummaryFile <- file.path(workFolder, "injectionSummary.rds") if (referenceSet == "ohdsiMethodsBenchmark") { negativeControls <- readRDS(system.file("ohdsiNegativeControls.rds", - package = "MethodEvaluation")) + package = "MethodEvaluation" + )) } else { negativeControls <- readRDS(system.file("ohdsiDevelopmentNegativeControls.rds", - package = "MethodEvaluation")) + package = "MethodEvaluation" + )) } if (!file.exists(injectionSummaryFile)) { - exposureOutcomePairs <- data.frame(exposureId = negativeControls$targetId, - outcomeId = negativeControls$outcomeId) + exposureOutcomePairs <- data.frame( + exposureId = negativeControls$targetId, + outcomeId = negativeControls$outcomeId + ) exposureOutcomePairs <- unique(exposureOutcomePairs) - + prior <- Cyclops::createPrior("laplace", exclude = 0, useCrossValidation = TRUE) - - control <- Cyclops::createControl(cvType = "auto", - startingVariance = 0.01, - noiseLevel = "quiet", - cvRepetitions = 1, - threads = min(c(10, maxCores))) - - covariateSettings <- FeatureExtraction::createCovariateSettings(useDemographicsAgeGroup = TRUE, - useDemographicsGender = TRUE, - useDemographicsIndexYear = TRUE, - useDemographicsIndexMonth = TRUE, - useConditionGroupEraLongTerm = TRUE, - useDrugGroupEraLongTerm = TRUE, - useProcedureOccurrenceLongTerm = TRUE, - useMeasurementLongTerm = TRUE, - useObservationLongTerm = TRUE, - useCharlsonIndex = TRUE, - useDcsi = TRUE, - useChads2Vasc = TRUE, - longTermStartDays = 365, - endDays = 0) - + + control <- Cyclops::createControl( + cvType = "auto", + startingVariance = 0.01, + noiseLevel = "quiet", + cvRepetitions = 1, + threads = min(c(10, maxCores)) + ) + + covariateSettings <- FeatureExtraction::createCovariateSettings( + useDemographicsAgeGroup = TRUE, + useDemographicsGender = TRUE, + useDemographicsIndexYear = TRUE, + useDemographicsIndexMonth = TRUE, + useConditionGroupEraLongTerm = TRUE, + useDrugGroupEraLongTerm = TRUE, + useProcedureOccurrenceLongTerm = TRUE, + useMeasurementLongTerm = TRUE, + useObservationLongTerm = TRUE, + useCharlsonIndex = TRUE, + useDcsi = TRUE, + useChads2Vasc = TRUE, + longTermStartDays = 365, + endDays = 0 + ) + result <- synthesizePositiveControls(connectionDetails, - cdmDatabaseSchema = cdmDatabaseSchema, - tempEmulationSchema = tempEmulationSchema, - exposureDatabaseSchema = exposureDatabaseSchema, - exposureTable = exposureTable, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - outputDatabaseSchema = outcomeDatabaseSchema, - outputTable = outcomeTable, - createOutputTable = FALSE, - outputIdOffset = 10000, - exposureOutcomePairs = exposureOutcomePairs, - firstExposureOnly = FALSE, - firstOutcomeOnly = TRUE, - removePeopleWithPriorOutcomes = TRUE, - modelType = "survival", - washoutPeriod = 365, - riskWindowStart = 0, - riskWindowEnd = 0, - endAnchor = "cohort end", - effectSizes = c(1.5, 2, 4), - precision = 0.01, - prior = prior, - control = control, - maxSubjectsForModel = 250000, - minOutcomeCountForModel = 100, - minOutcomeCountForInjection = 25, - workFolder = injectionFolder, - modelThreads = max(1, round(maxCores/8)), - generationThreads = min(6, maxCores), - covariateSettings = covariateSettings) + cdmDatabaseSchema = cdmDatabaseSchema, + tempEmulationSchema = tempEmulationSchema, + exposureDatabaseSchema = exposureDatabaseSchema, + exposureTable = exposureTable, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + outputDatabaseSchema = outcomeDatabaseSchema, + outputTable = outcomeTable, + createOutputTable = FALSE, + outputIdOffset = 10000, + exposureOutcomePairs = exposureOutcomePairs, + firstExposureOnly = FALSE, + firstOutcomeOnly = TRUE, + removePeopleWithPriorOutcomes = TRUE, + modelType = "survival", + washoutPeriod = 365, + riskWindowStart = 0, + riskWindowEnd = 0, + endAnchor = "cohort end", + effectSizes = c(1.5, 2, 4), + precision = 0.01, + prior = prior, + control = control, + maxSubjectsForModel = 250000, + minOutcomeCountForModel = 100, + minOutcomeCountForInjection = 25, + workFolder = injectionFolder, + modelThreads = max(1, round(maxCores / 8)), + generationThreads = min(6, maxCores), + covariateSettings = covariateSettings + ) saveRDS(result, injectionSummaryFile) } injectedSignals <- readRDS(injectionSummaryFile) injectedSignals$targetId <- injectedSignals$exposureId injectedSignals <- merge(injectedSignals, negativeControls) injectedSignals <- injectedSignals[injectedSignals$trueEffectSize != 0, ] - injectedSignals$outcomeName <- paste0(injectedSignals$outcomeName, - ", RR=", - injectedSignals$targetEffectSize) + injectedSignals$outcomeName <- paste0( + injectedSignals$outcomeName, + ", RR=", + injectedSignals$targetEffectSize + ) injectedSignals$oldOutcomeId <- injectedSignals$outcomeId injectedSignals$outcomeId <- injectedSignals$newOutcomeId negativeControls$targetEffectSize <- 1 @@ -526,26 +622,36 @@ synthesizeReferenceSetPositiveControls <- function(connectionDetails, negativeControls$oldOutcomeId <- negativeControls$outcomeId allControls <- rbind(negativeControls, injectedSignals[, names(negativeControls)]) exposureOutcomes <- data.frame() - exposureOutcomes <- rbind(exposureOutcomes, data.frame(exposureId = allControls$targetId, - outcomeId = allControls$outcomeId)) - exposureOutcomes <- rbind(exposureOutcomes, data.frame(exposureId = allControls$comparatorId, - outcomeId = allControls$outcomeId)) + exposureOutcomes <- rbind(exposureOutcomes, data.frame( + exposureId = allControls$targetId, + outcomeId = allControls$outcomeId + )) + exposureOutcomes <- rbind(exposureOutcomes, data.frame( + exposureId = allControls$comparatorId, + outcomeId = allControls$outcomeId + )) exposureOutcomes <- unique(exposureOutcomes) - mdrr <- computeMdrr(connectionDetails = connectionDetails, - cdmDatabaseSchema = cdmDatabaseSchema, - tempEmulationSchema = tempEmulationSchema, - exposureOutcomePairs = exposureOutcomes, - exposureDatabaseSchema = exposureDatabaseSchema, - exposureTable = exposureTable, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - cdmVersion = "5") - allControls <- merge(allControls, data.frame(targetId = mdrr$exposureId, - outcomeId = mdrr$outcomeId, - mdrrTarget = mdrr$mdrr)) - allControls <- merge(allControls, data.frame(comparatorId = mdrr$exposureId, - outcomeId = mdrr$outcomeId, - mdrrComparator = mdrr$mdrr), all.x = TRUE) + mdrr <- computeMdrr( + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + tempEmulationSchema = tempEmulationSchema, + exposureOutcomePairs = exposureOutcomes, + exposureDatabaseSchema = exposureDatabaseSchema, + exposureTable = exposureTable, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + cdmVersion = "5" + ) + allControls <- merge(allControls, data.frame( + targetId = mdrr$exposureId, + outcomeId = mdrr$outcomeId, + mdrrTarget = mdrr$mdrr + )) + allControls <- merge(allControls, data.frame( + comparatorId = mdrr$exposureId, + outcomeId = mdrr$outcomeId, + mdrrComparator = mdrr$mdrr + ), all.x = TRUE) readr::write_csv(allControls, summaryFileName) - ParallelLogger::logInfo("Positive and negative control summary written to ", summaryFileName) + ParallelLogger::logInfo("Positive and negative control summary written to ", summaryFileName) } diff --git a/R/Mdrr.R b/R/Mdrr.R index e1b3cd5..43989fa 100644 --- a/R/Mdrr.R +++ b/R/Mdrr.R @@ -85,9 +85,9 @@ #' } #' @export computeMdrr <- function(connectionDetails, - cdmDatabaseSchema, oracleTempSchema = NULL, tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), + cdmDatabaseSchema, exposureOutcomePairs, exposureDatabaseSchema = cdmDatabaseSchema, exposureTable = "drug_era", @@ -101,12 +101,18 @@ computeMdrr <- function(connectionDetails, if (is.null(exposureOutcomePairs$exposureId) && !is.null(exposureOutcomePairs$targetId)) { exposureOutcomePairs$exposureId <- exposureOutcomePairs$targetId } - if (is.null(exposureOutcomePairs$exposureId)) { - stop("exposureOutcomePairs is missing exposureId and targetId column") - } - if (is.null(exposureOutcomePairs$outcomeId)) { - stop("exposureOutcomePairs is missing outcomeId column") - } + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertClass(connectionDetails, "ConnectionDetails", add = errorMessages) + checkmate::assertCharacter(tempEmulationSchema, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::assertCharacter(cdmDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertDataFrame(exposureOutcomePairs, add = errorMessages) + checkmate::assertNames(colnames(exposureOutcomePairs), must.include = c("exposureId", "outcomeId"), add = errorMessages) + checkmate::assertCharacter(exposureDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(exposureTable, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeTable, len = 1, add = errorMessages) + checkmate::assertCharacter(cdmVersion, len = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) exposureTable <- tolower(exposureTable) outcomeTable <- tolower(outcomeTable) if (exposureTable == "drug_era") { diff --git a/R/MethodEvaluation.R b/R/MethodEvaluation.R index 05ca650..d08dc3d 100644 --- a/R/MethodEvaluation.R +++ b/R/MethodEvaluation.R @@ -3,13 +3,13 @@ # Copyright 2023 Observational Health Data Sciences and Informatics # # This file is part of MethodEvaluation -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -114,7 +114,7 @@ NULL #' The OHDSI Development Set - Negative Controls #' A set of 76 negative control outcomes, all for the exposures of ACE inhibitors (compared to -#' thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method +#' thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method #' Evaluation Benchmark, but follows the same principles. It is intended to be used when developing #' methods, leaving the Methods Benchark untouched until a final evaluation of the method, thus preventing #' 'training' on the evaluation set. The negative controls are borrowed from the LEGEND Hypertension study. diff --git a/R/Metrics.R b/R/Metrics.R index d3de789..86eacdd 100644 --- a/R/Metrics.R +++ b/R/Metrics.R @@ -44,7 +44,15 @@ computeMetrics <- function(logRr, seLogRr = NULL, ci95Lb = NULL, ci95Ub = NULL, p = NULL, trueLogRr) { # data <- EmpiricalCalibration::simulateControls(n = 50 * 3, mean = 0.25, sd = 0.25, trueLogRr = # log(c(1, 2, 4))); logRr <- data$logRr; seLogRr <- data$seLogRr; trueLogRr <- data$trueLogRr - + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertNumeric(logRr, min.len = 1, add = errorMessages) + checkmate::assertNumeric(seLogRr, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(ci95Lb, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(ci95Ub, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(p, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(trueLogRr, len = length(logRr), add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) + if (is.null(seLogRr) && is.null(ci95Lb)) { stop("Must specify either standard error or confidence interval") } @@ -70,14 +78,14 @@ computeMetrics <- function(logRr, seLogRr = NULL, ci95Lb = NULL, ci95Ub = NULL, } else { data$p <- p } - + idx <- is.na(data$logRr) | is.infinite(data$logRr) | is.na(data$seLogRr) | is.infinite(data$seLogRr) data$logRr[idx] <- 0 data$seLogRr[idx] <- 999 data$ci95Lb[idx] <- 0 data$ci95Ub[idx] <- 999 data$p[idx] <- 1 - + nonEstimable <- round(mean(data$seLogRr >= 99), 2) roc <- pROC::roc(data$trueLogRr > 0, data$logRr, algorithm = 3) auc <- round(pROC::auc(roc), 2) @@ -198,19 +206,19 @@ packageOhdsiBenchmarkResults <- function(estimates, checkmate::assertCharacter(databaseName, len = 1, add = errorMessages) checkmate::assertCharacter(exportFolder, len = 1, add = errorMessages) checkmate::reportAssertions(collection = errorMessages) - + if (!file.exists(exportFolder)) { dir.create(exportFolder, recursive = TRUE) } - + # Create full grid of controls (including those that did not make it in the database: if (referenceSet == "ohdsiMethodsBenchmark") { ohdsiNegativeControls <- readRDS(system.file("ohdsiNegativeControls.rds", - package = "MethodEvaluation" + package = "MethodEvaluation" )) } else { ohdsiNegativeControls <- readRDS(system.file("ohdsiDevelopmentNegativeControls.rds", - package = "MethodEvaluation" + package = "MethodEvaluation" )) } ohdsiNegativeControls$oldOutcomeId <- ohdsiNegativeControls$outcomeId @@ -238,7 +246,7 @@ packageOhdsiBenchmarkResults <- function(estimates, fullGrid$targetEffectSize[idx] ) allControls <- merge(controlSummary, fullGrid, all.y = TRUE) - + .packageBenchmarkResults( allControls = allControls, analysisRef = analysisRef, @@ -274,7 +282,7 @@ packageOhdsiBenchmarkResults <- function(estimates, by = join_by("analysisId") ) %>% mutate(database = databaseName) - + # Perform empirical calibration: # subset = subsets[[2]] calibrate <- function(subset) { @@ -304,7 +312,7 @@ packageOhdsiBenchmarkResults <- function(estimates, model = model ) null <- EmpiricalCalibration::fitNull(logRr = subsetMinusOne$logRr[subsetMinusOne$targetEffectSize == - 1], seLogRr = subsetMinusOne$seLogRr[subsetMinusOne$targetEffectSize == 1]) + 1], seLogRr = subsetMinusOne$seLogRr[subsetMinusOne$targetEffectSize == 1]) caliP <- EmpiricalCalibration::calibrateP( null = null, logRr = one$logRr, @@ -433,7 +441,7 @@ packageCustomBenchmarkResults <- function(estimates, checkmate::assertCharacter(databaseName, len = 1, add = errorMessages) checkmate::assertCharacter(exportFolder, len = 1, add = errorMessages) checkmate::reportAssertions(collection = errorMessages) - + trueEffecSizes <- c(1, unique(synthesisSummary$targetEffectSize)) negativeControls <- negativeControls %>% mutate(stratum = if_else(.data$type == "Outcome control", .data$targetId, .data$outcomeId)) %>% @@ -489,6 +497,15 @@ computeOhdsiBenchmarkMetrics <- function(exportFolder, trueEffectSize = "Overall", calibrated = FALSE, comparative = FALSE) { + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertCharacter(exportFolder, len = 1, add = errorMessages) + checkmate::assertNumeric(mdrr, len = 1, add = errorMessages) + checkmate::assertCharacter(stratum, len = 1, add = errorMessages) + checkmate::assertAtomic(trueEffectSize, len = 1, add = errorMessages) + checkmate::assertLogical(calibrated, len = 1, add = errorMessages) + checkmate::assertLogical(comparative, len = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) + # Load and prepare estimates of all methods files <- list.files(exportFolder, "estimates.*csv", full.names = TRUE) estimates <- lapply(files, read.csv) @@ -510,12 +527,12 @@ computeOhdsiBenchmarkMetrics <- function(exportFolder, estimates$calCi95Lb[idx] <- 0 estimates$calCi95Ub[idx] <- 999 estimates$calP[is.na(estimates$calP)] <- 1 - + # Load and prepare analysis refs files <- list.files(exportFolder, "analysisRef.*csv", full.names = TRUE) analysisRef <- lapply(files, read.csv) analysisRef <- do.call("rbind", analysisRef) - + # Apply selection criteria subset <- estimates if (mdrr != "All") { @@ -534,7 +551,7 @@ computeOhdsiBenchmarkMetrics <- function(exportFolder, subset$ci95Ub <- subset$calCi95Ub subset$p <- subset$calP } - + # Compute metrics combis <- unique(subset[, c("database", "method", "analysisId")]) if (trueEffectSize == "Overall") { @@ -566,7 +583,7 @@ computeOhdsiBenchmarkMetrics <- function(exportFolder, # trueRr <- input$trueRr computeMetrics <- function(i) { forEval <- subset[subset$method == combis$method[i] & subset$analysisId == combis$analysisId[i] & - subset$targetEffectSize == trueEffectSize, ] + subset$targetEffectSize == trueEffectSize, ] mse <- round(mean((forEval$logRr - log(forEval$trueEffectSize))^2), 2) coverage <- round( mean(forEval$ci95Lb < forEval$trueEffectSize & forEval$ci95Ub > forEval$trueEffectSize), @@ -580,7 +597,7 @@ computeOhdsiBenchmarkMetrics <- function(exportFolder, nonEstimable <- round(mean(forEval$seLogRr == 999), 2) } else { negAndPos <- subset[subset$method == combis$method[i] & subset$analysisId == combis$analysisId[i] & - (subset$targetEffectSize == trueEffectSize | subset$targetEffectSize == 1), ] + (subset$targetEffectSize == trueEffectSize | subset$targetEffectSize == 1), ] roc <- pROC::roc(negAndPos$targetEffectSize > 1, negAndPos$logRr, algorithm = 3) auc <- round(pROC::auc(roc), 2) type1 <- NA diff --git a/R/Plots.R b/R/Plots.R index 94a31df..2ee562c 100644 --- a/R/Plots.R +++ b/R/Plots.R @@ -3,13 +3,13 @@ # Copyright 2023 Observational Health Data Sciences and Informatics # # This file is part of MethodEvaluation -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,22 +29,32 @@ #' #' @export plotRocsInjectedSignals <- function(logRr, trueLogRr, showAucs, fileName = NULL) { + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertNumeric(logRr, min.len = 1, add = errorMessages) + checkmate::assertNumeric(trueLogRr, len = length(logRr), add = errorMessages) + checkmate::assertLogical(showAucs, len = 1, add = errorMessages) + checkmate::assertCharacter(fileName, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) + if (any(is.na(logRr))) { warning("Some estimates are NA, removing prior to computing AUCs") trueLogRr <- trueLogRr[!is.na(logRr)] logRr <- logRr[!is.na(logRr)] } trueLogRrLevels <- unique(trueLogRr) - if (all(trueLogRrLevels != 0)) + if (all(trueLogRrLevels != 0)) { stop("Requiring at least one true relative risk of 1") + } allData <- data.frame() aucs <- c() trueRrs <- c() for (trueLogRrLevel in trueLogRrLevels) { if (trueLogRrLevel != 0) { - data <- data.frame(logRr = logRr[trueLogRr == 0 | trueLogRr == trueLogRrLevel], - trueLogRr = trueLogRr[trueLogRr == 0 | trueLogRr == trueLogRrLevel]) + data <- data.frame( + logRr = logRr[trueLogRr == 0 | trueLogRr == trueLogRrLevel], + trueLogRr = trueLogRr[trueLogRr == 0 | trueLogRr == trueLogRrLevel] + ) data$truth <- data$trueLogRr != 0 roc <- pROC::roc(data$truth, data$logRr, algorithm = 3) @@ -52,19 +62,23 @@ plotRocsInjectedSignals <- function(logRr, trueLogRr, showAucs, fileName = NULL) aucs <- c(aucs, pROC::auc(roc)) trueRrs <- c(trueRrs, exp(trueLogRrLevel)) } - data <- data.frame(sens = roc$sensitivities, - fpRate = 1 - roc$specificities, - trueRr = exp(trueLogRrLevel)) + data <- data.frame( + sens = roc$sensitivities, + fpRate = 1 - roc$specificities, + trueRr = exp(trueLogRrLevel) + ) data <- data[order(data$sens, data$fpRate), ] allData <- rbind(allData, data) } } allData$trueRr <- as.factor(allData$trueRr) - plot <- ggplot2::ggplot(allData, ggplot2::aes(x = .data$fpRate, - y = .data$sens, - group = .data$trueRr, - color = .data$trueRr, - fill = .data$trueRr)) + + plot <- ggplot2::ggplot(allData, ggplot2::aes( + x = .data$fpRate, + y = .data$sens, + group = .data$trueRr, + color = .data$trueRr, + fill = .data$trueRr + )) + ggplot2::geom_abline(intercept = 0, slope = 1) + ggplot2::geom_line(alpha = 0.5, size = 1) + ggplot2::scale_x_continuous("1 - specificity") + @@ -74,19 +88,24 @@ plotRocsInjectedSignals <- function(logRr, trueLogRr, showAucs, fileName = NULL) aucs <- data.frame(auc = aucs, trueRr = trueRrs) aucs <- aucs[order(-aucs$trueRr), ] for (i in 1:nrow(aucs)) { - label <- paste0("True RR = ", - format(round(aucs$trueRr[i], 2), nsmall = 2), - ": AUC = ", - format(round(aucs$auc[i], 2), nsmall = 2)) - plot <- plot + ggplot2::geom_text(label = label, - x = 1, - y = (i - 1) * 0.1, - hjust = 1, - color = "#000000") + label <- paste0( + "True RR = ", + format(round(aucs$trueRr[i], 2), nsmall = 2), + ": AUC = ", + format(round(aucs$auc[i], 2), nsmall = 2) + ) + plot <- plot + ggplot2::geom_text( + label = label, + x = 1, + y = (i - 1) * 0.1, + hjust = 1, + color = "#000000" + ) } } - if (!is.null(fileName)) + if (!is.null(fileName)) { ggplot2::ggsave(fileName, plot, width = 5.5, height = 4.5, dpi = 400) + } return(plot) } @@ -107,11 +126,21 @@ plotRocsInjectedSignals <- function(logRr, trueLogRr, showAucs, fileName = NULL) #' #' @export plotCoverageInjectedSignals <- function(logRr, seLogRr, trueLogRr, region = 0.95, fileName = NULL) { - data <- data.frame(logRr = logRr, - logLb95Rr = logRr + qnorm((1 - region)/2) * seLogRr, - logUb95Rr = logRr + qnorm(1 - (1 - region)/2) * seLogRr, - trueLogRr = trueLogRr, - trueRr = round(exp(trueLogRr), 2)) + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertNumeric(logRr, min.len = 1, add = errorMessages) + checkmate::assertNumeric(seLogRr, len = length(logRr), add = errorMessages) + checkmate::assertNumeric(trueLogRr, len = length(logRr), add = errorMessages) + checkmate::assertNumeric(region, len = 1, lower = 0, upper = 1, add = errorMessages) + checkmate::assertCharacter(fileName, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) + + data <- data.frame( + logRr = logRr, + logLb95Rr = logRr + qnorm((1 - region) / 2) * seLogRr, + logUb95Rr = logRr + qnorm(1 - (1 - region) / 2) * seLogRr, + trueLogRr = trueLogRr, + trueRr = round(exp(trueLogRr), 2) + ) if (any(is.na(data$logRr))) { warning("Some estimates are NA, removing prior to computing coverage") data <- data[!is.na(data$logRr), ] @@ -119,15 +148,17 @@ plotCoverageInjectedSignals <- function(logRr, seLogRr, trueLogRr, region = 0.95 vizD <- data.frame() for (trueRr in unique(data$trueRr)) { subset <- data[data$trueRr == trueRr, ] - d <- data.frame(trueRr = trueRr, group = c("Below CI", - "Within CI", - "Above CI"), fraction = 0, pos = 0) + d <- data.frame(trueRr = trueRr, group = c( + "Below CI", + "Within CI", + "Above CI" + ), fraction = 0, pos = 0) d$fraction[1] <- mean(subset$trueLogRr < subset$logLb95Rr) d$fraction[2] <- mean(subset$trueLogRr >= subset$logLb95Rr & subset$trueLogRr <= subset$logUb95Rr) d$fraction[3] <- mean(subset$trueLogRr > subset$logUb95Rr) - d$pos[1] <- d$fraction[1]/2 - d$pos[2] <- d$fraction[1] + (d$fraction[2]/2) - d$pos[3] <- d$fraction[1] + d$fraction[2] + (d$fraction[3]/2) + d$pos[1] <- d$fraction[1] / 2 + d$pos[2] <- d$fraction[1] + (d$fraction[2] / 2) + d$pos[3] <- d$fraction[1] + d$fraction[2] + (d$fraction[3] / 2) vizD <- rbind(vizD, d) } vizD$pos <- sapply(vizD$pos, function(x) { @@ -138,23 +169,26 @@ plotCoverageInjectedSignals <- function(logRr, seLogRr, trueLogRr, region = 0.95 vizD$group <- factor(vizD$group, levels = c("Below CI", "Within CI", "Above CI")) theme <- ggplot2::element_text(colour = "#000000", size = 10) plot <- with(vizD, { - ggplot2::ggplot(vizD, ggplot2::aes(x = as.factor(trueRr), y = fraction)) + - ggplot2::geom_bar(ggplot2::aes(fill = group), stat = "identity", position = "stack", alpha = 0.8) + - ggplot2::scale_fill_manual(values = c("#174a9f", "#f9dd75", "#f15222")) + - ggplot2::geom_text(ggplot2::aes(label = label, y = pos), size = 3) + - ggplot2::scale_x_discrete("True relative risk") + - ggplot2::scale_y_continuous("Coverage") + - ggplot2::theme(panel.grid.minor = ggplot2::element_blank(), - panel.background = ggplot2::element_rect(fill = "#FAFAFA", colour = NA), - panel.grid.major = ggplot2::element_blank(), - axis.ticks = ggplot2::element_blank(), - axis.text.y = ggplot2::element_blank(), - axis.text.x = theme, - legend.key = ggplot2::element_blank(), - legend.position = "right") + ggplot2::ggplot(vizD, ggplot2::aes(x = as.factor(trueRr), y = fraction)) + + ggplot2::geom_bar(ggplot2::aes(fill = group), stat = "identity", position = "stack", alpha = 0.8) + + ggplot2::scale_fill_manual(values = c("#174a9f", "#f9dd75", "#f15222")) + + ggplot2::geom_text(ggplot2::aes(label = label, y = pos), size = 3) + + ggplot2::scale_x_discrete("True relative risk") + + ggplot2::scale_y_continuous("Coverage") + + ggplot2::theme( + panel.grid.minor = ggplot2::element_blank(), + panel.background = ggplot2::element_rect(fill = "#FAFAFA", colour = NA), + panel.grid.major = ggplot2::element_blank(), + axis.ticks = ggplot2::element_blank(), + axis.text.y = ggplot2::element_blank(), + axis.text.x = theme, + legend.key = ggplot2::element_blank(), + legend.position = "right" + ) }) - if (!is.null(fileName)) + if (!is.null(fileName)) { ggplot2::ggsave(fileName, plot, width = 5, height = 3.5, dpi = 400) + } return(plot) } @@ -163,11 +197,11 @@ plotCoverageInjectedSignals <- function(logRr, seLogRr, trueLogRr, region = 0.95 #' @param logRr A numeric vector of effect estimates on the log scale. #' @param seLogRr The standard error of the log of the effect estimates. Hint: often the standard #' error = (log() - log())/qnorm(0.025). If not provided the standard error will be inferred from +#' estimate>))/qnorm(0.025). If not provided the standard error will be inferred from #' the 95 percent confidence interval. -#' @param ci95Lb The lower bound of the 95 percent confidence interval. IF not provided it will be +#' @param ci95Lb The lower bound of the 95 percent confidence interval. IF not provided it will be #' inferred from the standard error. -#' @param ci95Ub The upper bound of the 95 percent confidence interval. IF not provided it will be +#' @param ci95Ub The upper bound of the 95 percent confidence interval. IF not provided it will be #' inferred from the standard error. #' @param trueLogRr A vector of the true effect sizes #' @param estimateType A character string to denote the effect size estimate type. Used for the x-axis @@ -175,19 +209,38 @@ plotCoverageInjectedSignals <- function(logRr, seLogRr, trueLogRr, region = 0.95 #' @param fileName Name of the file where the plot should be saved, for example 'plot.png'. See the #' function \code{ggsave} in the ggplot2 package for supported file formats. #' @param title An optional title to display above the plot. -#' +#' #' @return #' A Ggplot object. Use the \code{ggsave} function to save to file. -#' +#' #' @export -plotControls <- function(logRr, seLogRr = NULL, ci95Lb = NULL, ci95Ub = NULL, trueLogRr, estimateType = "relative risk", fileName = NULL, title) { +plotControls <- function(logRr, + seLogRr = NULL, + ci95Lb = NULL, + ci95Ub = NULL, + trueLogRr, + estimateType = "relative risk", + fileName = NULL, + title = NULL) { + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertNumeric(logRr, min.len = 1, add = errorMessages) + checkmate::assertNumeric(seLogRr, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(ci95Lb, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(ci95Ub, len = length(logRr), null.ok = TRUE, add = errorMessages) + checkmate::assertNumeric(trueLogRr, len = length(logRr), add = errorMessages) + checkmate::assertCharacter(estimateType, len = 1, add = errorMessages) + checkmate::assertCharacter(fileName, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::assertCharacter(title, null.ok = TRUE, len = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) if (is.null(seLogRr) && is.null(ci95Lb)) { stop("Must specify either standard error or confidence interval") } - data <- data.frame(logRr = logRr, - trueLogRr = trueLogRr) + data <- data.frame( + logRr = logRr, + trueLogRr = trueLogRr + ) if (is.null(seLogRr)) { - data$seLogRr <- (log(ci95Ub) - log(ci95Lb)) / (2*qnorm(0.975)) + data$seLogRr <- (log(ci95Ub) - log(ci95Lb)) / (2 * qnorm(0.975)) } else { data$seLogRr <- seLogRr } @@ -207,74 +260,88 @@ plotControls <- function(logRr, seLogRr = NULL, ci95Lb = NULL, ci95Ub = NULL, tr temp1$nLabel <- paste0(formatC(temp1$Significant, big.mark = ","), " estimates") temp1$Significant <- NULL - temp2$meanLabel <- paste0(formatC(100 * (1 - temp2$Significant), digits = 1, format = "f"), - "% of CIs includes ", - substr(as.character(temp2$Group), - start = 21, - stop = nchar(as.character(temp2$Group)))) + temp2$meanLabel <- paste0( + formatC(100 * (1 - temp2$Significant), digits = 1, format = "f"), + "% of CIs includes ", + substr(as.character(temp2$Group), + start = 21, + stop = nchar(as.character(temp2$Group)) + ) + ) temp2$Significant <- NULL dd <- merge(temp1, temp2) dd$tes <- as.numeric(substr(as.character(dd$Group), start = 21, - stop = nchar(as.character(dd$Group)))) + stop = nchar(as.character(dd$Group)) + )) breaks <- c(0.25, 0.5, 1, 2, 4, 6, 8) theme <- ggplot2::element_text(colour = "#000000", size = 14) themeRA <- ggplot2::element_text(colour = "#000000", size = 14, hjust = 1) - alpha <- 1 - min(0.95 * (nrow(data)/nrow(dd)/50000)^0.1, 0.95) + alpha <- 1 - min(0.95 * (nrow(data) / nrow(dd) / 50000)^0.1, 0.95) plot <- ggplot2::ggplot(data, ggplot2::aes(x = logRr, y = seLogRr)) + ggplot2::geom_vline(xintercept = log(breaks), colour = "#CCCCCC", lty = 1, size = 0.5) + - ggplot2::geom_abline(ggplot2::aes(intercept = (-log(.data$tes))/qnorm(0.025), slope = 1/qnorm(0.025)), - colour = rgb(0.8, 0, 0), - linetype = "dashed", - size = 1, - alpha = 0.5, - data = dd) + - ggplot2::geom_abline(ggplot2::aes(intercept = (-log(.data$tes))/qnorm(0.975), slope = 1/qnorm(0.975)), - colour = rgb(0.8, 0, 0), - linetype = "dashed", - size = 1, - alpha = 0.5, - data = dd) + + ggplot2::geom_abline(ggplot2::aes(intercept = (-log(.data$tes)) / qnorm(0.025), slope = 1 / qnorm(0.025)), + colour = rgb(0.8, 0, 0), + linetype = "dashed", + size = 1, + alpha = 0.5, + data = dd + ) + + ggplot2::geom_abline(ggplot2::aes(intercept = (-log(.data$tes)) / qnorm(0.975), slope = 1 / qnorm(0.975)), + colour = rgb(0.8, 0, 0), + linetype = "dashed", + size = 1, + alpha = 0.5, + data = dd + ) + ggplot2::geom_point(size = 2, color = rgb(0, 0, 0, alpha = 0.05), alpha = alpha, shape = 16) + ggplot2::geom_hline(yintercept = 0) + - ggplot2::geom_label(x = log(0.26), - y = 0.96, - alpha = 1, - hjust = "left", - ggplot2::aes(label = .data$nLabel), - size = 5, - data = dd) + - ggplot2::geom_label(x = log(0.26), - y = 0.8, - alpha = 1, - hjust = "left", - ggplot2::aes(label = .data$meanLabel), - size = 5, - data = dd) + + ggplot2::geom_label( + x = log(0.26), + y = 0.96, + alpha = 1, + hjust = "left", + ggplot2::aes(label = .data$nLabel), + size = 5, + data = dd + ) + + ggplot2::geom_label( + x = log(0.26), + y = 0.8, + alpha = 1, + hjust = "left", + ggplot2::aes(label = .data$meanLabel), + size = 5, + data = dd + ) + ggplot2::scale_x_continuous(paste("Estimated", estimateType), - limits = log(c(0.25, 10)), - breaks = log(breaks), - labels = breaks) + + limits = log(c(0.25, 10)), + breaks = log(breaks), + labels = breaks + ) + ggplot2::scale_y_continuous("Standard Error", limits = c(0, 1)) + ggplot2::facet_grid(. ~ Group) + - ggplot2::theme(panel.grid.minor = ggplot2::element_blank(), - panel.background = ggplot2::element_blank(), - panel.grid.major = ggplot2::element_blank(), - axis.ticks = ggplot2::element_blank(), - axis.text.y = themeRA, - axis.text.x = theme, - axis.title = theme, - legend.key = ggplot2::element_blank(), - strip.text.x = theme, - strip.text.y = theme, - strip.background = ggplot2::element_blank(), - legend.position = "none") - if (!missing(title)) { + ggplot2::theme( + panel.grid.minor = ggplot2::element_blank(), + panel.background = ggplot2::element_blank(), + panel.grid.major = ggplot2::element_blank(), + axis.ticks = ggplot2::element_blank(), + axis.text.y = themeRA, + axis.text.x = theme, + axis.title = theme, + legend.key = ggplot2::element_blank(), + strip.text.x = theme, + strip.text.y = theme, + strip.background = ggplot2::element_blank(), + legend.position = "none" + ) + if (!is.null(title)) { plot <- plot + ggplot2::ggtitle(title) } - if (!is.null(fileName)) + if (!is.null(fileName)) { ggplot2::ggsave(fileName, plot, width = 1.6 + 3 * nrow(dd), height = 2.8, dpi = 400) + } return(plot) } diff --git a/R/PositiveControlSynthesis.R b/R/PositiveControlSynthesis.R index 57411af..9072885 100644 --- a/R/PositiveControlSynthesis.R +++ b/R/PositiveControlSynthesis.R @@ -194,16 +194,38 @@ synthesizePositiveControls <- function(connectionDetails, warning("The 'oracleTempSchema' argument is deprecated. Use 'tempEmulationSchema' instead.") tempEmulationSchema <- oracleTempSchema } - if (min(effectSizes) < 1) { - stop("Effect sizes smaller than 1 are currently not supported") - } - if (modelType != "poisson" && modelType != "survival") { - stop(paste0( - "Unknown modelType '", - modelType, - "', please select either 'poisson' or 'survival'" - )) - } + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertClass(connectionDetails, "ConnectionDetails", add = errorMessages) + checkmate::assertCharacter(cdmDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(tempEmulationSchema, len = 1, null.ok = TRUE, add = errorMessages) + checkmate::assertCharacter(exposureDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(exposureTable, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeDatabaseSchema, len = 1, add = errorMessages) + checkmate::assertCharacter(outcomeTable, len = 1, add = errorMessages) + checkmate::assertLogical(createOutputTable, len = 1, add = errorMessages) + checkmate::assertDataFrame(exposureOutcomePairs, add = errorMessages) + checkmate::assertNames(colnames(exposureOutcomePairs), must.include = c("exposureId", "outcomeId"), add = errorMessages) + checkmate::assertChoice(modelType, c("poisson", "survival"), add = errorMessages) + checkmate::assertInt(minOutcomeCountForModel, lower = 0, add = errorMessages) + checkmate::assertInt(minOutcomeCountForInjection, lower = 0, add = errorMessages) + checkmate::assertInt(minModelCount, lower = 0, add = errorMessages) + checkmate::assertList(covariateSettings, add = errorMessages) + checkmate::assertClass(prior, "cyclopsPrior", add = errorMessages) + checkmate::assertClass(control, "cyclopsControl", add = errorMessages) + checkmate::assertLogical(firstExposureOnly, len = 1, add = errorMessages) + checkmate::assertInt(washoutPeriod, lower = 0, add = errorMessages) + checkmate::assertInt(riskWindowStart, add = errorMessages) + checkmate::assertInt(riskWindowEnd, add = errorMessages) + checkmate::assertLogical(removePeopleWithPriorOutcomes, len = 1, add = errorMessages) + checkmate::assertInt(maxSubjectsForModel, lower = 0, add = errorMessages) + checkmate::assertNumeric(effectSizes, lower = 1, min.len = 1, add = errorMessages) + checkmate::assertNumeric(precision, lower = 0, len = 1, add = errorMessages) + checkmate::assertInt(outputIdOffset, lower = 0, add = errorMessages) + checkmate::assertCharacter(workFolder, len = 1, add = errorMessages) + checkmate::assertCharacter(cdmVersion, len = 1, add = errorMessages) + checkmate::assertInt(modelThreads, lower = 1, add = errorMessages) + checkmate::assertInt(generationThreads, lower = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) if (!firstExposureOnly && addIntentToTreat) { stop("Cannot have addIntentToTreat = TRUE and firstExposureOnly = FALSE at the same time") } diff --git a/R/ShinyApps.R b/R/ShinyApps.R index db8292d..6045914 100644 --- a/R/ShinyApps.R +++ b/R/ShinyApps.R @@ -1,13 +1,13 @@ # Copyright 2023 Observational Health Data Sciences and Informatics # # This file is part of MethodEvaluation -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,6 +27,11 @@ #' #' @export launchMethodEvaluationApp <- function(exportFolder, launch.browser = TRUE) { + errorMessages <- checkmate::makeAssertCollection() + checkmate::assertCharacter(exportFolder, len = 1, add = errorMessages) + checkmate::assertLogical(launch.browser, len = 1, add = errorMessages) + checkmate::reportAssertions(collection = errorMessages) + ensure_installed("DT") appDir <- system.file("shinyApps", "MethodEvalViewer", package = "MethodEvaluation") shinySettings <- list(exportFolder = exportFolder) diff --git a/R/SignalInjection.R b/R/SignalInjection.R index a4d1cd1..0bc35cf 100644 --- a/R/SignalInjection.R +++ b/R/SignalInjection.R @@ -135,25 +135,29 @@ injectSignals <- function(connectionDetails, modelType = "poisson", minOutcomeCountForModel = 100, minOutcomeCountForInjection = 25, - covariateSettings = FeatureExtraction::createCovariateSettings(useDemographicsAgeGroup = TRUE, - useDemographicsGender = TRUE, - useDemographicsIndexYear = TRUE, - useDemographicsIndexMonth = TRUE, - useConditionGroupEraLongTerm = TRUE, - useDrugGroupEraLongTerm = TRUE, - useProcedureOccurrenceLongTerm = TRUE, - useMeasurementLongTerm = TRUE, - useObservationLongTerm = TRUE, - useCharlsonIndex = TRUE, - useDcsi = TRUE, - useChads2Vasc = TRUE, - longTermStartDays = 365, - endDays = 0), + covariateSettings = FeatureExtraction::createCovariateSettings( + useDemographicsAgeGroup = TRUE, + useDemographicsGender = TRUE, + useDemographicsIndexYear = TRUE, + useDemographicsIndexMonth = TRUE, + useConditionGroupEraLongTerm = TRUE, + useDrugGroupEraLongTerm = TRUE, + useProcedureOccurrenceLongTerm = TRUE, + useMeasurementLongTerm = TRUE, + useObservationLongTerm = TRUE, + useCharlsonIndex = TRUE, + useDcsi = TRUE, + useChads2Vasc = TRUE, + longTermStartDays = 365, + endDays = 0 + ), prior = Cyclops::createPrior("laplace", exclude = 0, useCrossValidation = TRUE), - control = Cyclops::createControl(cvType = "auto", - startingVariance = 0.1, - noiseLevel = "quiet", - threads = 10), + control = Cyclops::createControl( + cvType = "auto", + startingVariance = 0.1, + noiseLevel = "quiet", + threads = 10 + ), firstExposureOnly = FALSE, washoutPeriod = 183, riskWindowStart = 0, @@ -171,44 +175,46 @@ injectSignals <- function(connectionDetails, modelThreads = 1, generationThreads = 1) { .Deprecated("synthesizePositiveControls") - + if (addExposureDaysToEnd) { endAnchor <- "cohort end" } else { endAnchor <- "cohort start" } - - synthesizePositiveControls(connectionDetails = connectionDetails, - cdmDatabaseSchema = cdmDatabaseSchema, - oracleTempSchema = oracleTempSchema, - exposureDatabaseSchema = exposureDatabaseSchema, - exposureTable = exposureTable, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - outputDatabaseSchema = outputDatabaseSchema, - outputTable = outputTable, - createOutputTable = createOutputTable, - exposureOutcomePairs = exposureOutcomePairs, - modelType = modelType, - minOutcomeCountForModel = minOutcomeCountForModel, - minOutcomeCountForInjection = minOutcomeCountForInjection, - covariateSettings = covariateSettings, - prior = prior, - control = control, - firstExposureOnly = firstExposureOnly, - washoutPeriod = washoutPeriod, - riskWindowStart = riskWindowStart, - riskWindowEnd = riskWindowEnd, - endAnchor = endAnchor, - addIntentToTreat = addIntentToTreat, - firstOutcomeOnly = firstOutcomeOnly, - removePeopleWithPriorOutcomes = removePeopleWithPriorOutcomes, - maxSubjectsForModel = maxSubjectsForModel, - effectSizes = effectSizes, - precision = precision, - outputIdOffset = outputIdOffset, - workFolder = workFolder, - cdmVersion = cdmVersion, - modelThreads = modelThreads, - generationThreads = generationThreads) + + synthesizePositiveControls( + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + oracleTempSchema = oracleTempSchema, + exposureDatabaseSchema = exposureDatabaseSchema, + exposureTable = exposureTable, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + outputDatabaseSchema = outputDatabaseSchema, + outputTable = outputTable, + createOutputTable = createOutputTable, + exposureOutcomePairs = exposureOutcomePairs, + modelType = modelType, + minOutcomeCountForModel = minOutcomeCountForModel, + minOutcomeCountForInjection = minOutcomeCountForInjection, + covariateSettings = covariateSettings, + prior = prior, + control = control, + firstExposureOnly = firstExposureOnly, + washoutPeriod = washoutPeriod, + riskWindowStart = riskWindowStart, + riskWindowEnd = riskWindowEnd, + endAnchor = endAnchor, + addIntentToTreat = addIntentToTreat, + firstOutcomeOnly = firstOutcomeOnly, + removePeopleWithPriorOutcomes = removePeopleWithPriorOutcomes, + maxSubjectsForModel = maxSubjectsForModel, + effectSizes = effectSizes, + precision = precision, + outputIdOffset = outputIdOffset, + workFolder = workFolder, + cdmVersion = cdmVersion, + modelThreads = modelThreads, + generationThreads = generationThreads + ) } diff --git a/docs/404.html b/docs/404.html index 8f6ebf0..b546ad5 100644 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -42,7 +42,7 @@ Reference
  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/articles/OhdsiMethodsBenchmark.html b/docs/articles/OhdsiMethodsBenchmark.html index b2bd701..92a5b8f 100644 --- a/docs/articles/OhdsiMethodsBenchmark.html +++ b/docs/articles/OhdsiMethodsBenchmark.html @@ -33,7 +33,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -43,7 +43,7 @@ Reference
  • -
    vignettes/OhdsiMethodsBenchmark.Rmd @@ -96,259 +97,440 @@

    2022-01-21

    Introduction

    -

    When designing an observational study, there are many study designs to choose from, and many additional choices to make, and it is often unclear how these choices will affect the accuracy of the results. (e.g. If I match on propensity scores, will that lead to more or less bias than when I stratify? What about power?) The literature contains many papers evaluating one design choice at a time, but often with unsatisfactory scientific rigor; typically, a method is evaluated on one or two exemplar study from which we cannot generalize, or by using simulations which have an unclear relationship with the real world.

    -

    This vignette describes the OHDSI Methods Benchmark for evaluating population-level estimation methods, a benchmark that can inform on how a particular study design and set of analysis choices perform in general. The benchmark consists of a gold standard of research hypothesis where the truth is known, and a set of metrics for characterizing a methods performance when applied to the gold standard. We distinguish between two types of tasks: (1) estimation of the average effect of an exposure on an outcome relative to no exposure (effect estimation), and (2) estimation of the average effect of an exposure on an outcome relative to another exposure (comparative effect estimation). The benchmark allows evaluation of a method on either or both tasks.

    -

    This benchmark builds on previous efforts in EU-ADR, OMOP, and the WHO, adding the ability to evaluate methods on both tasks, and using synthetic positive controls as real positive controls have been observed to be problematic in the past.

    +

    When designing an observational study, there are many study designs +to choose from, and many additional choices to make, and it is often +unclear how these choices will affect the accuracy of the results. +(e.g. If I match on propensity scores, will that lead to more or less +bias than when I stratify? What about power?) The literature contains +many papers evaluating one design choice at a time, but often with +unsatisfactory scientific rigor; typically, a method is evaluated on one +or two exemplar study from which we cannot generalize, or by using +simulations which have an unclear relationship with the real world.

    +

    This vignette describes the OHDSI Methods Benchmark for evaluating +population-level estimation methods, a benchmark that can inform on how +a particular study design and set of analysis choices perform in +general. The benchmark consists of a gold standard of +research hypothesis where the truth is known, and a set of +metrics for characterizing a methods performance when +applied to the gold standard. We distinguish between two types of tasks: +(1) estimation of the average effect of an exposure on an outcome +relative to no exposure (effect estimation), and (2) estimation of the +average effect of an exposure on an outcome relative to another exposure +(comparative effect estimation). The benchmark allows evaluation of a +method on either or both tasks.

    +

    This benchmark builds on previous efforts in EU-ADR, OMOP, and the +WHO, adding the ability to evaluate methods on both tasks, and using +synthetic positive controls as real positive controls have been observed +to be problematic in the past.

    Gold standard

    -

    The gold standard comprises 800 entries, with each item specifying a target exposure, comparator exposure, outcome, nesting cohort, and true effect size. The true effect size refers to the absolute effect of the target on the outcome. Because the comparator is always believed to have no effect on the outcome, the true effect size also holds for the relative effect of the target compared to the comparator. Thus, each entry can be used for evaluating both effect estimation and comparative effect estimation. The nesting cohort identifies a more homogeneous subgroup of the population and can be used to evaluate methods such as the nested case-control design.

    -

    Of the total set, 200 entries are real negative controls with a presumed true relative risk of 1. We select these negative controls by first picking four diverse outcomes (acute pancreatitis, gastrointestinal bleeding, inflammatory bowel disease, and stroke) and four diverse exposures (ciprofloxacin, diclofenac, metformin, and sertraline). For each of the four outcomes, we create 25 entries with target and comparator exposures that are not believed to cause the outcome. To aid the creation of these entries, we generate candidate lists of negative controls for each of the four main outcomes and four main exposures using an automated procedure, drawing on literature, product labels, and spontaneous reports. These candidates are used to construct target-comparator-outcome triplets where neither the target nor the exposure causes the outcome, and the target and comparator were either previously compared in a randomized trial per ClinicalTrials.gov, or both had the same four-digit ATC code (same indication) but not the same five-digit ATC code (different class). These candidates are ranked on prevalence of the exposures and outcome and manually reviewed until 25 were approved per initial outcome or exposure. Nesting cohorts were selected by manually reviewing the most prevalent conditions and procedures on the first day of the target or comparator treatment.

    -

    The remaining 600 entries are positive controls, which were automatically derived from the 200 negative controls by adding synthetic additional outcomes during the target exposure until a desired incidence rate ratio was achieved between before and after injection of the synthetic outcomes. The target incidence rate ratios were 1.25, 2, and 4. To preserve (measured) confounding, predictive models were fitted for each outcome during target exposure and used to generate probabilities from which the synthetic outcomes were sampled.

    +

    The gold standard comprises 800 entries, with each item specifying a +target exposure, comparator exposure, outcome, nesting cohort, and true +effect size. The true effect size refers to the absolute effect of the +target on the outcome. Because the comparator is always believed to have +no effect on the outcome, the true effect size also holds for the +relative effect of the target compared to the comparator. Thus, each +entry can be used for evaluating both effect estimation and comparative +effect estimation. The nesting cohort identifies a more homogeneous +subgroup of the population and can be used to evaluate methods such as +the nested case-control design.

    +

    Of the total set, 200 entries are real negative controls with a +presumed true relative risk of 1. We select these negative controls by +first picking four diverse outcomes (acute pancreatitis, +gastrointestinal bleeding, inflammatory bowel disease, and stroke) and +four diverse exposures (ciprofloxacin, diclofenac, metformin, and +sertraline). For each of the four outcomes, we create 25 entries with +target and comparator exposures that are not believed to cause the +outcome. To aid the creation of these entries, we generate candidate +lists of negative controls for each of the four main outcomes and four +main exposures using an automated procedure, drawing on literature, +product labels, and spontaneous reports. These candidates are used to +construct target-comparator-outcome triplets where neither the target +nor the exposure causes the outcome, and the target and comparator were +either previously compared in a randomized trial per ClinicalTrials.gov, +or both had the same four-digit ATC code (same indication) but not the +same five-digit ATC code (different class). These candidates are ranked +on prevalence of the exposures and outcome and manually reviewed until +25 were approved per initial outcome or exposure. Nesting cohorts were +selected by manually reviewing the most prevalent conditions and +procedures on the first day of the target or comparator treatment.

    +

    The remaining 600 entries are positive controls, which were +automatically derived from the 200 negative controls by adding synthetic +additional outcomes during the target exposure until a desired incidence +rate ratio was achieved between before and after injection of the +synthetic outcomes. The target incidence rate ratios were 1.25, 2, and +4. To preserve (measured) confounding, predictive models were fitted for +each outcome during target exposure and used to generate probabilities +from which the synthetic outcomes were sampled.

    Metrics

    -

    Once a method has been used to produce estimates for the gold standard the following metrics are computed:

    +

    Once a method has been used to produce estimates for the gold +standard the following metrics are computed:

    • -AUC: the ability to discriminate between positive controls and negative controls.
      • +AUC: the ability to discriminate between positive +controls and negative controls.
    • -Coverage: how often the true effect size is within the 95% confidence interval.
      • +Coverage: how often the true effect size is within +the 95% confidence interval.
    • -Mean precision: Precision is computed as 1 / (standard error)2, higher precision means narrower confidence intervals. We use the geometric mean to account for the skewed distribution of the precision.
      • +Mean precision: Precision is computed as 1 / +(standard error)2, higher precision means narrower confidence intervals. +We use the geometric mean to account for the skewed distribution of the +precision.
    • -Mean squared error (MSE): Mean squared error between the log of the effect size point-estimate and the log of the true effect size.
      • +Mean squared error (MSE): Mean squared error +between the log of the effect size point-estimate and the log of the +true effect size.
    • -Type 1 error: For negative controls, how often was the null rejected (at alpha = 0.05). This is equivalent to the false positive rate and 1 - specificity.
      • +Type 1 error: For negative controls, how often was +the null rejected (at alpha = 0.05). This is equivalent to the false +positive rate and 1 - specificity.
    • -Type 2 error: For positive controls, how often was the null not rejected (at alpha = 0.05). This is equivalent to the false negative rate and 1 - sensitivity.
      • +Type 2 error: For positive controls, how often was +the null not rejected (at alpha = 0.05). This is equivalent to the false +negative rate and 1 - sensitivity.
    • -Non-estimable: For how many of the controls was the method unable to produce an estimate? There can be various reasons why an estimate cannot be produced, for example because there were no subjects left after propensity score matching, or because no subjects remained having the outcome.
      • +Non-estimable: For how many of the controls was the +method unable to produce an estimate? There can be various reasons why +an estimate cannot be produced, for example because there were no +subjects left after propensity score matching, or because no subjects +remained having the outcome.
    -

    The Benchmark computes these metrics both overall, as well as stratified by true effect size, by each of the four initial outcomes and four initial exposures, and by amount of data as reflected by the minimum detectable relative risk (MDRR). When a method cannot estimate an effect, it returns an estimate of 1 with an infinite confidence interval.

    -

    In prior work we described a method for empirically calibrating p-values and confidence intervals. Briefly, empirical calibration estimates a systematic error distribution based on the estimates produced for the negative and positive controls. This systematic error is then taken into consideration, in addition to the random error, when computing the confidence interval and p-value. We therefore also report the performance metrics listed above after calibration, using a leave-one-out approach: for each control the error distributions are fitted on all controls except the control being calibrated and its siblings. By siblings we mean the set of a negative control and the positive controls derived from that negative control.

    +

    The Benchmark computes these metrics both overall, as well as +stratified by true effect size, by each of the four initial outcomes and +four initial exposures, and by amount of data as reflected by the +minimum detectable relative risk (MDRR). When a method cannot estimate +an effect, it returns an estimate of 1 with an infinite confidence +interval.

    +

    In prior work we described a method for empirically calibrating +p-values and confidence intervals. Briefly, empirical calibration +estimates a systematic error distribution based on the estimates +produced for the negative and positive controls. This systematic error +is then taken into consideration, in addition to the random error, when +computing the confidence interval and p-value. We therefore also report +the performance metrics listed above after calibration, using a +leave-one-out approach: for each control the error distributions are +fitted on all controls except the control being calibrated and its +siblings. By siblings we mean the set of a negative control and the +positive controls derived from that negative control.

    Running the benchmark

    -

    This section will describe how to run methods against the benchmark on a particular observational healthcare database. The database needs to conform to the OMOP Common Data Model (CDM).

    +

    This section will describe how to run methods against the benchmark +on a particular observational healthcare database. The database needs to +conform to the OMOP Common Data Model (CDM).

    Specifying locations on the server and local file system

    -

    We need to specify the following locations on the database server:

    +

    We need to specify the following locations on the database +server:

      -
    • The database schema containing the observational healthcare data in CDM format. Only read access is required to this schema.
      • -
    • A database schema and table name where outcome cohorts can be instantiated. Write access is required to this schema.
      • -
    • A database schema and table name where nesting cohorts can be instantiated. Write access is required to this schema.
      • +
    • The database schema containing the observational healthcare data in +CDM format. Only read access is required to this +schema.
      • +
    • A database schema and table name where outcome +cohorts can be instantiated. Write access is required to this +schema.
      • +
    • A database schema and table name where nesting +cohorts can be instantiated. Write access is required to this +schema.
    • -On Oracle only: a schema where temporary tables can be created. Write access is required to this schema. This is required since Oracle doesn’t fully support temp tables like other database platforms.
      • +On Oracle only: a schema where temporary +tables can be created. Write access is required to this schema. +This is required since Oracle doesn’t fully support temp tables like +other database platforms.

    We also need to specify on the local file system:

      -
    • An output folder where all intermediary and final results can be written. This should not be a network drive, since that would severely slow down the analyses.
      • +
    • An output folder where all intermediary and final +results can be written. This should not be a network drive, since that +would severely slow down the analyses.
    -

    Below is example code showing we specify how to connect to the database server, and the various required locations. MethodEvaluation uses the DatabaseConnector package, which provides the createConnectionDetails function. Type ?createConnectionDetails for the specific settings required for the various database management systems (DBMS).

    +

    Below is example code showing we specify how to connect to the +database server, and the various required locations. +MethodEvaluation uses the DatabaseConnector +package, which provides the createConnectionDetails +function. Type ?createConnectionDetails for the specific +settings required for the various database management systems +(DBMS).

    -library(MethodEvaluation)
-connectionDetails <- createConnectionDetails(dbms = "postgresql", 
-                                             server = "localhost/ohdsi", 
-                                             user = "joe", 
-                                             password = "supersecret")
-
-cdmDatabaseSchema <- "my_cdm_data"
-oracleTempSchema <- NULL
-outcomeDatabaseSchema <- "scratch"
-outcomeTable <- "ohdsi_outcomes"
-nestingCohortDatabaseSchema <- "scratch"
-nestingCohortTable <- "ohdsi_nesting_cohorts"
-outputFolder <- "/home/benchmarkOutput"
-cdmVersion <- "5"
    -

    Note that for Microsoft SQL Server, database schemas need to specify both the database and the schema, so for example cdmDatabaseSchema <- "my_cdm_data.dbo".

    +library(MethodEvaluation) +connectionDetails <- createConnectionDetails( + dbms = "postgresql", + server = "localhost/ohdsi", + user = "joe", + password = "supersecret" +) + +cdmDatabaseSchema <- "my_cdm_data" +oracleTempSchema <- NULL +outcomeDatabaseSchema <- "scratch" +outcomeTable <- "ohdsi_outcomes" +nestingCohortDatabaseSchema <- "scratch" +nestingCohortTable <- "ohdsi_nesting_cohorts" +outputFolder <- "/home/benchmarkOutput" +cdmVersion <- "5"
    +

    Note that for Microsoft SQL Server, database schemas need to specify +both the database and the schema, so for example +cdmDatabaseSchema <- "my_cdm_data.dbo".

    Creating all necessary cohorts

    -

    In order to run our population-level effect estimation method, we will need to identify the exposures and outcomes of interest. To identify exposures we will simlpy use the predefined exposure eras in the drug_era table in the CDM. However, for the outcomes custom cohorts need to be generated. In addition, some methods such as the nested case-control design require nesting cohorts to be defined. The negative control outcome and nesting cohorts can be generated using the following command:

    +

    In order to run our population-level effect estimation method, we +will need to identify the exposures and outcomes of interest. To +identify exposures we will simlpy use the predefined exposure eras in +the drug_era table in the CDM. However, for the outcomes +custom cohorts need to be generated. In addition, some methods such as +the nested case-control design require nesting cohorts to be defined. +The negative control outcome and nesting cohorts can be generated using +the following command:

    -createReferenceSetCohorts(connectionDetails = connectionDetails,
-                          oracleTempSchema = oracleTempSchema,
-                          cdmDatabaseSchema = cdmDatabaseSchema,
-                          outcomeDatabaseSchema = outcomeDatabaseSchema,
-                          outcomeTable = outcomeTable,
-                          nestingDatabaseSchema = nestingCohortDatabaseSchema,
-                          nestingTable = nestingCohortTable,
-                          referenceSet = "ohdsiMethodsBenchmark")
    -

    This will automatically create the outcome and nesting cohort tables, and populate them.

    -

    The nest step is to create the positive controls. These are generated by taking the negative controls, and adding synthetic outcomes during exposed time to achieve predefined relative risks greater than one. The synthetic outcomes are sampled based on models that are fitted for each negative control outcome, using predictors extracted from the data. These positive control outcomes are added to the outcome table.

    +createReferenceSetCohorts( + connectionDetails = connectionDetails, + oracleTempSchema = oracleTempSchema, + cdmDatabaseSchema = cdmDatabaseSchema, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + nestingDatabaseSchema = nestingCohortDatabaseSchema, + nestingTable = nestingCohortTable, + referenceSet = "ohdsiMethodsBenchmark" +)
    +

    This will automatically create the outcome and nesting cohort tables, +and populate them.

    +

    The nest step is to create the positive controls. These are generated +by taking the negative controls, and adding synthetic outcomes during +exposed time to achieve predefined relative risks greater than one. The +synthetic outcomes are sampled based on models that are fitted for each +negative control outcome, using predictors extracted from the data. +These positive control outcomes are added to the outcome table.

    -synthesizeReferenceSetPositiveControls(connectionDetails = connectionDetails,
-                                       oracleTempSchema = oracleTempSchema,
-                                       cdmDatabaseSchema = cdmDatabaseSchema,
-                                       outcomeDatabaseSchema = outcomeDatabaseSchema,
-                                       outcomeTable = outcomeTable,
-                                       maxCores = 10,
-                                       workFolder = outputFolder,
-                                       summaryFileName = file.path(outputFolder, 
-                                                                   "allControls.csv"),
-                                       referenceSet = "ohdsiMethodsBenchmark")
    -

    Fitting the outcome models can take a while. It is recommend to set the maxCores argument to the number of CPU cores available in the machine to speed up the process. Note that at the end, a file will be created called allControls.csv in the output folder containing the information on the controls:

    +synthesizeReferenceSetPositiveControls( + connectionDetails = connectionDetails, + oracleTempSchema = oracleTempSchema, + cdmDatabaseSchema = cdmDatabaseSchema, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + maxCores = 10, + workFolder = outputFolder, + summaryFileName = file.path( + outputFolder, + "allControls.csv" + ), + referenceSet = "ohdsiMethodsBenchmark" +)
    +

    Fitting the outcome models can take a while. It is recommend to set +the maxCores argument to the number of CPU cores available +in the machine to speed up the process. Note that at the end, a file +will be created called allControls.csv in the output folder +containing the information on the controls:

    -allControls <- read.csv(file.path(outputFolder, "allControls.csv"))
-head(allControls)
    -
    #>  outcomeId comparatorId targetId targetName comparatorName nestingId                  nestingName        outcomeName             type targetEffectSize trueEffectSize trueEffectSizeFirstExposure oldOutcomeId mdrrTarget
-#> 1         1      1105775  1110942 omalizumab  Aminophylline  37203741 Bronchospasm and obstruction Acute pancreatitis Exposure control                1              1                           1            1   1.421960
-#> 2         1      1110942  1140088 Dyphylline     omalizumab  37203741 Bronchospasm and obstruction Acute pancreatitis Exposure control                1              1                           1            1   1.420173
-#> 3         1      1136422   943634 epinastine levocetirizine  36009773            Rhinitis allergic Acute pancreatitis Exposure control                1              1                           1            1   1.217681
-#> 4         1      1315865 40163718  prasugrel   fondaparinux  37622411              Phlebosclerosis Acute pancreatitis Exposure control                1              1                           1            1   1.196514
-#> 5         1      1315865  1350310 cilostazol   fondaparinux  37622411              Phlebosclerosis Acute pancreatitis Exposure control                1              1                           1            1   1.233415
-#> 6         1      1336926  1311276 vardenafil      tadalafil  36919202           Sexual dysfunction Acute pancreatitis Exposure control                1              1                           1            1   1.097112
-#>   mdrrComparator
-#> 1       1.177202
-#> 2       1.421960
-#> 3       1.085877
-#> 4       1.278911
-#> 5       1.278911
-#> 6       1.060737
    +allControls <- read.csv(file.path(outputFolder, "allControls.csv")) +head(allControls)
    +
    #>  outcomeId comparatorId targetId targetName comparatorName nestingId                  nestingName        outcomeName             type targetEffectSize trueEffectSize trueEffectSizeFirstExposure oldOutcomeId mdrrTarget
+#> 1         1      1105775  1110942 omalizumab  Aminophylline  37203741 Bronchospasm and obstruction Acute pancreatitis Exposure control                1              1                           1            1   1.421960
+#> 2         1      1110942  1140088 Dyphylline     omalizumab  37203741 Bronchospasm and obstruction Acute pancreatitis Exposure control                1              1                           1            1   1.420173
+#> 3         1      1136422   943634 epinastine levocetirizine  36009773            Rhinitis allergic Acute pancreatitis Exposure control                1              1                           1            1   1.217681
+#> 4         1      1315865 40163718  prasugrel   fondaparinux  37622411              Phlebosclerosis Acute pancreatitis Exposure control                1              1                           1            1   1.196514
+#> 5         1      1315865  1350310 cilostazol   fondaparinux  37622411              Phlebosclerosis Acute pancreatitis Exposure control                1              1                           1            1   1.233415
+#> 6         1      1336926  1311276 vardenafil      tadalafil  36919202           Sexual dysfunction Acute pancreatitis Exposure control                1              1                           1            1   1.097112
+#>   mdrrComparator
+#> 1       1.177202
+#> 2       1.421960
+#> 3       1.085877
+#> 4       1.278911
+#> 5       1.278911
+#> 6       1.060737

    Running the method to evaluate

    -

    Now that we have the controls in place, we can run the method we would like to evaluate. Here we will run the SelfControlledCohort package which implements the Self-Controlled Cohort (SCC) design. We first will first define some SCC analysis settings to evaluate. Here we define two analysis settings, one using the length of exposure as the time-at-risk, and one using the first 30 days after exposure start as the time-at-risk:

    +

    Now that we have the controls in place, we can run the method we +would like to evaluate. Here we will run the +SelfControlledCohort package which implements the +Self-Controlled Cohort (SCC) design. We first will first define some SCC +analysis settings to evaluate. Here we define two analysis settings, one +using the length of exposure as the time-at-risk, and one using the +first 30 days after exposure start as the time-at-risk:

    -library(SelfControlledCohort)
-runSccArgs1 <- createRunSelfControlledCohortArgs(addLengthOfExposureExposed = TRUE,
-                                                 riskWindowStartExposed = 0,
-                                                 riskWindowEndExposed = 0,
-                                                 riskWindowEndUnexposed = -1,
-                                                 addLengthOfExposureUnexposed = TRUE,
-                                                 riskWindowStartUnexposed = -1,
-                                                 washoutPeriod = 365)
-
-sccAnalysis1 <- createSccAnalysis(analysisId = 1,
-                                  description = "Length of exposure",
-                                  runSelfControlledCohortArgs = runSccArgs1)
-
-runSccArgs2 <- createRunSelfControlledCohortArgs(addLengthOfExposureExposed = FALSE,
-                                                 riskWindowStartExposed = 0,
-                                                 riskWindowEndExposed = 30,
-                                                 riskWindowEndUnexposed = -1,
-                                                 addLengthOfExposureUnexposed = FALSE,
-                                                 riskWindowStartUnexposed = -30,
-                                                 washoutPeriod = 365)
-
-sccAnalysis2 <- createSccAnalysis(analysisId = 2,
-                                  description = "30 days of each exposure",
-                                  runSelfControlledCohortArgs = runSccArgs2)
-sccAnalysisList <- list(sccAnalysis1, sccAnalysis2)
    -

    Next, we specify all exposure-outcome pairs we want to apply the SCC design to. Note that we only use the targetId and outcomeId fields of the controls, because the SCC design can only be used for effect estimation, and does not support nesting:

    +library(SelfControlledCohort) +runSccArgs1 <- createRunSelfControlledCohortArgs( + addLengthOfExposureExposed = TRUE, + riskWindowStartExposed = 0, + riskWindowEndExposed = 0, + riskWindowEndUnexposed = -1, + addLengthOfExposureUnexposed = TRUE, + riskWindowStartUnexposed = -1, + washoutPeriod = 365 +) + +sccAnalysis1 <- createSccAnalysis( + analysisId = 1, + description = "Length of exposure", + runSelfControlledCohortArgs = runSccArgs1 +) + +runSccArgs2 <- createRunSelfControlledCohortArgs( + addLengthOfExposureExposed = FALSE, + riskWindowStartExposed = 0, + riskWindowEndExposed = 30, + riskWindowEndUnexposed = -1, + addLengthOfExposureUnexposed = FALSE, + riskWindowStartUnexposed = -30, + washoutPeriod = 365 +) + +sccAnalysis2 <- createSccAnalysis( + analysisId = 2, + description = "30 days of each exposure", + runSelfControlledCohortArgs = runSccArgs2 +) +sccAnalysisList <- list(sccAnalysis1, sccAnalysis2)
    +

    Next, we specify all exposure-outcome pairs we want to apply the SCC +design to. Note that we only use the targetId and +outcomeId fields of the controls, because the SCC design +can only be used for effect estimation, and does not support +nesting:

    -allControls <- read.csv(file.path(outputFolder , "allControls.csv"))
-eos <- list()
-for (i in 1:nrow(allControls)) {
-  eos[[length(eos) + 1]] <- createExposureOutcome(exposureId = allControls$targetId[i],
-                                                  outcomeId = allControls$outcomeId[i])
-}
    -

    Then, we run the SCC method. The following command will execute the SCC with the specified analysis settings against the exposure-outcome pairs:

    +allControls <- read.csv(file.path(outputFolder, "allControls.csv")) +eos <- list() +for (i in 1:nrow(allControls)) { + eos[[length(eos) + 1]] <- createExposureOutcome( + exposureId = allControls$targetId[i], + outcomeId = allControls$outcomeId[i] + ) +} +

    Then, we run the SCC method. The following command will execute the +SCC with the specified analysis settings against the exposure-outcome +pairs:

    -sccResult <- runSccAnalyses(connectionDetails = connectionDetails,
-                            cdmDatabaseSchema = cdmDatabaseSchema,
-                            oracleTempSchema = oracleTempSchema,
-                            exposureTable = "drug_era",
-                            outcomeDatabaseSchema = outcomeDatabaseSchema,
-                            outcomeTable = outcomeTable,
-                            sccAnalysisList = sccAnalysisList,
-                            exposureOutcomeList = eos,
-                            outputFolder = outputFolder)
-sccSummary <- summarizeAnalyses(sccResult, outputFolder)
-write.csv(sccSummary, file.path(outputFolder, "sccSummary.csv"), row.names = FALSE)
    -

    This produces a file called sccSummary.csv which contains the effect size estimates for each analysisId-exposureId-outcomeId combination.

    +sccResult <- runSccAnalyses( + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + oracleTempSchema = oracleTempSchema, + exposureTable = "drug_era", + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + sccAnalysisList = sccAnalysisList, + exposureOutcomeList = eos, + outputFolder = outputFolder +) +sccSummary <- summarizeAnalyses(sccResult, outputFolder) +write.csv(sccSummary, file.path(outputFolder, "sccSummary.csv"), row.names = FALSE) +

    This produces a file called sccSummary.csv which contains +the effect size estimates for each analysisId-exposureId-outcomeId +combination.

    Packaging the evaluation results

    -

    Now that we have the estimates, we need to feed them back into the MethodEvaluation package. To do that we need to create two data frames, one specifying the effect size estimates, the other providing details on the various analyses.

    +

    Now that we have the estimates, we need to feed them back into the +MethodEvaluation package. To do that we need to create two +data frames, one specifying the effect size estimates, the other +providing details on the various analyses.

    First, we specify the estimates:

    -estimates <- readRDS(file.path(outputFolder, "sccSummary.csv"))
-estimates <- data.frame(analysisId = estimates$analysisId,
-                        targetId = estimates$exposureId,
-                        outcomeId = estimates$outcomeId,
-                        logRr = estimates$logRr,
-                        seLogRr = estimates$seLogRr,
-                        ci95Lb = estimates$irrLb95,
-                        ci95Ub = estimates$irrUb95)
-head(estimates)
    -
    #>   analysisId targetId outcomeId      logRr   seLogRr    ci95Lb    ci95Ub
-#> 1          1  1110942         1 -1.5000000        NA        NA 13.177433
-#> 2          1  1140088         1         NA        NA        NA        NA
-#> 3          1   943634         1         NA        NA        NA        NA
-#> 4          1 40163718         1 -0.4054651 0.6926191 0.1479317  2.234489
-#> 5          1  1350310         1 -0.2876821 0.7038381 0.1642789  2.592973
-#> 6          1  1311276         1 -0.1823216 0.5451585 0.2651910  2.247182
    +estimates <- readRDS(file.path(outputFolder, "sccSummary.csv")) +estimates <- data.frame( + analysisId = estimates$analysisId, + targetId = estimates$exposureId, + outcomeId = estimates$outcomeId, + logRr = estimates$logRr, + seLogRr = estimates$seLogRr, + ci95Lb = estimates$irrLb95, + ci95Ub = estimates$irrUb95 +) +head(estimates)
    +
    #>   analysisId targetId outcomeId      logRr   seLogRr    ci95Lb    ci95Ub
+#> 1          1  1110942         1 -1.5000000        NA        NA 13.177433
+#> 2          1  1140088         1         NA        NA        NA        NA
+#> 3          1   943634         1         NA        NA        NA        NA
+#> 4          1 40163718         1 -0.4054651 0.6926191 0.1479317  2.234489
+#> 5          1  1350310         1 -0.2876821 0.7038381 0.1642789  2.592973
+#> 6          1  1311276         1 -0.1823216 0.5451585 0.2651910  2.247182

    Next, we create the analysis reference:

    -# Create a reference of the analysis settings:
-analysisRef <- data.frame(method = "SelfControlledCohort",
-                          analysisId = c(1, 2),
-                          description = c("Length of exposure", 
-                                          "30 days of each exposure"),
-                          details = "",
-                          comparative = FALSE,
-                          nesting = FALSE,
-                          firstExposureOnly = FALSE)
-head(analysisRef)
    -
    #>                 method analysisId              description details comparative
-#> 1 SelfControlledCohort          1       Length of exposure               FALSE
-#> 2 SelfControlledCohort          2 30 days of each exposure               FALSE
-#>   nesting firstExposureOnly
-#> 1   FALSE             FALSE
-#> 2   FALSE             FALSE
    -

    Finally, we supply these two data frames, together with the data frame with controls to the packageOhdsiBenchmarkResults function:

    +# Create a reference of the analysis settings: +analysisRef <- data.frame( + method = "SelfControlledCohort", + analysisId = c(1, 2), + description = c( + "Length of exposure", + "30 days of each exposure" + ), + details = "", + comparative = FALSE, + nesting = FALSE, + firstExposureOnly = FALSE +) +head(analysisRef) +
    #>                 method analysisId              description details comparative
+#> 1 SelfControlledCohort          1       Length of exposure               FALSE
+#> 2 SelfControlledCohort          2 30 days of each exposure               FALSE
+#>   nesting firstExposureOnly
+#> 1   FALSE             FALSE
+#> 2   FALSE             FALSE
    +

    Finally, we supply these two data frames, together with the data +frame with controls to the packageOhdsiBenchmarkResults +function:

    -allControls <- read.csv(file.path(outputFolder , "allControls.csv"))
-packageOhdsiBenchmarkResults(estimates = estimates,
-                             controlSummary = allControls,
-                             analysisRef = analysisRef,
-                             databaseName = databaseName,
-                             exportFolder = file.path(outputFolder, "export"))
    -

    This will generate two CSV files in the export folder. Results from different methods and databases can all be written to the same export folder.

    +allControls <- read.csv(file.path(outputFolder, "allControls.csv")) +packageOhdsiBenchmarkResults( + estimates = estimates, + controlSummary = allControls, + analysisRef = analysisRef, + databaseName = databaseName, + exportFolder = file.path(outputFolder, "export") +) +

    This will generate two CSV files in the export folder. Results from +different methods and databases can all be written to the same export +folder.

    Computing metrics

    -

    There are two ways to view the method’s performance metrics: using a Shiny app, and by generating a data frame.

    +

    There are two ways to view the method’s performance metrics: using a +Shiny app, and by generating a data frame.

    Using the Shiny app

    -

    Run the follow command to launch the Shiny app and view the metrics:

    +

    Run the follow command to launch the Shiny app and view the +metrics:

    -exportFolder <- file.path(outputFolder, "export")
-launchMethodEvaluationApp(exportFolder)
    +exportFolder <- file.path(outputFolder, "export") +launchMethodEvaluationApp(exportFolder)

    This will launch a Shiny app as shown in Figure 1.

    Method Benchmark Shiny app

    -

    Note that the export folder can hold results from multiple methods and multiple databases.

    +

    Note that the export folder can hold results from multiple methods +and multiple databases.

    Generating a data frame

    Alternatively, we can output the metrics to a data frame:

    -exportFolder <- file.path(outputFolder, "export")
-metrics <- computeOhdsiBenchmarkMetrics(exportFolder, 
-                                        mdrr = 1.25, 
-                                        stratum = "All", 
-                                        trueEffectSize = "Overall", 
-                                        calibrated = FALSE, 
-                                        comparative = FALSE)
-head(metrics)
    -
    #>    database               method analysisId                                           description  auc coverage   meanP  mse type1 type2 nonEstimable
-#> 1     CCAE SelfControlledCohort          1                    Length of exposure 0.89     0.21 1048.97 0.30  0.66  0.08         0.01
-#> 2     CCAE SelfControlledCohort          2                        30 days of each exposure 0.87     0.41  422.25 0.14  0.55  0.11         0.00
    -

    See ?computeOhdsiBenchmarkMetrics for information on the various arguments.

    +exportFolder <- file.path(outputFolder, "export") +metrics <- computeOhdsiBenchmarkMetrics(exportFolder, + mdrr = 1.25, + stratum = "All", + trueEffectSize = "Overall", + calibrated = FALSE, + comparative = FALSE +) +head(metrics)
    +
    #>    database               method analysisId                                           description  auc coverage   meanP  mse type1 type2 nonEstimable
+#> 1     CCAE SelfControlledCohort          1                    Length of exposure 0.89     0.21 1048.97 0.30  0.66  0.08         0.01
+#> 2     CCAE SelfControlledCohort          2                        30 days of each exposure 0.87     0.41  422.25 0.14  0.55  0.11         0.00
    +

    See ?computeOhdsiBenchmarkMetrics for information on the +various arguments.

    @@ -371,7 +553,7 @@

    Generating a data frame

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/articles/index.html b/docs/articles/index.html index f046eff..01baaff 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference

  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/authors.html b/docs/authors.html index 83e29df..7d99f36 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    https://github.com/OHDSI/MethodEvaluation. +R package version 2.3.0, https://github.com/OHDSI/MethodEvaluation. 

    @Manual{,
   title = {MethodEvaluation: Package for Evaluation of Estimation Methods},
   author = {Martijn Schuemie},
-  year = {2022},
-  note = {R package version 2.2.0},
+  year = {2023},
+  note = {R package version 2.3.0},
   url = {https://github.com/OHDSI/MethodEvaluation},
 }
    @@ -97,7 +97,7 @@

    Citation

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/index.html b/docs/index.html index d0fb5b8..7f2cf83 100644 --- a/docs/index.html +++ b/docs/index.html @@ -40,7 +40,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -50,7 +50,7 @@ Reference
  • -

    - +

    Build Status codecov.io

    MethodEvaluation is part of HADES.

    @@ -127,8 +127,8 @@

    Installation
    -install.packages("remotes")
-remotes::install_github("ohdsi/MethodEvaluation")

    +install.packages("remotes") +remotes::install_github("ohdsi/MethodEvaluation")

    User Documentation @@ -213,13 +213,7 @@

    Developers

    -
    -

    Dev status

    -
      -
    • Build Status
      • -
    • codecov.io
      • -
    -
    + @@ -232,7 +226,7 @@

    Dev status

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/news/index.html b/docs/news/index.html index 171f205..3d05279 100644 --- a/docs/news/index.html +++ b/docs/news/index.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • - NEWS.md +
    +

    MethodEvaluation 2.3.0

    +

    Changes:

    +

    1. Using checkmate to check function input.

      2. +

    2. Adding packageCustomBenchmarkResults() to support custom methods benchmarks.

      3. +

    3. Fixing seeds and setting resetCoefficients = TRUE to ensure reproducibility of positive control synthesis.

      4. +

    MethodEvaluation 2.2.0

    Changes:

    -

    1. Updating from ‘oracleTempSchema’ to ‘tempEmulationSchema’ for newer versions of SqlRender.

      2. +

      1. Updating from oracleTempSchema to tempEmulationSchema for newer versions of SqlRender.

      2. Added unit tests.

    -

    MethodEvaluation 2.1.0

    +

    MethodEvaluation 2.1.0

    Changes:

    1. Added ‘ohdsiDevelopment’ reference set, intended to be used when developing methods. This set is based on 76 negative controls from the LEGEND Hypertension study.

    2. Now generating cohort counts when generating reference set cohorts.

      3. @@ -117,7 +124,7 @@

      MethodEv

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index fa1db2b..9cf782e 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -1,7 +1,7 @@ -pandoc: 2.16.2 -pkgdown: 2.0.2 +pandoc: 2.19.2 +pkgdown: 2.0.7 pkgdown_sha: ~ articles: OhdsiMethodsBenchmark: OhdsiMethodsBenchmark.html -last_built: 2022-01-21T07:31Z +last_built: 2023-04-13T13:56Z diff --git a/docs/pull_request_template.html b/docs/pull_request_template.html index 890c20f..ec4f9dd 100644 --- a/docs/pull_request_template.html +++ b/docs/pull_request_template.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/MethodEvaluation-package.html b/docs/reference/MethodEvaluation-package.html index 2f1e5c9..d21cfed 100644 --- a/docs/reference/MethodEvaluation-package.html +++ b/docs/reference/MethodEvaluation-package.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/computeMdrr.html b/docs/reference/computeMdrr.html index 4024588..9af7322 100644 --- a/docs/reference/computeMdrr.html +++ b/docs/reference/computeMdrr.html @@ -18,7 +18,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -27,7 +27,7 @@ Reference
  • - 
    getOption("sqlRenderTempEmulationSchema"),
-  exposureOutcomePairs,
-  exposureDatabaseSchema = cdmDatabaseSchema,
-  exposureTable = "drug_era",
-  outcomeDatabaseSchema = cdmDatabaseSchema,
-  outcomeTable = "condition_era",
-  cdmVersion = "5"
-)
    + 
    computeMdrr(
+  connectionDetails,
+  oracleTempSchema = NULL,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  cdmDatabaseSchema,
+  exposureOutcomePairs,
+  exposureDatabaseSchema = cdmDatabaseSchema,
+  exposureTable = "drug_era",
+  outcomeDatabaseSchema = cdmDatabaseSchema,
+  outcomeTable = "condition_era",
+  cdmVersion = "5"
+)
    @@ -88,47 +88,68 @@

    Arguments

    An R object of type ConnectionDetails created using the function createConnectionDetails in the DatabaseConnector package.

    -
    cdmDatabaseSchema
    -

    Name of database schema that contains OMOP CDM and vocabulary.

    + +
    oracleTempSchema

    DEPRECATED: use `tempEmulationSchema` instead.

    + +
    tempEmulationSchema

    Some database platforms like Oracle and Impala do not truly support temp tables. To emulate temp tables, provide a schema with write privileges where temp tables can be created.

    + + +
    cdmDatabaseSchema
    +

    Name of database schema that contains OMOP CDM and vocabulary.

    + +
    exposureOutcomePairs

    A data frame with at least two columns:

    • "exposureId" or "targetId" containing the drug_concept_ID or cohort_definition_id of the exposure variable

    • "outcomeId" containing the condition_concept_ID or cohort_definition_id of the outcome variable

    + +
    exposureDatabaseSchema

    The name of the database schema that is the location where the exposure data used to define the exposure cohorts is available. If exposureTable = DRUG_ERA, exposureDatabaseSchema is not used by assumed to be cdmSchema. Requires read permissions to this database.

    + +
    exposureTable

    The tablename that contains the exposure cohorts. If exposureTable <> DRUG_ERA, then expectation is exposureTable has format of COHORT table: COHORT_DEFINITION_ID, SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.

    + +
    outcomeDatabaseSchema

    The name of the database schema that is the location where the data used to define the outcome cohorts is available. If exposureTable = CONDITION_ERA, exposureDatabaseSchema is not used by assumed to be cdmSchema. Requires read permissions to this database.

    + +
    outcomeTable

    The tablename that contains the outcome cohorts. If outcomeTable <> CONDITION_OCCURRENCE, then expectation is outcomeTable has format of COHORT table: COHORT_DEFINITION_ID, SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.

    + +
    cdmVersion

    Define the OMOP CDM version used: currently support "4" and "5".

    +

    Value

    -

    A data frame containing the MDRRs for the given exposure-outcome pairs.

    + + +

    A data frame containing the MDRRs for the given exposure-outcome pairs.

    References

    @@ -138,16 +159,21 @@

    References

    Examples

    - 
    if (FALSE) {
-connectionDetails <- createConnectionDetails(dbms = "sql server",
-                                             server = "RNDUSRDHIT07.jnj.com")
-exposureOutcomePairs <- data.frame(exposureId = c(767410, 1314924, 907879),
-                                   outcomeId = c(444382, 79106, 138825))
-mdrrs <- computeMdrr(connectionDetails,
-                     "cdm_truven_mdcr",
-                     exposureOutcomePairs,
-                     outcomeTable = "condition_era")
-}
+    
    if (FALSE) {
+connectionDetails <- createConnectionDetails(
+  dbms = "sql server",
+  server = "RNDUSRDHIT07.jnj.com"
+)
+exposureOutcomePairs <- data.frame(
+  exposureId = c(767410, 1314924, 907879),
+  outcomeId = c(444382, 79106, 138825)
+)
+mdrrs <- computeMdrr(connectionDetails,
+  "cdm_truven_mdcr",
+  exposureOutcomePairs,
+  outcomeTable = "condition_era"
+)
+}
    @@ -162,7 +188,7 @@

    Examples

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/computeMetrics.html b/docs/reference/computeMetrics.html index c7f79f8..e8ffd01 100644 --- a/docs/reference/computeMetrics.html +++ b/docs/reference/computeMetrics.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • - 
    library(EmpiricalCalibration)
-data <- simulateControls(n = 50 * 3, trueLogRr = log(c(1, 2, 4)))
-computeMetrics(logRr = data$logRr, seLogRr = data$seLogRr, trueLogRr = data$trueLogRr)
+    
    library(EmpiricalCalibration)
+data <- simulateControls(n = 50 * 3, trueLogRr = log(c(1, 2, 4)))
+computeMetrics(logRr = data$logRr, seLogRr = data$seLogRr, trueLogRr = data$trueLogRr)
 #> Setting levels: control = FALSE, case = TRUE
 #> Setting direction: controls < cases
 #>          auc     coverage        meanP          mse        type1        type2 
-#>         1.00         0.75       135.70         0.02         0.30         0.01 
+#>         1.00         0.71       148.61         0.02         0.32         0.00 
 #> nonEstimable 
 #>         0.00 
-
+
    @@ -129,7 +140,7 @@

    Examples

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/computeOhdsiBenchmarkMetrics.html b/docs/reference/computeOhdsiBenchmarkMetrics.html index a86bbf0..76d97bb 100644 --- a/docs/reference/computeOhdsiBenchmarkMetrics.html +++ b/docs/reference/computeOhdsiBenchmarkMetrics.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    packageOhdsiBenchmarkResults function. This folder can contain results from various methods, analyses, and databases.

    + +
    mdrr

    The minimum detectable relative risk (MDRR). Only controls with this MDRR will be used to compute the performance metrics. Set to "All" to include all controls.

    + +
    stratum

    The stratum for which to compute the metrics, e.g. 'Acute Pancreatitis'. Set to 'All' to use all controls.

    + +
    trueEffectSize

    Should the analysis be limited to a specific true effect size? Set to "Overall" to include all.

    + +
    calibrated

    Should confidence intervals and p-values be empirically calibrated before computing the metrics?

    + +
    comparative

    Should the methods be evaluated on the task of comprative effect estimation? If FALSE, they will be evaluated on the task of effect estimation.

    +

    Value

    -

    A data frame with the various metrics per method - analysisId - database combination.

    + + +

    A data frame with the various metrics per method - analysisId - database combination.

    @@ -116,7 +129,7 @@

    Value

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/createReferenceSetCohorts.html b/docs/reference/createReferenceSetCohorts.html index a2a5852..0f9fb88 100644 --- a/docs/reference/createReferenceSetCohorts.html +++ b/docs/reference/createReferenceSetCohorts.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • - 
    getOption("sqlRenderTempEmulationSchema"),
-  cdmDatabaseSchema,
-  exposureDatabaseSchema = cdmDatabaseSchema,
-  exposureTable = "exposures",
-  outcomeDatabaseSchema = cdmDatabaseSchema,
-  outcomeTable = "outcomes",
-  nestingDatabaseSchema = cdmDatabaseSchema,
-  nestingTable = "nesting",
-  referenceSet = "ohdsiMethodsBenchmark",
-  workFolder
-)
    + 
    createReferenceSetCohorts(
+  connectionDetails,
+  oracleTempSchema = NULL,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  cdmDatabaseSchema,
+  exposureDatabaseSchema = cdmDatabaseSchema,
+  exposureTable = "exposures",
+  outcomeDatabaseSchema = cdmDatabaseSchema,
+  outcomeTable = "outcomes",
+  nestingDatabaseSchema = cdmDatabaseSchema,
+  nestingTable = "nesting",
+  referenceSet = "ohdsiMethodsBenchmark",
+  workFolder
+)
    @@ -88,46 +88,69 @@

    Arguments

    An R object of type ConnectionDetails created using the function createConnectionDetails in the DatabaseConnector package.

    + +
    oracleTempSchema

    DEPRECATED: use `tempEmulationSchema` instead.

    + +
    tempEmulationSchema

    Some database platforms like Oracle and Impala do not truly support temp tables. To emulate temp tables, provide a schema with write privileges where temp tables can be created.

    + +
    cdmDatabaseSchema

    A database schema containing health care data in the OMOP Commond Data Model. Note that for SQL Server, botth the database and schema should be specified, e.g. 'cdm_schema.dbo'.

    + +
    exposureDatabaseSchema
    -

    The name of the database schema where the exposure cohorts will be +

    The name of the database schema where the exposure cohorts will be created. Only needed if referenceSet = 'ohdsiDevelopment'. Note -that for SQL Server, both the database and schema should be specified, +that for SQL Server, both the database and schema should be specified, e.g. 'cdm_schema.dbo'.

    + +
    exposureTable
    -

    The name of the table that will be created to store the exposure +

    The name of the table that will be created to store the exposure cohorts. Only needed if referenceSet = 'ohdsiDevelopment'.

    + +
    outcomeDatabaseSchema

    The database schema where the target outcome table is located. Note that for SQL Server, both the database and schema should be specified, e.g. 'cdm_schema.dbo'

    + +
    outcomeTable

    The name of the table where the outcomes will be stored.

    + +
    nestingDatabaseSchema

    (For the OHDSI Methods Benchmark and OHDSI Development Set only) The -database schema where the nesting outcome table is located. Note that +database schema where the nesting outcome table is located. Note that for SQL Server, both the database and schema should be specified, e.g. 'cdm_schema.dbo'.

    + +
    nestingTable
    -

    (For the OHDSI Methods Benchmark and OHDSI Development Set only) The +

    (For the OHDSI Methods Benchmark and OHDSI Development Set only) The name of the table where the nesting cohorts will be stored.

    + +
    referenceSet

    The name of the reference set for which outcomes need to be created. -Currently supported are "omopReferenceSet", "euadrReferenceSet", +Currently supported are "omopReferenceSet", "euadrReferenceSet", "ohdsiMethodsBenchmark", and "ohdsiDevelopment".

    + +
    workFolder

    Name of local folder to place intermediary results; make sure to use forward slashes (/). Do not use a folder on a network drive since this greatly impacts performance.

    +

    Details

    @@ -135,7 +158,7 @@

    Details

    reference sets. The outcomes of interest are derives using information like diagnoses, procedures, and drug prescriptions. The outcomes are stored in a table on the database server.

    For the 'ohdsiMethodsBenchmark' reference set, the exposures are taken from the drug_era table, and -are therefore not generated as separate cohorts, and an exposure cohort table therefore needn't be supplied. +are therefore not generated as separate cohorts, and an exposure cohort table therefore needn't be supplied. For the 'ohdsiDevelopment' reference set, exposure cohorts will be generated.

    @@ -151,7 +174,7 @@

    Details

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/euadrReferenceSet.html b/docs/reference/euadrReferenceSet.html index dec75bc..cf2b9e3 100644 --- a/docs/reference/euadrReferenceSet.html +++ b/docs/reference/euadrReferenceSet.html @@ -32,7 +32,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -41,7 +41,7 @@ Reference
  • - 

    data(euadrReferenceSet)
    + 
    data(euadrReferenceSet)
    @@ -145,7 +145,7 @@

    References

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/index.html b/docs/reference/index.html index 1f07499..0caf076 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference

  • -

    The OHDSI Development Set - Negative Controls A set of 76 negative control outcomes, all for the exposures of ACE inhibitors (compared to -thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method +thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method Evaluation Benchmark, but follows the same principles. It is intended to be used when developing methods, leaving the Methods Benchark untouched until a final evaluation of the method, thus preventing 'training' on the evaluation set. The negative controls are borrowed from the LEGEND Hypertension study. @@ -130,6 +130,10 @@

    All functions + +

    packageCustomBenchmarkResults()

    + +

    Package results of a method on the OHDSI Methods Benchmark

    packageOhdsiBenchmarkResults()

    @@ -167,7 +171,7 @@

    All functions
    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/injectSignals.html b/docs/reference/injectSignals.html index 649b130..e0dc998 100644 --- a/docs/reference/injectSignals.html +++ b/docs/reference/injectSignals.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference

  • - 
    createCovariateSettings(useDemographicsAgeGroup
-    = TRUE, useDemographicsGender = TRUE, useDemographicsIndexYear = TRUE,
-    useDemographicsIndexMonth = TRUE, useConditionGroupEraLongTerm = TRUE,
-    useDrugGroupEraLongTerm = TRUE, useProcedureOccurrenceLongTerm = TRUE,
-    useMeasurementLongTerm = TRUE, useObservationLongTerm = TRUE, useCharlsonIndex =
-    TRUE, useDcsi = TRUE, useChads2Vasc = TRUE, longTermStartDays = 365, endDays = 0),
-  prior = Cyclops::createPrior("laplace", exclude = 0, useCrossValidation = TRUE),
-  control = Cyclops::createControl(cvType = "auto", startingVariance = 0.1, noiseLevel
-    = "quiet", threads = 10),
-  firstExposureOnly = FALSE,
-  washoutPeriod = 183,
-  riskWindowStart = 0,
-  riskWindowEnd = 0,
-  addExposureDaysToEnd = TRUE,
-  addIntentToTreat = FALSE,
-  firstOutcomeOnly = FALSE,
-  removePeopleWithPriorOutcomes = FALSE,
-  maxSubjectsForModel = 1e+05,
-  effectSizes = c(1, 1.25, 1.5, 2, 4),
-  precision = 0.01,
-  outputIdOffset = 1000,
-  workFolder = "./SignalInjectionTemp",
-  cdmVersion = "5",
-  modelThreads = 1,
-  generationThreads = 1
-)
    + 
    injectSignals(
+  connectionDetails,
+  cdmDatabaseSchema,
+  oracleTempSchema = cdmDatabaseSchema,
+  exposureDatabaseSchema = cdmDatabaseSchema,
+  exposureTable = "drug_era",
+  outcomeDatabaseSchema = cdmDatabaseSchema,
+  outcomeTable = "cohort",
+  outputDatabaseSchema = outcomeDatabaseSchema,
+  outputTable = outcomeTable,
+  createOutputTable = FALSE,
+  exposureOutcomePairs,
+  modelType = "poisson",
+  minOutcomeCountForModel = 100,
+  minOutcomeCountForInjection = 25,
+  covariateSettings = FeatureExtraction::createCovariateSettings(useDemographicsAgeGroup
+    = TRUE, useDemographicsGender = TRUE, useDemographicsIndexYear = TRUE,
+    useDemographicsIndexMonth = TRUE, useConditionGroupEraLongTerm = TRUE,
+    useDrugGroupEraLongTerm = TRUE, useProcedureOccurrenceLongTerm = TRUE,
+    useMeasurementLongTerm = TRUE, useObservationLongTerm = TRUE, useCharlsonIndex =
+    TRUE, useDcsi = TRUE, useChads2Vasc = TRUE, longTermStartDays = 365, endDays = 0),
+  prior = Cyclops::createPrior("laplace", exclude = 0, useCrossValidation = TRUE),
+  control = Cyclops::createControl(cvType = "auto", startingVariance = 0.1, noiseLevel =
+    "quiet", threads = 10),
+  firstExposureOnly = FALSE,
+  washoutPeriod = 183,
+  riskWindowStart = 0,
+  riskWindowEnd = 0,
+  addExposureDaysToEnd = TRUE,
+  addIntentToTreat = FALSE,
+  firstOutcomeOnly = FALSE,
+  removePeopleWithPriorOutcomes = FALSE,
+  maxSubjectsForModel = 1e+05,
+  effectSizes = c(1, 1.25, 1.5, 2, 4),
+  precision = 0.01,
+  outputIdOffset = 1000,
+  workFolder = "./SignalInjectionTemp",
+  cdmVersion = "5",
+  modelThreads = 1,
+  generationThreads = 1
+)
    @@ -116,92 +115,136 @@

    Arguments

    An R object of type ConnectionDetails created using the function createConnectionDetails in the DatabaseConnector package.

    + +
    cdmDatabaseSchema

    Name of database schema that contains OMOP CDM and vocabulary.

    + +
    oracleTempSchema

    For Oracle only: the name of the database schema where you want all temporary tables to be managed. Requires create/insert permissions to this database.

    + +
    exposureDatabaseSchema

    The name of the database schema that is the location where the exposure data used to define the exposure cohorts is available. If exposureTable = DRUG_ERA, exposureDatabaseSchema is not used by assumed to be cdmSchema. Requires read permissions to this database.

    + +
    exposureTable

    The table name that contains the exposure cohorts. If exposureTable <> DRUG_ERA, then expectation is exposureTable has format of COHORT table: cohort_concept_id, SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.

    + +
    outcomeDatabaseSchema

    The name of the database schema that is the location where the data used to define the outcome cohorts is available. If exposureTable = CONDITION_ERA, exposureDatabaseSchema is not used by assumed to be cdmSchema. Requires read permissions to this database.

    + +
    outcomeTable

    The table name that contains the outcome cohorts. When the table name is not CONDITION_ERA This table is expected to have the same format as the COHORT table: SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE, COHORT_CONCEPT_ID (CDM v4) or COHORT_DEFINITION_ID (CDM v5 and higher).

    + +
    outputDatabaseSchema

    The name of the database schema that is the location of the tables containing the new outcomesRequires write permissions to this database.

    + +
    outputTable

    The name of the table names that will contain the generated outcome cohorts.

    + +
    createOutputTable

    Should the output table be created prior to inserting the outcomes? If TRUE and the tables already exists, it will first be deleted. If FALSE, the table is assumed to exist and the outcomes will be inserted. Any existing outcomes with the same IDs will first be deleted.

    + +
    exposureOutcomePairs

    A data frame with at least two columns:

    • "exposureId" containing the drug_concept_ID or cohort_concept_id of the exposure variable

    • "outcomeId" containing the condition_concept_ID or cohort_concept_id of the outcome variable

    + +
    modelType

    Can be either "poisson" or "survival"

    + +
    minOutcomeCountForModel

    Minimum number of outcome events required to build a model.

    + +
    minOutcomeCountForInjection

    Minimum number of outcome events required to inject a signal.

    + +
    covariateSettings

    An object of type covariateSettings as created using the createCovariateSettings function in the FeatureExtraction package.

    + +
    prior

    The prior used to fit the outcome model. See createPrior for details.

    + +
    control

    The control object used to control the cross-validation used to determine the hyperparameters of the prior (if applicable). See createControl for details.

    + +
    firstExposureOnly

    Should signals be injected only for the first exposure? (ie. assuming an acute effect)

    + +
    washoutPeriod

    Number of days at the start of observation for which no signals will be injected, but will be used to determine whether exposure or outcome is the first one, and for extracting covariates to build the outcome model.

    + +
    riskWindowStart

    The start of the risk window relative to the start of the exposure (in days). When 0, risk is assumed to start on the first day of exposure.

    + +
    riskWindowEnd

    The end of the risk window relative to the start of the exposure. Note that typically the length of exposure is added to this number (when the addExposureDaysToEnd parameter is set to TRUE).

    + +
    addExposureDaysToEnd

    Should length of exposure be added to the risk window?

    + +
    addIntentToTreat

    If true, the signal will not only be injected in the primary time at risk, but also after the time at risk (up until the @@ -212,32 +255,53 @@

    Arguments

    preclude the controls to be used in self-controlled designs that consider the time after exposure. Requires firstExposureOnly = TRUE.

    + +
    firstOutcomeOnly

    Should only the first outcome per person be considered when modeling the outcome?

    + +
    removePeopleWithPriorOutcomes

    Remove people with prior outcomes?

    + +
    maxSubjectsForModel

    Maximum number of people used to fit an outcome model.

    + +
    effectSizes

    A numeric vector of effect sizes that should be inserted.

    + +
    precision

    The allowed ratio between target and injected signal size.

    + +
    outputIdOffset

    What should be the first new outcome ID that is to be created?

    + +
    workFolder

    Path to a folder where intermediate data will be stored.

    + +
    cdmVersion

    Define the OMOP CDM version used: currently support "4" and "5".

    + +
    modelThreads

    Number of parallel threads to use when fitting outcome models.

    + +
    generationThreads

    Number of parallel threads to use when generating outcomes.

    A data.frame listing all the drug-pairs in combination with requested effect sizes and the real inserted effect size (might be different from the requested effect size because of sampling error).

    +

    Details

    @@ -256,7 +320,7 @@

    Details

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/launchMethodEvaluationApp.html b/docs/reference/launchMethodEvaluationApp.html index da8e5ce..5c083ad 100644 --- a/docs/reference/launchMethodEvaluationApp.html +++ b/docs/reference/launchMethodEvaluationApp.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    packageOhdsiBenchmarkResults function to populate this folder.

    + +
    launch.browser

    Should the app be launched in your default browser, or in a Shiny window. Note: copying to clipboard will not work in a Shiny window.

    +

    Details

    @@ -96,7 +99,7 @@

    Details

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/ohdsiDevelopmentNegativeControls.html b/docs/reference/ohdsiDevelopmentNegativeControls.html index bfffe1d..c8d9d88 100644 --- a/docs/reference/ohdsiDevelopmentNegativeControls.html +++ b/docs/reference/ohdsiDevelopmentNegativeControls.html @@ -1,7 +1,7 @@ The OHDSI Development Set - Negative Controls A set of 76 negative control outcomes, all for the exposures of ACE inhibitors (compared to -thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method +thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method Evaluation Benchmark, but follows the same principles. It is intended to be used when developing methods, leaving the Methods Benchark untouched until a final evaluation of the method, thus preventing 'training' on the evaluation set. The negative controls are borrowed from the LEGEND Hypertension study. @@ -10,7 +10,7 @@ These negative controls can form the basis to generate positive controls using the injectSignals function. — ohdsiDevelopmentNegativeControls • MethodEvaluation MethodEvaluation - 2.2.0 + 2.3.0 @@ -53,7 +53,7 @@ Reference
  • - 

    data(ohdsiDevelopmentNegativeControls)
    + 
    data(ohdsiDevelopmentNegativeControls)
    @@ -160,7 +160,7 @@

    Format

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/ohdsiNegativeControls.html b/docs/reference/ohdsiNegativeControls.html index 0f274d3..aaf72df 100644 --- a/docs/reference/ohdsiNegativeControls.html +++ b/docs/reference/ohdsiNegativeControls.html @@ -47,7 +47,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -56,7 +56,7 @@ Reference
  • - 

    data(ohdsiNegativeControls)
    + 
    data(ohdsiNegativeControls)
    @@ -165,7 +165,7 @@

    Format

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/omopReferenceSet.html b/docs/reference/omopReferenceSet.html index ff6ad3d..05ad5c4 100644 --- a/docs/reference/omopReferenceSet.html +++ b/docs/reference/omopReferenceSet.html @@ -26,7 +26,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -35,7 +35,7 @@ Reference

  • - 

    data(omopReferenceSet)
    + 
    data(omopReferenceSet)
    @@ -133,7 +133,7 @@

    References

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/packageCustomBenchmarkResults.html b/docs/reference/packageCustomBenchmarkResults.html new file mode 100644 index 0000000..4de2a43 --- /dev/null +++ b/docs/reference/packageCustomBenchmarkResults.html @@ -0,0 +1,158 @@ + +Package results of a method on the OHDSI Methods Benchmark — packageCustomBenchmarkResults • MethodEvaluation + + +
    +
    + + + +
    +
    +
    +

    Package results of a method on the OHDSI Methods Benchmark

    + Source: R/Metrics.R + +
    + +
    +

    Stores the results of a method on the OHDSI Methods Benchmark in a standardized format, for example +for use in the Method Evaluation Shiny app.

    +
    + +
    + 
    packageCustomBenchmarkResults(
+  estimates,
+  negativeControls,
+  synthesisSummary,
+  analysisRef,
+  databaseName,
+  exportFolder
+)
    +
    + +
    +

    Arguments

    +
    estimates
    +

    A data frame containing the estimates. See details for required columns.

    + + +
    negativeControls
    +

    A data frame containing the negative controls. See details for required columns.

    + + +
    synthesisSummary
    +

    A data frame with the summary of the positive control synthesis as generated by the +synthesizePositiveControls function.

    + + +
    analysisRef
    +

    A file describing the various analyses that were performed. See details for +required columns.

    + + +
    databaseName
    +

    A character string to identify the database the method was executed on.

    + + +
    exportFolder
    +

    The folder where the output CSV files will written.

    + +
    +
    +

    Details

    +

    The estimates argument should have the following columns: "targetId", "outcomeId", +"analysisId", "logRr", "seLogRr", "ci95Lb", "ci95Ub". +The negativeControls argument should have the following columns: "targetId", "outcomeId", "type". +The analysisRef argument should have the following columns: "analysisId", "method", +"comparative", "nesting", "firstExposureOnly" +The targetId and outcomeId fields identify the specific control, and should +correspond to those in the negativeControls and synthesisSummary objects. +The type fields should be either "Outcome control" or "Exposure control". +The analysisId field is an integer that identifies a specific variant of the method. For +example, if the method is 'CohortMethod', analysisId = 1 could identify a set of settings using +propensity score matching, and analysisId = 2 could identify a set of settings using +stratification. +logRr, seLogRr, ci95Lb, and ci95Ub correspond to the log of the effect +estimate (e.g. the hazard ratio), the standard error, and the upper and lower bound of the effect +size estimate, as produced by the method. +method is a character string identifyign the method (e.g. "CohortMethod"). +comparative is a boolean indicating whether the analysis can also be considerd to perform +comparative effect estimation (comparing the target to the comparator). +nesting is a boolean indicating whether the analysis is nested in the nesting cohorts +identified in the gold standard. +firstExposureOnly is a boolean indicating whether only the first exposure was used in the +analysis.

    +
    + +
    + +
    + + +
    +

    Developed by Martijn Schuemie.

    +
    + +
    +

    Site built with pkgdown 2.0.7.

    +
    + +
    + + + + + + + + diff --git a/docs/reference/packageOhdsiBenchmarkResults.html b/docs/reference/packageOhdsiBenchmarkResults.html index 7e8ff2d..508735b 100644 --- a/docs/reference/packageOhdsiBenchmarkResults.html +++ b/docs/reference/packageOhdsiBenchmarkResults.html @@ -18,7 +18,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -27,7 +27,7 @@ Reference

  • -

    synthesizePositiveControls function.

    +synthesizeReferenceSetPositiveControls function.

    + +
    analysisRef

    A file describing the various analyses that were performed. See details for required columns.

    + +
    databaseName

    A character string to identify the database the method was executed on.

    + +
    exportFolder

    The folder where the output CSV files will written.

    + +
    referenceSet
    -

    The name of the reference set for which to package the results. Currently +

    The name of the reference set for which to package the results. Currently supported are "ohdsiMethodsBenchmark" and "ohdsiDevelopment".

    +

    Details

    @@ -132,7 +143,7 @@

    Details

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/plotControls.html b/docs/reference/plotControls.html index 835025c..44d7438 100644 --- a/docs/reference/plotControls.html +++ b/docs/reference/plotControls.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/plotCoverageInjectedSignals.html b/docs/reference/plotCoverageInjectedSignals.html index 53a3feb..cd11081 100644 --- a/docs/reference/plotCoverageInjectedSignals.html +++ b/docs/reference/plotCoverageInjectedSignals.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/plotRocsInjectedSignals.html b/docs/reference/plotRocsInjectedSignals.html index 6a1a295..1c2baee 100644 --- a/docs/reference/plotRocsInjectedSignals.html +++ b/docs/reference/plotRocsInjectedSignals.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • -

    pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/synthesizePositiveControls.html b/docs/reference/synthesizePositiveControls.html index cf2dee7..8fec975 100644 --- a/docs/reference/synthesizePositiveControls.html +++ b/docs/reference/synthesizePositiveControls.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • - 
    getOption("sqlRenderTempEmulationSchema"),
-  exposureDatabaseSchema = cdmDatabaseSchema,
-  exposureTable = "drug_era",
-  outcomeDatabaseSchema = cdmDatabaseSchema,
-  outcomeTable = "cohort",
-  outputDatabaseSchema = outcomeDatabaseSchema,
-  outputTable = outcomeTable,
-  createOutputTable = FALSE,
-  exposureOutcomePairs,
-  modelType = "poisson",
-  minOutcomeCountForModel = 100,
-  minOutcomeCountForInjection = 25,
-  minModelCount = 5,
- 
-    covariateSettings = FeatureExtraction::createCovariateSettings(useDemographicsAgeGroup
-    = TRUE, useDemographicsGender = TRUE, useDemographicsIndexYear = TRUE,
-    useDemographicsIndexMonth = TRUE, useConditionGroupEraLongTerm = TRUE,
-    useDrugGroupEraLongTerm = TRUE, useProcedureOccurrenceLongTerm = TRUE,
-    useMeasurementLongTerm = TRUE, useObservationLongTerm = TRUE, useCharlsonIndex =
-    TRUE, useDcsi = TRUE, useChads2Vasc = TRUE, longTermStartDays = 365, endDays = 0),
-  prior = Cyclops::createPrior("laplace", exclude = 0, useCrossValidation = TRUE),
-  control = Cyclops::createControl(cvType = "auto", startingVariance = 0.1, noiseLevel
-    = "quiet", threads = 10),
-  firstExposureOnly = FALSE,
-  washoutPeriod = 183,
-  riskWindowStart = 0,
-  riskWindowEnd = 0,
-  endAnchor = "cohort end",
-  addIntentToTreat = FALSE,
-  firstOutcomeOnly = FALSE,
-  removePeopleWithPriorOutcomes = FALSE,
-  maxSubjectsForModel = 1e+05,
-  effectSizes = c(1, 1.25, 1.5, 2, 4),
-  precision = 0.01,
-  outputIdOffset = 1000,
-  workFolder = "./SignalInjectionTemp",
-  cdmVersion = "5",
-  modelThreads = 1,
-  generationThreads = 1
-)
    + 
    synthesizePositiveControls(
+  connectionDetails,
+  cdmDatabaseSchema,
+  oracleTempSchema = NULL,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  exposureDatabaseSchema = cdmDatabaseSchema,
+  exposureTable = "drug_era",
+  outcomeDatabaseSchema = cdmDatabaseSchema,
+  outcomeTable = "cohort",
+  outputDatabaseSchema = outcomeDatabaseSchema,
+  outputTable = outcomeTable,
+  createOutputTable = FALSE,
+  exposureOutcomePairs,
+  modelType = "poisson",
+  minOutcomeCountForModel = 100,
+  minOutcomeCountForInjection = 25,
+  minModelCount = 5,
+  covariateSettings = FeatureExtraction::createCovariateSettings(useDemographicsAgeGroup
+    = TRUE, useDemographicsGender = TRUE, useDemographicsIndexYear = TRUE,
+    useDemographicsIndexMonth = TRUE, useConditionGroupEraLongTerm = TRUE,
+    useDrugGroupEraLongTerm = TRUE, useProcedureOccurrenceLongTerm = TRUE,
+    useMeasurementLongTerm = TRUE, useObservationLongTerm = TRUE, useCharlsonIndex =
+    TRUE, useDcsi = TRUE, useChads2Vasc = TRUE, longTermStartDays = 365, endDays = 0),
+  prior = Cyclops::createPrior("laplace", exclude = 0, useCrossValidation = TRUE),
+  control = Cyclops::createControl(cvType = "auto", startingVariance = 0.1, seed = 1,
+    resetCoefficients = TRUE, noiseLevel = "quiet", threads = 10),
+  firstExposureOnly = FALSE,
+  washoutPeriod = 183,
+  riskWindowStart = 0,
+  riskWindowEnd = 0,
+  endAnchor = "cohort end",
+  addIntentToTreat = FALSE,
+  firstOutcomeOnly = FALSE,
+  removePeopleWithPriorOutcomes = FALSE,
+  maxSubjectsForModel = 1e+05,
+  effectSizes = c(1, 1.25, 1.5, 2, 4),
+  precision = 0.01,
+  outputIdOffset = 1000,
+  workFolder = "./SignalInjectionTemp",
+  cdmVersion = "5",
+  modelThreads = 1,
+  generationThreads = 1
+)
    @@ -118,95 +117,143 @@

    Arguments

    An R object of type ConnectionDetails created using the function createConnectionDetails in the DatabaseConnector package.

    + +
    cdmDatabaseSchema

    Name of database schema that contains OMOP CDM and vocabulary.

    + +
    oracleTempSchema

    DEPRECATED: use `tempEmulationSchema` instead.

    + +
    tempEmulationSchema

    Some database platforms like Oracle and Impala do not truly support temp tables. To emulate temp tables, provide a schema with write privileges where temp tables can be created.

    + +
    exposureDatabaseSchema

    The name of the database schema that is the location where the exposure data used to define the exposure cohorts is available. If exposureTable = DRUG_ERA, exposureDatabaseSchema is not used by assumed to be cdmSchema. Requires read permissions to this database.

    + +
    exposureTable

    The table name that contains the exposure cohorts. If exposureTable <> DRUG_ERA, then expectation is exposureTable has format of COHORT table: cohort_concept_id, SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.

    + +
    outcomeDatabaseSchema

    The name of the database schema that is the location where the data used to define the outcome cohorts is available. If exposureTable = CONDITION_ERA, exposureDatabaseSchema is not used by assumed to be cdmSchema. Requires read permissions to this database.

    + +
    outcomeTable

    The table name that contains the outcome cohorts. When the table name is not CONDITION_ERA This table is expected to have the same format as the COHORT table: SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE, COHORT_CONCEPT_ID (CDM v4) or COHORT_DEFINITION_ID (CDM v5 and higher).

    + +
    outputDatabaseSchema

    The name of the database schema that is the location of the tables containing the new outcomesRequires write permissions to this database.

    + +
    outputTable

    The name of the table names that will contain the generated outcome cohorts.

    + +
    createOutputTable

    Should the output table be created prior to inserting the outcomes? If TRUE and the tables already exists, it will first be deleted. If FALSE, the table is assumed to exist and the outcomes will be inserted. Any existing outcomes with the same IDs will first be deleted.

    + +
    exposureOutcomePairs

    A data frame with at least two columns:

    • "exposureId" containing the drug_concept_ID or cohort_concept_id of the exposure variable

    • "outcomeId" containing the condition_concept_ID or cohort_concept_id of the outcome variable

    + +
    modelType

    Can be either "poisson" or "survival"

    + +
    minOutcomeCountForModel

    Minimum number of outcome events required to build a model.

    + +
    minOutcomeCountForInjection

    Minimum number of outcome events required to inject a signal.

    + +
    minModelCount
    -

    Minimum number of negative controls having enough outcomes to +

    Minimum number of negative controls having enough outcomes to fit an outcome model.

    + +
    covariateSettings

    An object of type covariateSettings as created using the createCovariateSettings function in the FeatureExtraction package.

    + +
    prior

    The prior used to fit the outcome model. See createPrior for details.

    + +
    control

    The control object used to control the cross-validation used to determine the hyperparameters of the prior (if applicable). See createControl for details.

    + +
    firstExposureOnly

    Should signals be injected only for the first exposure? (ie. assuming an acute effect)

    + +
    washoutPeriod

    Number of days at the start of observation for which no signals will be injected, but will be used to determine whether exposure or outcome is the first one, and for extracting covariates to build the outcome model.

    + +
    riskWindowStart

    The start of the risk window relative to the start of the exposure (in days). When 0, risk is assumed to start on the first day of exposure.

    + +
    riskWindowEnd

    The end of the risk window (in days) relative to the endAnchor.

    + +
    endAnchor

    The anchor point for the end of the risk window. Can be "cohort start" or "cohort end".

    + +
    addIntentToTreat

    If true, the signal will not only be injected in the primary time at risk, but also after the time at risk (up until the @@ -217,34 +264,57 @@

    Arguments

    preclude the controls to be used in self-controlled designs that consider the time after exposure. Requires firstExposureOnly = TRUE.

    + +
    firstOutcomeOnly

    Should only the first outcome per person be considered when modeling the outcome?

    + +
    removePeopleWithPriorOutcomes

    Remove people with prior outcomes?

    + +
    maxSubjectsForModel

    Maximum number of people used to fit an outcome model.

    + +
    effectSizes

    A numeric vector of effect sizes that should be inserted.

    + +
    precision

    The allowed ratio between target and injected signal size.

    + +
    outputIdOffset

    What should be the first new outcome ID that is to be created?

    + +
    workFolder

    Path to a folder where intermediate data will be stored.

    + +
    cdmVersion

    Define the OMOP CDM version used: currently support "4" and "5".

    + +
    modelThreads

    Number of parallel threads to use when fitting outcome models.

    + +
    generationThreads

    Number of parallel threads to use when generating outcomes.

    +

    Value

    -

    A data.frame listing all the drug-pairs in combination with requested effect sizes and the real + + +

    A data.frame listing all the drug-pairs in combination with requested effect sizes and the real inserted effect size (might be different from the requested effect size because of sampling error).

    @@ -255,7 +325,7 @@

    Details

    specified risk window of the drug: a Poisson model assuming multiple outcomes could occurr during a single exposure, and a survival model considering only one outcome per exposure. It is possible to use bulk import to insert the generated outcomes in the database. This requires -the environmental variable 'USE_MPP_BULK_LOAD' to be set to 'TRUE'. See +the environmental variable 'USE_MPP_BULK_LOAD' to be set to 'TRUE'. See ?DatabaseConnector::insertTable for details on how to configure the bulk upload.

    @@ -277,7 +347,7 @@

    References

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/reference/synthesizeReferenceSetPositiveControls.html b/docs/reference/synthesizeReferenceSetPositiveControls.html index f89b5bc..efa3eb8 100644 --- a/docs/reference/synthesizeReferenceSetPositiveControls.html +++ b/docs/reference/synthesizeReferenceSetPositiveControls.html @@ -17,7 +17,7 @@ MethodEvaluation - 2.2.0 + 2.3.0 @@ -26,7 +26,7 @@ Reference
  • - 
    getOption("sqlRenderTempEmulationSchema"),
-  outcomeDatabaseSchema = cdmDatabaseSchema,
-  outcomeTable = "cohort",
-  exposureDatabaseSchema = cdmDatabaseSchema,
-  exposureTable = "drug_era",
-  referenceSet = "ohdsiMethodsBenchmark",
-  maxCores = 1,
-  workFolder,
-  summaryFileName = file.path(workFolder, "allControls.csv")
-)
    + 
    synthesizeReferenceSetPositiveControls(
+  connectionDetails,
+  oracleTempSchema = NULL,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  cdmDatabaseSchema,
+  exposureDatabaseSchema = cdmDatabaseSchema,
+  exposureTable = "drug_era",
+  outcomeDatabaseSchema = cdmDatabaseSchema,
+  outcomeTable = "cohort",
+  referenceSet = "ohdsiMethodsBenchmark",
+  maxCores = 1,
+  workFolder,
+  summaryFileName = file.path(workFolder, "allControls.csv")
+)
    @@ -88,46 +88,69 @@

    Arguments

    An R object of type ConnectionDetails created using the function createConnectionDetails in the DatabaseConnector package.

    -
    cdmDatabaseSchema
    -

    A database schema containing health care data in the OMOP Commond -Data Model. Note that for SQL Server, botth the database and schema -should be specified, e.g. 'cdm_schema.dbo'

    + +
    oracleTempSchema

    DEPRECATED: use `tempEmulationSchema` instead.

    + +
    tempEmulationSchema

    Some database platforms like Oracle and Impala do not truly support temp tables. To emulate temp tables, provide a schema with write privileges where temp tables can be created.

    -
    outcomeDatabaseSchema
    -

    The database schema where the target outcome table is located. Note -that for SQL Server, both the database and schema should be -specified, e.g. 'cdm_schema.dbo'

    -
    outcomeTable
    -

    The name of the table where the outcomes will be stored.

    + + +
    cdmDatabaseSchema
    +

    A database schema containing health care data in the OMOP Commond +Data Model. Note that for SQL Server, botth the database and schema +should be specified, e.g. 'cdm_schema.dbo'

    + +
    exposureDatabaseSchema

    The name of the database schema that is the location where the exposure data used to define the exposure cohorts is available. If exposureTable = DRUG_ERA, exposureDatabaseSchema is not used and assumed to be cdmDatabaseSchema. Requires read permissions to this database.

    + +
    exposureTable

    The tablename that contains the exposure cohorts. If exposureTable <> DRUG_ERA, then expectation is exposureTable has format of COHORT table: COHORT_DEFINITION_ID, SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.

    + + +
    outcomeDatabaseSchema
    +

    The database schema where the target outcome table is located. Note +that for SQL Server, both the database and schema should be +specified, e.g. 'cdm_schema.dbo'

    + + +
    outcomeTable
    +

    The name of the table where the outcomes will be stored.

    + +
    referenceSet

    The name of the reference set for which positive controls need to be synthesized. Currently supported are "ohdsiMethodsBenchmark" and "ohdsiDevelopment".

    + +
    maxCores

    How many parallel cores should be used? If more cores are made available this can speed up the analyses.

    + +
    workFolder

    Name of local folder to place intermediary results; make sure to use forward slashes (/). Do not use a folder on a network drive since this greatly impacts performance.

    + +
    summaryFileName

    The name of the CSV file where to store the summary of the final set of positive and negative controls.

    +

    Details

    @@ -153,7 +176,7 @@

    Details

    -

    Site built with pkgdown 2.0.2.

    +

    Site built with pkgdown 2.0.7.

    diff --git a/docs/sitemap.xml b/docs/sitemap.xml index a5124fb..0e36318 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -78,6 +78,9 @@ /reference/omopReferenceSet.html + + /reference/packageCustomBenchmarkResults.html + /reference/packageOhdsiBenchmarkResults.html diff --git a/extras/MethodEvaluation.pdf b/extras/MethodEvaluation.pdf index 0f0a3b2..52dbd60 100644 Binary files a/extras/MethodEvaluation.pdf and b/extras/MethodEvaluation.pdf differ diff --git a/inst/doc/OhdsiMethodsBenchmark.pdf b/inst/doc/OhdsiMethodsBenchmark.pdf index ae50fd3..46b70a9 100644 Binary files a/inst/doc/OhdsiMethodsBenchmark.pdf and b/inst/doc/OhdsiMethodsBenchmark.pdf differ diff --git a/man/computeMdrr.Rd b/man/computeMdrr.Rd index 8f6e1fa..6d27f51 100644 --- a/man/computeMdrr.Rd +++ b/man/computeMdrr.Rd @@ -6,9 +6,9 @@ \usage{ computeMdrr( connectionDetails, - cdmDatabaseSchema, oracleTempSchema = NULL, tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), + cdmDatabaseSchema, exposureOutcomePairs, exposureDatabaseSchema = cdmDatabaseSchema, exposureTable = "drug_era", @@ -22,14 +22,14 @@ computeMdrr( function \code{createConnectionDetails} in the \code{DatabaseConnector} package.} -\item{cdmDatabaseSchema}{Name of database schema that contains OMOP CDM and vocabulary.} - \item{oracleTempSchema}{DEPRECATED: use `tempEmulationSchema` instead.} \item{tempEmulationSchema}{Some database platforms like Oracle and Impala do not truly support temp tables. To emulate temp tables, provide a schema with write privileges where temp tables can be created.} +\item{cdmDatabaseSchema}{Name of database schema that contains OMOP CDM and vocabulary.} + \item{exposureOutcomePairs}{A data frame with at least two columns: \itemize{ \item {"exposureId" or "targetId" containing the drug_concept_ID or diff --git a/man/createReferenceSetCohorts.Rd b/man/createReferenceSetCohorts.Rd index 6148ae1..b61193d 100644 --- a/man/createReferenceSetCohorts.Rd +++ b/man/createReferenceSetCohorts.Rd @@ -34,12 +34,12 @@ can be created.} Data Model. Note that for SQL Server, botth the database and schema should be specified, e.g. 'cdm_schema.dbo'.} -\item{exposureDatabaseSchema}{The name of the database schema where the exposure cohorts will be +\item{exposureDatabaseSchema}{The name of the database schema where the exposure cohorts will be created. Only needed if \code{referenceSet = 'ohdsiDevelopment'}. Note -that for SQL Server, both the database and schema should be specified, +that for SQL Server, both the database and schema should be specified, e.g. 'cdm_schema.dbo'.} -\item{exposureTable}{The name of the table that will be created to store the exposure +\item{exposureTable}{The name of the table that will be created to store the exposure cohorts. Only needed if \code{referenceSet = 'ohdsiDevelopment'}.} \item{outcomeDatabaseSchema}{The database schema where the target outcome table is located. Note @@ -49,15 +49,15 @@ specified, e.g. 'cdm_schema.dbo'} \item{outcomeTable}{The name of the table where the outcomes will be stored.} \item{nestingDatabaseSchema}{(For the OHDSI Methods Benchmark and OHDSI Development Set only) The -database schema where the nesting outcome table is located. Note that +database schema where the nesting outcome table is located. Note that for SQL Server, both the database and schema should be specified, e.g. 'cdm_schema.dbo'.} -\item{nestingTable}{(For the OHDSI Methods Benchmark and OHDSI Development Set only) The +\item{nestingTable}{(For the OHDSI Methods Benchmark and OHDSI Development Set only) The name of the table where the nesting cohorts will be stored.} \item{referenceSet}{The name of the reference set for which outcomes need to be created. -Currently supported are "omopReferenceSet", "euadrReferenceSet", +Currently supported are "omopReferenceSet", "euadrReferenceSet", "ohdsiMethodsBenchmark", and "ohdsiDevelopment".} \item{workFolder}{Name of local folder to place intermediary results; make sure to use @@ -73,6 +73,6 @@ reference sets. The outcomes of interest are derives using information like diag and drug prescriptions. The outcomes are stored in a table on the database server. For the 'ohdsiMethodsBenchmark' reference set, the exposures are taken from the drug_era table, and -are therefore not generated as separate cohorts, and an exposure cohort table therefore needn't be supplied. +are therefore not generated as separate cohorts, and an exposure cohort table therefore needn't be supplied. For the 'ohdsiDevelopment' reference set, exposure cohorts will be generated. } diff --git a/man/ohdsiDevelopmentNegativeControls.Rd b/man/ohdsiDevelopmentNegativeControls.Rd index 2e2619e..3611819 100644 --- a/man/ohdsiDevelopmentNegativeControls.Rd +++ b/man/ohdsiDevelopmentNegativeControls.Rd @@ -5,7 +5,7 @@ \alias{ohdsiDevelopmentNegativeControls} \title{The OHDSI Development Set - Negative Controls A set of 76 negative control outcomes, all for the exposures of ACE inhibitors (compared to -thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method +thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method Evaluation Benchmark, but follows the same principles. It is intended to be used when developing methods, leaving the Methods Benchark untouched until a final evaluation of the method, thus preventing 'training' on the evaluation set. The negative controls are borrowed from the LEGEND Hypertension study. @@ -29,7 +29,7 @@ data(ohdsiDevelopmentNegativeControls) \description{ The OHDSI Development Set - Negative Controls A set of 76 negative control outcomes, all for the exposures of ACE inhibitors (compared to -thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method +thiazides and thiazide-like diuretics). This set is a much small set than the he OHDSI Method Evaluation Benchmark, but follows the same principles. It is intended to be used when developing methods, leaving the Methods Benchark untouched until a final evaluation of the method, thus preventing 'training' on the evaluation set. The negative controls are borrowed from the LEGEND Hypertension study. diff --git a/man/plotControls.Rd b/man/plotControls.Rd index 64411cd..6dd4896 100644 --- a/man/plotControls.Rd +++ b/man/plotControls.Rd @@ -12,7 +12,7 @@ plotControls( trueLogRr, estimateType = "relative risk", fileName = NULL, - title + title = NULL ) } \arguments{ @@ -20,13 +20,13 @@ plotControls( \item{seLogRr}{The standard error of the log of the effect estimates. Hint: often the standard error = (log() - log())/qnorm(0.025). If not provided the standard error will be inferred from +estimate>))/qnorm(0.025). If not provided the standard error will be inferred from the 95 percent confidence interval.} -\item{ci95Lb}{The lower bound of the 95 percent confidence interval. IF not provided it will be +\item{ci95Lb}{The lower bound of the 95 percent confidence interval. IF not provided it will be inferred from the standard error.} -\item{ci95Ub}{The upper bound of the 95 percent confidence interval. IF not provided it will be +\item{ci95Ub}{The upper bound of the 95 percent confidence interval. IF not provided it will be inferred from the standard error.} \item{trueLogRr}{A vector of the true effect sizes} diff --git a/man/synthesizeReferenceSetPositiveControls.Rd b/man/synthesizeReferenceSetPositiveControls.Rd index 7e18d63..b966679 100644 --- a/man/synthesizeReferenceSetPositiveControls.Rd +++ b/man/synthesizeReferenceSetPositiveControls.Rd @@ -6,13 +6,13 @@ \usage{ synthesizeReferenceSetPositiveControls( connectionDetails, - cdmDatabaseSchema, oracleTempSchema = NULL, tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), - outcomeDatabaseSchema = cdmDatabaseSchema, - outcomeTable = "cohort", + cdmDatabaseSchema, exposureDatabaseSchema = cdmDatabaseSchema, exposureTable = "drug_era", + outcomeDatabaseSchema = cdmDatabaseSchema, + outcomeTable = "cohort", referenceSet = "ohdsiMethodsBenchmark", maxCores = 1, workFolder, @@ -24,21 +24,15 @@ synthesizeReferenceSetPositiveControls( function \code{createConnectionDetails} in the \code{DatabaseConnector} package.} -\item{cdmDatabaseSchema}{A database schema containing health care data in the OMOP Commond -Data Model. Note that for SQL Server, botth the database and schema -should be specified, e.g. 'cdm_schema.dbo'} - \item{oracleTempSchema}{DEPRECATED: use `tempEmulationSchema` instead.} \item{tempEmulationSchema}{Some database platforms like Oracle and Impala do not truly support temp tables. To emulate temp tables, provide a schema with write privileges where temp tables can be created.} -\item{outcomeDatabaseSchema}{The database schema where the target outcome table is located. Note -that for SQL Server, both the database and schema should be -specified, e.g. 'cdm_schema.dbo'} - -\item{outcomeTable}{The name of the table where the outcomes will be stored.} +\item{cdmDatabaseSchema}{A database schema containing health care data in the OMOP Commond +Data Model. Note that for SQL Server, botth the database and schema +should be specified, e.g. 'cdm_schema.dbo'} \item{exposureDatabaseSchema}{The name of the database schema that is the location where the exposure data used to define the exposure cohorts is available. If @@ -51,6 +45,12 @@ database.} table: COHORT_DEFINITION_ID, SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.} +\item{outcomeDatabaseSchema}{The database schema where the target outcome table is located. Note +that for SQL Server, both the database and schema should be +specified, e.g. 'cdm_schema.dbo'} + +\item{outcomeTable}{The name of the table where the outcomes will be stored.} + \item{referenceSet}{The name of the reference set for which positive controls need to be synthesized. Currently supported are "ohdsiMethodsBenchmark" and "ohdsiDevelopment".} diff --git a/tests/testthat/test-01.R b/tests/testthat/test-01.R index 93a2ec7..0b3f4f4 100644 --- a/tests/testthat/test-01.R +++ b/tests/testthat/test-01.R @@ -5,76 +5,96 @@ connectionDetails <- Eunomia::getEunomiaConnectionDetails() Eunomia::createCohorts(connectionDetails) test_that("Compute MDRR", { - result <- computeMdrr(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - exposureOutcomePairs = data.frame( - exposureId = c(1,2), - outcomeId = c(4,5)), - exposureTable = "cohort", - outcomeTable = "cohort") - + result <- computeMdrr( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + exposureOutcomePairs = data.frame( + exposureId = c(1, 2), + outcomeId = c(4, 5) + ), + exposureTable = "cohort", + outcomeTable = "cohort" + ) + expect_equal(ncol(result), 7) # TODO: need to check results (that are currently empty) }) test_that("Synthesize positive controls", { - expect_error( suppressWarnings( - synthesizePositiveControls(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - exposureOutcomePairs = data.frame( - exposureId = c(1), - outcomeId = c(4))) + synthesizePositiveControls( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + exposureOutcomePairs = data.frame( + exposureId = c(1), + outcomeId = c(4) + ) + ) ), - "Not enough negative controls") - + "Not enough negative controls" + ) + # TODO: need to check an actual synthesis - + expect_error( - suppressWarnings( - injectSignals(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - exposureOutcomePairs = data.frame( - exposureId = c(1), - outcomeId = c(4))) + suppressWarnings( + injectSignals( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + exposureOutcomePairs = data.frame( + exposureId = c(1), + outcomeId = c(4) + ) + ) ), - "Not enough negative controls") + "Not enough negative controls" + ) }) test_that("Create reference set cohort", { tempFolder <- tempfile() dir.create(tempFolder) - createReferenceSetCohorts(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - referenceSet = "omopReferenceSet", - workFolder = tempFolder) - + createReferenceSetCohorts( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + referenceSet = "omopReferenceSet", + workFolder = tempFolder + ) + # TODO Need a check for completion - - createReferenceSetCohorts(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - referenceSet = "ohdsiMethodsBenchmark", - workFolder = tempFolder) + + createReferenceSetCohorts( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + referenceSet = "ohdsiMethodsBenchmark", + workFolder = tempFolder + ) cohortCounts <- read.csv(file.path(tempFolder, "cohortCounts.csv")) controls <- readRDS(system.file("ohdsiNegativeControls.rds", - package = "MethodEvaluation")) %>% + package = "MethodEvaluation" + )) %>% filter(.data$outcomeId > 4) %>% - distinct(cohortId = .data$outcomeId) + distinct(cohortId = .data$outcomeId) expect_equal( nrow(cohortCounts %>% filter(type == "Outcome")), - nrow(controls) + 4) # adding c("acute_pancreatitis", "gi_bleed", "stroke", "ibd") - - createReferenceSetCohorts(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - referenceSet = "ohdsiDevelopment", - workFolder = tempFolder) + nrow(controls) + 4 + ) # adding c("acute_pancreatitis", "gi_bleed", "stroke", "ibd") + + createReferenceSetCohorts( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + referenceSet = "ohdsiDevelopment", + workFolder = tempFolder + ) cohortCounts <- read.csv(file.path(tempFolder, "cohortCounts.csv")) controls <- readRDS(system.file("ohdsiDevelopmentNegativeControls.rds", - package = "MethodEvaluation")) + package = "MethodEvaluation" + )) expect_equal( nrow(cohortCounts %>% filter(type == "Outcome")), - nrow(controls)) + nrow(controls) + ) unlink(tempFolder, recursive = TRUE) }) @@ -84,19 +104,25 @@ test_that("Synthesize positive control reference set cohort", { dir.create(tempFolder) expect_error( suppressWarnings( - synthesizeReferenceSetPositiveControls(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - referenceSet = "ohdsiMethodsBenchmark", - workFolder = tempFolder) + synthesizeReferenceSetPositiveControls( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + referenceSet = "ohdsiMethodsBenchmark", + workFolder = tempFolder + ) ), - "Not enough negative controls") + "Not enough negative controls" + ) expect_error( suppressWarnings( - synthesizeReferenceSetPositiveControls(connectionDetails = connectionDetails, - cdmDatabaseSchema = "main", - referenceSet = "ohdsiDevelopment", - workFolder = tempFolder) + synthesizeReferenceSetPositiveControls( + connectionDetails = connectionDetails, + cdmDatabaseSchema = "main", + referenceSet = "ohdsiDevelopment", + workFolder = tempFolder + ) ), - "Not enough negative controls") + "Not enough negative controls" + ) unlink(tempFolder, recursive = TRUE) }) diff --git a/tests/testthat/test-package.R b/tests/testthat/test-package.R index 54c95c9..8af7b94 100644 --- a/tests/testthat/test-package.R +++ b/tests/testthat/test-package.R @@ -2,38 +2,42 @@ library(testthat) library(dplyr) test_that("Package results", { - # packageOhdsiBenchmarkResults - + estimates <- readr::read_delim(col_names = TRUE, delim = ";", trim_ws = TRUE, file = " targetId ; outcomeId ; analysisId ; logRr ; seLogRr ; ci95Lb ; ci95Ub 1 ; 3 ; 1; 0 ; 0 ; 0 ; 0") - + controlSummary <- readr::read_delim(col_names = TRUE, delim = ";", trim_ws = TRUE, file = " targetId ; comparatorId ; outcomeId ; nestingId; targetName ; comparatorName ; outcomeName ; nestingName ; type ; targetEffectSize ; trueEffectSize ; trueEffectSizeFirstExposure ; oldOutcomeId ; mdrrTarget ; mdrrComparator - 1 ; 2 ; 3 ; 1 ; name1 ; name2 ; name3 ; nest ; fun ; 1.25 ; 1 ; 1 ; 1 ; 1 ; 1") - - analysisRef <- data.frame(method = "CohortMethod", - analysisId = 1, - description = "1-on-1 matching", - details = "", - comparative = TRUE, - nesting = FALSE, - firstExposureOnly = TRUE) + 1 ; 2 ; 3 ; 1 ; name1 ; name2 ; name3 ; nest ; fun ; 1.25 ; 1 ; 1 ; 1 ; 1 ; 1") + + analysisRef <- data.frame( + method = "CohortMethod", + analysisId = 1, + description = "1-on-1 matching", + details = "", + comparative = TRUE, + nesting = FALSE, + firstExposureOnly = TRUE + ) tempFolder <- tempfile() dir.create(tempFolder) - - packageOhdsiBenchmarkResults(estimates = estimates, - controlSummary = controlSummary, - analysisRef = analysisRef, - databaseName = "test", - exportFolder = tempFolder) + + packageOhdsiBenchmarkResults( + estimates = estimates, + controlSummary = controlSummary, + analysisRef = analysisRef, + databaseName = "test", + exportFolder = tempFolder + ) result <- read.csv(file.path(tempFolder, "estimates_CohortMethod_test.csv")) expect_equal(result$analysisId[1], 1) - + expect_error( metrics <- computeOhdsiBenchmarkMetrics(exportFolder = tempFolder), - "'response' must have two levels") - + "'response' must have two levels" + ) + unlink(tempFolder, recursive = TRUE) }) diff --git a/tests/testthat/test-plots.R b/tests/testthat/test-plots.R index f5dfcdd..92ea754 100644 --- a/tests/testthat/test-plots.R +++ b/tests/testthat/test-plots.R @@ -4,13 +4,15 @@ test_that("Plot ROCs", { set.seed(123) data <- EmpiricalCalibration::simulateControls(n = 50 * 3, trueLogRr = log(c(1, 2, 4))) fileName <- tempfile(fileext = ".png") - - plot <- plotRocsInjectedSignals(logRr = data$logRr, trueLogRr = data$trueLogRr, showAucs = TRUE, - fileName = fileName) - + + plot <- plotRocsInjectedSignals( + logRr = data$logRr, trueLogRr = data$trueLogRr, showAucs = TRUE, + fileName = fileName + ) + expect_false(is.null(plot)) expect_true(file.exists(fileName)) - + unlink(fileName) }) @@ -18,13 +20,15 @@ test_that("Plot coverage", { set.seed(123) data <- EmpiricalCalibration::simulateControls(n = 50 * 3, trueLogRr = log(c(1, 2, 4))) fileName <- tempfile(fileext = ".png") - - plot <- plotCoverageInjectedSignals(logRr = data$logRr, seLogRr = data$seLogRr, trueLogRr = data$trueLogRr, - fileName = fileName) - + + plot <- plotCoverageInjectedSignals( + logRr = data$logRr, seLogRr = data$seLogRr, trueLogRr = data$trueLogRr, + fileName = fileName + ) + expect_false(is.null(plot)) expect_true(file.exists(fileName)) - + unlink(fileName) }) @@ -32,12 +36,14 @@ test_that("Plot controls", { set.seed(123) data <- EmpiricalCalibration::simulateControls(n = 50 * 3, trueLogRr = log(c(1, 2, 4))) fileName <- tempfile(fileext = ".png") - - plot <- plotControls(logRr = data$logRr, seLogRr = data$seLogRr, trueLogRr = data$trueLogRr, - fileName = fileName) - + + plot <- plotControls( + logRr = data$logRr, seLogRr = data$seLogRr, trueLogRr = data$trueLogRr, + fileName = fileName + ) + expect_false(is.null(plot)) expect_true(file.exists(fileName)) - + unlink(fileName) }) diff --git a/vignettes/OhdsiMethodsBenchmark.Rmd b/vignettes/OhdsiMethodsBenchmark.Rmd index c47c06c..cf73a41 100644 --- a/vignettes/OhdsiMethodsBenchmark.Rmd +++ b/vignettes/OhdsiMethodsBenchmark.Rmd @@ -19,7 +19,8 @@ knitr::opts_chunk$set( cache = FALSE, comment = "#>", error = FALSE, - tidy = FALSE) + tidy = FALSE +) ``` # Introduction @@ -76,10 +77,12 @@ Below is example code showing we specify how to connect to the database server, ```{r tidy=FALSE,eval=FALSE} library(MethodEvaluation) -connectionDetails <- createConnectionDetails(dbms = "postgresql", - server = "localhost/ohdsi", - user = "joe", - password = "supersecret") +connectionDetails <- createConnectionDetails( + dbms = "postgresql", + server = "localhost/ohdsi", + user = "joe", + password = "supersecret" +) cdmDatabaseSchema <- "my_cdm_data" oracleTempSchema <- NULL @@ -98,14 +101,16 @@ Note that for Microsoft SQL Server, database schemas need to specify both the da In order to run our population-level effect estimation method, we will need to identify the exposures and outcomes of interest. To identify exposures we will simlpy use the predefined exposure eras in the `drug_era` table in the CDM. However, for the outcomes custom cohorts need to be generated. In addition, some methods such as the nested case-control design require nesting cohorts to be defined. The negative control outcome and nesting cohorts can be generated using the following command: ```{r tidy=FALSE,eval=FALSE} -createReferenceSetCohorts(connectionDetails = connectionDetails, - oracleTempSchema = oracleTempSchema, - cdmDatabaseSchema = cdmDatabaseSchema, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - nestingDatabaseSchema = nestingCohortDatabaseSchema, - nestingTable = nestingCohortTable, - referenceSet = "ohdsiMethodsBenchmark") +createReferenceSetCohorts( + connectionDetails = connectionDetails, + oracleTempSchema = oracleTempSchema, + cdmDatabaseSchema = cdmDatabaseSchema, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + nestingDatabaseSchema = nestingCohortDatabaseSchema, + nestingTable = nestingCohortTable, + referenceSet = "ohdsiMethodsBenchmark" +) ``` This will automatically create the outcome and nesting cohort tables, and populate them. @@ -113,16 +118,20 @@ This will automatically create the outcome and nesting cohort tables, and popula The nest step is to create the positive controls. These are generated by taking the negative controls, and adding synthetic outcomes during exposed time to achieve predefined relative risks greater than one. The synthetic outcomes are sampled based on models that are fitted for each negative control outcome, using predictors extracted from the data. These positive control outcomes are added to the outcome table. ```{r tidy=FALSE,eval=FALSE} -synthesizeReferenceSetPositiveControls(connectionDetails = connectionDetails, - oracleTempSchema = oracleTempSchema, - cdmDatabaseSchema = cdmDatabaseSchema, - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - maxCores = 10, - workFolder = outputFolder, - summaryFileName = file.path(outputFolder, - "allControls.csv"), - referenceSet = "ohdsiMethodsBenchmark") +synthesizeReferenceSetPositiveControls( + connectionDetails = connectionDetails, + oracleTempSchema = oracleTempSchema, + cdmDatabaseSchema = cdmDatabaseSchema, + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + maxCores = 10, + workFolder = outputFolder, + summaryFileName = file.path( + outputFolder, + "allControls.csv" + ), + referenceSet = "ohdsiMethodsBenchmark" +) ``` Fitting the outcome models can take a while. It is recommend to set the `maxCores` argument to the number of CPU cores available in the machine to speed up the process. Note that at the end, a file will be created called *allControls.csv* in the output folder containing the information on the controls: @@ -154,55 +163,67 @@ Now that we have the controls in place, we can run the method we would like to e ```{r tidy=FALSE,eval=FALSE} library(SelfControlledCohort) -runSccArgs1 <- createRunSelfControlledCohortArgs(addLengthOfExposureExposed = TRUE, - riskWindowStartExposed = 0, - riskWindowEndExposed = 0, - riskWindowEndUnexposed = -1, - addLengthOfExposureUnexposed = TRUE, - riskWindowStartUnexposed = -1, - washoutPeriod = 365) - -sccAnalysis1 <- createSccAnalysis(analysisId = 1, - description = "Length of exposure", - runSelfControlledCohortArgs = runSccArgs1) - -runSccArgs2 <- createRunSelfControlledCohortArgs(addLengthOfExposureExposed = FALSE, - riskWindowStartExposed = 0, - riskWindowEndExposed = 30, - riskWindowEndUnexposed = -1, - addLengthOfExposureUnexposed = FALSE, - riskWindowStartUnexposed = -30, - washoutPeriod = 365) - -sccAnalysis2 <- createSccAnalysis(analysisId = 2, - description = "30 days of each exposure", - runSelfControlledCohortArgs = runSccArgs2) +runSccArgs1 <- createRunSelfControlledCohortArgs( + addLengthOfExposureExposed = TRUE, + riskWindowStartExposed = 0, + riskWindowEndExposed = 0, + riskWindowEndUnexposed = -1, + addLengthOfExposureUnexposed = TRUE, + riskWindowStartUnexposed = -1, + washoutPeriod = 365 +) + +sccAnalysis1 <- createSccAnalysis( + analysisId = 1, + description = "Length of exposure", + runSelfControlledCohortArgs = runSccArgs1 +) + +runSccArgs2 <- createRunSelfControlledCohortArgs( + addLengthOfExposureExposed = FALSE, + riskWindowStartExposed = 0, + riskWindowEndExposed = 30, + riskWindowEndUnexposed = -1, + addLengthOfExposureUnexposed = FALSE, + riskWindowStartUnexposed = -30, + washoutPeriod = 365 +) + +sccAnalysis2 <- createSccAnalysis( + analysisId = 2, + description = "30 days of each exposure", + runSelfControlledCohortArgs = runSccArgs2 +) sccAnalysisList <- list(sccAnalysis1, sccAnalysis2) ``` Next, we specify all exposure-outcome pairs we want to apply the SCC design to. Note that we only use the `targetId` and `outcomeId` fields of the controls, because the SCC design can only be used for effect estimation, and does not support nesting: ```{r tidy=FALSE,eval=FALSE} -allControls <- read.csv(file.path(outputFolder , "allControls.csv")) +allControls <- read.csv(file.path(outputFolder, "allControls.csv")) eos <- list() for (i in 1:nrow(allControls)) { - eos[[length(eos) + 1]] <- createExposureOutcome(exposureId = allControls$targetId[i], - outcomeId = allControls$outcomeId[i]) + eos[[length(eos) + 1]] <- createExposureOutcome( + exposureId = allControls$targetId[i], + outcomeId = allControls$outcomeId[i] + ) } ``` Then, we run the SCC method. The following command will execute the SCC with the specified analysis settings against the exposure-outcome pairs: ```{r tidy=FALSE,eval=FALSE} -sccResult <- runSccAnalyses(connectionDetails = connectionDetails, - cdmDatabaseSchema = cdmDatabaseSchema, - oracleTempSchema = oracleTempSchema, - exposureTable = "drug_era", - outcomeDatabaseSchema = outcomeDatabaseSchema, - outcomeTable = outcomeTable, - sccAnalysisList = sccAnalysisList, - exposureOutcomeList = eos, - outputFolder = outputFolder) +sccResult <- runSccAnalyses( + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + oracleTempSchema = oracleTempSchema, + exposureTable = "drug_era", + outcomeDatabaseSchema = outcomeDatabaseSchema, + outcomeTable = outcomeTable, + sccAnalysisList = sccAnalysisList, + exposureOutcomeList = eos, + outputFolder = outputFolder +) sccSummary <- summarizeAnalyses(sccResult, outputFolder) write.csv(sccSummary, file.path(outputFolder, "sccSummary.csv"), row.names = FALSE) ``` @@ -217,13 +238,15 @@ First, we specify the estimates: ```{r tidy=FALSE,eval=FALSE} estimates <- readRDS(file.path(outputFolder, "sccSummary.csv")) -estimates <- data.frame(analysisId = estimates$analysisId, - targetId = estimates$exposureId, - outcomeId = estimates$outcomeId, - logRr = estimates$logRr, - seLogRr = estimates$seLogRr, - ci95Lb = estimates$irrLb95, - ci95Ub = estimates$irrUb95) +estimates <- data.frame( + analysisId = estimates$analysisId, + targetId = estimates$exposureId, + outcomeId = estimates$outcomeId, + logRr = estimates$logRr, + seLogRr = estimates$seLogRr, + ci95Lb = estimates$irrLb95, + ci95Ub = estimates$irrUb95 +) head(estimates) ``` ```{r echo=FALSE,message=FALSE} @@ -239,26 +262,32 @@ writeLines(" analysisId targetId outcomeId logRr seLogRr ci95Lb ci Next, we create the analysis reference: ```{r tidy=FALSE} # Create a reference of the analysis settings: -analysisRef <- data.frame(method = "SelfControlledCohort", - analysisId = c(1, 2), - description = c("Length of exposure", - "30 days of each exposure"), - details = "", - comparative = FALSE, - nesting = FALSE, - firstExposureOnly = FALSE) +analysisRef <- data.frame( + method = "SelfControlledCohort", + analysisId = c(1, 2), + description = c( + "Length of exposure", + "30 days of each exposure" + ), + details = "", + comparative = FALSE, + nesting = FALSE, + firstExposureOnly = FALSE +) head(analysisRef) ``` Finally, we supply these two data frames, together with the data frame with controls to the `packageOhdsiBenchmarkResults` function: ```{r tidy=FALSE,eval=FALSE} -allControls <- read.csv(file.path(outputFolder , "allControls.csv")) -packageOhdsiBenchmarkResults(estimates = estimates, - controlSummary = allControls, - analysisRef = analysisRef, - databaseName = databaseName, - exportFolder = file.path(outputFolder, "export")) +allControls <- read.csv(file.path(outputFolder, "allControls.csv")) +packageOhdsiBenchmarkResults( + estimates = estimates, + controlSummary = allControls, + analysisRef = analysisRef, + databaseName = databaseName, + exportFolder = file.path(outputFolder, "export") +) ``` This will generate two CSV files in the export folder. Results from different methods and databases can all be written to the same export folder. @@ -288,12 +317,13 @@ Alternatively, we can output the metrics to a data frame: ```{r tidy=FALSE,eval=FALSE} exportFolder <- file.path(outputFolder, "export") -metrics <- computeOhdsiBenchmarkMetrics(exportFolder, - mdrr = 1.25, - stratum = "All", - trueEffectSize = "Overall", - calibrated = FALSE, - comparative = FALSE) +metrics <- computeOhdsiBenchmarkMetrics(exportFolder, + mdrr = 1.25, + stratum = "All", + trueEffectSize = "Overall", + calibrated = FALSE, + comparative = FALSE +) head(metrics) ``` ```{r echo=FALSE,message=FALSE}