Failure Thresholds and How to Change Them
Clair Blacketer
-2023-10-18
+2023-11-04
Source:vignettes/Thresholds.rmd
Thresholds.rmdConvert results JSON file case convertJsonResultsFileCase()
Convert JSON results file case
+ Write database results to a JSON file
+ Function to write DQD results from a database table into a JSON file
+
+
+
+
+ Write DQD results database table to json
Write DQD results database table to json — writeDBResultsToJson • DataQualityDashboard
+
+
+
+
+
+
+
+
+ -
+
+
+-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Write DQD results database table to json
+ Source: R/writeDBResultsTo.R
+ writeDBResultsToJson.Rd
+
+
+
+ Write DQD results database table to json
+
+
+
+ writeDBResultsToJson(
+ connection,
+ resultsDatabaseSchema,
+ cdmDatabaseSchema,
+ writeTableName,
+ outputFolder,
+ outputFile
+)
+
+
+
+ Arguments
+ - connection
+A connection object
- resultsDatabaseSchema
+The fully qualified database name of the results schema
- cdmDatabaseSchema
+The fully qualified database name of the CDM schema
- writeTableName
+Name of DQD results table in the database to read from
- outputFolder
+The folder to output the json results file to
- outputFile
+The output filename of the json results file
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docs/reference/writeJsonResultsToCsv.html b/docs/reference/writeJsonResultsToCsv.html
index 3083ad75..3bbd7d1f 100644
--- a/docs/reference/writeJsonResultsToCsv.html
+++ b/docs/reference/writeJsonResultsToCsv.html
@@ -17,7 +17,7 @@
diff --git a/docs/reference/writeJsonResultsToTable.html b/docs/reference/writeJsonResultsToTable.html
index 43b428f9..d23c3f39 100644
--- a/docs/reference/writeJsonResultsToTable.html
+++ b/docs/reference/writeJsonResultsToTable.html
@@ -17,7 +17,7 @@
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 99f8a396..5ae0b4bc 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -117,6 +117,9 @@
/reference/viewDqDashboard.html
+
+ /reference/writeDBResultsToJson.html
+
/reference/writeJsonResultsToCsv.html
diff --git a/extras/DataQualityDashboard.pdf b/extras/DataQualityDashboard.pdf
index 24a693fa..eb9dfa03 100644
Binary files a/extras/DataQualityDashboard.pdf and b/extras/DataQualityDashboard.pdf differ
diff --git a/inst/doc/AddNewCheck.pdf b/inst/doc/AddNewCheck.pdf
index ff24ed57..fe53eb2d 100644
Binary files a/inst/doc/AddNewCheck.pdf and b/inst/doc/AddNewCheck.pdf differ
diff --git a/inst/doc/CheckStatusDefinitions.pdf b/inst/doc/CheckStatusDefinitions.pdf
index b158d7cb..762f2594 100644
Binary files a/inst/doc/CheckStatusDefinitions.pdf and b/inst/doc/CheckStatusDefinitions.pdf differ
diff --git a/inst/doc/CheckTypeDescriptions.pdf b/inst/doc/CheckTypeDescriptions.pdf
index 4fc209fa..ff299a02 100644
Binary files a/inst/doc/CheckTypeDescriptions.pdf and b/inst/doc/CheckTypeDescriptions.pdf differ
diff --git a/inst/doc/DataQualityDashboard.pdf b/inst/doc/DataQualityDashboard.pdf
index 9d1faceb..062c6e2b 100644
Binary files a/inst/doc/DataQualityDashboard.pdf and b/inst/doc/DataQualityDashboard.pdf differ
diff --git a/inst/doc/DqdForCohorts.pdf b/inst/doc/DqdForCohorts.pdf
index dc381e31..9025cf6f 100644
Binary files a/inst/doc/DqdForCohorts.pdf and b/inst/doc/DqdForCohorts.pdf differ
diff --git a/inst/doc/SqlOnly.pdf b/inst/doc/SqlOnly.pdf
index f9107177..85328c66 100644
Binary files a/inst/doc/SqlOnly.pdf and b/inst/doc/SqlOnly.pdf differ
diff --git a/inst/doc/Thresholds.pdf b/inst/doc/Thresholds.pdf
index bb044544..a26b1308 100644
Binary files a/inst/doc/Thresholds.pdf and b/inst/doc/Thresholds.pdf differ
diff --git a/inst/sql/sql_server/field_fk_class.sql b/inst/sql/sql_server/field_fk_class.sql
index e9d6871e..2505fefc 100755
--- a/inst/sql/sql_server/field_fk_class.sql
+++ b/inst/sql/sql_server/field_fk_class.sql
@@ -4,7 +4,7 @@ FK_CLASS
Drug era standard concepts, ingredients only
Parameters used in this template:
-cdmDatabaseSchema = @cdmDatabaseSchema
+schema = @schema
vocabDatabaseSchema = @vocabDatabaseSchema
cdmTableName = @cdmTableName
cdmFieldName = @cdmFieldName
@@ -32,7 +32,7 @@ FROM (
SELECT
'@cdmTableName.@cdmFieldName' AS violating_field,
cdmTable.*
- FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+ FROM @schema.@cdmTableName cdmTable
LEFT JOIN @vocabDatabaseSchema.concept co
ON cdmTable.@cdmFieldName = co.concept_id
{@cohort & '@runForCohort' == 'Yes'}?{
@@ -48,7 +48,7 @@ FROM (
(
SELECT
COUNT_BIG(*) AS num_rows
- FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+ FROM @schema.@cdmTableName cdmTable
{@cohort & '@runForCohort' == 'Yes'}?{
JOIN @cohortDatabaseSchema.@cohortTableName c
ON cdmTable.person_id = c.subject_id
diff --git a/inst/sql/sql_server/field_fk_domain.sql b/inst/sql/sql_server/field_fk_domain.sql
index ce596c47..a3e188f8 100755
--- a/inst/sql/sql_server/field_fk_domain.sql
+++ b/inst/sql/sql_server/field_fk_domain.sql
@@ -5,7 +5,7 @@ FIELD_FK_DOMAIN
all standard concept ids are part of specified domain
Parameters used in this template:
-cdmDatabaseSchema = @cdmDatabaseSchema
+schema = @schema
vocabDatabaseSchema = @vocabDatabaseSchema
cdmTableName = @cdmTableName
cdmFieldName = @cdmFieldName
@@ -32,7 +32,7 @@ FROM (
SELECT
'@cdmTableName.@cdmFieldName' AS violating_field,
cdmTable.*
- FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+ FROM @schema.@cdmTableName cdmTable
LEFT JOIN @vocabDatabaseSchema.concept co
ON cdmTable.@cdmFieldName = co.concept_id
{@cohort & '@runForCohort' == 'Yes'}?{
@@ -48,7 +48,7 @@ FROM (
(
SELECT
COUNT_BIG(*) AS num_rows
- FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+ FROM @schema.@cdmTableName cdmTable
{@cohort & '@runForCohort' == 'Yes'}?{
JOIN @cohortDatabaseSchema.@cohortTableName c
ON cdmTable.PERSON_ID = c.SUBJECT_ID
diff --git a/inst/sql/sql_server/field_plausible_temporal_after.sql b/inst/sql/sql_server/field_plausible_temporal_after.sql
index 8dd28eab..034ce0b8 100755
--- a/inst/sql/sql_server/field_plausible_temporal_after.sql
+++ b/inst/sql/sql_server/field_plausible_temporal_after.sql
@@ -4,7 +4,7 @@ PLAUSIBLE_TEMPORAL_AFTER
get number of records and the proportion to total number of eligible records with datetimes that do not occur on or after their corresponding datetimes
Parameters used in this template:
-cdmDatabaseSchema = @cdmDatabaseSchema
+schema = @schema
cdmTableName = @cdmTableName
cdmFieldName = @cdmFieldName
plausibleTemporalAfterTableName = @plausibleTemporalAfterTableName
@@ -33,9 +33,9 @@ FROM
SELECT
'@cdmTableName.@cdmFieldName' AS violating_field,
cdmTable.*
- FROM @cdmDatabaseSchema.@cdmTableName cdmTable
- {@cdmDatabaseSchema.@cdmTableName != @cdmDatabaseSchema.@plausibleTemporalAfterTableName}?{
- JOIN @cdmDatabaseSchema.@plausibleTemporalAfterTableName plausibleTable ON cdmTable.person_id = plausibleTable.person_id}
+ FROM @schema.@cdmTableName cdmTable
+ {@schema.@cdmTableName != @schema.@plausibleTemporalAfterTableName}?{
+ JOIN @schema.@plausibleTemporalAfterTableName plausibleTable ON cdmTable.person_id = plausibleTable.person_id}
{@cohort & '@runForCohort' == 'Yes'}?{
JOIN @cohortDatabaseSchema.@cohortTableName c ON cdmTable.person_id = c.subject_id
AND c.cohort_definition_id = @cohortDefinitionId
@@ -55,7 +55,7 @@ FROM
(
SELECT
COUNT_BIG(*) AS num_rows
- FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+ FROM @schema.@cdmTableName cdmTable
{@cohort & '@runForCohort' == 'Yes'}?{
JOIN @cohortDatabaseSchema.@cohortTableName c ON cdmTable.person_id = c.subject_id
AND c.cohort_definition_id = @cohortDefinitionId
diff --git a/man/writeDBResultsToJson.Rd b/man/writeDBResultsToJson.Rd
new file mode 100644
index 00000000..3efc6b03
--- /dev/null
+++ b/man/writeDBResultsToJson.Rd
@@ -0,0 +1,31 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/writeDBResultsTo.R
+\name{writeDBResultsToJson}
+\alias{writeDBResultsToJson}
+\title{Write DQD results database table to json}
+\usage{
+writeDBResultsToJson(
+ connection,
+ resultsDatabaseSchema,
+ cdmDatabaseSchema,
+ writeTableName,
+ outputFolder,
+ outputFile
+)
+}
+\arguments{
+\item{connection}{A connection object}
+
+\item{resultsDatabaseSchema}{The fully qualified database name of the results schema}
+
+\item{cdmDatabaseSchema}{The fully qualified database name of the CDM schema}
+
+\item{writeTableName}{Name of DQD results table in the database to read from}
+
+\item{outputFolder}{The folder to output the json results file to}
+
+\item{outputFile}{The output filename of the json results file}
+}
+\description{
+Write DQD results database table to json
+}
diff --git a/tests/testthat/test-executeDqChecks.R b/tests/testthat/test-executeDqChecks.R
index 01cb11b3..1cafeff2 100644
--- a/tests/testthat/test-executeDqChecks.R
+++ b/tests/testthat/test-executeDqChecks.R
@@ -328,3 +328,30 @@ test_that("Incremental insert SQL is valid.", {
DatabaseConnector::renderTranslateExecuteSql(connection, "DROP TABLE @database_schema.dqd_results;", database_schema = resultsDatabaseSchemaEunomia)
})
+
+test_that("Multiple cdm_source rows triggers warning.", {
+ outputFolder <- tempfile("dqd_")
+ on.exit(unlink(outputFolder, recursive = TRUE))
+
+ connectionDetailsEunomiaCS <- Eunomia::getEunomiaConnectionDetails()
+ connection <- DatabaseConnector::connect(connectionDetailsEunomiaCS)
+ on.exit(DatabaseConnector::disconnect(connection), add = TRUE)
+ DatabaseConnector::renderTranslateExecuteSql(connection, "INSERT INTO @database_schema.cdm_source VALUES ('foo',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);", database_schema = cdmDatabaseSchemaEunomia)
+
+ w <- capture_warnings(
+ results <- executeDqChecks(
+ connectionDetails = connectionDetailsEunomiaCS,
+ cdmDatabaseSchema = cdmDatabaseSchemaEunomia,
+ resultsDatabaseSchema = resultsDatabaseSchemaEunomia,
+ cdmSourceName = "Eunomia",
+ checkNames = "measurePersonCompleteness",
+ outputFolder = outputFolder,
+ writeToTable = F
+ )
+ )
+
+ expect_match(w, "Missing check names", all = FALSE)
+ expect_match(w, "The cdm_source table has", all = FALSE)
+
+ expect_true(nrow(results$CheckResults) > 1)
+})
diff --git a/tests/testthat/test-writeDBResultsTo.R b/tests/testthat/test-writeDBResultsTo.R
new file mode 100644
index 00000000..8d10e79a
--- /dev/null
+++ b/tests/testthat/test-writeDBResultsTo.R
@@ -0,0 +1,53 @@
+library(testthat)
+
+test_that("Write DB results to json", {
+ outputFolder <- tempfile("dqd_")
+ on.exit(unlink(outputFolder, recursive = TRUE))
+ connectionDetailsEunomia <- Eunomia::getEunomiaConnectionDetails()
+ cdmDatabaseSchemaEunomia <- "main"
+ resultsDatabaseSchemaEunomia <- "main"
+ writeTableName <- "dqd_db_results"
+
+ expect_warning(
+ results <- DataQualityDashboard::executeDqChecks(
+ connectionDetails = connectionDetailsEunomia,
+ cdmDatabaseSchema = cdmDatabaseSchemaEunomia,
+ resultsDatabaseSchema = resultsDatabaseSchemaEunomia,
+ cdmSourceName = "Eunomia",
+ checkNames = "measurePersonCompleteness",
+ outputFolder = outputFolder,
+ writeToTable = TRUE,
+ writeTableName = writeTableName
+ ),
+ regexp = "^Missing check names.*"
+ )
+
+ connection <- DatabaseConnector::connect(connectionDetailsEunomia)
+
+ testExportFile <- "dq-result-test.json"
+
+ DataQualityDashboard::writeDBResultsToJson(
+ connection,
+ resultsDatabaseSchemaEunomia,
+ cdmDatabaseSchemaEunomia,
+ writeTableName,
+ outputFolder,
+ testExportFile
+ )
+
+ on.exit(DatabaseConnector::disconnect(connection), add = TRUE)
+
+ # Check that file was exported properly
+ expect_true(file.exists(file.path(outputFolder, testExportFile)))
+
+ # Check that export length matches length of db table
+ results <- jsonlite::fromJSON(file.path(outputFolder, testExportFile))
+ table_rows <- DatabaseConnector::renderTranslateQuerySql(
+ connection,
+ sql = "select count(*) from @resultsDatabaseSchema.@writeTableName;",
+ resultsDatabaseSchema = resultsDatabaseSchemaEunomia,
+ writeTableName = writeTableName,
+ snakeCaseToCamelCase = TRUE
+ )
+ expect_true(length(results$CheckResults) == table_rows)
+})
diff --git a/tests/testthat/test-writeJsonResultsTo.R b/tests/testthat/test-writeJsonResultsTo.R
index 7aff258f..c9df7f0d 100644
--- a/tests/testthat/test-writeJsonResultsTo.R
+++ b/tests/testthat/test-writeJsonResultsTo.R
@@ -29,11 +29,11 @@ test_that("Write JSON results", {
connectionDetails = connectionDetailsEunomia,
resultsDatabaseSchema = resultsDatabaseSchemaEunomia,
jsonFilePath = jsonPath,
- writeTableName = "dqd_results"
+ writeTableName = "dqd_json_results"
)
connection <- DatabaseConnector::connect(connectionDetailsEunomia)
on.exit(DatabaseConnector::disconnect(connection), add = TRUE)
tableNames <- DatabaseConnector::getTableNames(connection = connection, databaseSchema = resultsDatabaseSchemaEunomia)
- expect_true("dqd_results" %in% tolower(tableNames))
- DatabaseConnector::renderTranslateExecuteSql(connection, "DROP TABLE @database_schema.dqd_results;", database_schema = resultsDatabaseSchemaEunomia)
+ expect_true("dqd_json_results" %in% tolower(tableNames))
+ DatabaseConnector::renderTranslateExecuteSql(connection, "DROP TABLE @database_schema.dqd_json_results;", database_schema = resultsDatabaseSchemaEunomia)
})
diff --git a/vignettes/SqlOnly.rmd b/vignettes/SqlOnly.rmd
index 9fabcce0..dcaad7f1 100644
--- a/vignettes/SqlOnly.rmd
+++ b/vignettes/SqlOnly.rmd
@@ -171,49 +171,18 @@ for (dqdSqlFile in dqdSqlFiles) {
)
}
-# Get results
-checkResults <- DatabaseConnector::querySql(
- c,
- SqlRender::render(
- "SELECT * FROM @resultsDatabaseSchema.@writeTableName",
- resultsDatabaseSchema = resultsDatabaseSchema,
- writeTableName = writeTableName
- ),
- snakeCaseToCamelCase = TRUE
-)
-DatabaseConnector::disconnect(c)
-
-# convert check ID column name to correct format
-colnames(checkResults)[colnames(checkResults) == "checkid"] ="checkId"
-
-# Get overview of DQD results
-library(DataQualityDashboard)
-overview <- DataQualityDashboard:::.summarizeResults(checkResults = checkResults)
-
-# Create results object, adding fake metadata
-result <- list(
- startTimestamp = Sys.time(),
- endTimestamp = Sys.time(),
- executionTime = "",
- Metadata = data.frame(
- cdmSourceName = cdmSourceName,
- cdmSourceAbbreviation = cdmSourceName,
- cdmHolder = "",
- sourceDescription = "",
- sourceDocumentationReference = "",
- cdmEtlReference = "",
- sourceReleaseDate = "",
- cdmReleaseDate = "",
- cdmVersion = cdmVersion,
- cdmVersionConceptId = 0,
- vocabularyVersion = "",
- dqdVersion = as.character(packageVersion("DataQualityDashboard"))
- ),
- Overview = overview,
- CheckResults = checkResults
-)
+# Extract results table to JSON file for viewing or secondary use
+
+DataQualityDashboard::writeDBResultsToJson(
+ c,
+ connectionDetails,
+ resultsDatabaseSchema,
+ cdmDatabaseSchema,
+ writeTableName,
+ jsonOutputFolder,
+ jsonOutputFile
+ )
-DataQualityDashboard:::.writeResultsToJson(result, jsonOutputFolder, jsonOutputFile)
jsonFilePath <- R.utils::getAbsolutePath(file.path(jsonOutputFolder, jsonOutputFile))
DataQualityDashboard::viewDqDashboard(jsonFilePath)
Convert JSON results file case
Write database results to a JSON file
+Function to write DQD results from a database table into a JSON file
+Write DQD results database table to json
+
+
+
+
+
+
+
+
+
+
+ -
+
+
+
- + + + + + +
+
+
+
+
+
+
+
+
+
+
+ Write DQD results database table to json
+ Source:R/writeDBResultsTo.R
+ writeDBResultsToJson.Rd
+
+
+ Write DQD results database table to json
+
+
+
+
+ writeDBResultsToJson(
+ connection,
+ resultsDatabaseSchema,
+ cdmDatabaseSchema,
+ writeTableName,
+ outputFolder,
+ outputFile
+)
+
+
+ Arguments
+- connection +
A connection object
+
+
+- resultsDatabaseSchema +
The fully qualified database name of the results schema
+
+
+- cdmDatabaseSchema +
The fully qualified database name of the CDM schema
+
+
+- writeTableName +
Name of DQD results table in the database to read from
+
+
+- outputFolder +
The folder to output the json results file to
+
+
+- outputFile +
The output filename of the json results file
+
+