diff --git a/NAMESPACE b/NAMESPACE index 690105a7..26cb2a43 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,6 +5,7 @@ export(executeDqChecks) export(listDqChecks) export(reEvaluateThresholds) export(viewDqDashboard) +export(writeDBResultsToJson) export(writeJsonResultsToCsv) export(writeJsonResultsToTable) import(DatabaseConnector) @@ -30,4 +31,4 @@ importFrom(tools,file_path_sans_ext) importFrom(utils,install.packages) importFrom(utils,menu) importFrom(utils,packageVersion) -importFrom(utils,write.table) +importFrom(utils,write.table) \ No newline at end of file diff --git a/R/writeDBResultsTo.R b/R/writeDBResultsTo.R new file mode 100644 index 00000000..66134ae5 --- /dev/null +++ b/R/writeDBResultsTo.R @@ -0,0 +1,82 @@ +# Copyright 2023 Observational Health Data Sciences and Informatics +# +# This file is part of DataQualityDashboard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#' Write DQD results database table to json +#' +#' @param connection A connection object +#' @param connectionDetails A connectionDetails object for connecting to the CDM database +#' @param resultsDatabaseSchema The fully qualified database name of the results schema +#' @param cdmDatabaseSchema The fully qualified database name of the CDM schema +#' @param writeTableName Name of DQD results table in the database to read from +#' @param outputFolder The folder to output the json results file to +#' @param outputFile The output filename of the json results file +#' +#' @export +#' + +writeDBResultsToJson <- function(connection, + connectionDetails, + resultsDatabaseSchema, + cdmDatabaseSchema, + writeTableName, + outputFolder, + outputFile) { + metadata <- DatabaseConnector::renderTranslateQuerySql( + connection, + sql = "select * from @cdmDatabaseSchema.cdm_source;", + cdmDatabaseSchema = cdmDatabaseSchema, + targetDialect = connectionDetails$dbms, + snakeCaseToCamelCase = TRUE + ) + + checkResults <- DatabaseConnector::renderTranslateQuerySql( + connection, + sql = "select * from @resultsDatabaseSchema.@writeTableName;", + resultsDatabaseSchema = resultsDatabaseSchema, + writeTableName = writeTableName, + targetDialect = connectionDetails$dbms, + snakeCaseToCamelCase = TRUE + ) + + # Quick patch for missing value issues related to SQL Only Implementation + checkResults["error"][checkResults["error"] == ''] <- NA + checkResults["warning"][checkResults["warning"] == ''] <- NA + checkResults["executionTime"][checkResults["executionTime"] == ''] <- '0 secs' + checkResults["queryText"][checkResults["queryText"] == ''] <- '[Generated via SQL Only]' + + overview <- .summarizeResults( + checkResults = checkResults + ) + + # Quick patch for non-camel-case column name + names(checkResults)[names(checkResults) == "checkid"] <- "checkId" + + allResults <- list( + startTimestamp = Sys.time(), + endTimestamp = Sys.time(), + executionTime = '0 secs', + CheckResults = checkResults, + Metadata = metadata, + Overview = overview + ) + + .writeResultsToJson( + allResults, + outputFolder, + outputFile + ) + +} \ No newline at end of file diff --git a/man/writeDBResultsToJson.Rd b/man/writeDBResultsToJson.Rd new file mode 100644 index 00000000..67f23ac4 --- /dev/null +++ b/man/writeDBResultsToJson.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/writeDBResultsTo.R +\name{writeDBResultsToJson} +\alias{writeDBResultsToJson} +\title{Write DQD results database table to json} +\usage{ +writeDBResultsToJson( + connection, + connectionDetails, + resultsDatabaseSchema, + cdmDatabaseSchema, + writeTableName, + outputFolder, + outputFile +) +} +\arguments{ +\item{connection}{A connection object} + +\item{connectionDetails}{A connectionDetails object for connecting to the CDM database} + +\item{resultsDatabaseSchema}{The fully qualified database name of the results schema} + +\item{cdmDatabaseSchema}{The fully qualified database name of the CDM schema} + +\item{writeTableName}{Name of table in the database to write results to} + +\item{outputFolder}{The output folder} + +\item{outputFile}{The output filename} +} +\description{ +Write DQD results database table to json +} \ No newline at end of file diff --git a/tests/testthat/test-writeDBResultsTo.R b/tests/testthat/test-writeDBResultsTo.R new file mode 100644 index 00000000..66159efc --- /dev/null +++ b/tests/testthat/test-writeDBResultsTo.R @@ -0,0 +1,54 @@ +library(testthat) + +test_that("Write DB results to json", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + connectionDetailsEunomia <- Eunomia::getEunomiaConnectionDetails() + cdmDatabaseSchemaEunomia <- "main" + resultsDatabaseSchemaEunomia <- "main" + writeTableName <- "dqdashboard_results" + + results <- DataQualityDashboard::executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkNames = "measurePersonCompleteness", + outputFolder = outputFolder, + writeToTable = TRUE, + writeTableName = writeTableName + ) + + + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + + testExportFile <- "dq-result-test.json" + + DataQualityDashboard::writeDBResultsToJson( + connection, + connectionDetailsEunomia, + resultsDatabaseSchemaEunomia, + cdmDatabaseSchemaEunomia, + writeTableName, + outputFolder, + testExportFile + ) + + on.exit(DatabaseConnector::disconnect(connection), add = TRUE) + + # Check that file was exported properly + expect_true(file.exists(file.path(outputFolder,testExportFile))) + + # Check that export length matches length of db table + results <- jsonlite::fromJSON(file.path(outputFolder,testExportFile)) + table_rows <- DatabaseConnector::renderTranslateQuerySql( + connection, + sql = "select count(*) from @resultsDatabaseSchema.@writeTableName;", + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + writeTableName = writeTableName, + targetDialect = connectionDetailsEunomia$dbms, + snakeCaseToCamelCase = TRUE + ) + expect_true(length(results$CheckResults) == table_rows) + +}) \ No newline at end of file diff --git a/vignettes/SqlOnly.rmd b/vignettes/SqlOnly.rmd index 9fabcce0..dcaad7f1 100644 --- a/vignettes/SqlOnly.rmd +++ b/vignettes/SqlOnly.rmd @@ -171,49 +171,18 @@ for (dqdSqlFile in dqdSqlFiles) { ) } -# Get results -checkResults <- DatabaseConnector::querySql( - c, - SqlRender::render( - "SELECT * FROM @resultsDatabaseSchema.@writeTableName", - resultsDatabaseSchema = resultsDatabaseSchema, - writeTableName = writeTableName - ), - snakeCaseToCamelCase = TRUE -) -DatabaseConnector::disconnect(c) - -# convert check ID column name to correct format -colnames(checkResults)[colnames(checkResults) == "checkid"] ="checkId" - -# Get overview of DQD results -library(DataQualityDashboard) -overview <- DataQualityDashboard:::.summarizeResults(checkResults = checkResults) - -# Create results object, adding fake metadata -result <- list( - startTimestamp = Sys.time(), - endTimestamp = Sys.time(), - executionTime = "", - Metadata = data.frame( - cdmSourceName = cdmSourceName, - cdmSourceAbbreviation = cdmSourceName, - cdmHolder = "", - sourceDescription = "", - sourceDocumentationReference = "", - cdmEtlReference = "", - sourceReleaseDate = "", - cdmReleaseDate = "", - cdmVersion = cdmVersion, - cdmVersionConceptId = 0, - vocabularyVersion = "", - dqdVersion = as.character(packageVersion("DataQualityDashboard")) - ), - Overview = overview, - CheckResults = checkResults -) +# Extract results table to JSON file for viewing or secondary use + +DataQualityDashboard::writeDBResultsToJson( + c, + connectionDetails, + resultsDatabaseSchema, + cdmDatabaseSchema, + writeTableName, + jsonOutputFolder, + jsonOutputFile + ) -DataQualityDashboard:::.writeResultsToJson(result, jsonOutputFolder, jsonOutputFile) jsonFilePath <- R.utils::getAbsolutePath(file.path(jsonOutputFolder, jsonOutputFile)) DataQualityDashboard::viewDqDashboard(jsonFilePath)