release prep

OHDSI · Jul 13, 2024 · b26c91e · b26c91e
1 parent 54921ab
commit b26c91e
Show file tree

Hide file tree

Showing 88 changed files with 1,206 additions and 175 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: DataQualityDashboard
 Type: Package
 Title: Execute and View Data Quality Checks on OMOP CDM Database
-Version: 2.6.0
-Date: 2024-02-21
+Version: 2.6.1
+Date: 2024-07-12
 Authors@R: c(
   person("Katy", "Sadowski", email = "[email protected]", role = c("aut", "cre")),
   person("Clair", "Blacketer", role = c("aut")),

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,35 @@
+DataQualityDashboard 2.6.1
+==========================
+This release includes: 
+
+### Bugfixes
+
+- Checks
+  - `plausibleStartBeforeEnd` was failing if SOURCE_RELEASE_DATE was before CDM_RELEASE_DATE in the CDM_SOURCE table. This is the opposite of the correct logic!  The check is now updated to fail if the CDM_RELEASE_DATE is before the SOURCE_RELEASE_DATE
+  - `plausibleTemporalAfter` was throwing a syntax error in BigQuery due to the format of a hardcoded date in the SQL query.  This query has now been updated to be compliant with SqlRender and the issue has been resolved
+- A dependency issue was causing `viewDqDashboard` to error out in newer versions of R.  This has now been resolved
+- `SqlOnly` mode was failing due to the format of the new check `plausibleGenderUseDescendants`, which takes multiple concepts as an input.  This has now been fixed
+
+### New Results Field
+
+- A new field has been added to the DQD results output - `executionTimeSeconds`.  This field stores the execution time in seconds of each check in numeric format.  (The existing `executionTime` field stores execution time as a string, making it difficult to use in analysis.)
+
+### Check Threshold Updates
+
+The default thresholds for 2 checks were discovered to be inconsistently populated and occasionally set to illogical levels.  These have now been fixed as detailed below.
+
+- The default thresholds for `sourceValueCompleteness` have been updated as follows:
+  - 10% for `_source_value` columns in condition_occurrence, measurement, procedure_occurrence, drug_exposure, and visit_occurrence tables
+  - 100% for all other `_source_value` columns
+- The default thresholds for `sourceConceptRecordCompleteness` have been updated as follows:
+  - 10% for `_source_concept_id` columns in condition_occurrence, drug_exposure, measurement, procedure_occurrence, device_exposure, and observation tables
+  - 100% for all other `_source_concept_id` columns
+
+### New Documentation
+We have continued (and nearly completed) our initiative to add more comprehensive user documentation at the data quality check level.  A dedicated documentation page is being created for each check type.  Each check's page includes detailed information about how its result is generated and what to do if it fails.  Guidance is provided for both ETL developers and data users.
+
+Check out the newly added pages [here](https://ohdsi.github.io/DataQualityDashboard/articles/checkIndex.html) and please reach out with feedback as we continue improving our documentation!
+
 DataQualityDashboard 2.6.0
 ==========================
 This release includes: 

diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #
@@ -151,8 +151,12 @@
   checkResults$notApplicable <- NA
   checkResults$notApplicableReason <- NA
 
-  conditionOccurrenceIsMissing <- missingTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsMissing)
-  conditionOccurrenceIsEmpty <- emptyTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsEmpty)
+  conditionOccurrenceIsMissing <- missingTables %>%
+    dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>%
+    dplyr::pull(.data$tableIsMissing)
+  conditionOccurrenceIsEmpty <- emptyTables %>%
+    dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>%
+    dplyr::pull(.data$tableIsEmpty)
   for (i in seq_len(nrow(checkResults))) {
     # Special rule for measureConditionEraCompleteness, which should be notApplicable if CONDITION_OCCURRENCE is empty
     if (checkResults[i, "checkName"] == "measureConditionEraCompleteness") {
@@ -178,7 +182,7 @@
           .data$tableIsEmpty ~ sprintf("Table %s is empty.", .data$cdmTableName),
           .data$fieldIsEmpty ~ sprintf("Field %s.%s is not populated.", .data$cdmTableName, .data$cdmFieldName),
           .data$conceptIsMissing ~ sprintf("%s=%s is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$cdmTableName),
-          .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) #nolint
+          .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) # nolint
         ),
         NA
       ),

diff --git a/R/executeDqChecks.R b/R/executeDqChecks.R
@@ -327,7 +327,7 @@ executeDqChecks <- function(connectionDetails,
       startTimestamp = startTime,
       endTimestamp = endTime,
       executionTime = sprintf("%.0f %s", delta, attr(delta, "units")),
-      #new variable executionTimeSeconds added to store execution time in seconds
+      # new variable executionTimeSeconds added to store execution time in seconds
       executionTimeSeconds = as.numeric(delta),
       CheckResults = checkResults,
       Metadata = metadata,

diff --git a/R/sqlOnly.R b/R/sqlOnly.R
@@ -232,7 +232,7 @@
       )
     } else if (checkLevel == "CONCEPT") {
       if (is.na(unitConceptId) &&
-          grepl(",", conceptId)) {
+        grepl(",", conceptId)) {
         thresholdFilter <- sprintf(
           "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
                                   conceptChecks$cdmFieldName == '%s' &
@@ -243,7 +243,7 @@
           conceptId
         )
       } else if (is.na(unitConceptId) &&
-                 !grepl(",", conceptId)) {
+        !grepl(",", conceptId)) {
         thresholdFilter <- sprintf(
           "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
                                   conceptChecks$cdmFieldName == '%s' &

diff --git a/docs/404.html b/docs/404.html
diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html
diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html
diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html
diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html
diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html
diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html
diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html
diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html