Skip to content

Commit

Permalink
Introduce time interval testing functionality and enhance the impleme…
Browse files Browse the repository at this point in the history
…ntation of the datediff() function.

- Modify the cdm_v5.0.sql file to incorporate additional datetime fields.
- Include timeIntervalUnit parameters to handle various scenarios such as hours/minutes/seconds.
- Write tests to validate the functionality under different time interval units, including hours/minutes/seconds.
  • Loading branch information
jcnamendiOdysseus committed Jan 25, 2024
1 parent 2f97201 commit d7cdc0d
Show file tree
Hide file tree
Showing 30 changed files with 745 additions and 51 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@
<dependency>
<groupId>org.ohdsi.sql</groupId>
<artifactId>SqlRender</artifactId>
<version>1.6.8</version>
<version>1.16.1-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,21 @@ protected String getQueryTemplate() {
return CONDITION_ERA_TEMPLATE;
}

/**
* Add params timeIntervalUnit to check for hours/minutes/seconds situation
* @param column
* @param timeIntervalUnit
* @return
*/
@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.condition_concept_id";
case ERA_OCCURRENCES:
return "C.condition_occurrence_count";
case DURATION:
return "(DATEDIFF(d,C.start_date, C.end_date))";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
default:
throw new IllegalArgumentException("Invalid CriteriaColumn for Condition Era:" + column.toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,19 @@ protected String getQueryTemplate() {
return CONDITION_OCCURRENCE_TEMPLATE;
}

/**
* Add params timeIntervalUnit to check for hours/minutes/seconds situation
* @param column
* @param timeIntervalUnit
* @return
*/
@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.condition_concept_id";
case DURATION:
return "(DATEDIFF(d,C.start_date, C.end_date))";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
default:
throw new IllegalArgumentException("Invalid CriteriaColumn for Condition Occurrence:" + column.toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public String getCriteriaSql(T criteria, BuilderOptions options) {
.filter((column) -> !this.getDefaultColumns().contains(column))
.collect(Collectors.toList());
if (filteredColumns.size() > 0) {
query = StringUtils.replace(query, "@additionalColumns", ", " + this.getAdditionalColumns(filteredColumns));
query = StringUtils.replace(query, "@additionalColumns", ", " + this.getAdditionalColumnsWithTimeInterval(filteredColumns, criteria.intervalUnit));
} else {
query = StringUtils.replace(query, "@additionalColumns", "");
}
Expand All @@ -46,16 +46,24 @@ public String getCriteriaSql(T criteria, BuilderOptions options) {
return query;
}

protected abstract String getTableColumnForCriteriaColumn(CriteriaColumn column);
protected abstract String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit);

protected String getAdditionalColumns(List<CriteriaColumn> columns) {
String cols = String.join(", ", columns.stream()
.map((column) -> {
return String.format("%s as %s", getTableColumnForCriteriaColumn(column), column.columnName());
return String.format("%s as %s", getTableColumnForCriteriaColumn(column, null), column.columnName());
}).collect(Collectors.toList()));
return cols;
}

protected String getAdditionalColumnsWithTimeInterval(List<CriteriaColumn> columns, String timeIntervalUnit) {
String cols = String.join(", ", columns.stream()
.map((column) -> {
return String.format("%s as %s", getTableColumnForCriteriaColumn(column, timeIntervalUnit), column.columnName());
}).collect(Collectors.toList()));
return cols;
}

protected abstract Set<CriteriaColumn> getDefaultColumns();

protected String embedSelectClauses(String query, List<String> selectClauses) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "coalesce(C.cause_concept_id,0)";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,15 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.device_concept_id";
case QUANTITY:
return "C.quantity";
case DURATION:
return "DATEDIFF(d,c.start_date, c.end_date)";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
// return "DATEDIFF(d,c.start_date, c.end_date)";
default:
throw new IllegalArgumentException("Invalid CriteriaColumn for Device Exposure:" + column.toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.drug_concept_id";
case DURATION:
return "DATEDIFF(d, C.start_date, C.end_date)";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
case UNIT:
return "C.unit_concept_id";
case VALUE_AS_NUMBER:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.drug_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,14 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DAYS_SUPPLY:
return "C.days_supply";
case DOMAIN_CONCEPT:
return "C.drug_concept_id";
case DURATION:
return "DATEDIFF(d, C.start_date, C.end_date)";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
case QUANTITY:
return "C.quantity";
case REFILLS:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.region_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.measurement_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.period_type_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.observation_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.payer_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.procedure_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.specimen_concept_id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.visit_detail_concept_id";
case DURATION:
return "DATEDIFF(d, C.start_date, C.end_date)";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
default:
throw new IllegalArgumentException("Invalid CriteriaColumn for Visit Detail:" + column.toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ protected String getQueryTemplate() {
}

@Override
protected String getTableColumnForCriteriaColumn(CriteriaColumn column) {
protected String getTableColumnForCriteriaColumn(CriteriaColumn column, String timeIntervalUnit) {
switch (column) {
case DOMAIN_CONCEPT:
return "C.visit_concept_id";
case DURATION:
return "DATEDIFF(d, C.start_date, C.end_date)";
return String.format("DATEDIFF(%s,c.start_date, c.end_date)", StringUtils.isEmpty(timeIntervalUnit) ? "d" : timeIntervalUnit);
default:
throw new IllegalArgumentException("Invalid CriteriaColumn for Visit Occurrence:" + column.toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
import org.ohdsi.circe.AbstractDatabaseTest;
import org.ohdsi.circe.cohortdefinition.CohortExpressionQueryBuilder;
import org.ohdsi.circe.cohortdefinition.ConditionOccurrence;
import org.ohdsi.circe.cohortdefinition.CorelatedCriteria;
import org.ohdsi.circe.cohortdefinition.CriteriaGroup;
import org.ohdsi.circe.cohortdefinition.Occurrence;
import org.ohdsi.circe.cohortdefinition.*;
import org.ohdsi.sql.SqlRender;
import org.ohdsi.sql.SqlTranslate;
import org.slf4j.Logger;
Expand Down Expand Up @@ -193,6 +189,53 @@ public void distinctVisitTest() throws Exception {

}

@Test
public void distinctVisitTestWithTimeInterval() throws Exception {
final CohortExpressionQueryBuilder queryBuilder = new CohortExpressionQueryBuilder();
final String RESULTS_SCHEMA = "distinct_visit";
final String[] testDataSetsPrep = new String[] { "/datasets/vocabulary.json",
"/corelatedcriteria/distinctVisit_PREP.json"};

// Load expected data from an XML dataset
final String[] testDataSetsVerify = new String[] {"/corelatedcriteria/distinctVisit_VERIFY.json"};
final IDataSet expectedDataSet = DataSetFactory.createDataSet(testDataSetsVerify);

// prepare results schema for the specified options.resultSchema
prepareSchema(RESULTS_SCHEMA, RESULTS_DDL_PATH);

final IDatabaseConnection dbUnitCon = getConnection();

// load test data into DB.
final IDataSet dsPrep = DataSetFactory.createDataSet(testDataSetsPrep);
DatabaseOperation.CLEAN_INSERT.execute(dbUnitCon, dsPrep); // clean load of the DB. Careful, clean means "delete the old stuff"

/// build inclusion query for Group Criteria
CriteriaGroup cg = new CriteriaGroup();
cg.type= "ALL";
CorelatedCriteria cc = new CorelatedCriteria();
cc.criteria = new ConditionOccurrence(); // find any condition occurence
cc.criteria.intervalUnit = IntervalUnit.HOUR.getName();
cc.startWindow = CriteriaUtils.getPrior365Window();
cc.occurrence = CriteriaUtils.getDistinctCount(CriteriaColumn.VISIT_ID, Occurrence.AT_LEAST, 2);
cg.criteriaList = new CorelatedCriteria[] { cc };

// translate to PG
String eventTable = String.format(CriteriaUtils.EVENT_TABLE_TEMPLATE, RESULTS_SCHEMA + ".cohort", "cdm", 1);
String countQuery = queryBuilder.getCriteriaGroupQuery(cg, eventTable);
String translatedCountQuery = SqlRender.renderSql(SqlTranslate.translateSql(countQuery, "postgresql"),
new String[] {"cdm_database_schema", "indexId"},
new String[] {"cdm", "0"});

// Validate results
// perform inclusion query
final ITable actualInclusion = dbUnitCon.createQueryTable(RESULTS_SCHEMA + ".output", translatedCountQuery);
final ITable expectedInclusion = expectedDataSet.getTable(RESULTS_SCHEMA + ".output");

// Assert actual database table match expected table
Assertion.assertEquals(expectedInclusion, actualInclusion);

}

@Test
public void distinctDefaultTest() throws Exception {
final CohortExpressionQueryBuilder queryBuilder = new CohortExpressionQueryBuilder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,22 +115,6 @@ public void windowConditionEraTest() throws Exception {

}

@Test
public void windowConditionEraTestInHourInterval() throws Exception {

final String resultsSchema = "window_condition_era";
final String[] testDataSetsPrep = new String[]{"/datasets/vocabulary.json",
"/windowcriteria/windowConditionEra_PREP.json"};
final String[] testDataSetsVerify = new String[]{"/windowcriteria/windowConditionEra_VERIFY.json"};
WindowedCriteria wc = new WindowedCriteria();
wc.criteria = new ConditionEra(); // find any condition era
wc.startWindow = CriteriaUtils.getPrior365WindowTimeUnitInterval(IntervalUnit.HOUR.getName());
List<CriteriaColumn> additionalColumns = Arrays.asList(CriteriaColumn.START_DATE, CriteriaColumn.END_DATE, CriteriaColumn.DOMAIN_CONCEPT,
CriteriaColumn.ERA_OCCURRENCES, CriteriaColumn.DURATION);
this.performWindowTest(wc, resultsSchema, testDataSetsPrep, testDataSetsVerify, additionalColumns);

}

@Test
public void windowConditionOccurrenceTest() throws Exception {

Expand Down Expand Up @@ -181,10 +165,11 @@ public void windowDeathTestInSecondInterval() throws Exception {

final String resultsSchema = "window_death";
final String[] testDataSetsPrep = new String[]{"/datasets/vocabulary.json",
"/windowcriteria/windowDeath_PREP.json"};
final String[] testDataSetsVerify = new String[]{"/windowcriteria/windowDeath_VERIFY.json"};
"/windowcriteria/windowDeathTimeInterval_PREP.json"};
final String[] testDataSetsVerify = new String[]{"/windowcriteria/windowDeathTimeInterval_VERIFY.json"};
WindowedCriteria wc = new WindowedCriteria();
wc.criteria = new Death(); // find any death
wc.criteria.intervalUnit = IntervalUnit.SECOND.getName();
wc.startWindow = CriteriaUtils.getPrior365WindowTimeUnitInterval(IntervalUnit.SECOND.getName());
List<CriteriaColumn> additionalColumns = Arrays.asList(CriteriaColumn.START_DATE, CriteriaColumn.END_DATE, CriteriaColumn.DOMAIN_CONCEPT, CriteriaColumn.DURATION);
this.performWindowTest(wc, resultsSchema, testDataSetsPrep, testDataSetsVerify, additionalColumns);
Expand All @@ -206,6 +191,20 @@ public void windowDeviceExposureTest() throws Exception {

}

@Test
public void windowDeviceExposureTestInHourInterval() throws Exception {
final String resultsSchema = "window_device_exposure";
final String[] testDataSetsPrep = new String[]{"/datasets/vocabulary.json",
"/windowcriteria/windowDeviceExposureTimeInterval_PREP.json"};
final String[] testDataSetsVerify = new String[]{"/windowcriteria/windowDeviceExposureTimeInterval_VERIFY.json"};
WindowedCriteria wc = new WindowedCriteria();
wc.criteria = new DeviceExposure(); // find any device exposure
wc.criteria.intervalUnit = IntervalUnit.HOUR.getName();
wc.startWindow = CriteriaUtils.getPrior365WindowTimeUnitInterval(IntervalUnit.HOUR.getName());
List<CriteriaColumn> additionalColumns = Arrays.asList(CriteriaColumn.START_DATE, CriteriaColumn.END_DATE, CriteriaColumn.DOMAIN_CONCEPT, CriteriaColumn.QUANTITY, CriteriaColumn.DURATION);
this.performWindowTest(wc, resultsSchema, testDataSetsPrep, testDataSetsVerify, additionalColumns);
}

@Test
public void windowDoseEraTest() throws Exception {

Expand Down Expand Up @@ -254,6 +253,20 @@ public void windowDrugExposureTest() throws Exception {

}

@Test
public void windowDrugExposureTestInMinuteInterval() throws Exception {
final String resultsSchema = "window_drug_exposure";
final String[] testDataSetsPrep = new String[]{"/datasets/vocabulary.json", "/windowcriteria/windowDrugExposureTimeInterval_PREP.json"};
final String[] testDataSetsVerify = new String[]{"/windowcriteria/windowDrugExposureTimeInterval_VERIFY.json"};
WindowedCriteria wc = new WindowedCriteria();
wc.criteria = new DrugExposure(); // find any drug exposure
wc.criteria.intervalUnit = IntervalUnit.MINUTE.getName();
wc.startWindow = CriteriaUtils.getPrior365WindowTimeUnitInterval(IntervalUnit.MINUTE.getName());
List<CriteriaColumn> additionalColumns = Arrays.asList(CriteriaColumn.START_DATE, CriteriaColumn.END_DATE,
CriteriaColumn.DOMAIN_CONCEPT, CriteriaColumn.REFILLS, CriteriaColumn.QUANTITY, CriteriaColumn.DAYS_SUPPLY, CriteriaColumn.DURATION);
this.performWindowTest(wc, resultsSchema, testDataSetsPrep, testDataSetsVerify, additionalColumns);
}

@Test
public void windowMeasurementTest() throws Exception {

Expand Down
Loading

0 comments on commit d7cdc0d

Please sign in to comment.