From 0bcee7fa448f975101e22818d11c912596db6f0d Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Mon, 9 Dec 2024 17:13:25 +0100 Subject: [PATCH 01/30] WIP --- .../AbstractJdbcEventAnalyticsManager.java | 2 +- .../data/JdbcEnrollmentAnalyticsManager.java | 130 +++++++++++++++++- ...Make_ou_event_and_enrollment_not_null.xxx} | 0 3 files changed, 125 insertions(+), 7 deletions(-) rename dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/{V2_42_28__Make_ou_event_and_enrollment_not_null.sql => V2_42_28__Make_ou_event_and_enrollment_not_null.xxx} (100%) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 55a58883bd63..6ffca56fae5b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -169,7 +169,7 @@ public abstract class AbstractJdbcEventAnalyticsManager { * @param params the {@link EventQueryParams}. * @param maxLimit the configurable max limit of records. */ - private String getPagingClause(EventQueryParams params, int maxLimit) { + protected String getPagingClause(EventQueryParams params, int maxLimit) { String sql = ""; if (params.isPaging()) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 8fc75e4e7660..98668e202993 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -35,6 +35,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; +import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; @@ -42,6 +43,8 @@ import static org.hisp.dhis.util.DateUtils.toMediumDate; import com.google.common.collect.Sets; + +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -64,6 +67,7 @@ import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.FallbackCoordinateFieldType; import org.hisp.dhis.common.Grid; +import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.OrganisationUnitSelectionMode; import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.common.ValueStatus; @@ -76,6 +80,7 @@ import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramIndicatorService; +import org.hisp.dhis.util.DateUtils; import org.locationtech.jts.util.Assert; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.InvalidResultSetAccessException; @@ -139,17 +144,22 @@ public JdbcEnrollmentAnalyticsManager( @Override public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { - String sql = - params.isAggregatedEnrollments() - ? getAggregatedEnrollmentsSql(grid.getHeaders(), params) - : getAggregatedEnrollmentsSql(params, maxLimit); + String sql; + if (params.isAggregatedEnrollments()) { + sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); + } else if (true) { // We need to add this method to EventQueryParams + sql = buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); // Handle first item like original + } else { + sql = getAggregatedEnrollmentsSql(params, maxLimit); + } if (params.analyzeOnly()) { withExceptionHandling( - () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); } else { withExceptionHandling( - () -> getEnrollments(params, grid, sql, maxLimit == 0), params.isMultipleQueries()); + () -> getEnrollments(params, grid, sql, maxLimit == 0), + params.isMultipleQueries()); } } @@ -788,4 +798,112 @@ private String createOrderType(int offset) { return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "asc"); } } + + // New methods // + + private String getLatestEventsCTE(EventQueryParams params, QueryItem item) { + String eventTableName = "analytics_event_" + params.getProgram().getUid().toLowerCase(); + String excludingScheduledCondition = "eventstatus != 'SCHEDULE'"; + + return String.format( + "with LatestEvents as ( " + + "select enrollment, " + + "%s as value, " + // will contain the data element column + "eventstatus, " + + "row_number() over (partition by enrollment order by occurreddate desc, created desc) as rn " + + "from %s " + + "where %s " + + "and ps = '%s' " + + ") ", + getColumnName(item), + eventTableName, + excludingScheduledCondition, + item.getProgramStage().getUid() + ); + } + + private String getEventDataClauseWithCTE(QueryItem item) { + String column = item.getItem().getUid(); + String programStage = item.getProgramStage().getUid(); + + return String.format( + "le.value as \"%s.%s\", " + + "(le.value is not null) as \"%s.%s.exists\", " + + "le.eventstatus as \"%s.%s.status\"", + programStage, column, + programStage, column, + programStage, column + ); + } + + private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { + String enrollmentTable = "analytics_enrollment_" + params.getProgram().getUid().toLowerCase(); + + String sql = getLatestEventsCTE(params, item) + + "select ax.enrollment, " + + "ax.trackedentity, " + + "ax.enrollmentdate, " + + "ax.occurreddate, " + + "ax.storedby, " + + "ax.createdbydisplayname, " + + "ax.lastupdatedbydisplayname, " + + "ax.lastupdated, " + + "ax.enrollmentgeometry, " + + "ax.longitude, " + + "ax.latitude, " + + "ax.ouname, " + + "ax.ounamehierarchy, " + + "ax.oucode, " + + "ax.enrollmentstatus, " + + "ax.ou, " + + getEventDataClauseWithCTE(item) + " " + + String.format("from %s as ax ", enrollmentTable) + + "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1 " + + "where " + getWhereClauseWithCTE(params, item); + + // Add paging + sql += getPagingClause(params, 5000); //maxlimit + + return sql; + } + + private String getColumnName(QueryItem item) { + if (item.getItem() == null) { + throw new IllegalQueryException("Item cannot be null"); + } + + // For data elements, we use the UID directly + if (item.getItem().getDimensionItemType() == DATA_ELEMENT) { + return sqlBuilder.quote(item.getItem().getUid()); + } + + + throw new IllegalQueryException( + String.format("Item type '%s' not supported for CTE queries", + item.getItem().getDimensionItemType())); + } + + private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { + List conditions = new ArrayList<>(); + + // Date range conditions + if (params.getStartDate() != null && params.getEndDate() != null) { + conditions.add(String.format( + "ax.enrollmentdate >= '%s' and ax.enrollmentdate < '%s'", + DateUtils.toMediumDate(params.getStartDate()), + DateUtils.toMediumDate(params.getEndDate()) + )); + } + + // Organization unit conditions + if (!params.getOrganisationUnits().isEmpty()) { + String orgUnit = params.getOrganisationUnits().get(0).getUid(); + conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); + } + + // Value conditions - similar to your example + conditions.add("le.value is null and (le.enrollment is not null)"); + + return String.join(" and ", conditions); + } } diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.sql b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.xxx similarity index 100% rename from dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.sql rename to dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.xxx From 813c5cd6674dd88b0ffe3ef222b83c2cf7f7910a Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 14:17:55 +0100 Subject: [PATCH 02/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 95 ++++++++++++++----- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 98668e202993..e5e38aebe806 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -838,33 +838,55 @@ private String getEventDataClauseWithCTE(QueryItem item) { private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { String enrollmentTable = "analytics_enrollment_" + params.getProgram().getUid().toLowerCase(); - - String sql = getLatestEventsCTE(params, item) + + boolean useCTE = params.hasProgramStage(); // Decides whether we use the CTE logic. + + String cteClause = ""; + String joinClause = ""; + String whereClause = ""; + + if (useCTE) { + // Generate the CTE if `params.hasProgramStage()` is true + cteClause = getLatestEventsCTE(params, item); + + // Add the join logic for CTE results + joinClause = "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1"; + + // Use the corresponding WHERE clause for CTE logic + whereClause = getWhereClauseWithCTE(params, item); + } else { + // Use the WHERE clause for non-CTE queries + whereClause = getWhereClauseWithoutCTE(params); + } + + String selectClause = String.format( "select ax.enrollment, " + - "ax.trackedentity, " + - "ax.enrollmentdate, " + - "ax.occurreddate, " + - "ax.storedby, " + - "ax.createdbydisplayname, " + - "ax.lastupdatedbydisplayname, " + - "ax.lastupdated, " + - "ax.enrollmentgeometry, " + - "ax.longitude, " + - "ax.latitude, " + - "ax.ouname, " + - "ax.ounamehierarchy, " + - "ax.oucode, " + - "ax.enrollmentstatus, " + - "ax.ou, " + - getEventDataClauseWithCTE(item) + " " + - String.format("from %s as ax ", enrollmentTable) + - "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1 " + - "where " + getWhereClauseWithCTE(params, item); - - // Add paging - sql += getPagingClause(params, 5000); //maxlimit + "ax.trackedentity, " + + "ax.enrollmentdate, " + + "ax.occurreddate, " + + "ax.storedby, " + + "ax.createdbydisplayname, " + + "ax.lastupdatedbydisplayname, " + + "ax.lastupdated, " + + "ax.enrollmentgeometry, " + + "ax.longitude, " + + "ax.latitude, " + + "ax.ouname, " + + "ax.ounamehierarchy, " + + "ax.oucode, " + + "ax.enrollmentstatus, " + + "ax.ou" + + (useCTE ? ", " + getEventDataClauseWithCTE(item) : "") + ); - return sql; + return String.format( + "%s %s from %s as ax %s where %s %s", + cteClause, // Optional: The CTE declaration + selectClause, // The SELECT clause with/without CTE data + enrollmentTable, // The main enrollment table + joinClause, // Optional: Join clause for CTE + whereClause, // WHERE clause for filters + getPagingClause(params, 5000) // Add the paging clause + ); } private String getColumnName(QueryItem item) { @@ -883,6 +905,29 @@ private String getColumnName(QueryItem item) { item.getItem().getDimensionItemType())); } + private String getWhereClauseWithoutCTE(EventQueryParams params) { + List conditions = new ArrayList<>(); + + // Date range conditions + if (params.getStartDate() != null && params.getEndDate() != null) { + conditions.add(String.format( + "enrollmentdate >= '%s' and enrollmentdate < '%s'", + DateUtils.toMediumDate(params.getStartDate()), + DateUtils.toMediumDate(params.getEndDate()) + )); + } + + // Organization unit conditions + if (!params.getOrganisationUnits().isEmpty()) { + String orgUnit = params.getOrganisationUnits().get(0).getUid(); + conditions.add(String.format("uidlevel1 = '%s'", orgUnit)); + } + + // Add more conditions as necessary + + return String.join(" and ", conditions); + } + private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { List conditions = new ArrayList<>(); From 6e085ee48e4e24a3cf0f00a846a119da0963a7c1 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 15:22:04 +0100 Subject: [PATCH 03/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 122 ++++++++---------- 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index e5e38aebe806..430bfee538d1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -27,30 +27,7 @@ */ package org.hisp.dhis.analytics.event.data; -import static java.util.stream.Collectors.joining; -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; -import static org.hisp.dhis.analytics.DataType.BOOLEAN; -import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; -import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; -import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; -import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; -import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; -import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; -import static org.hisp.dhis.util.DateUtils.toMediumDate; - import com.google.common.collect.Sets; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; @@ -88,6 +65,28 @@ import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; +import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; +import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; +import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; +import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; +import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; +import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; +import static org.hisp.dhis.util.DateUtils.toMediumDate; + /** * @author Markus Bekken */ @@ -148,7 +147,9 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { if (params.isAggregatedEnrollments()) { sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); } else if (true) { // We need to add this method to EventQueryParams - sql = buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); // Handle first item like original + sql = params.getItems().isEmpty() + ? buildEnrollmentQueryWithoutCTE(params) + : buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); } else { sql = getAggregatedEnrollmentsSql(params, maxLimit); } @@ -822,71 +823,62 @@ private String getLatestEventsCTE(EventQueryParams params, QueryItem item) { ); } - private String getEventDataClauseWithCTE(QueryItem item) { - String column = item.getItem().getUid(); - String programStage = item.getProgramStage().getUid(); + private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { + String enrollmentTable = String.format("analytics_enrollment_%s", params.getProgram().getUid().toLowerCase()); + + String selectClause = getSelectClause(params); // Reuse to dynamically generate SELECT columns + String whereClause = getWhereClauseWithoutCTE(params); // Filters on time range, organization, etc. + String pagingClause = getPagingClause(params, 5000); // Add pagination + + // Compose the SQL query return String.format( - "le.value as \"%s.%s\", " + - "(le.value is not null) as \"%s.%s.exists\", " + - "le.eventstatus as \"%s.%s.status\"", - programStage, column, - programStage, column, - programStage, column + "%s from %s as ax where %s %s", + selectClause, // Dynamic SELECT clause + enrollmentTable, // Main enrollment table + whereClause, // Filters/conditions + pagingClause // Pagination ); } private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { - String enrollmentTable = "analytics_enrollment_" + params.getProgram().getUid().toLowerCase(); - boolean useCTE = params.hasProgramStage(); // Decides whether we use the CTE logic. + // Enrollment table name + String enrollmentTable = String.format("analytics_enrollment_%s", params.getProgram().getUid().toLowerCase()); + boolean useCTE = params.hasProgramStage(); // Toggle on CTE usage String cteClause = ""; String joinClause = ""; String whereClause = ""; if (useCTE) { - // Generate the CTE if `params.hasProgramStage()` is true + // Build the CTE for event queries if programStage filtering is required cteClause = getLatestEventsCTE(params, item); - // Add the join logic for CTE results + // Event data join clause for CTE joinClause = "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1"; - // Use the corresponding WHERE clause for CTE logic + // WHERE clause specific to CTE whereClause = getWhereClauseWithCTE(params, item); } else { - // Use the WHERE clause for non-CTE queries + // WHERE clause for non-CTE logic whereClause = getWhereClauseWithoutCTE(params); } - String selectClause = String.format( - "select ax.enrollment, " + - "ax.trackedentity, " + - "ax.enrollmentdate, " + - "ax.occurreddate, " + - "ax.storedby, " + - "ax.createdbydisplayname, " + - "ax.lastupdatedbydisplayname, " + - "ax.lastupdated, " + - "ax.enrollmentgeometry, " + - "ax.longitude, " + - "ax.latitude, " + - "ax.ouname, " + - "ax.ounamehierarchy, " + - "ax.oucode, " + - "ax.enrollmentstatus, " + - "ax.ou" + - (useCTE ? ", " + getEventDataClauseWithCTE(item) : "") - ); + // Use the original getSelectClause to ensure consistent SELECT clause generation + String selectClause = getSelectClause(params); - return String.format( + // Construct the final SQL query + String sql = String.format( "%s %s from %s as ax %s where %s %s", - cteClause, // Optional: The CTE declaration - selectClause, // The SELECT clause with/without CTE data - enrollmentTable, // The main enrollment table - joinClause, // Optional: Join clause for CTE - whereClause, // WHERE clause for filters - getPagingClause(params, 5000) // Add the paging clause + cteClause, // Optional: CTE declaration + selectClause, // The SELECT clause (dynamic columns handled) + enrollmentTable, // Main enrollment table + joinClause, // Optional: CTE join clause + whereClause, // WHERE clause (dynamic filtering logic) + getPagingClause(params, 5000) // Pagination logic ); + + return sql; } private String getColumnName(QueryItem item) { From ad73bea6b9c61544e4f39b53b050ee95c87eb8b1 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 17:41:06 +0100 Subject: [PATCH 04/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 175 +++++++++--------- 1 file changed, 91 insertions(+), 84 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 430bfee538d1..4488f0b4f91a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -146,12 +146,10 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { String sql; if (params.isAggregatedEnrollments()) { sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); - } else if (true) { // We need to add this method to EventQueryParams - sql = params.getItems().isEmpty() - ? buildEnrollmentQueryWithoutCTE(params) - : buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); + } else if (!params.getItems().isEmpty() && shouldUseCTE(params)) { + sql = buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); } else { - sql = getAggregatedEnrollmentsSql(params, maxLimit); + sql = buildEnrollmentQueryWithoutCTE(params); } if (params.analyzeOnly()) { @@ -164,6 +162,14 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { } } + private boolean shouldUseCTE(EventQueryParams params) { + if (params.getItems().isEmpty()) { + return false; + } + QueryItem item = params.getItems().get(0); + return item.hasProgram() && item.hasProgramStage(); + } + /** * Adds enrollments to the given grid based on the given parameters and SQL statement. * @@ -803,104 +809,107 @@ private String createOrderType(int offset) { // New methods // private String getLatestEventsCTE(EventQueryParams params, QueryItem item) { - String eventTableName = "analytics_event_" + params.getProgram().getUid().toLowerCase(); - String excludingScheduledCondition = "eventstatus != 'SCHEDULE'"; + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); + String columnName = quote(item.getItem().getUid()); return String.format( "with LatestEvents as ( " + - "select enrollment, " + - "%s as value, " + // will contain the data element column - "eventstatus, " + - "row_number() over (partition by enrollment order by occurreddate desc, created desc) as rn " + + "select " + + " enrollment, " + + " %s as value, " + + " eventstatus, " + + " row_number() over (partition by enrollment " + + " order by occurreddate desc, created desc) as rn " + "from %s " + - "where %s " + - "and ps = '%s' " + + "where eventstatus != 'SCHEDULE' " + + " and ps = '%s' " + ") ", - getColumnName(item), + columnName, eventTableName, - excludingScheduledCondition, item.getProgramStage().getUid() ); } - private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { - String enrollmentTable = String.format("analytics_enrollment_%s", params.getProgram().getUid().toLowerCase()); - - String selectClause = getSelectClause(params); // Reuse to dynamically generate SELECT columns - String whereClause = getWhereClauseWithoutCTE(params); // Filters on time range, organization, etc. - String pagingClause = getPagingClause(params, 5000); // Add pagination + String selectClause = getSelectClause(params); - // Compose the SQL query return String.format( - "%s from %s as ax where %s %s", - selectClause, // Dynamic SELECT clause - enrollmentTable, // Main enrollment table - whereClause, // Filters/conditions - pagingClause // Pagination + "%s " + + "from %s as ax " + + "where %s %s", + selectClause, + params.getTableName(), + getWhereClauseWithoutCTE(params), + getPagingClause(params, 5000) ); } private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { - // Enrollment table name - String enrollmentTable = String.format("analytics_enrollment_%s", params.getProgram().getUid().toLowerCase()); - boolean useCTE = params.hasProgramStage(); // Toggle on CTE usage - - String cteClause = ""; - String joinClause = ""; - String whereClause = ""; - - if (useCTE) { - // Build the CTE for event queries if programStage filtering is required - cteClause = getLatestEventsCTE(params, item); + // 1. Build CTE + String cteClause = getLatestEventsCTE(params, item); + + // 2. Get select columns using existing mechanism but override the item columns + String selectBase = getBasicSelectColumns(); // New method for enrollment, trackedentity, etc. + String selectWithCTE = String.format( + "%s, " + + "le.value as \"%s.%s\", " + + "(le.value is not null) as \"%s.%s.exists\", " + + "le.eventstatus as \"%s.%s.status\"", + selectBase, + item.getProgramStage().getUid(), item.getItem().getUid(), + item.getProgramStage().getUid(), item.getItem().getUid(), + item.getProgramStage().getUid(), item.getItem().getUid() + ); - // Event data join clause for CTE - joinClause = "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1"; + // 3. Build the main query + String sql = String.format( + "%s select %s " + + "from %s as ax " + + "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1 ", + cteClause, + selectWithCTE, + params.getTableName() + ); - // WHERE clause specific to CTE - whereClause = getWhereClauseWithCTE(params, item); - } else { - // WHERE clause for non-CTE logic - whereClause = getWhereClauseWithoutCTE(params); + // 4. Add where clause + String whereClause = getWhereClauseWithCTE(params, item); + if (!whereClause.isEmpty()) { + sql += "where " + whereClause; } - // Use the original getSelectClause to ensure consistent SELECT clause generation - String selectClause = getSelectClause(params); - - // Construct the final SQL query - String sql = String.format( - "%s %s from %s as ax %s where %s %s", - cteClause, // Optional: CTE declaration - selectClause, // The SELECT clause (dynamic columns handled) - enrollmentTable, // Main enrollment table - joinClause, // Optional: CTE join clause - whereClause, // WHERE clause (dynamic filtering logic) - getPagingClause(params, 5000) // Pagination logic - ); + // 5. Add paging + sql += " " + getPagingClause(params, 5000); return sql; } - private String getColumnName(QueryItem item) { - if (item.getItem() == null) { - throw new IllegalQueryException("Item cannot be null"); - } - - // For data elements, we use the UID directly - if (item.getItem().getDimensionItemType() == DATA_ELEMENT) { - return sqlBuilder.quote(item.getItem().getUid()); - } + private String getBasicSelectColumns() { + return String.join(",", + "ax.enrollment", + "trackedentity", + "enrollmentdate", + "occurreddate", + "storedby", + "createdbydisplayname", + "lastupdatedbydisplayname", + "lastupdated", + "ST_AsGeoJSON(enrollmentgeometry)", + "longitude", + "latitude", + "ouname", + "ounamehierarchy", + "oucode", + "enrollmentstatus", + "ax.\"ou\"" + ); + } - throw new IllegalQueryException( - String.format("Item type '%s' not supported for CTE queries", - item.getItem().getDimensionItemType())); - } private String getWhereClauseWithoutCTE(EventQueryParams params) { List conditions = new ArrayList<>(); - // Date range conditions + // Add date range conditions if (params.getStartDate() != null && params.getEndDate() != null) { conditions.add(String.format( "enrollmentdate >= '%s' and enrollmentdate < '%s'", @@ -909,21 +918,25 @@ private String getWhereClauseWithoutCTE(EventQueryParams params) { )); } - // Organization unit conditions + // Add organization unit conditions if (!params.getOrganisationUnits().isEmpty()) { String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("uidlevel1 = '%s'", orgUnit)); + conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); } - // Add more conditions as necessary - return String.join(" and ", conditions); } private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { List conditions = new ArrayList<>(); - // Date range conditions + // Add organization unit conditions + if (!params.getOrganisationUnits().isEmpty()) { + String orgUnit = params.getOrganisationUnits().get(0).getUid(); + conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); + } + + // Add date range conditions if (params.getStartDate() != null && params.getEndDate() != null) { conditions.add(String.format( "ax.enrollmentdate >= '%s' and ax.enrollmentdate < '%s'", @@ -932,13 +945,7 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { )); } - // Organization unit conditions - if (!params.getOrganisationUnits().isEmpty()) { - String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); - } - - // Value conditions - similar to your example + // Add value conditions using CTE conditions.add("le.value is null and (le.enrollment is not null)"); return String.join(" and ", conditions); From 029ac331ee6c23da4fa7285977d1baac2538123c Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 17:57:16 +0100 Subject: [PATCH 05/30] WIP --- .../AbstractJdbcEventAnalyticsManager.java | 2 +- .../data/JdbcEnrollmentAnalyticsManager.java | 43 ++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 6ffca56fae5b..92b61ec69ed0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -191,7 +191,7 @@ protected String getPagingClause(EventQueryParams params, int maxLimit) { * * @param params the {@link EventQueryParams}. */ - private String getSortClause(EventQueryParams params) { + protected String getSortClause(EventQueryParams params) { String sql = ""; if (params.isSorting()) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 4488f0b4f91a..f91bc585281d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -848,27 +848,35 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it // 1. Build CTE String cteClause = getLatestEventsCTE(params, item); - // 2. Get select columns using existing mechanism but override the item columns - String selectBase = getBasicSelectColumns(); // New method for enrollment, trackedentity, etc. + // 2. Get select columns + String selectBase = getBasicSelectColumns(); + + // Get the offset from the program stage params + int offset = item.getProgramStageOffset(); + String offsetSuffix = offset != 0 ? "[" + offset + "]" : ""; + String columnPrefix = item.getProgramStage().getUid() + offsetSuffix; + String selectWithCTE = String.format( "%s, " + "le.value as \"%s.%s\", " + - "(le.value is not null) as \"%s.%s.exists\", " + + // Changed how we calculate exists to match original behavior + "(le.enrollment is not null) as \"%s.%s.exists\", " + "le.eventstatus as \"%s.%s.status\"", selectBase, - item.getProgramStage().getUid(), item.getItem().getUid(), - item.getProgramStage().getUid(), item.getItem().getUid(), - item.getProgramStage().getUid(), item.getItem().getUid() + columnPrefix, item.getItem().getUid(), + columnPrefix, item.getItem().getUid(), + columnPrefix, item.getItem().getUid() ); // 3. Build the main query String sql = String.format( "%s select %s " + "from %s as ax " + - "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = 1 ", + "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = %d ", cteClause, selectWithCTE, - params.getTableName() + params.getTableName(), + Math.abs(offset) + 1 ); // 4. Add where clause @@ -877,7 +885,10 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it sql += "where " + whereClause; } - // 5. Add paging + // 5. Add order by + sql += getSortClause(params); + + // 6. Add paging sql += " " + getPagingClause(params, 5000); return sql; @@ -945,9 +956,19 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { )); } - // Add value conditions using CTE - conditions.add("le.value is null and (le.enrollment is not null)"); + // Add item filters from the original params + String itemFilters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + if (!itemFilters.isEmpty()) { + conditions.add(itemFilters); + } return String.join(" and ", conditions); } + + protected String getSortClause(EventQueryParams params) { + if (params.isSorting()) { + return super.getSortClause(params); + } + return ""; + } } From ed5afd9650397ca2d20d2dcb6388774e5b0db361 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 18:04:45 +0100 Subject: [PATCH 06/30] WIP --- .../AbstractJdbcEventAnalyticsManager.java | 2 +- .../data/JdbcEnrollmentAnalyticsManager.java | 66 ++++++++++++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 92b61ec69ed0..a04481e76348 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1173,7 +1173,7 @@ private String joinSql(Stream conditions, Collector sqlConditionByGroup = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index f91bc585281d..c71251b3f120 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -859,7 +859,6 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it String selectWithCTE = String.format( "%s, " + "le.value as \"%s.%s\", " + - // Changed how we calculate exists to match original behavior "(le.enrollment is not null) as \"%s.%s.exists\", " + "le.eventstatus as \"%s.%s.status\"", selectBase, @@ -868,30 +867,28 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it columnPrefix, item.getItem().getUid() ); + StringBuilder sql = new StringBuilder(); + // 3. Build the main query - String sql = String.format( - "%s select %s " + - "from %s as ax " + - "left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = %d ", - cteClause, - selectWithCTE, - params.getTableName(), - Math.abs(offset) + 1 - ); + sql.append(cteClause) + .append(" select ").append(selectWithCTE) + .append(" from ").append(params.getTableName()).append(" as ax") + .append(" left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = ") + .append(Math.abs(offset) + 1); - // 4. Add where clause + // 4. Add where clause - ensure only one WHERE keyword String whereClause = getWhereClauseWithCTE(params, item); if (!whereClause.isEmpty()) { - sql += "where " + whereClause; + sql.append(" where ").append(whereClause); } // 5. Add order by - sql += getSortClause(params); + sql.append(getSortClause(params)); // 6. Add paging - sql += " " + getPagingClause(params, 5000); + sql.append(" ").append(getPagingClause(params, 5000)); - return sql; + return sql.toString(); } private String getBasicSelectColumns() { @@ -956,12 +953,20 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { )); } - // Add item filters from the original params - String itemFilters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + // Replace subquery conditions with CTE equivalents + conditions.add("(le.value is null and le.enrollment is not null)"); + + // Add any additional filters without the WHERE keyword + SqlHelper hlp = new SqlHelper(); + String itemFilters = getQueryItemsAndFiltersWhereClause(params, hlp) + .replaceAll("(?i)\\s*where\\s+and\\s+", "") // Remove "where and" + .replaceAll("(?i)\\s*where\\s+", ""); // Remove "where" + if (!itemFilters.isEmpty()) { conditions.add(itemFilters); } + // Join all conditions with AND return String.join(" and ", conditions); } @@ -971,4 +976,31 @@ protected String getSortClause(EventQueryParams params) { } return ""; } + + protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, SqlHelper hlp) { + if (params.isEnhancedCondition()) { + return getItemsSqlForEnhancedConditions(params, hlp); + } + + List conditions = new ArrayList<>(); + + // Add conditions based on the CTE instead of subqueries + for (QueryItem item : params.getItems()) { + if (item.hasFilter()) { + conditions.add("(le.value is null and le.enrollment is not null)"); + } + } + + for (QueryItem item : params.getItemFilters()) { + if (item.hasFilter()) { + conditions.add("(le.value is null and le.enrollment is not null)"); + } + } + + if (conditions.isEmpty()) { + return ""; + } + + return conditions.stream().collect(joining(" and ")); + } } From 2f75486231f614c88cff729b78b08433eea37bcc Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 18:15:10 +0100 Subject: [PATCH 07/30] WIP --- .../analytics/event/EventQueryParams.java | 10 +++ .../data/JdbcEnrollmentAnalyticsManager.java | 86 ++++++++++++++++--- 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 4d6f0683f691..ebf508b9aa9f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -678,6 +678,16 @@ public boolean hasAnalyticsVariables() { return hasProgramIndicatorDimension() && getProgramIndicator().hasAnalyticsVariables(); } + public boolean hasEnrollmentDateCriteria() { + // Check if enrollmentDate parameter exists in the query + return !getDimensionOrFilterItems("enrollmentDate").isEmpty(); + } + + public String getEnrollmentDateCriteria() { + // Get the enrollment date value (year) + return getDimensionOrFilterItems("enrollmentDate").get(0).getDimensionItem(); + } + public boolean useIndividualQuery() { return this.hasAnalyticsVariables() || this.hasNonDefaultBoundaries(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index c71251b3f120..baae225e56b3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -46,6 +46,7 @@ import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.OrganisationUnitSelectionMode; +import org.hisp.dhis.common.QueryFilter; import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.common.ValueStatus; import org.hisp.dhis.common.ValueType; @@ -944,8 +945,14 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); } - // Add date range conditions - if (params.getStartDate() != null && params.getEndDate() != null) { + // Add enrollment date conditions + if (params.hasEnrollmentDateCriteria()) { // We might need to add this method to EventQueryParams + String year = params.getEnrollmentDateCriteria(); // We might need to add this method + conditions.add(String.format( + "ax.enrollmentdate >= '%s-01-01' and ax.enrollmentdate < '%s-01-01'", + year, (Integer.parseInt(year) + 1) + )); + } else if (params.getStartDate() != null && params.getEndDate() != null) { conditions.add(String.format( "ax.enrollmentdate >= '%s' and ax.enrollmentdate < '%s'", DateUtils.toMediumDate(params.getStartDate()), @@ -953,23 +960,33 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { )); } - // Replace subquery conditions with CTE equivalents - conditions.add("(le.value is null and le.enrollment is not null)"); + // Handle NV (null value) filter + if (hasNullValueFilter(params)) { + conditions.add("le.value is null"); + conditions.add("le.enrollment is not null"); + } - // Add any additional filters without the WHERE keyword + // Add any additional filters SqlHelper hlp = new SqlHelper(); String itemFilters = getQueryItemsAndFiltersWhereClause(params, hlp) - .replaceAll("(?i)\\s*where\\s+and\\s+", "") // Remove "where and" - .replaceAll("(?i)\\s*where\\s+", ""); // Remove "where" + .replaceAll("(?i)\\s*where\\s+and\\s+", "") + .replaceAll("(?i)\\s*where\\s+", ""); if (!itemFilters.isEmpty()) { conditions.add(itemFilters); } - // Join all conditions with AND return String.join(" and ", conditions); } + + private boolean hasNullValueFilter(EventQueryParams params) { + return params.getItems().stream() + .anyMatch(item -> item.hasFilter() && + item.getFilters().stream() + .anyMatch(filter -> "NV".equals(filter.getFilter()))); + } + protected String getSortClause(EventQueryParams params) { if (params.isSorting()) { return super.getSortClause(params); @@ -984,23 +1001,64 @@ protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, Sql List conditions = new ArrayList<>(); - // Add conditions based on the CTE instead of subqueries + // Handle item filters using CTE for (QueryItem item : params.getItems()) { if (item.hasFilter()) { - conditions.add("(le.value is null and le.enrollment is not null)"); + conditions.addAll(getItemConditions(item)); } } + // Handle item filters for (QueryItem item : params.getItemFilters()) { if (item.hasFilter()) { - conditions.add("(le.value is null and le.enrollment is not null)"); + conditions.addAll(getItemConditions(item)); } } - if (conditions.isEmpty()) { - return ""; + return conditions.isEmpty() ? "" : String.join(" and ", conditions); + } + + private List getItemConditions(QueryItem item) { + List conditions = new ArrayList<>(); + + for (QueryFilter filter : item.getFilters()) { + if ("NV".equals(filter.getFilter())) { + // Special handling for null values + conditions.add("le.value is null"); + } else { + String operator = filter.getSqlOperator(); + if (operator.equals("in")) { + // Handle IN clause differently based on value type + String value = getSqlFilter(filter, item); + if (item.isNumeric()) { + conditions.add("le.value in (" + value + ")"); + } else { + // For text values, keep the quotes + conditions.add("le.value in (" + value + ")"); + } + } else { + String value = getSqlFilter(filter, item); + conditions.add(String.format("le.value %s %s", + operator, + value)); + } + } } - return conditions.stream().collect(joining(" and ")); + return conditions; + } + + protected String getSqlFilter(QueryFilter filter, QueryItem item) { + String value = filter.getFilter(); + + if ("NV".equals(value)) { + return "null"; + } + + if (item.isNumeric()) { + return value; // Don't quote numeric values + } else { + return sqlBuilder.singleQuote(value); // Quote text values + } } } From ae60cd9d687f7b1b9eefbdcbab0b3dde9ad1b0cc Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 10 Dec 2024 18:30:41 +0100 Subject: [PATCH 08/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 99 ++++++------------- 1 file changed, 32 insertions(+), 67 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index baae225e56b3..9713a72e629a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -73,6 +73,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang3.StringUtils.EMPTY; @@ -110,7 +111,7 @@ public class JdbcEnrollmentAnalyticsManager extends AbstractJdbcEventAnalyticsMa private static final List COLUMNS = List.of( - EnrollmentAnalyticsColumnName.ENROLLMENT_COLUMN_NAME, + "ax." + EnrollmentAnalyticsColumnName.ENROLLMENT_COLUMN_NAME, EnrollmentAnalyticsColumnName.TRACKED_ENTITY_COLUMN_NAME, EnrollmentAnalyticsColumnName.ENROLLMENT_DATE_COLUMN_NAME, EnrollmentAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME, @@ -846,74 +847,45 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { } private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { - // 1. Build CTE - String cteClause = getLatestEventsCTE(params, item); + StringBuilder sql = new StringBuilder(); + + // 1. Build CTE if needed + if (item != null && item.hasProgramStage()) { + sql.append(getLatestEventsCTE(params, item)); + } // 2. Get select columns - String selectBase = getBasicSelectColumns(); - - // Get the offset from the program stage params - int offset = item.getProgramStageOffset(); - String offsetSuffix = offset != 0 ? "[" + offset + "]" : ""; - String columnPrefix = item.getProgramStage().getUid() + offsetSuffix; - - String selectWithCTE = String.format( - "%s, " + - "le.value as \"%s.%s\", " + - "(le.enrollment is not null) as \"%s.%s.exists\", " + - "le.eventstatus as \"%s.%s.status\"", - selectBase, - columnPrefix, item.getItem().getUid(), - columnPrefix, item.getItem().getUid(), - columnPrefix, item.getItem().getUid() - ); + String selectClause = getSelectClause(params); + sql.append(selectClause); - StringBuilder sql = new StringBuilder(); + // 3. From clause + sql.append(" from ").append(params.getTableName()).append(" as ax"); - // 3. Build the main query - sql.append(cteClause) - .append(" select ").append(selectWithCTE) - .append(" from ").append(params.getTableName()).append(" as ax") - .append(" left join LatestEvents le on ax.enrollment = le.enrollment and le.rn = ") - .append(Math.abs(offset) + 1); + // 4. Join clause if needed + if (item != null && item.hasProgramStage()) { + int offset = item.getProgramStageOffset(); + sql.append(" left join LatestEvents le on ax.enrollment = le.enrollment") + .append(" and le.rn = ").append(Math.abs(offset) + 1); + } - // 4. Add where clause - ensure only one WHERE keyword + // 5. Where clause String whereClause = getWhereClauseWithCTE(params, item); if (!whereClause.isEmpty()) { sql.append(" where ").append(whereClause); } - // 5. Add order by - sql.append(getSortClause(params)); + // 6. Order by - ensure proper spacing + String sortClause = getSortClause(params); + if (!sortClause.isEmpty()) { + sql.append(" ").append(sortClause); + } - // 6. Add paging + // 7. Limit clause sql.append(" ").append(getPagingClause(params, 5000)); return sql.toString(); } - private String getBasicSelectColumns() { - return String.join(",", - "ax.enrollment", - "trackedentity", - "enrollmentdate", - "occurreddate", - "storedby", - "createdbydisplayname", - "lastupdatedbydisplayname", - "lastupdated", - "ST_AsGeoJSON(enrollmentgeometry)", - "longitude", - "latitude", - "ouname", - "ounamehierarchy", - "oucode", - "enrollmentstatus", - "ax.\"ou\"" - ); - } - - private String getWhereClauseWithoutCTE(EventQueryParams params) { List conditions = new ArrayList<>(); @@ -945,9 +917,9 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); } - // Add enrollment date conditions - if (params.hasEnrollmentDateCriteria()) { // We might need to add this method to EventQueryParams - String year = params.getEnrollmentDateCriteria(); // We might need to add this method + // Add date range conditions + if (params.hasEnrollmentDateCriteria()) { + String year = params.getEnrollmentDateCriteria(); conditions.add(String.format( "ax.enrollmentdate >= '%s-01-01' and ax.enrollmentdate < '%s-01-01'", year, (Integer.parseInt(year) + 1) @@ -966,17 +938,10 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { conditions.add("le.enrollment is not null"); } - // Add any additional filters - SqlHelper hlp = new SqlHelper(); - String itemFilters = getQueryItemsAndFiltersWhereClause(params, hlp) - .replaceAll("(?i)\\s*where\\s+and\\s+", "") - .replaceAll("(?i)\\s*where\\s+", ""); - - if (!itemFilters.isEmpty()) { - conditions.add(itemFilters); - } - - return String.join(" and ", conditions); + // Join conditions with AND, but don't add extra spaces or concatenate with order by + return conditions.isEmpty() ? "" : conditions.stream() + .filter(StringUtils::isNotBlank) + .collect(Collectors.joining(" and ")); } From 8e5409dbb4f7ecb6dede6b3486e6dcf43378bf47 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 11 Dec 2024 11:15:19 +0100 Subject: [PATCH 09/30] WIP --- .../analytics/event/EventQueryParams.java | 51 +++++-- .../data/JdbcEnrollmentAnalyticsManager.java | 132 +++++++++++++++--- 2 files changed, 158 insertions(+), 25 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index ebf508b9aa9f..d116d69bcac0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -678,15 +678,6 @@ public boolean hasAnalyticsVariables() { return hasProgramIndicatorDimension() && getProgramIndicator().hasAnalyticsVariables(); } - public boolean hasEnrollmentDateCriteria() { - // Check if enrollmentDate parameter exists in the query - return !getDimensionOrFilterItems("enrollmentDate").isEmpty(); - } - - public String getEnrollmentDateCriteria() { - // Get the enrollment date value (year) - return getDimensionOrFilterItems("enrollmentDate").get(0).getDimensionItem(); - } public boolean useIndividualQuery() { return this.hasAnalyticsVariables() || this.hasNonDefaultBoundaries(); @@ -846,6 +837,48 @@ public boolean hasDataIdScheme() { return dataIdScheme != null; } + /** + * Returns true if enrollment date criteria exists in dimensions or filters. + */ + public boolean hasEnrollmentDateCriteria() { + return getDimensionOrFilterItems("enrollmentDate").size() > 0 || + getDimensionOrFilterItems("enrollmentdate").size() > 0; // Try both cases + } + + /** + * Returns the enrollment date criteria years. + */ + public String[] getEnrollmentDateCriteria() { + List items = getDimensionOrFilterItems("enrollmentDate"); + if (items.isEmpty()) { + items = getDimensionOrFilterItems("enrollmentdate"); + } + return items.stream() + .map(item -> item.getDimensionItem()) + .toArray(String[]::new); + } + + /** + * Returns true if incident date criteria exists in dimensions or filters. + */ + public boolean hasIncidentDateCriteria() { + return getDimensionOrFilterItems("incidentDate").size() > 0 || + getDimensionOrFilterItems("incidentdate").size() > 0; // Try both cases + } + + /** + * Returns the incident date criteria years. + */ + public String[] getIncidentDateCriteria() { + List items = getDimensionOrFilterItems("incidentDate"); + if (items.isEmpty()) { + items = getDimensionOrFilterItems("incidentdate"); + } + return items.stream() + .map(item -> item.getDimensionItem()) + .toArray(String[]::new); + } + /** * Returns a negative integer in case of ascending sort order, a positive in case of descending * sort order and 0 in case of no sort order. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 9713a72e629a..1c6d4bb12fac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -30,6 +30,7 @@ import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; @@ -38,13 +39,13 @@ import org.hisp.dhis.analytics.table.EnrollmentAnalyticsColumnName; import org.hisp.dhis.analytics.table.EventAnalyticsColumnName; import org.hisp.dhis.category.CategoryOption; +import org.hisp.dhis.common.DateRange; import org.hisp.dhis.common.DimensionItemType; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.FallbackCoordinateFieldType; import org.hisp.dhis.common.Grid; -import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.OrganisationUnitSelectionMode; import org.hisp.dhis.common.QueryFilter; import org.hisp.dhis.common.QueryItem; @@ -154,6 +155,8 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { sql = buildEnrollmentQueryWithoutCTE(params); } + System.out.println("SQL: " + sql); + if (params.analyzeOnly()) { withExceptionHandling( () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); @@ -833,17 +836,75 @@ private String getLatestEventsCTE(EventQueryParams params, QueryItem item) { } private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { - String selectClause = getSelectClause(params); + StringBuilder sql = new StringBuilder(); - return String.format( - "%s " + - "from %s as ax " + - "where %s %s", - selectClause, - params.getTableName(), - getWhereClauseWithoutCTE(params), - getPagingClause(params, 5000) - ); + // 1. Select clause + sql.append(getSelectClause(params)); + + // 2. From clause + sql.append(" from ").append(params.getTableName()).append(" as ax"); + + // 3. Where clause + List conditions = new ArrayList<>(); + + // Add organization unit condition + if (!params.getOrganisationUnits().isEmpty()) { + String orgUnit = params.getOrganisationUnits().get(0).getUid(); + conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); + } + + // Use TimeField and DateRange instead of the dimension approach + if (!params.getTimeDateRanges().isEmpty()) { + for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { + String column = getColumnForTimeField(entry.getKey()); + if (column != null) { + List dateConditions = new ArrayList<>(); + for (DateRange range : entry.getValue()) { + dateConditions.add(String.format( + "(%s >= '%s' and %s < '%s')", + column, DateUtils.toMediumDate(range.getStartDate()), + column, DateUtils.toMediumDate(range.getEndDate()) + )); + } + conditions.add("(" + String.join(" or ", dateConditions) + ")"); + } + } + } + + if (!conditions.isEmpty()) { + sql.append(" where ").append(String.join(" and ", conditions)); + } + + // 4. Order by + if (params.getAsc() != null && !params.getAsc().isEmpty()) { + sql.append(" order by"); + boolean first = true; + for (QueryItem item : params.getAsc()) { + if (!first) { + sql.append(","); + } + String columnName = item.getItemId().equals("incidentdate") ? + "occurreddate" : item.getItemId(); + sql.append(" ").append(quote(columnName)).append(" asc nulls last"); + first = false; + } + } + + // 5. Paging + sql.append(" ").append(getPagingClause(params, 5000)); + + return sql.toString(); + } + + private String getColumnForTimeField(TimeField timeField) { + switch (timeField) { + case ENROLLMENT_DATE: + return "enrollmentdate"; + case INCIDENT_DATE: + return "occurreddate"; + default: + return null; + } } private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { @@ -886,6 +947,43 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it return sql.toString(); } + private String buildYearlyDateRangeCondition(String column, String[] years) { + List yearConditions = new ArrayList<>(); + for (String year : years) { + yearConditions.add(String.format( + "(%s >= '%s-01-01' and %s < '%s-02-01')", + column, year, + column, year + )); + } + // Note: The entire set of year conditions should be wrapped in parentheses + return "(" + String.join(" or ", yearConditions) + ")"; + } + + + private String getOrderByColumn(String itemId) { + // Map the column names exactly as they appear in the original query + switch (itemId.toLowerCase()) { + case "incidentdate": + return "occurreddate"; // Note: maps to occurreddate + case "ouname": + return "ouname"; + default: + return itemId; + } + } + + private String buildDateRangeCondition(String column, String[] years) { + List yearConditions = new ArrayList<>(); + for (String year : years) { + yearConditions.add(String.format( + "(%s >= '%s-01-01' and %s < '%s-02-01')", + column, year, + column, year + )); + } + return String.join(" or ", yearConditions); + } private String getWhereClauseWithoutCTE(EventQueryParams params) { List conditions = new ArrayList<>(); @@ -919,11 +1017,13 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { // Add date range conditions if (params.hasEnrollmentDateCriteria()) { - String year = params.getEnrollmentDateCriteria(); - conditions.add(String.format( - "ax.enrollmentdate >= '%s-01-01' and ax.enrollmentdate < '%s-01-01'", - year, (Integer.parseInt(year) + 1) - )); + conditions.add(buildYearlyDateRangeCondition("ax.enrollmentdate", + params.getEnrollmentDateCriteria())); + } + + if (params.hasIncidentDateCriteria()) { + conditions.add(buildYearlyDateRangeCondition("ax.occurreddate", + params.getIncidentDateCriteria())); } else if (params.getStartDate() != null && params.getEndDate() != null) { conditions.add(String.format( "ax.enrollmentdate >= '%s' and ax.enrollmentdate < '%s'", From ee1ea4e28377b31eb642ec7d25c2d88c8f6ee1fb Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 11 Dec 2024 11:17:07 +0100 Subject: [PATCH 10/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 1c6d4bb12fac..f0bb838380bc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -1012,24 +1012,25 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { // Add organization unit conditions if (!params.getOrganisationUnits().isEmpty()) { String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); - } - - // Add date range conditions - if (params.hasEnrollmentDateCriteria()) { - conditions.add(buildYearlyDateRangeCondition("ax.enrollmentdate", - params.getEnrollmentDateCriteria())); + conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); } - if (params.hasIncidentDateCriteria()) { - conditions.add(buildYearlyDateRangeCondition("ax.occurreddate", - params.getIncidentDateCriteria())); - } else if (params.getStartDate() != null && params.getEndDate() != null) { - conditions.add(String.format( - "ax.enrollmentdate >= '%s' and ax.enrollmentdate < '%s'", - DateUtils.toMediumDate(params.getStartDate()), - DateUtils.toMediumDate(params.getEndDate()) - )); + // Handle date ranges using TimeField and DateRange + if (!params.getTimeDateRanges().isEmpty()) { + for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { + String column = getColumnForTimeField(entry.getKey()); + if (column != null) { + List dateConditions = new ArrayList<>(); + for (DateRange range : entry.getValue()) { + dateConditions.add(String.format( + "(%s >= '%s' and %s < '%s')", + "ax." + column, DateUtils.toMediumDate(range.getStartDate()), + "ax." + column, DateUtils.toMediumDate(range.getEndDate()) + )); + } + conditions.add("(" + String.join(" or ", dateConditions) + ")"); + } + } } // Handle NV (null value) filter @@ -1038,7 +1039,7 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { conditions.add("le.enrollment is not null"); } - // Join conditions with AND, but don't add extra spaces or concatenate with order by + // Join conditions with AND return conditions.isEmpty() ? "" : conditions.stream() .filter(StringUtils::isNotBlank) .collect(Collectors.joining(" and ")); From 50c0d10d28648bf0160b9e5d078db724adbfb107 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 11 Dec 2024 11:26:37 +0100 Subject: [PATCH 11/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 71 ++----------------- 1 file changed, 5 insertions(+), 66 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index f0bb838380bc..1d97b7910f07 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -897,14 +897,11 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { } private String getColumnForTimeField(TimeField timeField) { - switch (timeField) { - case ENROLLMENT_DATE: - return "enrollmentdate"; - case INCIDENT_DATE: - return "occurreddate"; - default: - return null; - } + return switch (timeField) { + case ENROLLMENT_DATE -> "enrollmentdate"; + case INCIDENT_DATE -> "occurreddate"; + default -> null; + }; } private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { @@ -947,64 +944,6 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it return sql.toString(); } - private String buildYearlyDateRangeCondition(String column, String[] years) { - List yearConditions = new ArrayList<>(); - for (String year : years) { - yearConditions.add(String.format( - "(%s >= '%s-01-01' and %s < '%s-02-01')", - column, year, - column, year - )); - } - // Note: The entire set of year conditions should be wrapped in parentheses - return "(" + String.join(" or ", yearConditions) + ")"; - } - - - private String getOrderByColumn(String itemId) { - // Map the column names exactly as they appear in the original query - switch (itemId.toLowerCase()) { - case "incidentdate": - return "occurreddate"; // Note: maps to occurreddate - case "ouname": - return "ouname"; - default: - return itemId; - } - } - - private String buildDateRangeCondition(String column, String[] years) { - List yearConditions = new ArrayList<>(); - for (String year : years) { - yearConditions.add(String.format( - "(%s >= '%s-01-01' and %s < '%s-02-01')", - column, year, - column, year - )); - } - return String.join(" or ", yearConditions); - } - - private String getWhereClauseWithoutCTE(EventQueryParams params) { - List conditions = new ArrayList<>(); - - // Add date range conditions - if (params.getStartDate() != null && params.getEndDate() != null) { - conditions.add(String.format( - "enrollmentdate >= '%s' and enrollmentdate < '%s'", - DateUtils.toMediumDate(params.getStartDate()), - DateUtils.toMediumDate(params.getEndDate()) - )); - } - - // Add organization unit conditions - if (!params.getOrganisationUnits().isEmpty()) { - String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("ax.uidlevel1 = '%s'", orgUnit)); - } - - return String.join(" and ", conditions); - } private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { List conditions = new ArrayList<>(); From 52740037c39081d35f0d773d308c55620d81c806 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 11 Dec 2024 11:54:27 +0100 Subject: [PATCH 12/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 135 ++++++++++++++---- 1 file changed, 104 insertions(+), 31 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 1d97b7910f07..d3b7ff3ce3cb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -904,46 +904,127 @@ private String getColumnForTimeField(TimeField timeField) { }; } + private String getBasicSelectColumns() { + return String.join(",", + "ax.enrollment", + "trackedentity", + "enrollmentdate", + "occurreddate", + "storedby", + "createdbydisplayname", + "lastupdatedbydisplayname", + "lastupdated", + "ST_AsGeoJSON(enrollmentgeometry)", + "longitude", + "latitude", + "ouname", + "ounamehierarchy", + "oucode", + "enrollmentstatus", + "ax.\"ou\"" + ); + } + private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { StringBuilder sql = new StringBuilder(); - // 1. Build CTE if needed - if (item != null && item.hasProgramStage()) { - sql.append(getLatestEventsCTE(params, item)); - } + // 1. Build CTE + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); + String columnName = quote(item.getItem().getUid()); + int offset = item.getProgramStageOffset(); + String offsetSuffix = offset != 0 ? "[" + offset + "]" : ""; + String columnPrefix = item.getProgramStage().getUid() + offsetSuffix; + + sql.append(String.format( + "with LatestEvents as ( " + + "select " + + " enrollment, " + + " %s as value, " + + " eventstatus, " + + " row_number() over (partition by enrollment order by occurreddate desc, created desc) as rn " + + "from %s " + + "where eventstatus != 'SCHEDULE' " + + " and ps = '%s' " + + ") ", + columnName, + eventTableName, + item.getProgramStage().getUid() + )); + + // 2. Select columns + sql.append(String.format( + "select %s, " + + "le.value as \"%s.%s\", " + + "(le.enrollment is not null) as \"%s.%s.exists\", " + + "le.eventstatus as \"%s.%s.status\" ", + getBasicSelectColumns(), + columnPrefix, item.getItem().getUid(), + columnPrefix, item.getItem().getUid(), + columnPrefix, item.getItem().getUid() + )); + + // 3. From clause with join + sql.append(" from ").append(params.getTableName()).append(" as ax") + .append(" left join LatestEvents le on ax.enrollment = le.enrollment") + .append(" and le.rn = ").append(Math.abs(offset) + 1); + + // 4. Where clause + List conditions = new ArrayList<>(); - // 2. Get select columns - String selectClause = getSelectClause(params); - sql.append(selectClause); + // Add org unit condition + if (!params.getOrganisationUnits().isEmpty()) { + String orgUnit = params.getOrganisationUnits().get(0).getUid(); + conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); + } - // 3. From clause - sql.append(" from ").append(params.getTableName()).append(" as ax"); + // Add date conditions + if (!params.getTimeDateRanges().isEmpty()) { + for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { + String column = getColumnForTimeField(entry.getKey()); + if (column != null) { + List dateConditions = new ArrayList<>(); + for (DateRange range : entry.getValue()) { + dateConditions.add(String.format( + "(%s >= '%s' and %s < '%s')", + "ax." + column, + DateUtils.toMediumDate(range.getStartDate()), + "ax." + column, + DateUtils.toMediumDate(range.getEndDate()) + )); + } + conditions.add("(" + String.join(" or ", dateConditions) + ")"); + } + } + } - // 4. Join clause if needed - if (item != null && item.hasProgramStage()) { - int offset = item.getProgramStageOffset(); - sql.append(" left join LatestEvents le on ax.enrollment = le.enrollment") - .append(" and le.rn = ").append(Math.abs(offset) + 1); + // Add NV filter condition + if (hasNullValueFilter(params)) { + conditions.add("le.value is null"); + conditions.add("le.enrollment is not null"); } - // 5. Where clause - String whereClause = getWhereClauseWithCTE(params, item); - if (!whereClause.isEmpty()) { - sql.append(" where ").append(whereClause); + if (!conditions.isEmpty()) { + sql.append(" where ").append(String.join(" and ", conditions)); } - // 6. Order by - ensure proper spacing - String sortClause = getSortClause(params); - if (!sortClause.isEmpty()) { - sql.append(" ").append(sortClause); + // Add order by + if (params.isSorting()) { + sql.append(" ").append(getSortClause(params)); } - // 7. Limit clause + // Add limit sql.append(" ").append(getPagingClause(params, 5000)); return sql.toString(); } + private boolean hasNullValueFilter(EventQueryParams params) { + return params.getItems().stream() + .anyMatch(item -> item.hasFilter() && + item.getFilters().stream() + .anyMatch(filter -> "NV".equals(filter.getFilter()))); + } + private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { List conditions = new ArrayList<>(); @@ -984,14 +1065,6 @@ private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { .collect(Collectors.joining(" and ")); } - - private boolean hasNullValueFilter(EventQueryParams params) { - return params.getItems().stream() - .anyMatch(item -> item.hasFilter() && - item.getFilters().stream() - .anyMatch(filter -> "NV".equals(filter.getFilter()))); - } - protected String getSortClause(EventQueryParams params) { if (params.isSorting()) { return super.getSortClause(params); From fa832e46c43c5b759bdb4cbaaa84da867c410f11 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 11 Dec 2024 15:16:07 +0100 Subject: [PATCH 13/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 39 ++----------------- 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index d3b7ff3ce3cb..815cb2c94219 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -969,42 +969,9 @@ private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem it .append(" and le.rn = ").append(Math.abs(offset) + 1); // 4. Where clause - List conditions = new ArrayList<>(); - - // Add org unit condition - if (!params.getOrganisationUnits().isEmpty()) { - String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); - } - - // Add date conditions - if (!params.getTimeDateRanges().isEmpty()) { - for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { - String column = getColumnForTimeField(entry.getKey()); - if (column != null) { - List dateConditions = new ArrayList<>(); - for (DateRange range : entry.getValue()) { - dateConditions.add(String.format( - "(%s >= '%s' and %s < '%s')", - "ax." + column, - DateUtils.toMediumDate(range.getStartDate()), - "ax." + column, - DateUtils.toMediumDate(range.getEndDate()) - )); - } - conditions.add("(" + String.join(" or ", dateConditions) + ")"); - } - } - } - - // Add NV filter condition - if (hasNullValueFilter(params)) { - conditions.add("le.value is null"); - conditions.add("le.enrollment is not null"); - } - - if (!conditions.isEmpty()) { - sql.append(" where ").append(String.join(" and ", conditions)); + String whereClause = getWhereClauseWithCTE(params, item); + if (!whereClause.isEmpty()) { + sql.append(" where ").append(whereClause); } // Add order by From 8a22fd79054188a8d29ea08958740e9201192bcf Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 11 Dec 2024 15:30:42 +0100 Subject: [PATCH 14/30] WIP - fix missing time range condition --- .../analytics/event/data/JdbcEnrollmentAnalyticsManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 815cb2c94219..621434220d40 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -996,6 +996,11 @@ private boolean hasNullValueFilter(EventQueryParams params) { private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { List conditions = new ArrayList<>(); + String timeFieldSql = timeFieldSqlRenderer.renderPeriodTimeFieldSql(params); + if (StringUtils.isNotBlank(timeFieldSql)) { + conditions.add(timeFieldSql); + } + // Add organization unit conditions if (!params.getOrganisationUnits().isEmpty()) { String orgUnit = params.getOrganisationUnits().get(0).getUid(); From 301cbfefcc2552282ceae8af684219fb4a16a6ef Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Thu, 12 Dec 2024 09:39:30 +0100 Subject: [PATCH 15/30] WIP - revert flybase change --- ...ll.xxx => V2_42_28__Make_ou_event_and_enrollment_not_null.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/{V2_42_28__Make_ou_event_and_enrollment_not_null.xxx => V2_42_28__Make_ou_event_and_enrollment_not_null.sql} (100%) diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.xxx b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.sql similarity index 100% rename from dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.xxx rename to dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.sql From fe5e108acaea548213362f6c2456cc88d948cb88 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Thu, 12 Dec 2024 12:48:55 +0100 Subject: [PATCH 16/30] WIP --- .../analytics/event/data/AbstractJdbcEventAnalyticsManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index a04481e76348..2142d0566a62 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1385,7 +1385,7 @@ protected String getCoalesce(List fields, String defaultColumnName) { String args = fields.stream() .filter(f -> f != null && !f.isBlank()) - .map(f -> sqlBuilder.quoteAx(f)) + .map(sqlBuilder::quoteAx) .collect(Collectors.joining(",")); return args.isEmpty() ? defaultColumnName : "coalesce(" + args + ")"; From caeb7bc98f58e606f14633b171f2758196540623 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Fri, 13 Dec 2024 14:55:02 +0100 Subject: [PATCH 17/30] WIP --- .../analytics/event/EventQueryParams.java | 33 +- .../data/JdbcEnrollmentAnalyticsManager.java | 423 +++++++++--------- 2 files changed, 211 insertions(+), 245 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index d116d69bcac0..589ba6cbb498 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -678,7 +678,6 @@ public boolean hasAnalyticsVariables() { return hasProgramIndicatorDimension() && getProgramIndicator().hasAnalyticsVariables(); } - public boolean useIndividualQuery() { return this.hasAnalyticsVariables() || this.hasNonDefaultBoundaries(); } @@ -837,46 +836,34 @@ public boolean hasDataIdScheme() { return dataIdScheme != null; } - /** - * Returns true if enrollment date criteria exists in dimensions or filters. - */ + /** Returns true if enrollment date criteria exists in dimensions or filters. */ public boolean hasEnrollmentDateCriteria() { - return getDimensionOrFilterItems("enrollmentDate").size() > 0 || - getDimensionOrFilterItems("enrollmentdate").size() > 0; // Try both cases + return getDimensionOrFilterItems("enrollmentDate").size() > 0 + || getDimensionOrFilterItems("enrollmentdate").size() > 0; // Try both cases } - /** - * Returns the enrollment date criteria years. - */ + /** Returns the enrollment date criteria years. */ public String[] getEnrollmentDateCriteria() { List items = getDimensionOrFilterItems("enrollmentDate"); if (items.isEmpty()) { items = getDimensionOrFilterItems("enrollmentdate"); } - return items.stream() - .map(item -> item.getDimensionItem()) - .toArray(String[]::new); + return items.stream().map(item -> item.getDimensionItem()).toArray(String[]::new); } - /** - * Returns true if incident date criteria exists in dimensions or filters. - */ + /** Returns true if incident date criteria exists in dimensions or filters. */ public boolean hasIncidentDateCriteria() { - return getDimensionOrFilterItems("incidentDate").size() > 0 || - getDimensionOrFilterItems("incidentdate").size() > 0; // Try both cases + return getDimensionOrFilterItems("incidentDate").size() > 0 + || getDimensionOrFilterItems("incidentdate").size() > 0; // Try both cases } - /** - * Returns the incident date criteria years. - */ + /** Returns the incident date criteria years. */ public String[] getIncidentDateCriteria() { List items = getDimensionOrFilterItems("incidentDate"); if (items.isEmpty()) { items = getDimensionOrFilterItems("incidentdate"); } - return items.stream() - .map(item -> item.getDimensionItem()) - .toArray(String[]::new); + return items.stream().map(item -> item.getDimensionItem()).toArray(String[]::new); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 621434220d40..f0f76205926b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -27,7 +27,30 @@ */ package org.hisp.dhis.analytics.event.data; +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; +import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; +import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; +import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; +import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; +import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; +import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; +import static org.hisp.dhis.util.DateUtils.toMediumDate; + import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.TimeField; @@ -67,29 +90,6 @@ import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.joining; -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; -import static org.hisp.dhis.analytics.DataType.BOOLEAN; -import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; -import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; -import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; -import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; -import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; -import static org.hisp.dhis.util.DateUtils.toMediumDate; - /** * @author Markus Bekken */ @@ -159,11 +159,10 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { if (params.analyzeOnly()) { withExceptionHandling( - () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); + () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); } else { withExceptionHandling( - () -> getEnrollments(params, grid, sql, maxLimit == 0), - params.isMultipleQueries()); + () -> getEnrollments(params, grid, sql, maxLimit == 0), params.isMultipleQueries()); } } @@ -800,6 +799,18 @@ private String createOffset(int offset) { } } + private int createOffset2(int offset) { + if (offset == 0) { + return 0; + } + + if (offset < 0) { + return (-1 * offset); + } else { + return (offset - 1); + } + } + private String createOrderType(int offset) { if (offset == 0) { return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "desc"); @@ -813,28 +824,6 @@ private String createOrderType(int offset) { // New methods // - private String getLatestEventsCTE(EventQueryParams params, QueryItem item) { - String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); - String columnName = quote(item.getItem().getUid()); - - return String.format( - "with LatestEvents as ( " + - "select " + - " enrollment, " + - " %s as value, " + - " eventstatus, " + - " row_number() over (partition by enrollment " + - " order by occurreddate desc, created desc) as rn " + - "from %s " + - "where eventstatus != 'SCHEDULE' " + - " and ps = '%s' " + - ") ", - columnName, - eventTableName, - item.getProgramStage().getUid() - ); - } - private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { StringBuilder sql = new StringBuilder(); @@ -860,11 +849,13 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { if (column != null) { List dateConditions = new ArrayList<>(); for (DateRange range : entry.getValue()) { - dateConditions.add(String.format( + dateConditions.add( + String.format( "(%s >= '%s' and %s < '%s')", - column, DateUtils.toMediumDate(range.getStartDate()), - column, DateUtils.toMediumDate(range.getEndDate()) - )); + column, + DateUtils.toMediumDate(range.getStartDate()), + column, + DateUtils.toMediumDate(range.getEndDate()))); } conditions.add("(" + String.join(" or ", dateConditions) + ")"); } @@ -883,8 +874,8 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { if (!first) { sql.append(","); } - String columnName = item.getItemId().equals("incidentdate") ? - "occurreddate" : item.getItemId(); + String columnName = + item.getItemId().equals("incidentdate") ? "occurreddate" : item.getItemId(); sql.append(" ").append(quote(columnName)).append(" asc nulls last"); first = false; } @@ -897,205 +888,193 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { } private String getColumnForTimeField(TimeField timeField) { - return switch (timeField) { - case ENROLLMENT_DATE -> "enrollmentdate"; - case INCIDENT_DATE -> "occurreddate"; - default -> null; - }; + return switch (timeField) { + case ENROLLMENT_DATE -> "enrollmentdate"; + case INCIDENT_DATE -> "occurreddate"; + default -> null; + }; } private String getBasicSelectColumns() { - return String.join(",", - "ax.enrollment", - "trackedentity", - "enrollmentdate", - "occurreddate", - "storedby", - "createdbydisplayname", - "lastupdatedbydisplayname", - "lastupdated", - "ST_AsGeoJSON(enrollmentgeometry)", - "longitude", - "latitude", - "ouname", - "ounamehierarchy", - "oucode", - "enrollmentstatus", - "ax.\"ou\"" - ); + return String.join( + ",", + "ax.enrollment", + "trackedentity", + "enrollmentdate", + "occurreddate", + "storedby", + "createdbydisplayname", + "lastupdatedbydisplayname", + "lastupdated", + "ST_AsGeoJSON(enrollmentgeometry)", + "longitude", + "latitude", + "ouname", + "ounamehierarchy", + "oucode", + "enrollmentstatus", + "ax.\"ou\""); } - private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { - StringBuilder sql = new StringBuilder(); + private String resolveOrderByOffset(int offset) { - // 1. Build CTE - String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); - String columnName = quote(item.getItem().getUid()); - int offset = item.getProgramStageOffset(); - String offsetSuffix = offset != 0 ? "[" + offset + "]" : ""; - String columnPrefix = item.getProgramStage().getUid() + offsetSuffix; - - sql.append(String.format( - "with LatestEvents as ( " + - "select " + - " enrollment, " + - " %s as value, " + - " eventstatus, " + - " row_number() over (partition by enrollment order by occurreddate desc, created desc) as rn " + - "from %s " + - "where eventstatus != 'SCHEDULE' " + - " and ps = '%s' " + - ") ", - columnName, - eventTableName, - item.getProgramStage().getUid() - )); - - // 2. Select columns - sql.append(String.format( - "select %s, " + - "le.value as \"%s.%s\", " + - "(le.enrollment is not null) as \"%s.%s.exists\", " + - "le.eventstatus as \"%s.%s.status\" ", - getBasicSelectColumns(), - columnPrefix, item.getItem().getUid(), - columnPrefix, item.getItem().getUid(), - columnPrefix, item.getItem().getUid() - )); - - // 3. From clause with join - sql.append(" from ").append(params.getTableName()).append(" as ax") - .append(" left join LatestEvents le on ax.enrollment = le.enrollment") - .append(" and le.rn = ").append(Math.abs(offset) + 1); - - // 4. Where clause - String whereClause = getWhereClauseWithCTE(params, item); - if (!whereClause.isEmpty()) { - sql.append(" where ").append(whereClause); + if (offset <= 0) { + return "desc"; } + return "asc"; + } - // Add order by - if (params.isSorting()) { - sql.append(" ").append(getSortClause(params)); - } + private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { - // Add limit - sql.append(" ").append(getPagingClause(params, 5000)); + List items = params.getItems(); - return sql.toString(); - } + // Determine the program and stage UIDs + String stageUid = item.getProgramStage().getUid(); + String itemUid = item.getItem().getUid(); - private boolean hasNullValueFilter(EventQueryParams params) { - return params.getItems().stream() - .anyMatch(item -> item.hasFilter() && - item.getFilters().stream() - .anyMatch(filter -> "NV".equals(filter.getFilter()))); - } + var offset = createOffset2(item.getProgramStageOffset()); + // The column from the event analytics table we are interested in + String columnName = quote(itemUid); - private String getWhereClauseWithCTE(EventQueryParams params, QueryItem item) { - List conditions = new ArrayList<>(); + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); - String timeFieldSql = timeFieldSqlRenderer.renderPeriodTimeFieldSql(params); - if (StringUtils.isNotBlank(timeFieldSql)) { - conditions.add(timeFieldSql); + String order = resolveOrderByOffset(item.getProgramStageOffset()); + + + + // Build the CTE that ranks all events for each enrollment + String allRankedEventsCTE = + String.format( + "with AllRankedEvents as (" + + " select enrollment, %s as value, eventstatus, " + + " row_number() over (partition by enrollment order by occurreddate %s, created %s) as rn " + + " from %s " + + " where eventstatus != 'SCHEDULE' " + + " and ps = '%s' " + + ") ", + columnName, order, order, eventTableName, stageUid); + + StringBuilder selectBuilder = new StringBuilder(getBasicSelectColumns()); + StringBuilder joinBuilder = new StringBuilder(); + + int rn = offset + 1; + String alias = "le"; + + joinBuilder + .append(" left join AllRankedEvents ") + .append(alias) + .append(" on ax.enrollment = ") + .append(alias) + .append(".enrollment and ") + .append(alias) + .append(".rn = ") + .append(rn) + .append(" "); + + String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; + String valueCol = quote(stageUid + offsetLabel + "." + itemUid); + String existsCol = quote(stageUid + offsetLabel + "." + itemUid + ".exists"); + String statusCol = quote(stageUid + offsetLabel + "." + itemUid + ".status"); + + selectBuilder + .append(", ") + .append(alias) + .append(".value as ") + .append(valueCol) + // Use (alias.enrollment is not null) for .exists logic + .append(", (") + .append(alias) + .append(".enrollment is not null) as ") + .append(existsCol) + .append(", ") + .append(alias) + .append(".eventstatus as ") + .append(statusCol); + + // FROM clause with all joins + String fromClause = " from " + params.getTableName() + " as ax " + joinBuilder; + + // WHERE clause: from params and original conditions + // 5. Integrate the original where logic + // The getWhereClause method returns conditions that typically start with "and". + // We need to insert them properly after a "where". + String baseConditions = getWhereClause(params).trim(); + + String whereClause = ""; + + if (baseConditions.startsWith("and")) { + whereClause = " " + baseConditions.replaceFirst("and", "where"); + } else { + whereClause = baseConditions; } - // Add organization unit conditions - if (!params.getOrganisationUnits().isEmpty()) { - String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); - } + // ORDER BY and LIMIT/OFFSET (paging) + String orderByClause = getSortClause(params); + String pagingClause = getPagingClause(params, 5000); + + // Combine all parts into the final SQL + return allRankedEventsCTE + + "select " + + selectBuilder + + fromClause + + whereClause + + orderByClause + + pagingClause; + } - // Handle date ranges using TimeField and DateRange - if (!params.getTimeDateRanges().isEmpty()) { - for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { - String column = getColumnForTimeField(entry.getKey()); - if (column != null) { - List dateConditions = new ArrayList<>(); - for (DateRange range : entry.getValue()) { - dateConditions.add(String.format( - "(%s >= '%s' and %s < '%s')", - "ax." + column, DateUtils.toMediumDate(range.getStartDate()), - "ax." + column, DateUtils.toMediumDate(range.getEndDate()) - )); - } - conditions.add("(" + String.join(" or ", dateConditions) + ")"); - } - } - } + private List determineOffsetsNeeded(EventQueryParams params, QueryItem item) { - // Handle NV (null value) filter - if (hasNullValueFilter(params)) { - conditions.add("le.value is null"); - conditions.add("le.enrollment is not null"); - } + // var dimensions = params.getDimensions(); + // get all dimensions with offset -> CWaAcQYKVpq[-1].fyjPqlHE7Dn + // extract dimension + // pass dimension to getRepeatableStageParams(x) - // Join conditions with AND - return conditions.isEmpty() ? "" : conditions.stream() - .filter(StringUtils::isNotBlank) - .collect(Collectors.joining(" and ")); - } + // item.setRepeatableStageParams(getRepeatableStageParams(dimension)); - protected String getSortClause(EventQueryParams params) { - if (params.isSorting()) { - return super.getSortClause(params); - } - return ""; - } + Set offsets = new HashSet<>(); - protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, SqlHelper hlp) { - if (params.isEnhancedCondition()) { - return getItemsSqlForEnhancedConditions(params, hlp); + // If no repeatable stage params, assume only top event + if (!item.hasProgramStage()) { + offsets.add(0); + return new ArrayList<>(offsets); } - List conditions = new ArrayList<>(); + int startIndex = item.getProgramStageOffset(); + int count = item.hasRepeatableStageParams() ? item.getRepeatableStageParams().getCount() : 1; - // Handle item filters using CTE - for (QueryItem item : params.getItems()) { - if (item.hasFilter()) { - conditions.addAll(getItemConditions(item)); + if (startIndex == 0) { + // If startIndex is 0, we start from the top event + // count=1 means just the top event [0] + // count>1 means top event plus more recent events behind it: [0,1,2,...] + offsets.add(0); + for (int i = 1; i < count; i++) { + offsets.add(i); } - } - - // Handle item filters - for (QueryItem item : params.getItemFilters()) { - if (item.hasFilter()) { - conditions.addAll(getItemConditions(item)); + } else if (startIndex < 0) { + // Negative startIndex means starting behind the top event + offsets.add(0); // Always include the latest event + int x = Math.abs(startIndex); + // For count=1 and startIndex=-1, we get [0,1] + // For count=2 and startIndex=-1, we get [0,1,2] + // For count=1 and startIndex=-2, we get [0,2] + for (int i = x; i < x + count; i++) { + offsets.add(i); } + } else { + // If positive offsets are not used by original logic, handle as needed. + // For safety, include top event at least. + offsets.add(0); } - return conditions.isEmpty() ? "" : String.join(" and ", conditions); + return new ArrayList<>(offsets); } - private List getItemConditions(QueryItem item) { - List conditions = new ArrayList<>(); - - for (QueryFilter filter : item.getFilters()) { - if ("NV".equals(filter.getFilter())) { - // Special handling for null values - conditions.add("le.value is null"); - } else { - String operator = filter.getSqlOperator(); - if (operator.equals("in")) { - // Handle IN clause differently based on value type - String value = getSqlFilter(filter, item); - if (item.isNumeric()) { - conditions.add("le.value in (" + value + ")"); - } else { - // For text values, keep the quotes - conditions.add("le.value in (" + value + ")"); - } - } else { - String value = getSqlFilter(filter, item); - conditions.add(String.format("le.value %s %s", - operator, - value)); - } - } + protected String getSortClause(EventQueryParams params) { + if (params.isSorting()) { + return super.getSortClause(params); } - - return conditions; + return ""; } protected String getSqlFilter(QueryFilter filter, QueryItem item) { From 44c038a66278a50cf6de0a7acb476cd9f93fa73b Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Fri, 13 Dec 2024 15:29:50 +0100 Subject: [PATCH 18/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 251 +++++++++--------- 1 file changed, 126 insertions(+), 125 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index f0f76205926b..4e7a64958977 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -926,148 +926,149 @@ private String resolveOrderByOffset(int offset) { private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { + StringBuilder sql = new StringBuilder(); List items = params.getItems(); - // Determine the program and stage UIDs - String stageUid = item.getProgramStage().getUid(); - String itemUid = item.getItem().getUid(); - - var offset = createOffset2(item.getProgramStageOffset()); - - // The column from the event analytics table we are interested in - String columnName = quote(itemUid); - - String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); - - String order = resolveOrderByOffset(item.getProgramStageOffset()); - - - - // Build the CTE that ranks all events for each enrollment - String allRankedEventsCTE = - String.format( - "with AllRankedEvents as (" - + " select enrollment, %s as value, eventstatus, " - + " row_number() over (partition by enrollment order by occurreddate %s, created %s) as rn " - + " from %s " - + " where eventstatus != 'SCHEDULE' " - + " and ps = '%s' " - + ") ", - columnName, order, order, eventTableName, stageUid); - - StringBuilder selectBuilder = new StringBuilder(getBasicSelectColumns()); - StringBuilder joinBuilder = new StringBuilder(); - - int rn = offset + 1; - String alias = "le"; - - joinBuilder - .append(" left join AllRankedEvents ") - .append(alias) - .append(" on ax.enrollment = ") - .append(alias) - .append(".enrollment and ") - .append(alias) - .append(".rn = ") - .append(rn) - .append(" "); - - String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; - String valueCol = quote(stageUid + offsetLabel + "." + itemUid); - String existsCol = quote(stageUid + offsetLabel + "." + itemUid + ".exists"); - String statusCol = quote(stageUid + offsetLabel + "." + itemUid + ".status"); - - selectBuilder - .append(", ") - .append(alias) - .append(".value as ") - .append(valueCol) - // Use (alias.enrollment is not null) for .exists logic - .append(", (") - .append(alias) - .append(".enrollment is not null) as ") - .append(existsCol) - .append(", ") - .append(alias) - .append(".eventstatus as ") - .append(statusCol); - - // FROM clause with all joins - String fromClause = " from " + params.getTableName() + " as ax " + joinBuilder; - - // WHERE clause: from params and original conditions - // 5. Integrate the original where logic - // The getWhereClause method returns conditions that typically start with "and". - // We need to insert them properly after a "where". - String baseConditions = getWhereClause(params).trim(); - - String whereClause = ""; - - if (baseConditions.startsWith("and")) { - whereClause = " " + baseConditions.replaceFirst("and", "where"); - } else { - whereClause = baseConditions; + // 1. Build CTEs for each unique program stage + offset combination + sql.append(buildAllRankedEventsCTEs(items)); + + // 2. Build main query + sql.append("select "); + + // 2.1 Add basic columns + sql.append(getBasicSelectColumns()); + + // 2.2 Add value columns for each item + String valueColumns = buildValueColumns(items); + if (!valueColumns.isEmpty()) { + sql.append(", ").append(valueColumns); + } + + // 2.3 Add FROM clause with all necessary joins + sql.append(buildFromClauseWithJoins(params, items)); + + // 2.4 Add WHERE clause + String whereClause = getWhereClause(params).trim(); + if (!whereClause.isEmpty()) { + // Remove any leading "and" and ensure only one "where" keyword + whereClause = whereClause.replaceFirst("^and\\s+", ""); + whereClause = whereClause.replaceFirst("^where\\s+", ""); + if (!whereClause.isEmpty()) { + sql.append(" where ").append(whereClause); + } } - // ORDER BY and LIMIT/OFFSET (paging) - String orderByClause = getSortClause(params); - String pagingClause = getPagingClause(params, 5000); - - // Combine all parts into the final SQL - return allRankedEventsCTE - + "select " - + selectBuilder - + fromClause - + whereClause - + orderByClause - + pagingClause; + // 2.5 Add ORDER BY and paging + sql.append(getSortClause(params)); + sql.append(getPagingClause(params, 5000)); + + return sql.toString(); } - private List determineOffsetsNeeded(EventQueryParams params, QueryItem item) { + private String buildAllRankedEventsCTEs(List items) { + StringBuilder ctes = new StringBuilder(); + Set processedCombinations = new HashSet<>(); + + for (QueryItem item : items) { + if (!item.hasProgramStage()) { + continue; + } - // var dimensions = params.getDimensions(); - // get all dimensions with offset -> CWaAcQYKVpq[-1].fyjPqlHE7Dn - // extract dimension - // pass dimension to getRepeatableStageParams(x) + String stageUid = item.getProgramStage().getUid(); + int offset = createOffset2(item.getProgramStageOffset()); + String order = resolveOrderByOffset(item.getProgramStageOffset()); - // item.setRepeatableStageParams(getRepeatableStageParams(dimension)); + // Create unique key for this combination to avoid duplicate CTEs + String key = stageUid + "_" + offset + "_" + order; + if (processedCombinations.contains(key)) { + continue; + } - Set offsets = new HashSet<>(); + if (!ctes.isEmpty()) { + ctes.append(",\n"); + } - // If no repeatable stage params, assume only top event - if (!item.hasProgramStage()) { - offsets.add(0); - return new ArrayList<>(offsets); + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); + String columnName = quote(item.getItem().getUid()); + + ctes.append(String.format( + """ + RankedEvents_%s as ( + select enrollment, %s as value, eventstatus, + row_number() over (partition by enrollment order by occurreddate %s, created %s) as rn + from %s + where eventstatus != 'SCHEDULE' + and ps = '%s' + ) + """, + key, columnName, order, order, eventTableName, stageUid + )); + + processedCombinations.add(key); } - int startIndex = item.getProgramStageOffset(); - int count = item.hasRepeatableStageParams() ? item.getRepeatableStageParams().getCount() : 1; + return ctes.length() > 0 ? "with " + ctes.toString() + "\n" : ""; + } + + private String buildValueColumns(List items) { + StringBuilder columns = new StringBuilder(); - if (startIndex == 0) { - // If startIndex is 0, we start from the top event - // count=1 means just the top event [0] - // count>1 means top event plus more recent events behind it: [0,1,2,...] - offsets.add(0); - for (int i = 1; i < count; i++) { - offsets.add(i); + for (QueryItem item : items) { + if (!item.hasProgramStage()) { + continue; } - } else if (startIndex < 0) { - // Negative startIndex means starting behind the top event - offsets.add(0); // Always include the latest event - int x = Math.abs(startIndex); - // For count=1 and startIndex=-1, we get [0,1] - // For count=2 and startIndex=-1, we get [0,1,2] - // For count=1 and startIndex=-2, we get [0,2] - for (int i = x; i < x + count; i++) { - offsets.add(i); + + String stageUid = item.getProgramStage().getUid(); + int offset = createOffset2(item.getProgramStageOffset()); + String key = stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + + String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; + String alias = "re_" + key; + + if (!columns.isEmpty()) { + columns.append(",\n"); } - } else { - // If positive offsets are not used by original logic, handle as needed. - // For safety, include top event at least. - offsets.add(0); + + // Add value column + columns.append(String.format("%s.value as %s", + alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid()))); + + // Add exists column + columns.append(String.format(",\n(%s.enrollment is not null) as %s", + alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".exists"))); + + // Add status column + columns.append(String.format(",\n%s.eventstatus as %s", + alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".status"))); + } + + return columns.toString(); + } + + private String buildFromClauseWithJoins(EventQueryParams params, List items) { + StringBuilder fromClause = new StringBuilder(); + + // Start with base table + fromClause.append("\nfrom ").append(params.getTableName()).append(" as ax"); + + // Add joins for each item + for (QueryItem item : items) { + if (!item.hasProgramStage()) { + continue; + } + + String stageUid = item.getProgramStage().getUid(); + int offset = createOffset2(item.getProgramStageOffset()); + String key = stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + String alias = "re_" + key; + + fromClause.append(String.format( + "\nleft join RankedEvents_%s %s on ax.enrollment = %s.enrollment and %s.rn = %d", + key, alias, alias, alias, offset + 1 + )); } - return new ArrayList<>(offsets); + return fromClause.toString(); } protected String getSortClause(EventQueryParams params) { From 6c33e0cf4b1830090d6b8433afed2a15040503ae Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Fri, 13 Dec 2024 19:03:59 +0100 Subject: [PATCH 19/30] WIP - handle PI --- .../dhis/analytics/common/CTEContext.java | 48 ++++++++++ .../ProgramIndicatorSubqueryBuilder.java | 15 +++ .../AbstractJdbcEventAnalyticsManager.java | 36 +++++++ .../data/JdbcEnrollmentAnalyticsManager.java | 85 +++++++++++++---- ...efaultProgramIndicatorSubqueryBuilder.java | 93 +++++++++++++++++-- 5 files changed, 249 insertions(+), 28 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java new file mode 100644 index 000000000000..c85123d1f3f5 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -0,0 +1,48 @@ +package org.hisp.dhis.analytics.common; + + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +public class CTEContext { + private final Map cteDefinitions = new LinkedHashMap<>(); + private final Map columnMappings = new HashMap<>(); + + public void addCTE(String cteName, String cteDefinition) { + cteDefinitions.put(cteName, cteDefinition); + } + + public void addColumnMapping(String originalColumn, String cteReference) { + columnMappings.put(originalColumn, cteReference); + } + + public String getCTEDefinition() { + if (cteDefinitions.isEmpty()) { + return ""; + } + + StringBuilder sb = new StringBuilder("WITH "); + boolean first = true; + for (Map.Entry entry : cteDefinitions.entrySet()) { + if (!first) { + sb.append(", "); + } + sb.append(entry.getKey()) + .append(" AS (") + .append(entry.getValue()) + .append(")"); + first = false; + } + return sb.toString(); + } + + public Set getCTENames() { + return cteDefinitions.keySet(); + } + + public String getColumnMapping(String columnId) { + return columnMappings.getOrDefault(columnId, columnId); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java index 411757dc52a6..745c3bb29cba 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java @@ -101,4 +101,19 @@ String getAggregateClauseForProgramIndicator( AnalyticsType outerSqlEntity, Date earliestStartDate, Date latestDate); + + void contributeCTE( + ProgramIndicator programIndicator, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext); + + void contributeCTE( + ProgramIndicator programIndicator, + RelationshipType relationshipType, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 2142d0566a62..28af157a0e0c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -91,6 +91,7 @@ import org.hisp.dhis.analytics.EventOutputType; import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.util.AnalyticsUtils; @@ -1391,6 +1392,41 @@ protected String getCoalesce(List fields, String defaultColumnName) { return args.isEmpty() ? defaultColumnName : "coalesce(" + args + ")"; } + protected List getSelectColumnsWithCTE(EventQueryParams params, CTEContext cteContext) { + List columns = new ArrayList<>(); + + // Mirror the logic of addDimensionSelectColumns + // (We can copy this logic from the superclass since it's about dimensions) + addDimensionSelectColumns(columns, params, false); // assuming we keep this protected method + + // Mirror the logic of addItemSelectColumns but with CTE references + for (QueryItem queryItem : params.getItems()) { + if (queryItem.isProgramIndicator()) { + // For program indicators, use CTE reference + String piUid = queryItem.getItem().getUid(); + String cteReference = cteContext.getColumnMapping(piUid); + columns.add(cteReference + " as \"" + piUid + "\""); + } else if (ValueType.COORDINATE == queryItem.getValueType()) { + // Handle coordinates + columns.add(getCoordinateColumn(queryItem).asSql()); + } else if (ValueType.ORGANISATION_UNIT == queryItem.getValueType()) { + // Handle org units + if (params.getCoordinateFields().stream() + .anyMatch(f -> queryItem.getItem().getUid().equals(f))) { + columns.add(getCoordinateColumn(queryItem, OU_GEOMETRY_COL_SUFFIX).asSql()); + } else { + columns.add(getOrgUnitQueryItemColumnAndAlias(params, queryItem).asSql()); + } + } else { + // Handle other types as before + ColumnAndAlias columnAndAlias = getColumnAndAlias(queryItem, false, ""); + columns.add(columnAndAlias.asSql()); + } + } + + return columns; + } + /** * Returns a select SQL clause for the given query. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 4e7a64958977..0e0d4a501e3a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -55,6 +55,7 @@ import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; @@ -81,6 +82,7 @@ import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.AnalyticsType; +import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.util.DateUtils; import org.locationtech.jts.util.Assert; @@ -152,7 +154,7 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { } else if (!params.getItems().isEmpty() && shouldUseCTE(params)) { sql = buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); } else { - sql = buildEnrollmentQueryWithoutCTE(params); + sql = buildEnrollmentQueryWithCte2(params); } System.out.println("SQL: " + sql); @@ -824,16 +826,61 @@ private String createOrderType(int offset) { // New methods // - private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { + private String buildEnrollmentQueryWithCte2(EventQueryParams params) { + // Create CTEContext to collect all CTEs + CTEContext cteContext = new CTEContext(); StringBuilder sql = new StringBuilder(); - // 1. Select clause - sql.append(getSelectClause(params)); + // 1. Process all program indicators to generate CTEs before building the main query + for (QueryItem queryItem : params.getItems()) { + if (queryItem.isProgramIndicator()) { + ProgramIndicator pi = (ProgramIndicator) queryItem.getItem(); + + if (queryItem.hasRelationshipType()) { + programIndicatorSubqueryBuilder.contributeCTE( + pi, + queryItem.getRelationshipType(), + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); + } else { + programIndicatorSubqueryBuilder.contributeCTE( + pi, + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); + } + } + } - // 2. From clause + // 2. Add WITH clause if we have any CTEs + String cteDefinitions = cteContext.getCTEDefinition(); + if (!cteDefinitions.isEmpty()) { + sql.append(cteDefinitions).append("\n"); + } + + // 3. Select clause using our new method + List selectCols = ListUtils.distinctUnion( + params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, + getSelectColumnsWithCTE(params, cteContext) + ); + sql.append("SELECT ").append(String.join(",\n ", selectCols)); + + // 4. From clause sql.append(" from ").append(params.getTableName()).append(" as ax"); - // 3. Where clause + // 5. Add joins for each CTE + for (String cteName : cteContext.getCTENames()) { + sql.append("\nLEFT JOIN ") + .append(cteName) + .append(" ON ax.enrollment = ") + .append(cteName) + .append(".enrollment"); + } + + // 6. Where clause List conditions = new ArrayList<>(); // Add organization unit condition @@ -842,7 +889,7 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); } - // Use TimeField and DateRange instead of the dimension approach + // Add date range conditions if (!params.getTimeDateRanges().isEmpty()) { for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { String column = getColumnForTimeField(entry.getKey()); @@ -850,12 +897,12 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { List dateConditions = new ArrayList<>(); for (DateRange range : entry.getValue()) { dateConditions.add( - String.format( - "(%s >= '%s' and %s < '%s')", - column, - DateUtils.toMediumDate(range.getStartDate()), - column, - DateUtils.toMediumDate(range.getEndDate()))); + String.format( + "(%s >= '%s' and %s < '%s')", + column, + DateUtils.toMediumDate(range.getStartDate()), + column, + DateUtils.toMediumDate(range.getEndDate()))); } conditions.add("(" + String.join(" or ", dateConditions) + ")"); } @@ -863,30 +910,32 @@ private String buildEnrollmentQueryWithoutCTE(EventQueryParams params) { } if (!conditions.isEmpty()) { - sql.append(" where ").append(String.join(" and ", conditions)); + sql.append("\nWHERE ").append(String.join(" AND ", conditions)); } - // 4. Order by + // 7. Order by if (params.getAsc() != null && !params.getAsc().isEmpty()) { - sql.append(" order by"); + sql.append("\nORDER BY"); boolean first = true; for (QueryItem item : params.getAsc()) { if (!first) { sql.append(","); } String columnName = - item.getItemId().equals("incidentdate") ? "occurreddate" : item.getItemId(); + item.getItemId().equals("incidentdate") ? "occurreddate" : item.getItemId(); sql.append(" ").append(quote(columnName)).append(" asc nulls last"); first = false; } } - // 5. Paging + // 8. Paging sql.append(" ").append(getPagingClause(params, 5000)); return sql.toString(); } + + private String getColumnForTimeField(TimeField timeField) { return switch (timeField) { case ENROLLMENT_DATE -> "enrollmentdate"; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java index 13b83e647c82..b1589025131e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java @@ -38,6 +38,7 @@ import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.DataType; +import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.commons.util.TextUtils; @@ -77,6 +78,75 @@ public String getAggregateClauseForProgramIndicator( programIndicator, relationshipType, outerSqlEntity, earliestStartDate, latestDate); } + @Override + public void contributeCTE(ProgramIndicator programIndicator, AnalyticsType outerSqlEntity, Date earliestStartDate, Date latestDate, CTEContext cteContext) { + contributeCTE(programIndicator, null, outerSqlEntity, earliestStartDate, latestDate, cteContext); + } + + @Override + public void contributeCTE(ProgramIndicator programIndicator, RelationshipType relationshipType, AnalyticsType outerSqlEntity, Date earliestStartDate, Date latestDate, CTEContext cteContext) { + // Generate a unique CTE name for this program indicator + String cteName = "pi_" + programIndicator.getUid().toLowerCase(); + + // Build the CTE definition + StringBuilder cteSql = new StringBuilder(); + + // Define aggregation function + String function = TextUtils.emptyIfEqual( + programIndicator.getAggregationTypeFallback().getValue(), + AggregationType.CUSTOM.getValue()); + + // Start building the SELECT part of CTE + cteSql.append("SELECT ") + .append(SUBQUERY_TABLE_ALIAS) + .append(".enrollment, "); + + // Add the aggregation expression + cteSql.append(function) + .append("(") + .append(getProgramIndicatorSql( + programIndicator.getExpression(), + NUMERIC, + programIndicator, + earliestStartDate, + latestDate)) + .append(") as value"); + + // Add FROM clause + cteSql.append(getFrom(programIndicator)); + + // Add WHERE clause + String where = getWhere(outerSqlEntity, programIndicator, relationshipType); + if (!where.isEmpty()) { + cteSql.append(where); + } + + // Add program indicator filter if present + if (!Strings.isNullOrEmpty(programIndicator.getFilter())) { + cteSql.append(where.isBlank() ? " WHERE " : " AND ") + .append("(") + .append(getProgramIndicatorSql( + programIndicator.getFilter(), + BOOLEAN, + programIndicator, + earliestStartDate, + latestDate)) + .append(")"); + } + + // Add GROUP BY clause for the enrollment + cteSql.append(" GROUP BY ") + .append(SUBQUERY_TABLE_ALIAS) + .append(".enrollment"); + + // Register the CTE and its column mapping + cteContext.addCTE(cteName, cteSql.toString()); + cteContext.addColumnMapping( + programIndicator.getUid(), + cteName + ".value" + ); + } + /** * Generate a subquery based on the result of a Program Indicator and an (optional) Relationship * Type @@ -165,20 +235,23 @@ private String getWhere( RelationshipType relationshipType) { String condition = ""; if (relationshipType != null) { - condition = - RelationshipTypeJoinGenerator.generate( - SUBQUERY_TABLE_ALIAS, relationshipType, programIndicator.getAnalyticsType()); + condition = RelationshipTypeJoinGenerator.generate( + SUBQUERY_TABLE_ALIAS, + relationshipType, + programIndicator.getAnalyticsType()); } else { - if (AnalyticsType.ENROLLMENT == outerSqlEntity) { - condition = "enrollment = ax.enrollment"; - } else { - if (AnalyticsType.EVENT == programIndicator.getAnalyticsType()) { - condition = "event = ax.event"; - } + // Remove the reference to the outer query's enrollment + // We'll handle the join in the main query + if (AnalyticsType.ENROLLMENT == programIndicator.getAnalyticsType()) { + // No condition needed, we'll join on enrollment in the main query + condition = ""; + } else if (AnalyticsType.EVENT == programIndicator.getAnalyticsType()) { + // Handle event type if needed + condition = ""; } } - return isNotBlank(condition) ? " WHERE " + condition : condition; + return !condition.isEmpty() ? " WHERE " + condition : ""; } /** From 00d8671671b49755dec7edcd876d893ddf90cc97 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Mon, 16 Dec 2024 13:11:13 +0100 Subject: [PATCH 20/30] PI query works --- .../AbstractJdbcEventAnalyticsManager.java | 9 +- .../data/JdbcEnrollmentAnalyticsManager.java | 161 +++++++++++++++--- .../event/data/JdbcEventAnalyticsManager.java | 7 + 3 files changed, 151 insertions(+), 26 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 28af157a0e0c..3f28851933cd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1396,8 +1396,7 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte List columns = new ArrayList<>(); // Mirror the logic of addDimensionSelectColumns - // (We can copy this logic from the superclass since it's about dimensions) - addDimensionSelectColumns(columns, params, false); // assuming we keep this protected method + addDimensionSelectColumns(columns, params, false); // Mirror the logic of addItemSelectColumns but with CTE references for (QueryItem queryItem : params.getItems()) { @@ -1417,6 +1416,9 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte } else { columns.add(getOrgUnitQueryItemColumnAndAlias(params, queryItem).asSql()); } + } else if (queryItem.hasProgramStage()) { + // Handle program stage items with CTE + columns.add(getColumnWithCte(queryItem, "", cteContext)); } else { // Handle other types as before ColumnAndAlias columnAndAlias = getColumnAndAlias(queryItem, false, ""); @@ -1427,6 +1429,8 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte return columns; } + + /** * Returns a select SQL clause for the given query. * @@ -1434,6 +1438,7 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte */ protected abstract String getSelectClause(EventQueryParams params); + protected abstract String getColumnWithCte(QueryItem item, String suffix, CTEContext cteContext); /** * Generates the SQL for the from-clause. Generally this means which analytics table to get data * from. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 0e0d4a501e3a..6a5ebce80109 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -602,6 +602,106 @@ protected ColumnAndAlias getCoordinateColumn(QueryItem item, String suffix) { return ColumnAndAlias.EMPTY; } + protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteContext) { + String colName = item.getItemName(); + String alias = EMPTY; + + if (item.hasProgramStage()) { + assertProgram(item); + colName = quote(colName + suffix); + + // Generate CTE name based on program stage and item + String cteName = String.format("ps_%s_%s", + item.getProgramStage().getUid().toLowerCase(), + item.getItem().getUid().toLowerCase()); + + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid(); + String excludingScheduledCondition = + eventTableName + ".eventstatus != '" + EventStatus.SCHEDULE + "'"; + + if (item.getProgramStage().getRepeatable() + && item.hasRepeatableStageParams() + && !item.getRepeatableStageParams().simpleStageValueExpected()) { + + String cteSql = String.format( + "SELECT enrollment, json_agg(t1) as value FROM (" + + " SELECT %s, %s " + // Removed extra %s placeholders + " FROM %s " + + " WHERE %s AND ps = '%s' %s %s %s %s" + + ") as t1 GROUP BY enrollment", + colName, + String.join(", ", + EventAnalyticsColumnName.ENROLLMENT_OCCURRED_DATE_COLUMN_NAME, + EventAnalyticsColumnName.SCHEDULED_DATE_COLUMN_NAME, + EventAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME), + eventTableName, + excludingScheduledCondition, + item.getProgramStage().getUid(), + getExecutionDateFilter( + item.getRepeatableStageParams().getStartDate(), + item.getRepeatableStageParams().getEndDate()), + createOrderType(item.getProgramStageOffset()), + createOffset(item.getProgramStageOffset()), + getLimit(item.getRepeatableStageParams().getCount()) + ); + + cteContext.addCTE(cteName, cteSql); + return cteName + ".value"; + + } else if (item.getProgramStage().getRepeatable() && item.hasRepeatableStageParams()) { + String cteSql = String.format( + "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + + "FROM %s " + + "WHERE %s AND ps = '%s' %s %s %s " + + "ORDER BY enrollment, %s", + colName, + eventTableName, + excludingScheduledCondition, + item.getProgramStage().getUid(), + getExecutionDateFilter( + item.getRepeatableStageParams().getStartDate(), + item.getRepeatableStageParams().getEndDate()), + createOrderType(item.getProgramStageOffset()), + createOffset(item.getProgramStageOffset()), + createOrderType(item.getProgramStageOffset()).replace("ORDER BY ", "") + ); + + cteContext.addCTE(cteName, cteSql); + return cteName + ".value"; + } + + if (item.getItem().getDimensionItemType() == DATA_ELEMENT + && item.getProgramStage() != null) { + alias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); + } + + String cteSql = String.format( + "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + + "FROM %s " + + "WHERE %s AND ps = '%s' AND %s IS NOT NULL %s %s " + + "ORDER BY enrollment, %s", + colName, + eventTableName, + excludingScheduledCondition, + item.getProgramStage().getUid(), + colName, + createOrderType(item.getProgramStageOffset()), + createOffset(item.getProgramStageOffset()), + createOrderType(item.getProgramStageOffset()).replace("ORDER BY ", "") + ); + + cteContext.addCTE(cteName, cteSql); + return cteName + ".value" + (alias.isEmpty() ? "" : " as " + alias); + } + + // Non-program stage cases remain unchanged + if (isOrganizationUnitProgramAttribute(item)) { + return quoteAlias(colName + suffix); + } else { + return quoteAlias(colName); + } + } + /** * Creates a column "selector" for the given item name. The suffix will be appended as part of the * item name. The column selection is based on events analytics tables. @@ -827,11 +927,10 @@ private String createOrderType(int offset) { // New methods // private String buildEnrollmentQueryWithCte2(EventQueryParams params) { - // Create CTEContext to collect all CTEs CTEContext cteContext = new CTEContext(); StringBuilder sql = new StringBuilder(); - // 1. Process all program indicators to generate CTEs before building the main query + // 1. Process all program indicators to generate CTEs for (QueryItem queryItem : params.getItems()) { if (queryItem.isProgramIndicator()) { ProgramIndicator pi = (ProgramIndicator) queryItem.getItem(); @@ -853,6 +952,34 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { cteContext); } } + else if (queryItem.hasProgramStage()) { + // Generate CTE for program stage items + String cteName = String.format("ps_%s_%s", + queryItem.getProgramStage().getUid().toLowerCase(), + queryItem.getItem().getUid().toLowerCase()); + + String colName = quote(queryItem.getItemName()); + String eventTableName = ANALYTICS_EVENT + queryItem.getProgram().getUid(); + + String cteSql = String.format( + "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + + "FROM %s " + + "WHERE eventstatus != 'SCHEDULE' " + + "AND ps = '%s' " + + "AND %s IS NOT NULL " + + "ORDER BY enrollment, occurreddate DESC, created DESC", + colName, + eventTableName, + queryItem.getProgramStage().getUid(), + colName); + + cteContext.addCTE(cteName, cteSql); + + String alias = quote(queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid()); + cteContext.addColumnMapping( + queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(), + cteName + ".value as " + alias); + } } // 2. Add WITH clause if we have any CTEs @@ -869,7 +996,7 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { sql.append("SELECT ").append(String.join(",\n ", selectCols)); // 4. From clause - sql.append(" from ").append(params.getTableName()).append(" as ax"); + sql.append("\nFROM ").append(params.getTableName()).append(" AS ax"); // 5. Add joins for each CTE for (String cteName : cteContext.getCTENames()) { @@ -896,13 +1023,12 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { if (column != null) { List dateConditions = new ArrayList<>(); for (DateRange range : entry.getValue()) { - dateConditions.add( - String.format( - "(%s >= '%s' and %s < '%s')", - column, - DateUtils.toMediumDate(range.getStartDate()), - column, - DateUtils.toMediumDate(range.getEndDate()))); + dateConditions.add(String.format( + "(%s >= '%s' and %s < '%s')", + column, + DateUtils.toMediumDate(range.getStartDate()), + column, + DateUtils.toMediumDate(range.getEndDate()))); } conditions.add("(" + String.join(" or ", dateConditions) + ")"); } @@ -914,19 +1040,7 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { } // 7. Order by - if (params.getAsc() != null && !params.getAsc().isEmpty()) { - sql.append("\nORDER BY"); - boolean first = true; - for (QueryItem item : params.getAsc()) { - if (!first) { - sql.append(","); - } - String columnName = - item.getItemId().equals("incidentdate") ? "occurreddate" : item.getItemId(); - sql.append(" ").append(quote(columnName)).append(" asc nulls last"); - first = false; - } - } + sql.append(getSortClause(params)); // 8. Paging sql.append(" ").append(getPagingClause(params, 5000)); @@ -935,7 +1049,6 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { } - private String getColumnForTimeField(TimeField timeField) { return switch (timeField) { case ENROLLMENT_DATE -> "enrollmentdate"; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 667a139a2209..91bdefa52326 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -63,6 +63,7 @@ import org.hisp.dhis.analytics.Rectangle; import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; @@ -357,6 +358,12 @@ protected String getSelectClause(EventQueryParams params) { return "select " + StringUtils.join(selectCols, ",") + " "; } + @Override + protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteContext) { + // TODO: Implement + return ""; + } + /** * Returns a from SQL clause for the given analytics table partition. If the query has a * non-default time field specified, a join with the {@code date period structure} resource table From 42edd673a84c5e336ae67fbe508aeca6cfa769f7 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Mon, 16 Dec 2024 16:24:18 +0100 Subject: [PATCH 21/30] WIP --- .../data/JdbcEnrollmentAnalyticsManager.java | 70 ++++++----------- ...efaultProgramIndicatorSubqueryBuilder.java | 76 +++++++++---------- 2 files changed, 57 insertions(+), 89 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 6a5ebce80109..f1de11e97c32 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -152,12 +152,13 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { if (params.isAggregatedEnrollments()) { sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); } else if (!params.getItems().isEmpty() && shouldUseCTE(params)) { + // TODO no longer needed sql = buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); } else { sql = buildEnrollmentQueryWithCte2(params); } - System.out.println("SQL: " + sql); + System.out.println("SQL: " + sql); // FIXME: Remove debug line if (params.analyzeOnly()) { withExceptionHandling( @@ -604,7 +605,7 @@ protected ColumnAndAlias getCoordinateColumn(QueryItem item, String suffix) { protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteContext) { String colName = item.getItemName(); - String alias = EMPTY; + String alias; if (item.hasProgramStage()) { assertProgram(item); @@ -624,9 +625,9 @@ protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteC && !item.getRepeatableStageParams().simpleStageValueExpected()) { String cteSql = String.format( - "SELECT enrollment, json_agg(t1) as value FROM (" + - " SELECT %s, %s " + // Removed extra %s placeholders - " FROM %s " + + "SELECT enrollment, COALESCE(json_agg(t1), '[]') as value FROM (" + + " SELECT %s, %s, %s, %s" + + " FROM %s" + " WHERE %s AND ps = '%s' %s %s %s %s" + ") as t1 GROUP BY enrollment", colName, @@ -646,52 +647,25 @@ protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteC ); cteContext.addCTE(cteName, cteSql); - return cteName + ".value"; + alias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); + return cteName + ".value as " + alias; - } else if (item.getProgramStage().getRepeatable() && item.hasRepeatableStageParams()) { + } else { String cteSql = String.format( "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + "FROM %s " + - "WHERE %s AND ps = '%s' %s %s %s " + - "ORDER BY enrollment, %s", + "WHERE %s AND ps = '%s' " + + "ORDER BY enrollment, occurreddate DESC, created DESC", colName, eventTableName, excludingScheduledCondition, - item.getProgramStage().getUid(), - getExecutionDateFilter( - item.getRepeatableStageParams().getStartDate(), - item.getRepeatableStageParams().getEndDate()), - createOrderType(item.getProgramStageOffset()), - createOffset(item.getProgramStageOffset()), - createOrderType(item.getProgramStageOffset()).replace("ORDER BY ", "") + item.getProgramStage().getUid() ); cteContext.addCTE(cteName, cteSql); - return cteName + ".value"; + String columnAlias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); + return cteName + ".value as " + columnAlias; } - - if (item.getItem().getDimensionItemType() == DATA_ELEMENT - && item.getProgramStage() != null) { - alias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); - } - - String cteSql = String.format( - "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + - "FROM %s " + - "WHERE %s AND ps = '%s' AND %s IS NOT NULL %s %s " + - "ORDER BY enrollment, %s", - colName, - eventTableName, - excludingScheduledCondition, - item.getProgramStage().getUid(), - colName, - createOrderType(item.getProgramStageOffset()), - createOffset(item.getProgramStageOffset()), - createOrderType(item.getProgramStageOffset()).replace("ORDER BY ", "") - ); - - cteContext.addCTE(cteName, cteSql); - return cteName + ".value" + (alias.isEmpty() ? "" : " as " + alias); } // Non-program stage cases remain unchanged @@ -962,12 +936,14 @@ else if (queryItem.hasProgramStage()) { String eventTableName = ANALYTICS_EVENT + queryItem.getProgram().getUid(); String cteSql = String.format( - "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + - "FROM %s " + - "WHERE eventstatus != 'SCHEDULE' " + - "AND ps = '%s' " + - "AND %s IS NOT NULL " + - "ORDER BY enrollment, occurreddate DESC, created DESC", + """ + -- Generate CTE for program stage items + SELECT DISTINCT ON (enrollment) enrollment, %s as value + FROM %s + WHERE eventstatus != 'SCHEDULE' + AND ps = '%s' + -- AND %s IS NOT NULL + ORDER BY enrollment, occurreddate DESC, created DESC""", colName, eventTableName, queryItem.getProgramStage().getUid(), @@ -1040,7 +1016,7 @@ else if (queryItem.hasProgramStage()) { } // 7. Order by - sql.append(getSortClause(params)); + sql.append(" " ).append(getSortClause(params)); // 8. Paging sql.append(" ").append(getPagingClause(params, 5000)); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java index b1589025131e..763b0fd35215 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java @@ -84,60 +84,47 @@ public void contributeCTE(ProgramIndicator programIndicator, AnalyticsType outer } @Override - public void contributeCTE(ProgramIndicator programIndicator, RelationshipType relationshipType, AnalyticsType outerSqlEntity, Date earliestStartDate, Date latestDate, CTEContext cteContext) { + public void contributeCTE( + ProgramIndicator programIndicator, + RelationshipType relationshipType, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext) { + // Generate a unique CTE name for this program indicator String cteName = "pi_" + programIndicator.getUid().toLowerCase(); - // Build the CTE definition - StringBuilder cteSql = new StringBuilder(); - // Define aggregation function String function = TextUtils.emptyIfEqual( programIndicator.getAggregationTypeFallback().getValue(), AggregationType.CUSTOM.getValue()); - // Start building the SELECT part of CTE - cteSql.append("SELECT ") - .append(SUBQUERY_TABLE_ALIAS) - .append(".enrollment, "); - - // Add the aggregation expression - cteSql.append(function) - .append("(") - .append(getProgramIndicatorSql( + String cteSql = String.format( + "SELECT e.enrollment, " + + "COALESCE(%s(%s), 0) as value " + + "FROM analytics_enrollment_%s e " + + "LEFT JOIN ( " + + " SELECT enrollment, eventstatus " + + " FROM %s as subax " + + " WHERE %s " + + ") t ON t.enrollment = e.enrollment " + + "GROUP BY e.enrollment", + function, + getProgramIndicatorSql( programIndicator.getExpression(), NUMERIC, programIndicator, earliestStartDate, - latestDate)) - .append(") as value"); - - // Add FROM clause - cteSql.append(getFrom(programIndicator)); - - // Add WHERE clause - String where = getWhere(outerSqlEntity, programIndicator, relationshipType); - if (!where.isEmpty()) { - cteSql.append(where); - } - - // Add program indicator filter if present - if (!Strings.isNullOrEmpty(programIndicator.getFilter())) { - cteSql.append(where.isBlank() ? " WHERE " : " AND ") - .append("(") - .append(getProgramIndicatorSql( - programIndicator.getFilter(), - BOOLEAN, - programIndicator, - earliestStartDate, - latestDate)) - .append(")"); - } - - // Add GROUP BY clause for the enrollment - cteSql.append(" GROUP BY ") - .append(SUBQUERY_TABLE_ALIAS) - .append(".enrollment"); + latestDate), + programIndicator.getProgram().getUid().toLowerCase(), + getTableName(programIndicator), + getProgramIndicatorSql( + programIndicator.getFilter(), + BOOLEAN, + programIndicator, + earliestStartDate, + latestDate)); // Register the CTE and its column mapping cteContext.addCTE(cteName, cteSql.toString()); @@ -147,6 +134,11 @@ public void contributeCTE(ProgramIndicator programIndicator, RelationshipType re ); } + + private String getTableName(ProgramIndicator programIndicator) { + return "analytics_event_" + programIndicator.getProgram().getUid().toLowerCase(); + } + /** * Generate a subquery based on the result of a Program Indicator and an (optional) Relationship * Type From 334d2877ddb39d3ce841f44c4f38df70f6c2af13 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Mon, 16 Dec 2024 18:17:12 +0100 Subject: [PATCH 22/30] WIP - filtering #1 --- .../AbstractJdbcEventAnalyticsManager.java | 4 +- .../data/JdbcEnrollmentAnalyticsManager.java | 99 ++++++++----------- 2 files changed, 43 insertions(+), 60 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 3f28851933cd..6f4f8685394b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1114,13 +1114,13 @@ protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, Sql List orConditions = repeatableConditionsByIdentifier.values().stream() .map(sameGroup -> joinSql(sameGroup, OR_JOINER)) - .collect(toList()); + .toList(); // Non-repeatable conditions List andConditions = asSqlCollection(itemsByRepeatableFlag.get(false), params) .map(IdentifiableSql::getSql) - .collect(toList()); + .toList(); if (orConditions.isEmpty() && andConditions.isEmpty()) { return StringUtils.EMPTY; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index f1de11e97c32..16b565d55383 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -27,30 +27,7 @@ */ package org.hisp.dhis.analytics.event.data; -import static java.util.stream.Collectors.joining; -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; -import static org.hisp.dhis.analytics.DataType.BOOLEAN; -import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; -import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; -import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; -import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; -import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; -import static org.hisp.dhis.util.DateUtils.toMediumDate; - import com.google.common.collect.Sets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.TimeField; @@ -63,7 +40,6 @@ import org.hisp.dhis.analytics.table.EnrollmentAnalyticsColumnName; import org.hisp.dhis.analytics.table.EventAnalyticsColumnName; import org.hisp.dhis.category.CategoryOption; -import org.hisp.dhis.common.DateRange; import org.hisp.dhis.common.DimensionItemType; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; @@ -84,7 +60,6 @@ import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramIndicatorService; -import org.hisp.dhis.util.DateUtils; import org.locationtech.jts.util.Assert; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.InvalidResultSetAccessException; @@ -92,6 +67,29 @@ import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; +import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; +import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; +import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; +import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; +import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; +import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; +import static org.hisp.dhis.util.DateUtils.toMediumDate; + /** * @author Markus Bekken */ @@ -312,6 +310,7 @@ public long getEnrollmentCount(EventQueryParams params) { sql += getFromClause(params); sql += getWhereClause(params); + sql += addFiltersToWhereClause(params); long count = 0; @@ -449,8 +448,8 @@ protected String getWhereClause(EventQueryParams params) { // --------------------------------------------------------------------- // Query items and filters // --------------------------------------------------------------------- - - sql += getQueryItemsAndFiltersWhereClause(params, hlp); + // USE addFiltersToWhereClause(params) + //sql += getQueryItemsAndFiltersWhereClause(params, hlp); // --------------------------------------------------------------------- // Filter expression @@ -514,6 +513,19 @@ protected String getWhereClause(EventQueryParams params) { return sql; } + private String addFiltersToWhereClause(EventQueryParams params) { + return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + } + + private String addFiltersToWhereClause(EventQueryParams params, CTEContext cteContext) { + SqlHelper hlp = new SqlHelper(); + if (params.isEnhancedCondition()) { + return getItemsSqlForEnhancedConditions(params, hlp); + } + + return ""; + } + @Override protected String getSelectClause(EventQueryParams params) { List selectCols = @@ -983,37 +995,8 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value .append(".enrollment"); } - // 6. Where clause - List conditions = new ArrayList<>(); - - // Add organization unit condition - if (!params.getOrganisationUnits().isEmpty()) { - String orgUnit = params.getOrganisationUnits().get(0).getUid(); - conditions.add(String.format("ax.\"uidlevel1\" = '%s'", orgUnit)); - } - - // Add date range conditions - if (!params.getTimeDateRanges().isEmpty()) { - for (Map.Entry> entry : params.getTimeDateRanges().entrySet()) { - String column = getColumnForTimeField(entry.getKey()); - if (column != null) { - List dateConditions = new ArrayList<>(); - for (DateRange range : entry.getValue()) { - dateConditions.add(String.format( - "(%s >= '%s' and %s < '%s')", - column, - DateUtils.toMediumDate(range.getStartDate()), - column, - DateUtils.toMediumDate(range.getEndDate()))); - } - conditions.add("(" + String.join(" or ", dateConditions) + ")"); - } - } - } - - if (!conditions.isEmpty()) { - sql.append("\nWHERE ").append(String.join(" AND ", conditions)); - } + sql.append(" ").append(getWhereClause(params)); + sql.append(" ").append(addFiltersToWhereClause(params, cteContext)); // 7. Order by sql.append(" " ).append(getSortClause(params)); From 5732b3e1944b70dec8f7a9d31e400be7fe85f93d Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 17 Dec 2024 14:25:21 +0100 Subject: [PATCH 23/30] WIP - filtering ok --- .../AbstractJdbcEventAnalyticsManager.java | 4 +- .../data/JdbcEnrollmentAnalyticsManager.java | 193 +++++++++++++++--- 2 files changed, 167 insertions(+), 30 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 6f4f8685394b..9c1e514b410a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1238,7 +1238,7 @@ private String getIdentifier(QueryItem queryItem) { @Getter @Builder - private static class IdentifiableSql { + public static class IdentifiableSql { private final String identifier; private final String sql; @@ -1251,7 +1251,7 @@ private static class IdentifiableSql { * @param filter the {@link QueryFilter}. * @param params the {@link EventQueryParams}. */ - private String toSql(QueryItem item, QueryFilter filter, EventQueryParams params) { + protected String toSql(QueryItem item, QueryFilter filter, EventQueryParams params) { String field = item.hasAggregationType() ? getSelectSql(filter, item, params) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 16b565d55383..5e29d96d0174 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -30,7 +30,6 @@ import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; @@ -49,6 +48,8 @@ import org.hisp.dhis.common.OrganisationUnitSelectionMode; import org.hisp.dhis.common.QueryFilter; import org.hisp.dhis.common.QueryItem; +import org.hisp.dhis.common.QueryOperator; +import org.hisp.dhis.common.RequestTypeAware; import org.hisp.dhis.common.ValueStatus; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -75,7 +76,10 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; @@ -445,12 +449,6 @@ protected String getWhereClause(EventQueryParams params) { sql += "and ps = '" + params.getProgramStage().getUid() + "' "; } - // --------------------------------------------------------------------- - // Query items and filters - // --------------------------------------------------------------------- - // USE addFiltersToWhereClause(params) - //sql += getQueryItemsAndFiltersWhereClause(params, hlp); - // --------------------------------------------------------------------- // Filter expression // --------------------------------------------------------------------- @@ -517,13 +515,103 @@ private String addFiltersToWhereClause(EventQueryParams params) { return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); } - private String addFiltersToWhereClause(EventQueryParams params, CTEContext cteContext) { - SqlHelper hlp = new SqlHelper(); - if (params.isEnhancedCondition()) { - return getItemsSqlForEnhancedConditions(params, hlp); + private String addCteFiltersToWhereClause(EventQueryParams params) { + StringBuilder whereClause = new StringBuilder(); + + // Iterate over each filter and apply the correct condition + for (QueryItem item : Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + .filter(QueryItem::hasFilter) + .toList()) { + + String cteName = "filter_" + getIdentifier(item).replace('.', '_').toLowerCase(); + + for (QueryFilter filter : item.getFilters()) { + if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly + whereClause.append(" AND ").append(cteName).append(".value IS NULL"); + } else { + String operator = getSqlOperator(filter); + String value = getSqlFilterValue(filter, item); + whereClause.append(" AND ").append(cteName).append(".value ") + .append(operator).append(" ").append(value); + } + } } - return ""; + return whereClause.toString(); + } + + private String getSqlOperator(QueryFilter filter) { + return switch (filter.getOperator()) { + case EQ -> "="; + case NEQ -> "!="; + case GT -> ">"; + case LT -> "<"; + case GE -> ">="; + case LE -> "<="; + case IN -> "IN"; + default -> throw new IllegalArgumentException("Unsupported operator: " + filter.getOperator()); + }; + } + + private String getSqlFilterValue(QueryFilter filter, QueryItem item) { + if ("NV".equals(filter.getFilter())) { + return "NULL"; // Special case for 'null' filters + } + + // Handle IN operator: wrap the value(s) in parentheses + if (filter.getOperator() == QueryOperator.IN) { + String[] values = filter.getFilter().split(","); // Support multiple values + String quotedValues = Arrays.stream(values) + .map(value -> item.isNumeric() ? value : sqlBuilder.singleQuote(value)) + .collect(Collectors.joining(", ")); + return "(" + quotedValues + ")"; + } + + // Handle text and numeric values + return item.isNumeric() ? filter.getFilter() : sqlBuilder.singleQuote(filter.getFilter()); + } + + private String buildFilterCteSql(List queryItems, EventQueryParams params) { + return queryItems.stream() + .map(item -> { + // Determine the correct table: event table or enrollment table + String tableName = item.hasProgramStage() + ? "analytics_event_" + item.getProgram().getUid().toLowerCase() // Event table for program stage + : params.getTableName(); // Enrollment table + + String columnName = quote(item.getItemName()); // Raw column name without alias + String programStageCondition = item.hasProgramStage() + ? "AND ps = '" + item.getProgramStage().getUid() + "'" + : ""; // Add program stage filter if available + + // Generate the CTE SQL + return String.format(""" + SELECT DISTINCT ON (enrollment) enrollment, %s AS value + FROM %s + WHERE eventstatus != 'SCHEDULE' %s + ORDER BY enrollment, occurreddate DESC, created DESC""", + columnName, tableName, programStageCondition + ); + }) + .collect(Collectors.joining("\nUNION ALL\n")); + } + + private Stream asSqlCollection(List queryItems, EventQueryParams params) { + return Optional.ofNullable(queryItems).orElse(List.of()).stream() + .map(queryItem -> { + String sql = queryItem.getFilters().stream() + .map(filter -> toSql(queryItem, filter, params)) + .collect(Collectors.joining(" AND ")); + return IdentifiableSql.builder() + .identifier(getIdentifier(queryItem)) + .sql(sql) + .build(); + }); + } + + private String getIdentifier(QueryItem queryItem) { + String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; + return stage + "." + queryItem.getItemId(); } @Override @@ -912,11 +1000,8 @@ private String createOrderType(int offset) { // New methods // - private String buildEnrollmentQueryWithCte2(EventQueryParams params) { + private CTEContext getCteDefinitions(EventQueryParams params) { CTEContext cteContext = new CTEContext(); - StringBuilder sql = new StringBuilder(); - - // 1. Process all program indicators to generate CTEs for (QueryItem queryItem : params.getItems()) { if (queryItem.isProgramIndicator()) { ProgramIndicator pi = (ProgramIndicator) queryItem.getItem(); @@ -969,6 +1054,64 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value cteName + ".value as " + alias); } } + return cteContext; + } + + private String buildFilterCteSql(QueryItem item, EventQueryParams params) { + String tableName = params.getTableName(); + String columnName = getColumn(item); + String filterCondition = item.getFilters().stream() + .map(filter -> toSql(item, filter, params)) + .collect(Collectors.joining(" AND ")); + + return String.format( + "SELECT enrollment, %s AS value " + + "FROM %s " + + "WHERE %s", + columnName, tableName, filterCondition + ); + } + + private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) { + // Combine items and item filters + List queryItems = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + .filter(QueryItem::hasFilter) + .toList(); + + // Group query items by repeatable and non-repeatable stages + Map> itemsByRepeatableFlag = + queryItems.stream() + .collect(groupingBy( + queryItem -> queryItem.hasRepeatableStageParams() + && params.getEndpointItem() == RequestTypeAware.EndpointItem.ENROLLMENT + )); + + // Process repeatable stage filters + itemsByRepeatableFlag.getOrDefault(true, List.of()).stream() + .collect(groupingBy(this::getIdentifier)) + .forEach((identifier, items) -> { + String cteName = "filter_" + identifier.replace('.', '_').toLowerCase(); + String cteSql = buildFilterCteSql(items, params); + cteContext.addCTE(cteName, cteSql); + }); + + // Process non-repeatable stage filters + itemsByRepeatableFlag.getOrDefault(false, List.of()).forEach(queryItem -> { + String cteName = "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); + String cteSql = buildFilterCteSql(List.of(queryItem), params); + cteContext.addCTE(cteName, cteSql); + }); + } + + private String buildEnrollmentQueryWithCte2(EventQueryParams params) { + + StringBuilder sql = new StringBuilder(); + + // 1. Process all program indicators to generate CTEs + CTEContext cteContext = getCteDefinitions(params); + + // 1.1. Generate additional CTEs for filters + generateFilterCTEs(params, cteContext); // 2. Add WITH clause if we have any CTEs String cteDefinitions = cteContext.getCTEDefinition(); @@ -976,7 +1119,7 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value sql.append(cteDefinitions).append("\n"); } - // 3. Select clause using our new method + // 3. Select clause List selectCols = ListUtils.distinctUnion( params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, getSelectColumnsWithCTE(params, cteContext) @@ -995,8 +1138,10 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value .append(".enrollment"); } - sql.append(" ").append(getWhereClause(params)); - sql.append(" ").append(addFiltersToWhereClause(params, cteContext)); + // 6. Where clause + sql.append(" ").append(getWhereClause(params)) + .append(" ") + .append(addCteFiltersToWhereClause(params)); // 7. Order by sql.append(" " ).append(getSortClause(params)); @@ -1008,14 +1153,6 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value } - private String getColumnForTimeField(TimeField timeField) { - return switch (timeField) { - case ENROLLMENT_DATE -> "enrollmentdate"; - case INCIDENT_DATE -> "occurreddate"; - default -> null; - }; - } - private String getBasicSelectColumns() { return String.join( ",", @@ -1128,7 +1265,7 @@ private String buildAllRankedEventsCTEs(List items) { processedCombinations.add(key); } - return ctes.length() > 0 ? "with " + ctes.toString() + "\n" : ""; + return !ctes.isEmpty() ? "with " + ctes + "\n" : ""; } private String buildValueColumns(List items) { From b247679c102f5b152ff870bac97c0385ebb7920d Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Tue, 17 Dec 2024 15:27:48 +0100 Subject: [PATCH 24/30] WIP - removed dead code --- .../data/JdbcEnrollmentAnalyticsManager.java | 54 +------------------ 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 5e29d96d0174..53ea7026f6f7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -153,9 +153,6 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { String sql; if (params.isAggregatedEnrollments()) { sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); - } else if (!params.getItems().isEmpty() && shouldUseCTE(params)) { - // TODO no longer needed - sql = buildEnrollmentQueryWithCTE(params, params.getItems().get(0)); } else { sql = buildEnrollmentQueryWithCte2(params); } @@ -171,14 +168,6 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { } } - private boolean shouldUseCTE(EventQueryParams params) { - if (params.getItems().isEmpty()) { - return false; - } - QueryItem item = params.getItems().get(0); - return item.hasProgram() && item.hasProgramStage(); - } - /** * Adds enrollments to the given grid based on the given parameters and SQL statement. * @@ -330,7 +319,7 @@ public long getEnrollmentCount(EventQueryParams params) { withExceptionHandling( () -> jdbcTemplate.queryForObject(finalSqlValue, Long.class), params.isMultipleQueries()) - .orElse(0l); + .orElse(0L); } return count; @@ -1182,47 +1171,6 @@ private String resolveOrderByOffset(int offset) { return "asc"; } - private String buildEnrollmentQueryWithCTE(EventQueryParams params, QueryItem item) { - - StringBuilder sql = new StringBuilder(); - List items = params.getItems(); - - // 1. Build CTEs for each unique program stage + offset combination - sql.append(buildAllRankedEventsCTEs(items)); - - // 2. Build main query - sql.append("select "); - - // 2.1 Add basic columns - sql.append(getBasicSelectColumns()); - - // 2.2 Add value columns for each item - String valueColumns = buildValueColumns(items); - if (!valueColumns.isEmpty()) { - sql.append(", ").append(valueColumns); - } - - // 2.3 Add FROM clause with all necessary joins - sql.append(buildFromClauseWithJoins(params, items)); - - // 2.4 Add WHERE clause - String whereClause = getWhereClause(params).trim(); - if (!whereClause.isEmpty()) { - // Remove any leading "and" and ensure only one "where" keyword - whereClause = whereClause.replaceFirst("^and\\s+", ""); - whereClause = whereClause.replaceFirst("^where\\s+", ""); - if (!whereClause.isEmpty()) { - sql.append(" where ").append(whereClause); - } - } - - // 2.5 Add ORDER BY and paging - sql.append(getSortClause(params)); - sql.append(getPagingClause(params, 5000)); - - return sql.toString(); - } - private String buildAllRankedEventsCTEs(List items) { StringBuilder ctes = new StringBuilder(); Set processedCombinations = new HashSet<>(); From d8c46706d0ea0f1c94b47ee82bce6bf5f29bba5a Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 18 Dec 2024 11:01:29 +0100 Subject: [PATCH 25/30] WIP - spotless --- .../dhis/analytics/common/CTEContext.java | 87 +++-- .../ProgramIndicatorSubqueryBuilder.java | 22 +- .../AbstractJdbcEventAnalyticsManager.java | 5 +- .../data/JdbcEnrollmentAnalyticsManager.java | 323 ++++++++++-------- ...efaultProgramIndicatorSubqueryBuilder.java | 80 ++--- 5 files changed, 286 insertions(+), 231 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index c85123d1f3f5..cc69a4919067 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -1,48 +1,71 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package org.hisp.dhis.analytics.common; - import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; public class CTEContext { - private final Map cteDefinitions = new LinkedHashMap<>(); - private final Map columnMappings = new HashMap<>(); + private final Map cteDefinitions = new LinkedHashMap<>(); + private final Map columnMappings = new HashMap<>(); - public void addCTE(String cteName, String cteDefinition) { - cteDefinitions.put(cteName, cteDefinition); - } + public void addCTE(String cteName, String cteDefinition) { + cteDefinitions.put(cteName, cteDefinition); + } - public void addColumnMapping(String originalColumn, String cteReference) { - columnMappings.put(originalColumn, cteReference); - } + public void addColumnMapping(String originalColumn, String cteReference) { + columnMappings.put(originalColumn, cteReference); + } - public String getCTEDefinition() { - if (cteDefinitions.isEmpty()) { - return ""; - } - - StringBuilder sb = new StringBuilder("WITH "); - boolean first = true; - for (Map.Entry entry : cteDefinitions.entrySet()) { - if (!first) { - sb.append(", "); - } - sb.append(entry.getKey()) - .append(" AS (") - .append(entry.getValue()) - .append(")"); - first = false; - } - return sb.toString(); + public String getCTEDefinition() { + if (cteDefinitions.isEmpty()) { + return ""; } - public Set getCTENames() { - return cteDefinitions.keySet(); + StringBuilder sb = new StringBuilder("WITH "); + boolean first = true; + for (Map.Entry entry : cteDefinitions.entrySet()) { + if (!first) { + sb.append(", "); + } + sb.append(entry.getKey()).append(" AS (").append(entry.getValue()).append(")"); + first = false; } + return sb.toString(); + } - public String getColumnMapping(String columnId) { - return columnMappings.getOrDefault(columnId, columnId); - } + public Set getCTENames() { + return cteDefinitions.keySet(); + } + + public String getColumnMapping(String columnId) { + return columnMappings.getOrDefault(columnId, columnId); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java index 745c3bb29cba..4796431d1607 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java @@ -103,17 +103,17 @@ String getAggregateClauseForProgramIndicator( Date latestDate); void contributeCTE( - ProgramIndicator programIndicator, - AnalyticsType outerSqlEntity, - Date earliestStartDate, - Date latestDate, - CTEContext cteContext); + ProgramIndicator programIndicator, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext); void contributeCTE( - ProgramIndicator programIndicator, - RelationshipType relationshipType, - AnalyticsType outerSqlEntity, - Date earliestStartDate, - Date latestDate, - CTEContext cteContext); + ProgramIndicator programIndicator, + RelationshipType relationshipType, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 9c1e514b410a..0b88484ca5bd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1411,7 +1411,7 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte } else if (ValueType.ORGANISATION_UNIT == queryItem.getValueType()) { // Handle org units if (params.getCoordinateFields().stream() - .anyMatch(f -> queryItem.getItem().getUid().equals(f))) { + .anyMatch(f -> queryItem.getItem().getUid().equals(f))) { columns.add(getCoordinateColumn(queryItem, OU_GEOMETRY_COL_SUFFIX).asSql()); } else { columns.add(getOrgUnitQueryItemColumnAndAlias(params, queryItem).asSql()); @@ -1429,8 +1429,6 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte return columns; } - - /** * Returns a select SQL clause for the given query. * @@ -1439,6 +1437,7 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte protected abstract String getSelectClause(EventQueryParams params); protected abstract String getColumnWithCte(QueryItem item, String suffix, CTEContext cteContext); + /** * Generates the SQL for the from-clause. Generally this means which analytics table to get data * from. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 53ea7026f6f7..bd69e3e8094f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -27,7 +27,32 @@ */ package org.hisp.dhis.analytics.event.data; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; +import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; +import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; +import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; +import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; +import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; +import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; +import static org.hisp.dhis.util.DateUtils.toMediumDate; + import com.google.common.collect.Sets; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; @@ -68,32 +93,6 @@ import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.joining; -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; -import static org.hisp.dhis.analytics.DataType.BOOLEAN; -import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; -import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; -import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; -import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; -import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; -import static org.hisp.dhis.util.DateUtils.toMediumDate; - /** * @author Markus Bekken */ @@ -508,7 +507,8 @@ private String addCteFiltersToWhereClause(EventQueryParams params) { StringBuilder whereClause = new StringBuilder(); // Iterate over each filter and apply the correct condition - for (QueryItem item : Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + for (QueryItem item : + Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) .toList()) { @@ -520,8 +520,13 @@ private String addCteFiltersToWhereClause(EventQueryParams params) { } else { String operator = getSqlOperator(filter); String value = getSqlFilterValue(filter, item); - whereClause.append(" AND ").append(cteName).append(".value ") - .append(operator).append(" ").append(value); + whereClause + .append(" AND ") + .append(cteName) + .append(".value ") + .append(operator) + .append(" ") + .append(value); } } } @@ -530,16 +535,17 @@ private String addCteFiltersToWhereClause(EventQueryParams params) { } private String getSqlOperator(QueryFilter filter) { - return switch (filter.getOperator()) { - case EQ -> "="; - case NEQ -> "!="; - case GT -> ">"; - case LT -> "<"; - case GE -> ">="; - case LE -> "<="; - case IN -> "IN"; - default -> throw new IllegalArgumentException("Unsupported operator: " + filter.getOperator()); - }; + return switch (filter.getOperator()) { + case EQ -> "="; + case NEQ -> "!="; + case GT -> ">"; + case LT -> "<"; + case GE -> ">="; + case LE -> "<="; + case IN -> "IN"; + default -> + throw new IllegalArgumentException("Unsupported operator: " + filter.getOperator()); + }; } private String getSqlFilterValue(QueryFilter filter, QueryItem item) { @@ -550,7 +556,8 @@ private String getSqlFilterValue(QueryFilter filter, QueryItem item) { // Handle IN operator: wrap the value(s) in parentheses if (filter.getOperator() == QueryOperator.IN) { String[] values = filter.getFilter().split(","); // Support multiple values - String quotedValues = Arrays.stream(values) + String quotedValues = + Arrays.stream(values) .map(value -> item.isNumeric() ? value : sqlBuilder.singleQuote(value)) .collect(Collectors.joining(", ")); return "(" + quotedValues + ")"; @@ -562,39 +569,48 @@ private String getSqlFilterValue(QueryFilter filter, QueryItem item) { private String buildFilterCteSql(List queryItems, EventQueryParams params) { return queryItems.stream() - .map(item -> { + .map( + item -> { // Determine the correct table: event table or enrollment table - String tableName = item.hasProgramStage() - ? "analytics_event_" + item.getProgram().getUid().toLowerCase() // Event table for program stage + String tableName = + item.hasProgramStage() + ? "analytics_event_" + + item.getProgram() + .getUid() + .toLowerCase() // Event table for program stage : params.getTableName(); // Enrollment table String columnName = quote(item.getItemName()); // Raw column name without alias - String programStageCondition = item.hasProgramStage() + String programStageCondition = + item.hasProgramStage() ? "AND ps = '" + item.getProgramStage().getUid() + "'" : ""; // Add program stage filter if available // Generate the CTE SQL - return String.format(""" + return String.format( + """ SELECT DISTINCT ON (enrollment) enrollment, %s AS value FROM %s WHERE eventstatus != 'SCHEDULE' %s ORDER BY enrollment, occurreddate DESC, created DESC""", - columnName, tableName, programStageCondition - ); + columnName, tableName, programStageCondition); }) - .collect(Collectors.joining("\nUNION ALL\n")); + .collect(Collectors.joining("\nUNION ALL\n")); } - private Stream asSqlCollection(List queryItems, EventQueryParams params) { + private Stream asSqlCollection( + List queryItems, EventQueryParams params) { return Optional.ofNullable(queryItems).orElse(List.of()).stream() - .map(queryItem -> { - String sql = queryItem.getFilters().stream() + .map( + queryItem -> { + String sql = + queryItem.getFilters().stream() .map(filter -> toSql(queryItem, filter, params)) .collect(Collectors.joining(" AND ")); return IdentifiableSql.builder() - .identifier(getIdentifier(queryItem)) - .sql(sql) - .build(); + .identifier(getIdentifier(queryItem)) + .sql(sql) + .build(); }); } @@ -701,55 +717,57 @@ protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteC colName = quote(colName + suffix); // Generate CTE name based on program stage and item - String cteName = String.format("ps_%s_%s", - item.getProgramStage().getUid().toLowerCase(), - item.getItem().getUid().toLowerCase()); + String cteName = + String.format( + "ps_%s_%s", + item.getProgramStage().getUid().toLowerCase(), item.getItem().getUid().toLowerCase()); String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid(); String excludingScheduledCondition = - eventTableName + ".eventstatus != '" + EventStatus.SCHEDULE + "'"; + eventTableName + ".eventstatus != '" + EventStatus.SCHEDULE + "'"; if (item.getProgramStage().getRepeatable() - && item.hasRepeatableStageParams() - && !item.getRepeatableStageParams().simpleStageValueExpected()) { - - String cteSql = String.format( - "SELECT enrollment, COALESCE(json_agg(t1), '[]') as value FROM (" + - " SELECT %s, %s, %s, %s" + - " FROM %s" + - " WHERE %s AND ps = '%s' %s %s %s %s" + - ") as t1 GROUP BY enrollment", + && item.hasRepeatableStageParams() + && !item.getRepeatableStageParams().simpleStageValueExpected()) { + + String cteSql = + String.format( + "SELECT enrollment, COALESCE(json_agg(t1), '[]') as value FROM (" + + " SELECT %s, %s, %s, %s" + + " FROM %s" + + " WHERE %s AND ps = '%s' %s %s %s %s" + + ") as t1 GROUP BY enrollment", colName, - String.join(", ", - EventAnalyticsColumnName.ENROLLMENT_OCCURRED_DATE_COLUMN_NAME, - EventAnalyticsColumnName.SCHEDULED_DATE_COLUMN_NAME, - EventAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME), + String.join( + ", ", + EventAnalyticsColumnName.ENROLLMENT_OCCURRED_DATE_COLUMN_NAME, + EventAnalyticsColumnName.SCHEDULED_DATE_COLUMN_NAME, + EventAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME), eventTableName, excludingScheduledCondition, item.getProgramStage().getUid(), getExecutionDateFilter( - item.getRepeatableStageParams().getStartDate(), - item.getRepeatableStageParams().getEndDate()), + item.getRepeatableStageParams().getStartDate(), + item.getRepeatableStageParams().getEndDate()), createOrderType(item.getProgramStageOffset()), createOffset(item.getProgramStageOffset()), - getLimit(item.getRepeatableStageParams().getCount()) - ); + getLimit(item.getRepeatableStageParams().getCount())); cteContext.addCTE(cteName, cteSql); alias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); return cteName + ".value as " + alias; } else { - String cteSql = String.format( - "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + - "FROM %s " + - "WHERE %s AND ps = '%s' " + - "ORDER BY enrollment, occurreddate DESC, created DESC", + String cteSql = + String.format( + "SELECT DISTINCT ON (enrollment) enrollment, %s as value " + + "FROM %s " + + "WHERE %s AND ps = '%s' " + + "ORDER BY enrollment, occurreddate DESC, created DESC", colName, eventTableName, excludingScheduledCondition, - item.getProgramStage().getUid() - ); + item.getProgramStage().getUid()); cteContext.addCTE(cteName, cteSql); String columnAlias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); @@ -997,31 +1015,33 @@ private CTEContext getCteDefinitions(EventQueryParams params) { if (queryItem.hasRelationshipType()) { programIndicatorSubqueryBuilder.contributeCTE( - pi, - queryItem.getRelationshipType(), - getAnalyticsType(), - params.getEarliestStartDate(), - params.getLatestEndDate(), - cteContext); + pi, + queryItem.getRelationshipType(), + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); } else { programIndicatorSubqueryBuilder.contributeCTE( - pi, - getAnalyticsType(), - params.getEarliestStartDate(), - params.getLatestEndDate(), - cteContext); + pi, + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); } - } - else if (queryItem.hasProgramStage()) { + } else if (queryItem.hasProgramStage()) { // Generate CTE for program stage items - String cteName = String.format("ps_%s_%s", + String cteName = + String.format( + "ps_%s_%s", queryItem.getProgramStage().getUid().toLowerCase(), queryItem.getItem().getUid().toLowerCase()); String colName = quote(queryItem.getItemName()); String eventTableName = ANALYTICS_EVENT + queryItem.getProgram().getUid(); - String cteSql = String.format( + String cteSql = + String.format( """ -- Generate CTE for program stage items SELECT DISTINCT ON (enrollment) enrollment, %s as value @@ -1030,17 +1050,15 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value AND ps = '%s' -- AND %s IS NOT NULL ORDER BY enrollment, occurreddate DESC, created DESC""", - colName, - eventTableName, - queryItem.getProgramStage().getUid(), - colName); + colName, eventTableName, queryItem.getProgramStage().getUid(), colName); cteContext.addCTE(cteName, cteSql); - String alias = quote(queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid()); + String alias = + quote(queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid()); cteContext.addColumnMapping( - queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(), - cteName + ".value as " + alias); + queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(), + cteName + ".value as " + alias); } } return cteContext; @@ -1049,47 +1067,52 @@ SELECT DISTINCT ON (enrollment) enrollment, %s as value private String buildFilterCteSql(QueryItem item, EventQueryParams params) { String tableName = params.getTableName(); String columnName = getColumn(item); - String filterCondition = item.getFilters().stream() + String filterCondition = + item.getFilters().stream() .map(filter -> toSql(item, filter, params)) .collect(Collectors.joining(" AND ")); return String.format( - "SELECT enrollment, %s AS value " + - "FROM %s " + - "WHERE %s", - columnName, tableName, filterCondition - ); + "SELECT enrollment, %s AS value " + "FROM %s " + "WHERE %s", + columnName, tableName, filterCondition); } private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) { // Combine items and item filters - List queryItems = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + List queryItems = + Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) .toList(); // Group query items by repeatable and non-repeatable stages Map> itemsByRepeatableFlag = - queryItems.stream() - .collect(groupingBy( - queryItem -> queryItem.hasRepeatableStageParams() - && params.getEndpointItem() == RequestTypeAware.EndpointItem.ENROLLMENT - )); + queryItems.stream() + .collect( + groupingBy( + queryItem -> + queryItem.hasRepeatableStageParams() + && params.getEndpointItem() + == RequestTypeAware.EndpointItem.ENROLLMENT)); // Process repeatable stage filters itemsByRepeatableFlag.getOrDefault(true, List.of()).stream() - .collect(groupingBy(this::getIdentifier)) - .forEach((identifier, items) -> { + .collect(groupingBy(this::getIdentifier)) + .forEach( + (identifier, items) -> { String cteName = "filter_" + identifier.replace('.', '_').toLowerCase(); String cteSql = buildFilterCteSql(items, params); cteContext.addCTE(cteName, cteSql); }); // Process non-repeatable stage filters - itemsByRepeatableFlag.getOrDefault(false, List.of()).forEach(queryItem -> { - String cteName = "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); - String cteSql = buildFilterCteSql(List.of(queryItem), params); - cteContext.addCTE(cteName, cteSql); - }); + itemsByRepeatableFlag + .getOrDefault(false, List.of()) + .forEach( + queryItem -> { + String cteName = "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); + String cteSql = buildFilterCteSql(List.of(queryItem), params); + cteContext.addCTE(cteName, cteSql); + }); } private String buildEnrollmentQueryWithCte2(EventQueryParams params) { @@ -1109,10 +1132,10 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { } // 3. Select clause - List selectCols = ListUtils.distinctUnion( + List selectCols = + ListUtils.distinctUnion( params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, - getSelectColumnsWithCTE(params, cteContext) - ); + getSelectColumnsWithCTE(params, cteContext)); sql.append("SELECT ").append(String.join(",\n ", selectCols)); // 4. From clause @@ -1121,19 +1144,20 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { // 5. Add joins for each CTE for (String cteName : cteContext.getCTENames()) { sql.append("\nLEFT JOIN ") - .append(cteName) - .append(" ON ax.enrollment = ") - .append(cteName) - .append(".enrollment"); + .append(cteName) + .append(" ON ax.enrollment = ") + .append(cteName) + .append(".enrollment"); } // 6. Where clause - sql.append(" ").append(getWhereClause(params)) - .append(" ") - .append(addCteFiltersToWhereClause(params)); + sql.append(" ") + .append(getWhereClause(params)) + .append(" ") + .append(addCteFiltersToWhereClause(params)); // 7. Order by - sql.append(" " ).append(getSortClause(params)); + sql.append(" ").append(getSortClause(params)); // 8. Paging sql.append(" ").append(getPagingClause(params, 5000)); @@ -1141,7 +1165,6 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { return sql.toString(); } - private String getBasicSelectColumns() { return String.join( ",", @@ -1197,7 +1220,8 @@ private String buildAllRankedEventsCTEs(List items) { String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); String columnName = quote(item.getItem().getUid()); - ctes.append(String.format( + ctes.append( + String.format( """ RankedEvents_%s as ( select enrollment, %s as value, eventstatus, @@ -1207,8 +1231,7 @@ private String buildAllRankedEventsCTEs(List items) { and ps = '%s' ) """, - key, columnName, order, order, eventTableName, stageUid - )); + key, columnName, order, order, eventTableName, stageUid)); processedCombinations.add(key); } @@ -1226,7 +1249,8 @@ private String buildValueColumns(List items) { String stageUid = item.getProgramStage().getUid(); int offset = createOffset2(item.getProgramStageOffset()); - String key = stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + String key = + stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; String alias = "re_" + key; @@ -1236,15 +1260,21 @@ private String buildValueColumns(List items) { } // Add value column - columns.append(String.format("%s.value as %s", + columns.append( + String.format( + "%s.value as %s", alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid()))); // Add exists column - columns.append(String.format(",\n(%s.enrollment is not null) as %s", + columns.append( + String.format( + ",\n(%s.enrollment is not null) as %s", alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".exists"))); // Add status column - columns.append(String.format(",\n%s.eventstatus as %s", + columns.append( + String.format( + ",\n%s.eventstatus as %s", alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".status"))); } @@ -1265,13 +1295,14 @@ private String buildFromClauseWithJoins(EventQueryParams params, List String stageUid = item.getProgramStage().getUid(); int offset = createOffset2(item.getProgramStageOffset()); - String key = stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + String key = + stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); String alias = "re_" + key; - fromClause.append(String.format( + fromClause.append( + String.format( "\nleft join RankedEvents_%s %s on ax.enrollment = %s.enrollment and %s.rn = %d", - key, alias, alias, alias, offset + 1 - )); + key, alias, alias, alias, offset + 1)); } return fromClause.toString(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java index 763b0fd35215..74402819889d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java @@ -27,7 +27,6 @@ */ package org.hisp.dhis.analytics.event.data.programindicator; -import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.hisp.dhis.analytics.DataType.BOOLEAN; import static org.hisp.dhis.analytics.DataType.NUMERIC; @@ -79,62 +78,66 @@ public String getAggregateClauseForProgramIndicator( } @Override - public void contributeCTE(ProgramIndicator programIndicator, AnalyticsType outerSqlEntity, Date earliestStartDate, Date latestDate, CTEContext cteContext) { - contributeCTE(programIndicator, null, outerSqlEntity, earliestStartDate, latestDate, cteContext); + public void contributeCTE( + ProgramIndicator programIndicator, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext) { + contributeCTE( + programIndicator, null, outerSqlEntity, earliestStartDate, latestDate, cteContext); } @Override public void contributeCTE( - ProgramIndicator programIndicator, - RelationshipType relationshipType, - AnalyticsType outerSqlEntity, - Date earliestStartDate, - Date latestDate, - CTEContext cteContext) { + ProgramIndicator programIndicator, + RelationshipType relationshipType, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CTEContext cteContext) { // Generate a unique CTE name for this program indicator String cteName = "pi_" + programIndicator.getUid().toLowerCase(); // Define aggregation function - String function = TextUtils.emptyIfEqual( + String function = + TextUtils.emptyIfEqual( programIndicator.getAggregationTypeFallback().getValue(), AggregationType.CUSTOM.getValue()); - String cteSql = String.format( - "SELECT e.enrollment, " + - "COALESCE(%s(%s), 0) as value " + - "FROM analytics_enrollment_%s e " + - "LEFT JOIN ( " + - " SELECT enrollment, eventstatus " + - " FROM %s as subax " + - " WHERE %s " + - ") t ON t.enrollment = e.enrollment " + - "GROUP BY e.enrollment", + String cteSql = + String.format( + "SELECT e.enrollment, " + + "COALESCE(%s(%s), 0) as value " + + "FROM analytics_enrollment_%s e " + + "LEFT JOIN ( " + + " SELECT enrollment, eventstatus " + + " FROM %s as subax " + + " WHERE %s " + + ") t ON t.enrollment = e.enrollment " + + "GROUP BY e.enrollment", function, getProgramIndicatorSql( - programIndicator.getExpression(), - NUMERIC, - programIndicator, - earliestStartDate, - latestDate), + programIndicator.getExpression(), + NUMERIC, + programIndicator, + earliestStartDate, + latestDate), programIndicator.getProgram().getUid().toLowerCase(), getTableName(programIndicator), getProgramIndicatorSql( - programIndicator.getFilter(), - BOOLEAN, - programIndicator, - earliestStartDate, - latestDate)); + programIndicator.getFilter(), + BOOLEAN, + programIndicator, + earliestStartDate, + latestDate)); // Register the CTE and its column mapping cteContext.addCTE(cteName, cteSql.toString()); - cteContext.addColumnMapping( - programIndicator.getUid(), - cteName + ".value" - ); + cteContext.addColumnMapping(programIndicator.getUid(), cteName + ".value"); } - private String getTableName(ProgramIndicator programIndicator) { return "analytics_event_" + programIndicator.getProgram().getUid().toLowerCase(); } @@ -227,10 +230,9 @@ private String getWhere( RelationshipType relationshipType) { String condition = ""; if (relationshipType != null) { - condition = RelationshipTypeJoinGenerator.generate( - SUBQUERY_TABLE_ALIAS, - relationshipType, - programIndicator.getAnalyticsType()); + condition = + RelationshipTypeJoinGenerator.generate( + SUBQUERY_TABLE_ALIAS, relationshipType, programIndicator.getAnalyticsType()); } else { // Remove the reference to the outer query's enrollment // We'll handle the join in the main query From 75f3a5709f70f927ccc5eb61932d757835c8ec65 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 18 Dec 2024 14:05:12 +0100 Subject: [PATCH 26/30] WIP - Fix basic filters --- .../dhis/analytics/common/CTEContext.java | 4 + .../hisp/dhis/analytics/common/CTEUtils.java | 19 +++++ .../data/JdbcEnrollmentAnalyticsManager.java | 82 +++++++++---------- 3 files changed, 62 insertions(+), 43 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index cc69a4919067..3d8f8b14ee6e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -68,4 +68,8 @@ public Set getCTENames() { public String getColumnMapping(String columnId) { return columnMappings.getOrDefault(columnId, columnId); } + + public boolean containsCteFilter(String cteFilterName) { + return cteDefinitions.containsKey(cteFilterName); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java new file mode 100644 index 000000000000..08eb967609d8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java @@ -0,0 +1,19 @@ +package org.hisp.dhis.analytics.common; + +import org.hisp.dhis.common.QueryItem; + +public class CTEUtils { + + public static String createFilterName(QueryItem queryItem) { + return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); + } + + public static String createFilterNameByIdentifier(String identifier) { + return "filter_" + identifier.replace('.', '_').toLowerCase(); + } + + public static String getIdentifier(QueryItem queryItem) { + String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; + return stage + "." + queryItem.getItemId(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index bd69e3e8094f..9453ba02cbfe 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -32,6 +32,7 @@ import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.common.CTEUtils.createFilterNameByIdentifier; import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; @@ -57,6 +58,7 @@ import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; +import org.hisp.dhis.analytics.common.CTEUtils; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; @@ -503,7 +505,7 @@ private String addFiltersToWhereClause(EventQueryParams params) { return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); } - private String addCteFiltersToWhereClause(EventQueryParams params) { + private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext cteContext) { StringBuilder whereClause = new StringBuilder(); // Iterate over each filter and apply the correct condition @@ -512,21 +514,32 @@ private String addCteFiltersToWhereClause(EventQueryParams params) { .filter(QueryItem::hasFilter) .toList()) { - String cteName = "filter_" + getIdentifier(item).replace('.', '_').toLowerCase(); - - for (QueryFilter filter : item.getFilters()) { - if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly - whereClause.append(" AND ").append(cteName).append(".value IS NULL"); - } else { - String operator = getSqlOperator(filter); - String value = getSqlFilterValue(filter, item); - whereClause - .append(" AND ") - .append(cteName) - .append(".value ") - .append(operator) - .append(" ") - .append(value); + String cteName = CTEUtils.createFilterName(item); + + if (cteContext.containsCteFilter(cteName)) { + for (QueryFilter filter : item.getFilters()) { + if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly + whereClause.append(" AND ").append(cteName).append(".value IS NULL"); + } else { + String operator = getSqlOperator(filter); + String value = getSqlFilterValue(filter, item); + whereClause + .append(" AND ") + .append(cteName) + .append(".value ") + .append(operator) + .append(" ") + .append(value); + } + } + } else { + // If the filter is not part of the CTE, apply it directly to the enrollment table + // using the standard where clause method + String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + if (StringUtils.isNotBlank(filters) && filters.trim().startsWith("where")) { + // remove the 'where' keyword + filters = filters.trim().substring(5); + whereClause.append("and ").append(filters); } } } @@ -598,26 +611,6 @@ SELECT DISTINCT ON (enrollment) enrollment, %s AS value .collect(Collectors.joining("\nUNION ALL\n")); } - private Stream asSqlCollection( - List queryItems, EventQueryParams params) { - return Optional.ofNullable(queryItems).orElse(List.of()).stream() - .map( - queryItem -> { - String sql = - queryItem.getFilters().stream() - .map(filter -> toSql(queryItem, filter, params)) - .collect(Collectors.joining(" AND ")); - return IdentifiableSql.builder() - .identifier(getIdentifier(queryItem)) - .sql(sql) - .build(); - }); - } - - private String getIdentifier(QueryItem queryItem) { - String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; - return stage + "." + queryItem.getItemId(); - } @Override protected String getSelectClause(EventQueryParams params) { @@ -1009,6 +1002,7 @@ private String createOrderType(int offset) { private CTEContext getCteDefinitions(EventQueryParams params) { CTEContext cteContext = new CTEContext(); + for (QueryItem queryItem : params.getItems()) { if (queryItem.isProgramIndicator()) { ProgramIndicator pi = (ProgramIndicator) queryItem.getItem(); @@ -1029,7 +1023,7 @@ private CTEContext getCteDefinitions(EventQueryParams params) { params.getLatestEndDate(), cteContext); } - } else if (queryItem.hasProgramStage()) { + } else if (queryItem.hasProgramStage() && queryItem.hasProgram()) { // Generate CTE for program stage items String cteName = String.format( @@ -1096,10 +1090,10 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) // Process repeatable stage filters itemsByRepeatableFlag.getOrDefault(true, List.of()).stream() - .collect(groupingBy(this::getIdentifier)) + .collect(groupingBy(CTEUtils::getIdentifier)) .forEach( (identifier, items) -> { - String cteName = "filter_" + identifier.replace('.', '_').toLowerCase(); + String cteName = createFilterNameByIdentifier(identifier); String cteSql = buildFilterCteSql(items, params); cteContext.addCTE(cteName, cteSql); }); @@ -1109,9 +1103,11 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) .getOrDefault(false, List.of()) .forEach( queryItem -> { - String cteName = "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); - String cteSql = buildFilterCteSql(List.of(queryItem), params); - cteContext.addCTE(cteName, cteSql); + if (queryItem.hasProgram() && queryItem.hasProgramStage()) { + String cteName = CTEUtils.createFilterName(queryItem); + String cteSql = buildFilterCteSql(List.of(queryItem), params); + cteContext.addCTE(cteName, cteSql); + } }); } @@ -1154,7 +1150,7 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { sql.append(" ") .append(getWhereClause(params)) .append(" ") - .append(addCteFiltersToWhereClause(params)); + .append(addCteFiltersToWhereClause(params, cteContext)); // 7. Order by sql.append(" ").append(getSortClause(params)); From 83eb127591bb7a64cb0ba44987b48cb18ef71836 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 18 Dec 2024 17:56:33 +0100 Subject: [PATCH 27/30] WIP - Re-fixed row context logic --- .../dhis/analytics/common/CTEContext.java | 13 + .../hisp/dhis/analytics/common/CTEUtils.java | 47 ++- .../analytics/common/RowContextUtils.java | 67 ++++ .../AbstractJdbcEventAnalyticsManager.java | 2 +- .../data/JdbcEnrollmentAnalyticsManager.java | 359 +++++++++--------- 5 files changed, 299 insertions(+), 189 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index 3d8f8b14ee6e..82c4eacff75c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -31,10 +31,12 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import lombok.Getter; public class CTEContext { private final Map cteDefinitions = new LinkedHashMap<>(); private final Map columnMappings = new HashMap<>(); + @Getter private final Map rowContextReferences = new HashMap<>(); public void addCTE(String cteName, String cteDefinition) { cteDefinitions.put(cteName, cteDefinition); @@ -44,6 +46,17 @@ public void addColumnMapping(String originalColumn, String cteReference) { columnMappings.put(originalColumn, cteReference); } + /** + * Adds a mapping between a row context column and the CTE name that it references. + * + * @param alias The alias of the row context column, for instance "EPEcjy3FWmI.lJTx9EZ1dk1" + * @param cteName The name of the CTE that the row context column references, for instance + * "ps_epecjy3fwmi_ljtx9ez1dk1" + */ + public void addRowContextColumnMapping(String alias, String cteName) { + rowContextReferences.put(alias, cteName); + } + public String getCTEDefinition() { if (cteDefinitions.isEmpty()) { return ""; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java index 08eb967609d8..43511d865685 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java @@ -1,19 +1,46 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package org.hisp.dhis.analytics.common; import org.hisp.dhis.common.QueryItem; public class CTEUtils { - public static String createFilterName(QueryItem queryItem) { - return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); - } + public static String createFilterName(QueryItem queryItem) { + return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); + } - public static String createFilterNameByIdentifier(String identifier) { - return "filter_" + identifier.replace('.', '_').toLowerCase(); - } + public static String createFilterNameByIdentifier(String identifier) { + return "filter_" + identifier.replace('.', '_').toLowerCase(); + } - public static String getIdentifier(QueryItem queryItem) { - String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; - return stage + "." + queryItem.getItemId(); - } + public static String getIdentifier(QueryItem queryItem) { + String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; + return stage + "." + queryItem.getItemId(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java new file mode 100644 index 000000000000..d6341b17682f --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.hisp.dhis.db.sql.SqlBuilder; + +public class RowContextUtils { + + public static List getRowContextColumns(CTEContext cteContext, SqlBuilder sqlBuilder) { + List columns = new ArrayList<>(); + Map rowCtxRefs = cteContext.getRowContextReferences(); + for (String aliases : rowCtxRefs.keySet()) { + columns.add(getStatusColumn(aliases, rowCtxRefs.get(aliases), sqlBuilder)); + columns.add(getExistsColumn(aliases, rowCtxRefs.get(aliases), sqlBuilder)); + } + + return columns; + } + + public static List getRowContextWhereClauses(CTEContext cteContext) { + List whereClauses = new ArrayList<>(); + Map rowCtxRefs = cteContext.getRowContextReferences(); + for (String alias : rowCtxRefs.values()) { + whereClauses.add("%s.value is null".formatted(alias)); + whereClauses.add("%s.exists_flag = true".formatted(alias)); + } + return whereClauses; + } + + private static String getExistsColumn( + String aliases, String cteReference, SqlBuilder sqlBuilder) { + return "coalesce(%s.exists_flag, false) as %s" + .formatted(cteReference, sqlBuilder.quote(aliases + ".status.exists")); + } + + private static String getStatusColumn(String alias, String cteReference, SqlBuilder sqlBuilder) { + return "%s_status.status as %s".formatted(cteReference, sqlBuilder.quote(alias)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 0b88484ca5bd..f4ff6f8f2c49 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -414,7 +414,7 @@ private void addItemSelectColumns( * @param queryItem * @return true when eligible for row context */ - private boolean rowContextAllowedAndNeeded(EventQueryParams params, QueryItem queryItem) { + protected boolean rowContextAllowedAndNeeded(EventQueryParams params, QueryItem queryItem) { return params.getEndpointItem() == ENROLLMENT && params.isRowContext() && queryItem.hasProgramStage() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 9453ba02cbfe..41d686faa31a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -47,11 +47,9 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @@ -60,6 +58,7 @@ import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.CTEUtils; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; +import org.hisp.dhis.analytics.common.RowContextUtils; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.table.AbstractJdbcTableManager; @@ -155,7 +154,7 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { if (params.isAggregatedEnrollments()) { sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); } else { - sql = buildEnrollmentQueryWithCte2(params); + sql = buildEnrollmentQueryWithCte(params); } System.out.println("SQL: " + sql); // FIXME: Remove debug line @@ -524,25 +523,31 @@ private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext ct String operator = getSqlOperator(filter); String value = getSqlFilterValue(filter, item); whereClause - .append(" AND ") - .append(cteName) - .append(".value ") - .append(operator) - .append(" ") - .append(value); + .append(" AND ") + .append(cteName) + .append(".value ") + .append(operator) + .append(" ") + .append(value); } } } else { // If the filter is not part of the CTE, apply it directly to the enrollment table // using the standard where clause method - String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); if (StringUtils.isNotBlank(filters) && filters.trim().startsWith("where")) { // remove the 'where' keyword - filters = filters.trim().substring(5); - whereClause.append("and ").append(filters); + filters = filters.trim().substring(5); + whereClause.append("and ").append(filters); } } } + // Handle the row context case + List rowContextColumns = RowContextUtils.getRowContextWhereClauses(cteContext); + if (!StringUtils.isEmpty(whereClause)) { + whereClause.append(" AND "); + } + whereClause.append(String.join(" AND ", rowContextColumns)); return whereClause.toString(); } @@ -611,7 +616,6 @@ SELECT DISTINCT ON (enrollment) enrollment, %s AS value .collect(Collectors.joining("\nUNION ALL\n")); } - @Override protected String getSelectClause(EventQueryParams params) { List selectCols = @@ -1035,42 +1039,54 @@ private CTEContext getCteDefinitions(EventQueryParams params) { String eventTableName = ANALYTICS_EVENT + queryItem.getProgram().getUid(); String cteSql = - String.format( - """ + """ -- Generate CTE for program stage items - SELECT DISTINCT ON (enrollment) enrollment, %s as value + SELECT DISTINCT ON (enrollment) enrollment, %s as value, %s FROM %s WHERE eventstatus != 'SCHEDULE' AND ps = '%s' - -- AND %s IS NOT NULL - ORDER BY enrollment, occurreddate DESC, created DESC""", - colName, eventTableName, queryItem.getProgramStage().getUid(), colName); + ORDER BY enrollment, occurreddate DESC, created DESC""" + .formatted( + colName, + rowContextAllowedAndNeeded(params, queryItem) ? "true as exists_flag" : "", + eventTableName, + queryItem.getProgramStage().getUid()); cteContext.addCTE(cteName, cteSql); - - String alias = - quote(queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid()); + String unquotedAlias = + queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(); cteContext.addColumnMapping( queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(), - cteName + ".value as " + alias); + cteName + ".value as " + quote(unquotedAlias)); + + if (rowContextAllowedAndNeeded( + params, queryItem)) { // TODO original condition && !isEmpty(columnAndAlias.alias)) { + + String statusCteSql = + """ + select + distinct on (enrollment) + enrollment, + eventstatus as status + from + %s + where + eventstatus != 'SCHEDULE' + and ps = '%s' + order by + enrollment, + occurreddate desc, + created desc + """ + .formatted(eventTableName, queryItem.getProgramStage().getUid()); + cteContext.addCTE(cteName + "_status", statusCteSql); + cteContext.addRowContextColumnMapping(unquotedAlias, cteName); + } } } return cteContext; } - private String buildFilterCteSql(QueryItem item, EventQueryParams params) { - String tableName = params.getTableName(); - String columnName = getColumn(item); - String filterCondition = - item.getFilters().stream() - .map(filter -> toSql(item, filter, params)) - .collect(Collectors.joining(" AND ")); - - return String.format( - "SELECT enrollment, %s AS value " + "FROM %s " + "WHERE %s", - columnName, tableName, filterCondition); - } - private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) { // Combine items and item filters List queryItems = @@ -1111,7 +1127,7 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) }); } - private String buildEnrollmentQueryWithCte2(EventQueryParams params) { + private String buildEnrollmentQueryWithCte(EventQueryParams params) { StringBuilder sql = new StringBuilder(); @@ -1131,7 +1147,8 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { List selectCols = ListUtils.distinctUnion( params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, - getSelectColumnsWithCTE(params, cteContext)); + getSelectColumnsWithCTE(params, cteContext), + getRowContextColumns(cteContext)); sql.append("SELECT ").append(String.join(",\n ", selectCols)); // 4. From clause @@ -1161,148 +1178,134 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { return sql.toString(); } - private String getBasicSelectColumns() { - return String.join( - ",", - "ax.enrollment", - "trackedentity", - "enrollmentdate", - "occurreddate", - "storedby", - "createdbydisplayname", - "lastupdatedbydisplayname", - "lastupdated", - "ST_AsGeoJSON(enrollmentgeometry)", - "longitude", - "latitude", - "ouname", - "ounamehierarchy", - "oucode", - "enrollmentstatus", - "ax.\"ou\""); - } - - private String resolveOrderByOffset(int offset) { - - if (offset <= 0) { - return "desc"; - } - return "asc"; - } - - private String buildAllRankedEventsCTEs(List items) { - StringBuilder ctes = new StringBuilder(); - Set processedCombinations = new HashSet<>(); - - for (QueryItem item : items) { - if (!item.hasProgramStage()) { - continue; - } - - String stageUid = item.getProgramStage().getUid(); - int offset = createOffset2(item.getProgramStageOffset()); - String order = resolveOrderByOffset(item.getProgramStageOffset()); - - // Create unique key for this combination to avoid duplicate CTEs - String key = stageUid + "_" + offset + "_" + order; - if (processedCombinations.contains(key)) { - continue; - } - - if (!ctes.isEmpty()) { - ctes.append(",\n"); - } - - String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); - String columnName = quote(item.getItem().getUid()); - - ctes.append( - String.format( - """ - RankedEvents_%s as ( - select enrollment, %s as value, eventstatus, - row_number() over (partition by enrollment order by occurreddate %s, created %s) as rn - from %s - where eventstatus != 'SCHEDULE' - and ps = '%s' - ) - """, - key, columnName, order, order, eventTableName, stageUid)); - - processedCombinations.add(key); - } - - return !ctes.isEmpty() ? "with " + ctes + "\n" : ""; - } - - private String buildValueColumns(List items) { - StringBuilder columns = new StringBuilder(); - - for (QueryItem item : items) { - if (!item.hasProgramStage()) { - continue; - } - - String stageUid = item.getProgramStage().getUid(); - int offset = createOffset2(item.getProgramStageOffset()); - String key = - stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); - - String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; - String alias = "re_" + key; - - if (!columns.isEmpty()) { - columns.append(",\n"); - } - - // Add value column - columns.append( - String.format( - "%s.value as %s", - alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid()))); - - // Add exists column - columns.append( - String.format( - ",\n(%s.enrollment is not null) as %s", - alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".exists"))); - - // Add status column - columns.append( - String.format( - ",\n%s.eventstatus as %s", - alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".status"))); - } - - return columns.toString(); + private List getRowContextColumns(CTEContext cteContext) { + return RowContextUtils.getRowContextColumns(cteContext, sqlBuilder); } - private String buildFromClauseWithJoins(EventQueryParams params, List items) { - StringBuilder fromClause = new StringBuilder(); - - // Start with base table - fromClause.append("\nfrom ").append(params.getTableName()).append(" as ax"); - - // Add joins for each item - for (QueryItem item : items) { - if (!item.hasProgramStage()) { - continue; - } - - String stageUid = item.getProgramStage().getUid(); - int offset = createOffset2(item.getProgramStageOffset()); - String key = - stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); - String alias = "re_" + key; - - fromClause.append( - String.format( - "\nleft join RankedEvents_%s %s on ax.enrollment = %s.enrollment and %s.rn = %d", - key, alias, alias, alias, offset + 1)); - } - - return fromClause.toString(); - } + // private String resolveOrderByOffset(int offset) { + // + // if (offset <= 0) { + // return "desc"; + // } + // return "asc"; + // } + + // private String buildAllRankedEventsCTEs(List items) { + // StringBuilder ctes = new StringBuilder(); + // Set processedCombinations = new HashSet<>(); + // + // for (QueryItem item : items) { + // if (!item.hasProgramStage()) { + // continue; + // } + // + // String stageUid = item.getProgramStage().getUid(); + // int offset = createOffset2(item.getProgramStageOffset()); + // String order = resolveOrderByOffset(item.getProgramStageOffset()); + // + // // Create unique key for this combination to avoid duplicate CTEs + // String key = stageUid + "_" + offset + "_" + order; + // if (processedCombinations.contains(key)) { + // continue; + // } + // + // if (!ctes.isEmpty()) { + // ctes.append(",\n"); + // } + // + // String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); + // String columnName = quote(item.getItem().getUid()); + // + // ctes.append( + // String.format( + // """ + // RankedEvents_%s as ( + // select enrollment, %s as value, eventstatus, + // row_number() over (partition by enrollment order by occurreddate %s, created + // %s) as rn + // from %s + // where eventstatus != 'SCHEDULE' + // and ps = '%s' + // ) + // """, + // key, columnName, order, order, eventTableName, stageUid)); + // + // processedCombinations.add(key); + // } + // + // return !ctes.isEmpty() ? "with " + ctes + "\n" : ""; + // } + // + // private String buildValueColumns(List items) { + // StringBuilder columns = new StringBuilder(); + // + // for (QueryItem item : items) { + // if (!item.hasProgramStage()) { + // continue; + // } + // + // String stageUid = item.getProgramStage().getUid(); + // int offset = createOffset2(item.getProgramStageOffset()); + // String key = + // stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + // + // String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; + // String alias = "re_" + key; + // + // if (!columns.isEmpty()) { + // columns.append(",\n"); + // } + // + // // Add value column + // columns.append( + // String.format( + // "%s.value as %s", + // alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid()))); + // + // // Add exists column + // columns.append( + // String.format( + // ",\n(%s.enrollment is not null) as %s", + // alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + + // ".exists"))); + // + // // Add status column + // columns.append( + // String.format( + // ",\n%s.eventstatus as %s", + // alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + + // ".status"))); + // } + // + // return columns.toString(); + // } + // + // private String buildFromClauseWithJoins(EventQueryParams params, List items) { + // StringBuilder fromClause = new StringBuilder(); + // + // // Start with base table + // fromClause.append("\nfrom ").append(params.getTableName()).append(" as ax"); + // + // // Add joins for each item + // for (QueryItem item : items) { + // if (!item.hasProgramStage()) { + // continue; + // } + // + // String stageUid = item.getProgramStage().getUid(); + // int offset = createOffset2(item.getProgramStageOffset()); + // String key = + // stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + // String alias = "re_" + key; + // + // fromClause.append( + // String.format( + // "\nleft join RankedEvents_%s %s on ax.enrollment = %s.enrollment and %s.rn = %d", + // key, alias, alias, alias, offset + 1)); + // } + // + // return fromClause.toString(); + // } protected String getSortClause(EventQueryParams params) { if (params.isSorting()) { From 63a3f2a6a76b95a9fbb57dc4a82cfbcbd7836ba6 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 18 Dec 2024 18:07:09 +0100 Subject: [PATCH 28/30] WIP - Fix sql gen --- .../data/JdbcEnrollmentAnalyticsManager.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 41d686faa31a..09adfaf515f7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -542,12 +542,19 @@ private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext ct } } } + // Handle the row context case List rowContextColumns = RowContextUtils.getRowContextWhereClauses(cteContext); - if (!StringUtils.isEmpty(whereClause)) { - whereClause.append(" AND "); + if (rowContextColumns.isEmpty()) { + return whereClause.toString(); + } else { + if (whereClause.isEmpty()) { + whereClause.append(" where "); + } else { + whereClause.append(" AND "); + } + whereClause.append(String.join(" AND ", rowContextColumns)); } - whereClause.append(String.join(" AND ", rowContextColumns)); return whereClause.toString(); } @@ -1041,14 +1048,14 @@ private CTEContext getCteDefinitions(EventQueryParams params) { String cteSql = """ -- Generate CTE for program stage items - SELECT DISTINCT ON (enrollment) enrollment, %s as value, %s + SELECT DISTINCT ON (enrollment) enrollment, %s as value%s FROM %s WHERE eventstatus != 'SCHEDULE' AND ps = '%s' ORDER BY enrollment, occurreddate DESC, created DESC""" .formatted( colName, - rowContextAllowedAndNeeded(params, queryItem) ? "true as exists_flag" : "", + rowContextAllowedAndNeeded(params, queryItem) ? " ,true as exists_flag" : "", eventTableName, queryItem.getProgramStage().getUid()); From d8119c0f80c73ac5218b48f13209f02faefb08c3 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Fri, 20 Dec 2024 16:19:22 +0100 Subject: [PATCH 29/30] WIP - various fixes --- .../dhis/analytics/common/CTEContext.java | 128 ++++++- .../analytics/common/RowContextUtils.java | 2 +- .../AbstractJdbcEventAnalyticsManager.java | 8 +- .../data/JdbcEnrollmentAnalyticsManager.java | 348 +++++++++--------- ...efaultProgramIndicatorSubqueryBuilder.java | 3 +- 5 files changed, 293 insertions(+), 196 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index 82c4eacff75c..6695d77aa334 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -27,23 +27,46 @@ */ package org.hisp.dhis.analytics.common; +import lombok.Getter; +import org.apache.commons.text.RandomStringGenerator; +import org.hisp.dhis.common.QueryItem; +import org.hisp.dhis.program.ProgramIndicator; +import org.hisp.dhis.program.ProgramStage; + import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; -import lombok.Getter; public class CTEContext { - private final Map cteDefinitions = new LinkedHashMap<>(); - private final Map columnMappings = new HashMap<>(); + private final Map cteDefinitions = new LinkedHashMap<>(); @Getter private final Map rowContextReferences = new HashMap<>(); - public void addCTE(String cteName, String cteDefinition) { - cteDefinitions.put(cteName, cteDefinition); + + public CteDefinitionWithOffset getDefinitionByItemUid(String itemUid) { + return cteDefinitions.get(itemUid); + } + + public void addCTE(ProgramStage programStage, QueryItem item, String cteDefinition, int offset) { + cteDefinitions.put(item.getItem().getUid(), + new CteDefinitionWithOffset(programStage.getUid(), cteDefinition, offset)); + } + + public void addCTE(ProgramStage programStage, QueryItem item, String cteDefinition, int offset, boolean isRowContext) { + cteDefinitions.put(item.getItem().getUid(), new CteDefinitionWithOffset(programStage.getUid(), cteDefinition, offset, isRowContext)); } - public void addColumnMapping(String originalColumn, String cteReference) { - columnMappings.put(originalColumn, cteReference); + /** + * Adds a CTE definition to the context. + * @param programIndicator The program indicator + * @param cteDefinition The CTE definition (the SQL query) + */ + public void addCTE(ProgramIndicator programIndicator, String cteDefinition) { + cteDefinitions.put(programIndicator.getUid(), new CteDefinitionWithOffset(programIndicator.getUid(), cteDefinition)); + } + + public void addCTEFilter(String name, String ctedefinition) { + cteDefinitions.put(name, new CteDefinitionWithOffset(name, ctedefinition, true)); } /** @@ -64,25 +87,100 @@ public String getCTEDefinition() { StringBuilder sb = new StringBuilder("WITH "); boolean first = true; - for (Map.Entry entry : cteDefinitions.entrySet()) { + for (Map.Entry entry : cteDefinitions.entrySet()) { if (!first) { sb.append(", "); } - sb.append(entry.getKey()).append(" AS (").append(entry.getValue()).append(")"); + CteDefinitionWithOffset cteDef = entry.getValue(); + sb.append(cteDef.asCteName(entry.getKey())).append(" AS (").append(entry.getValue().cteDefinition).append(")"); first = false; } return sb.toString(); } - + // Rename to item uid public Set getCTENames() { return cteDefinitions.keySet(); } - - public String getColumnMapping(String columnId) { - return columnMappings.getOrDefault(columnId, columnId); - } - + public boolean containsCteFilter(String cteFilterName) { return cteDefinitions.containsKey(cteFilterName); } + + @Getter + public static class CteDefinitionWithOffset { + // The program stage uid + private final String programStageUid; + // The program indicator uid + private String programIndicatorUid; + // The CTE definition (the SQL query) + private final String cteDefinition; + // The calculated offset + private final int offset; + // The alias of the CTE + private final String alias; + // Whether the CTE is a row context (TODO this need a better explanation) + private boolean isRowContext; + // Whether the CTE is a program indicator + private boolean isProgramIndicator = false; + // Whether the CTE is a filter + private boolean isFilter = false; + private final static String PS_PREFIX = "ps"; + private final static String PI_PREFIX = "pi"; + + public CteDefinitionWithOffset(String programStageUid, String cteDefinition, int offset) { + this.programStageUid = programStageUid; + this.cteDefinition = cteDefinition; + this.offset = offset; + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build() + .generate(5); + this.isRowContext = false; + } + + public CteDefinitionWithOffset(String programStageUid, String cteDefinition, int offset, boolean isRowContext) { + this(programStageUid, cteDefinition, offset); + this.isRowContext = isRowContext; + } + + public CteDefinitionWithOffset(String programIndicatorUid, String cteDefinition) { + this.cteDefinition = cteDefinition; + this.programIndicatorUid = programIndicatorUid; + this.programStageUid = null; + this.offset = -999; + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build() + .generate(5); + this.isRowContext = false; + this.isProgramIndicator = true; + } + + public CteDefinitionWithOffset(String cteFilterName, String cteDefinition, boolean isFilter) { + this.cteDefinition = cteDefinition; + this.programIndicatorUid = null; + this.programStageUid = null; + this.offset = -999; + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build() + .generate(5); + this.isRowContext = false; + this.isProgramIndicator = false; + this.isFilter = isFilter; + } + + /** + * + * @param uid the uid of an dimension item or ProgramIndicator + * @return the name of the CTE + */ + public String asCteName(String uid) { + if (isProgramIndicator) { + return "%s_%s".formatted(PI_PREFIX, programIndicatorUid.toLowerCase()); + } + if (isFilter) { + return "%s".formatted(uid.toLowerCase()); + } + return "%s_%s_%s".formatted(PS_PREFIX, programStageUid.toLowerCase(), uid.toLowerCase()); + } + + public boolean isProgramStage() { + return !isFilter && !isProgramIndicator; + } + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java index d6341b17682f..d2753a7547e2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java @@ -50,7 +50,7 @@ public static List getRowContextWhereClauses(CTEContext cteContext) { Map rowCtxRefs = cteContext.getRowContextReferences(); for (String alias : rowCtxRefs.values()) { whereClauses.add("%s.value is null".formatted(alias)); - whereClauses.add("%s.exists_flag = true".formatted(alias)); + //whereClauses.add("%s.exists_flag = true".formatted(alias)); } return whereClauses; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index f4ff6f8f2c49..2e22c86905cd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -92,6 +92,8 @@ import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; +import org.hisp.dhis.analytics.common.CTEContext.CteDefinitionWithOffset; +import org.hisp.dhis.analytics.common.CTEUtils; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.util.AnalyticsUtils; @@ -1403,8 +1405,10 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte if (queryItem.isProgramIndicator()) { // For program indicators, use CTE reference String piUid = queryItem.getItem().getUid(); - String cteReference = cteContext.getColumnMapping(piUid); - columns.add(cteReference + " as \"" + piUid + "\""); + CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(piUid); + // ugaee.value as "CH6wamtY9kK", + String col = "%s.value as %s".formatted(cteDef.getAlias(), piUid); + columns.add(col); } else if (ValueType.COORDINATE == queryItem.getValueType()) { // Handle coordinates columns.add(getCoordinateColumn(queryItem).asSql()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 09adfaf515f7..88d51872d3b1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -44,6 +44,8 @@ import static org.hisp.dhis.util.DateUtils.toMediumDate; import com.google.common.collect.Sets; + +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -56,6 +58,7 @@ import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; +import org.hisp.dhis.analytics.common.CTEContext.CteDefinitionWithOffset; import org.hisp.dhis.analytics.common.CTEUtils; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.common.RowContextUtils; @@ -504,6 +507,15 @@ private String addFiltersToWhereClause(EventQueryParams params) { return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); } + private String addRowContextFilters(CTEContext cteContext) { + + List rowContextColumns = RowContextUtils.getRowContextWhereClauses(cteContext); + if (!rowContextColumns.isEmpty()) { + return String.join(" AND ", rowContextColumns); + } + return EMPTY; + } + private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext cteContext) { StringBuilder whereClause = new StringBuilder(); @@ -516,15 +528,16 @@ private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext ct String cteName = CTEUtils.createFilterName(item); if (cteContext.containsCteFilter(cteName)) { + CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(cteName); for (QueryFilter filter : item.getFilters()) { if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly - whereClause.append(" AND ").append(cteName).append(".value IS NULL"); + whereClause.append(" AND ").append(cteDef.getAlias()).append(".value IS NULL"); } else { String operator = getSqlOperator(filter); String value = getSqlFilterValue(filter, item); whereClause .append(" AND ") - .append(cteName) + .append(cteDef.getAlias()) .append(".value ") .append(operator) .append(" ") @@ -543,19 +556,6 @@ private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext ct } } - // Handle the row context case - List rowContextColumns = RowContextUtils.getRowContextWhereClauses(cteContext); - if (rowContextColumns.isEmpty()) { - return whereClause.toString(); - } else { - if (whereClause.isEmpty()) { - whereClause.append(" where "); - } else { - whereClause.append(" AND "); - } - whereClause.append(String.join(" AND ", rowContextColumns)); - } - return whereClause.toString(); } @@ -713,78 +713,32 @@ protected ColumnAndAlias getCoordinateColumn(QueryItem item, String suffix) { } protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteContext) { + List columns = new ArrayList<>(); String colName = item.getItemName(); - String alias; - if (item.hasProgramStage()) { - assertProgram(item); - colName = quote(colName + suffix); + CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(item.getItem().getUid()); - // Generate CTE name based on program stage and item - String cteName = - String.format( - "ps_%s_%s", - item.getProgramStage().getUid().toLowerCase(), item.getItem().getUid().toLowerCase()); + String alias = getAlias(item).orElse(null); - String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid(); - String excludingScheduledCondition = - eventTableName + ".eventstatus != '" + EventStatus.SCHEDULE + "'"; + // ed."lJTx9EZ1dk1" as "EPEcjy3FWmI[-1].lJTx9EZ1dk1" - if (item.getProgramStage().getRepeatable() - && item.hasRepeatableStageParams() - && !item.getRepeatableStageParams().simpleStageValueExpected()) { + columns.add(""" + %s.%s as %s + """.formatted(cteDef.getAlias(), quote(colName), quote(alias))); + if (cteDef.isRowContext()) { + // Add additional status and exists columns for row context + // (ed."lJTx9EZ1dk1" IS NOT NULL) as "EPEcjy3FWmI[-1].lJTx9EZ1dk1.exists", + // ed.eventstatus as "EPEcjy3FWmI[-1].lJTx9EZ1dk1.status" + columns.add(""" + (%s.%s IS NOT NULL) as %s + """.formatted(cteDef.getAlias(), quote(colName), quote(alias + ".exists"))); - String cteSql = - String.format( - "SELECT enrollment, COALESCE(json_agg(t1), '[]') as value FROM (" - + " SELECT %s, %s, %s, %s" - + " FROM %s" - + " WHERE %s AND ps = '%s' %s %s %s %s" - + ") as t1 GROUP BY enrollment", - colName, - String.join( - ", ", - EventAnalyticsColumnName.ENROLLMENT_OCCURRED_DATE_COLUMN_NAME, - EventAnalyticsColumnName.SCHEDULED_DATE_COLUMN_NAME, - EventAnalyticsColumnName.OCCURRED_DATE_COLUMN_NAME), - eventTableName, - excludingScheduledCondition, - item.getProgramStage().getUid(), - getExecutionDateFilter( - item.getRepeatableStageParams().getStartDate(), - item.getRepeatableStageParams().getEndDate()), - createOrderType(item.getProgramStageOffset()), - createOffset(item.getProgramStageOffset()), - getLimit(item.getRepeatableStageParams().getCount())); - - cteContext.addCTE(cteName, cteSql); - alias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); - return cteName + ".value as " + alias; + columns.add(""" + %s.eventstatus as %s + """.formatted(cteDef.getAlias(), quote(alias + ".status"))); - } else { - String cteSql = - String.format( - "SELECT DISTINCT ON (enrollment) enrollment, %s as value " - + "FROM %s " - + "WHERE %s AND ps = '%s' " - + "ORDER BY enrollment, occurreddate DESC, created DESC", - colName, - eventTableName, - excludingScheduledCondition, - item.getProgramStage().getUid()); - - cteContext.addCTE(cteName, cteSql); - String columnAlias = quote(item.getProgramStage().getUid() + "." + item.getItem().getUid()); - return cteName + ".value as " + columnAlias; - } - } - - // Non-program stage cases remain unchanged - if (isOrganizationUnitProgramAttribute(item)) { - return quoteAlias(colName + suffix); - } else { - return quoteAlias(colName); } + return String.join(", ", columns); } /** @@ -986,18 +940,6 @@ private String createOffset(int offset) { } } - private int createOffset2(int offset) { - if (offset == 0) { - return 0; - } - - if (offset < 0) { - return (-1 * offset); - } else { - return (offset - 1); - } - } - private String createOrderType(int offset) { if (offset == 0) { return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "desc"); @@ -1011,89 +953,112 @@ private String createOrderType(int offset) { // New methods // - private CTEContext getCteDefinitions(EventQueryParams params) { - CTEContext cteContext = new CTEContext(); - - for (QueryItem queryItem : params.getItems()) { - if (queryItem.isProgramIndicator()) { - ProgramIndicator pi = (ProgramIndicator) queryItem.getItem(); - - if (queryItem.hasRelationshipType()) { - programIndicatorSubqueryBuilder.contributeCTE( + private void handleProgramIndicatorCte(QueryItem item, CTEContext cteContext, EventQueryParams params) { + ProgramIndicator pi = (ProgramIndicator) item.getItem(); + if (item.hasRelationshipType()) { + programIndicatorSubqueryBuilder.contributeCTE( pi, - queryItem.getRelationshipType(), + item.getRelationshipType(), getAnalyticsType(), params.getEarliestStartDate(), params.getLatestEndDate(), cteContext); - } else { - programIndicatorSubqueryBuilder.contributeCTE( + } else { + programIndicatorSubqueryBuilder.contributeCTE( pi, getAnalyticsType(), params.getEarliestStartDate(), params.getLatestEndDate(), cteContext); - } - } else if (queryItem.hasProgramStage() && queryItem.hasProgram()) { - // Generate CTE for program stage items - String cteName = - String.format( - "ps_%s_%s", - queryItem.getProgramStage().getUid().toLowerCase(), - queryItem.getItem().getUid().toLowerCase()); - - String colName = quote(queryItem.getItemName()); - String eventTableName = ANALYTICS_EVENT + queryItem.getProgram().getUid(); - - String cteSql = - """ - -- Generate CTE for program stage items - SELECT DISTINCT ON (enrollment) enrollment, %s as value%s - FROM %s - WHERE eventstatus != 'SCHEDULE' - AND ps = '%s' - ORDER BY enrollment, occurreddate DESC, created DESC""" - .formatted( - colName, - rowContextAllowedAndNeeded(params, queryItem) ? " ,true as exists_flag" : "", - eventTableName, - queryItem.getProgramStage().getUid()); - - cteContext.addCTE(cteName, cteSql); - String unquotedAlias = - queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(); - cteContext.addColumnMapping( - queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(), - cteName + ".value as " + quote(unquotedAlias)); - - if (rowContextAllowedAndNeeded( - params, queryItem)) { // TODO original condition && !isEmpty(columnAndAlias.alias)) { - - String statusCteSql = - """ - select - distinct on (enrollment) - enrollment, - eventstatus as status - from - %s - where - eventstatus != 'SCHEDULE' - and ps = '%s' - order by - enrollment, - occurreddate desc, - created desc - """ - .formatted(eventTableName, queryItem.getProgramStage().getUid()); - cteContext.addCTE(cteName + "_status", statusCteSql); - cteContext.addRowContextColumnMapping(unquotedAlias, cteName); + } + } + + private CTEContext getCteDefinitions(EventQueryParams params) { + CTEContext cteContext = new CTEContext(); + + for (QueryItem item : params.getItems()) { + + String itemId = item.getItem().getUid(); + + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid(); + if (item.isProgramIndicator()) { + handleProgramIndicatorCte(item, cteContext, params); + } else if(item.hasProgramStage()) { + // TODO what is this condition would be good to give it a name + if (item.getProgramStage().getRepeatable() + && item.hasRepeatableStageParams() + && !item.getRepeatableStageParams().simpleStageValueExpected()) { + + // TODO: Implement repeatable stage items + log.warn("Repeatable stage items are not yet supported"); + // TODO what is this condition - would be good to give it a name + } else if (item.getProgramStage().getRepeatable() && item.hasRepeatableStageParams()) { + String colName = quote(item.getItemName()); + boolean hasEventStatusColumn = rowContextAllowedAndNeeded(params, item); + + var cteSql = """ + SELECT + enrollment, + %s,%s + ROW_NUMBER() OVER ( + PARTITION BY enrollment + ORDER BY occurreddate DESC, created DESC + ) as rn + FROM %s + WHERE eventstatus != 'SCHEDULE' + AND ps = '%s' + """.formatted( + colName, + hasEventStatusColumn ? " eventstatus," : "", + eventTableName, + item.getProgramStage().getUid()); + + cteContext.addCTE( + item.getProgramStage(), + item, + cteSql, + createOffset2(item.getProgramStageOffset()), + hasEventStatusColumn + ); + } else { + + // Generate CTE for program stage items + String colName = quote(item.getItemName()); + + String cteSql = + """ + -- Generate CTE for program stage items + SELECT DISTINCT ON (enrollment) enrollment, %s as value%s + FROM %s + WHERE eventstatus != 'SCHEDULE' + AND ps = '%s' + ORDER BY enrollment, occurreddate DESC, created DESC %s %s""" + .formatted( + colName, + rowContextAllowedAndNeeded(params, item) ? " ,true as exists_flag" : "", + eventTableName, + item.getProgramStage().getUid(), + createOffset(item.getProgramStageOffset()), LIMIT_1); + + cteContext.addCTE(item.getProgramStage(), item, cteSql, createOffset2(item.getProgramStageOffset())); } } } return cteContext; } + private int createOffset2(int offset) { + if (offset == 0) { + return 0; + } + + if (offset < 0) { + return (-1 * offset); + } else { + return (offset - 1); + } + } + private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) { // Combine items and item filters List queryItems = @@ -1118,7 +1083,7 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) (identifier, items) -> { String cteName = createFilterNameByIdentifier(identifier); String cteSql = buildFilterCteSql(items, params); - cteContext.addCTE(cteName, cteSql); + cteContext.addCTEFilter(cteName, cteSql); }); // Process non-repeatable stage filters @@ -1129,13 +1094,13 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) if (queryItem.hasProgram() && queryItem.hasProgramStage()) { String cteName = CTEUtils.createFilterName(queryItem); String cteSql = buildFilterCteSql(List.of(queryItem), params); - cteContext.addCTE(cteName, cteSql); + cteContext.addCTEFilter(cteName, cteSql); } }); } private String buildEnrollmentQueryWithCte(EventQueryParams params) { - + // LUCIANO // StringBuilder sql = new StringBuilder(); // 1. Process all program indicators to generate CTEs @@ -1154,27 +1119,35 @@ private String buildEnrollmentQueryWithCte(EventQueryParams params) { List selectCols = ListUtils.distinctUnion( params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, - getSelectColumnsWithCTE(params, cteContext), - getRowContextColumns(cteContext)); - sql.append("SELECT ").append(String.join(",\n ", selectCols)); + getSelectColumnsWithCTE(params, cteContext)); + sql.append("SELECT ").append(String.join(",\n", selectCols)); // 4. From clause sql.append("\nFROM ").append(params.getTableName()).append(" AS ax"); // 5. Add joins for each CTE - for (String cteName : cteContext.getCTENames()) { - sql.append("\nLEFT JOIN ") - .append(cteName) - .append(" ON ax.enrollment = ") - .append(cteName) - .append(".enrollment"); + for (String itemUid : cteContext.getCTENames()) { + CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(itemUid); + String join = """ + LEFT JOIN %s %s + ON + %s.enrollment = ax.enrollment + """.formatted(cteDef.asCteName(itemUid), cteDef.getAlias(), cteDef.getAlias()); + sql.append("\n").append(join); + if (cteDef.isProgramStage()) { + // equivalent to original OFFSET 1 LIMIT 1 but more efficient + // TODO use constant instead of hardcoded 'rn' column name + String offset = " AND %s.rn = %s".formatted(cteDef.getAlias(), cteDef.getOffset() + 1); + sql.append(offset); + } } // 6. Where clause - sql.append(" ") - .append(getWhereClause(params)) - .append(" ") - .append(addCteFiltersToWhereClause(params, cteContext)); + List conditions = collectWhereConditions(params, cteContext); + if (!conditions.isEmpty()) { + sql.append(" WHERE ") + .append(String.join(" AND ", conditions)); + } // 7. Order by sql.append(" ").append(getSortClause(params)); @@ -1185,10 +1158,33 @@ private String buildEnrollmentQueryWithCte(EventQueryParams params) { return sql.toString(); } - private List getRowContextColumns(CTEContext cteContext) { - return RowContextUtils.getRowContextColumns(cteContext, sqlBuilder); + private List collectWhereConditions(EventQueryParams params, CTEContext cteContext) { + + List conditions = new ArrayList<>(); + + String baseWhereClause = getWhereClause(params).trim(); + String cteFilters = addCteFiltersToWhereClause(params, cteContext).trim(); + String rowContextFilters = addRowContextFilters(cteContext).trim(); + + // Add non-empty conditions + if (!baseWhereClause.isEmpty()) { + // Remove leading WHERE if present + conditions.add(baseWhereClause.replaceFirst("(?i)^WHERE\\s+", "")); + } + if (!cteFilters.isEmpty()) { + conditions.add(cteFilters.replaceFirst("(?i)^AND\\s+", "")); + } + if (!rowContextFilters.isEmpty()) { + conditions.add(rowContextFilters); + } + + return conditions; } +// private List getRowContextColumns(CTEContext cteContext) { +// return RowContextUtils.getRowContextColumns(cteContext, sqlBuilder); +// } + // private String resolveOrderByOffset(int offset) { // // if (offset <= 0) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java index 74402819889d..18f7c002c8da 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java @@ -134,8 +134,7 @@ public void contributeCTE( latestDate)); // Register the CTE and its column mapping - cteContext.addCTE(cteName, cteSql.toString()); - cteContext.addColumnMapping(programIndicator.getUid(), cteName + ".value"); + cteContext.addCTE(programIndicator, cteSql); } private String getTableName(ProgramIndicator programIndicator) { From 33c9c3dc1d54f0b356791f30231d8c76d15c45da Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Fri, 20 Dec 2024 16:20:14 +0100 Subject: [PATCH 30/30] WIP - spotless --- .../dhis/analytics/common/CTEContext.java | 56 +++++---- .../analytics/common/RowContextUtils.java | 2 +- .../AbstractJdbcEventAnalyticsManager.java | 1 - .../data/JdbcEnrollmentAnalyticsManager.java | 111 ++++++++++-------- 4 files changed, 94 insertions(+), 76 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index 6695d77aa334..cb2c7cde6592 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -27,42 +27,51 @@ */ package org.hisp.dhis.analytics.common; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; import lombok.Getter; import org.apache.commons.text.RandomStringGenerator; import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramStage; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - public class CTEContext { private final Map cteDefinitions = new LinkedHashMap<>(); @Getter private final Map rowContextReferences = new HashMap<>(); - public CteDefinitionWithOffset getDefinitionByItemUid(String itemUid) { return cteDefinitions.get(itemUid); } public void addCTE(ProgramStage programStage, QueryItem item, String cteDefinition, int offset) { - cteDefinitions.put(item.getItem().getUid(), - new CteDefinitionWithOffset(programStage.getUid(), cteDefinition, offset)); + cteDefinitions.put( + item.getItem().getUid(), + new CteDefinitionWithOffset(programStage.getUid(), cteDefinition, offset)); } - public void addCTE(ProgramStage programStage, QueryItem item, String cteDefinition, int offset, boolean isRowContext) { - cteDefinitions.put(item.getItem().getUid(), new CteDefinitionWithOffset(programStage.getUid(), cteDefinition, offset, isRowContext)); + public void addCTE( + ProgramStage programStage, + QueryItem item, + String cteDefinition, + int offset, + boolean isRowContext) { + cteDefinitions.put( + item.getItem().getUid(), + new CteDefinitionWithOffset(programStage.getUid(), cteDefinition, offset, isRowContext)); } /** * Adds a CTE definition to the context. + * * @param programIndicator The program indicator * @param cteDefinition The CTE definition (the SQL query) */ public void addCTE(ProgramIndicator programIndicator, String cteDefinition) { - cteDefinitions.put(programIndicator.getUid(), new CteDefinitionWithOffset(programIndicator.getUid(), cteDefinition)); + cteDefinitions.put( + programIndicator.getUid(), + new CteDefinitionWithOffset(programIndicator.getUid(), cteDefinition)); } public void addCTEFilter(String name, String ctedefinition) { @@ -92,16 +101,20 @@ public String getCTEDefinition() { sb.append(", "); } CteDefinitionWithOffset cteDef = entry.getValue(); - sb.append(cteDef.asCteName(entry.getKey())).append(" AS (").append(entry.getValue().cteDefinition).append(")"); + sb.append(cteDef.asCteName(entry.getKey())) + .append(" AS (") + .append(entry.getValue().cteDefinition) + .append(")"); first = false; } return sb.toString(); } + // Rename to item uid public Set getCTENames() { return cteDefinitions.keySet(); } - + public boolean containsCteFilter(String cteFilterName) { return cteDefinitions.containsKey(cteFilterName); } @@ -124,19 +137,19 @@ public static class CteDefinitionWithOffset { private boolean isProgramIndicator = false; // Whether the CTE is a filter private boolean isFilter = false; - private final static String PS_PREFIX = "ps"; - private final static String PI_PREFIX = "pi"; + private static final String PS_PREFIX = "ps"; + private static final String PI_PREFIX = "pi"; public CteDefinitionWithOffset(String programStageUid, String cteDefinition, int offset) { this.programStageUid = programStageUid; this.cteDefinition = cteDefinition; this.offset = offset; - this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build() - .generate(5); + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); this.isRowContext = false; } - public CteDefinitionWithOffset(String programStageUid, String cteDefinition, int offset, boolean isRowContext) { + public CteDefinitionWithOffset( + String programStageUid, String cteDefinition, int offset, boolean isRowContext) { this(programStageUid, cteDefinition, offset); this.isRowContext = isRowContext; } @@ -146,8 +159,7 @@ public CteDefinitionWithOffset(String programIndicatorUid, String cteDefinition) this.programIndicatorUid = programIndicatorUid; this.programStageUid = null; this.offset = -999; - this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build() - .generate(5); + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); this.isRowContext = false; this.isProgramIndicator = true; } @@ -157,15 +169,13 @@ public CteDefinitionWithOffset(String cteFilterName, String cteDefinition, boole this.programIndicatorUid = null; this.programStageUid = null; this.offset = -999; - this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build() - .generate(5); + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); this.isRowContext = false; this.isProgramIndicator = false; this.isFilter = isFilter; } /** - * * @param uid the uid of an dimension item or ProgramIndicator * @return the name of the CTE */ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java index d2753a7547e2..e2a898f5582b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java @@ -50,7 +50,7 @@ public static List getRowContextWhereClauses(CTEContext cteContext) { Map rowCtxRefs = cteContext.getRowContextReferences(); for (String alias : rowCtxRefs.values()) { whereClauses.add("%s.value is null".formatted(alias)); - //whereClauses.add("%s.exists_flag = true".formatted(alias)); + // whereClauses.add("%s.exists_flag = true".formatted(alias)); } return whereClauses; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 2e22c86905cd..6264ea17cfd2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -93,7 +93,6 @@ import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.CTEContext.CteDefinitionWithOffset; -import org.hisp.dhis.analytics.common.CTEUtils; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.util.AnalyticsUtils; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 88d51872d3b1..05829e26844f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -44,7 +44,6 @@ import static org.hisp.dhis.util.DateUtils.toMediumDate; import com.google.common.collect.Sets; - import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -722,21 +721,26 @@ protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteC // ed."lJTx9EZ1dk1" as "EPEcjy3FWmI[-1].lJTx9EZ1dk1" - columns.add(""" + columns.add( + """ %s.%s as %s - """.formatted(cteDef.getAlias(), quote(colName), quote(alias))); + """ + .formatted(cteDef.getAlias(), quote(colName), quote(alias))); if (cteDef.isRowContext()) { // Add additional status and exists columns for row context // (ed."lJTx9EZ1dk1" IS NOT NULL) as "EPEcjy3FWmI[-1].lJTx9EZ1dk1.exists", // ed.eventstatus as "EPEcjy3FWmI[-1].lJTx9EZ1dk1.status" - columns.add(""" + columns.add( + """ (%s.%s IS NOT NULL) as %s - """.formatted(cteDef.getAlias(), quote(colName), quote(alias + ".exists"))); + """ + .formatted(cteDef.getAlias(), quote(colName), quote(alias + ".exists"))); - columns.add(""" + columns.add( + """ %s.eventstatus as %s - """.formatted(cteDef.getAlias(), quote(alias + ".status"))); - + """ + .formatted(cteDef.getAlias(), quote(alias + ".status"))); } return String.join(", ", columns); } @@ -953,23 +957,24 @@ private String createOrderType(int offset) { // New methods // - private void handleProgramIndicatorCte(QueryItem item, CTEContext cteContext, EventQueryParams params) { + private void handleProgramIndicatorCte( + QueryItem item, CTEContext cteContext, EventQueryParams params) { ProgramIndicator pi = (ProgramIndicator) item.getItem(); if (item.hasRelationshipType()) { programIndicatorSubqueryBuilder.contributeCTE( - pi, - item.getRelationshipType(), - getAnalyticsType(), - params.getEarliestStartDate(), - params.getLatestEndDate(), - cteContext); + pi, + item.getRelationshipType(), + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); } else { programIndicatorSubqueryBuilder.contributeCTE( - pi, - getAnalyticsType(), - params.getEarliestStartDate(), - params.getLatestEndDate(), - cteContext); + pi, + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); } } @@ -983,11 +988,11 @@ private CTEContext getCteDefinitions(EventQueryParams params) { String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid(); if (item.isProgramIndicator()) { handleProgramIndicatorCte(item, cteContext, params); - } else if(item.hasProgramStage()) { + } else if (item.hasProgramStage()) { // TODO what is this condition would be good to give it a name if (item.getProgramStage().getRepeatable() - && item.hasRepeatableStageParams() - && !item.getRepeatableStageParams().simpleStageValueExpected()) { + && item.hasRepeatableStageParams() + && !item.getRepeatableStageParams().simpleStageValueExpected()) { // TODO: Implement repeatable stage items log.warn("Repeatable stage items are not yet supported"); @@ -996,7 +1001,8 @@ private CTEContext getCteDefinitions(EventQueryParams params) { String colName = quote(item.getItemName()); boolean hasEventStatusColumn = rowContextAllowedAndNeeded(params, item); - var cteSql = """ + var cteSql = + """ SELECT enrollment, %s,%s @@ -1007,40 +1013,42 @@ private CTEContext getCteDefinitions(EventQueryParams params) { FROM %s WHERE eventstatus != 'SCHEDULE' AND ps = '%s' - """.formatted( - colName, - hasEventStatusColumn ? " eventstatus," : "", - eventTableName, - item.getProgramStage().getUid()); + """ + .formatted( + colName, + hasEventStatusColumn ? " eventstatus," : "", + eventTableName, + item.getProgramStage().getUid()); cteContext.addCTE( - item.getProgramStage(), - item, - cteSql, - createOffset2(item.getProgramStageOffset()), - hasEventStatusColumn - ); + item.getProgramStage(), + item, + cteSql, + createOffset2(item.getProgramStageOffset()), + hasEventStatusColumn); } else { // Generate CTE for program stage items String colName = quote(item.getItemName()); String cteSql = - """ + """ -- Generate CTE for program stage items SELECT DISTINCT ON (enrollment) enrollment, %s as value%s FROM %s WHERE eventstatus != 'SCHEDULE' AND ps = '%s' ORDER BY enrollment, occurreddate DESC, created DESC %s %s""" - .formatted( - colName, - rowContextAllowedAndNeeded(params, item) ? " ,true as exists_flag" : "", - eventTableName, - item.getProgramStage().getUid(), - createOffset(item.getProgramStageOffset()), LIMIT_1); - - cteContext.addCTE(item.getProgramStage(), item, cteSql, createOffset2(item.getProgramStageOffset())); + .formatted( + colName, + rowContextAllowedAndNeeded(params, item) ? " ,true as exists_flag" : "", + eventTableName, + item.getProgramStage().getUid(), + createOffset(item.getProgramStageOffset()), + LIMIT_1); + + cteContext.addCTE( + item.getProgramStage(), item, cteSql, createOffset2(item.getProgramStageOffset())); } } } @@ -1128,11 +1136,13 @@ private String buildEnrollmentQueryWithCte(EventQueryParams params) { // 5. Add joins for each CTE for (String itemUid : cteContext.getCTENames()) { CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(itemUid); - String join = """ + String join = + """ LEFT JOIN %s %s ON %s.enrollment = ax.enrollment - """.formatted(cteDef.asCteName(itemUid), cteDef.getAlias(), cteDef.getAlias()); + """ + .formatted(cteDef.asCteName(itemUid), cteDef.getAlias(), cteDef.getAlias()); sql.append("\n").append(join); if (cteDef.isProgramStage()) { // equivalent to original OFFSET 1 LIMIT 1 but more efficient @@ -1145,8 +1155,7 @@ private String buildEnrollmentQueryWithCte(EventQueryParams params) { // 6. Where clause List conditions = collectWhereConditions(params, cteContext); if (!conditions.isEmpty()) { - sql.append(" WHERE ") - .append(String.join(" AND ", conditions)); + sql.append(" WHERE ").append(String.join(" AND ", conditions)); } // 7. Order by @@ -1181,9 +1190,9 @@ private List collectWhereConditions(EventQueryParams params, CTEContext return conditions; } -// private List getRowContextColumns(CTEContext cteContext) { -// return RowContextUtils.getRowContextColumns(cteContext, sqlBuilder); -// } + // private List getRowContextColumns(CTEContext cteContext) { + // return RowContextUtils.getRowContextColumns(cteContext, sqlBuilder); + // } // private String resolveOrderByOffset(int offset) { //