diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index 3d8f8b14ee6..82c4eacff75 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -31,10 +31,12 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import lombok.Getter; public class CTEContext { private final Map cteDefinitions = new LinkedHashMap<>(); private final Map columnMappings = new HashMap<>(); + @Getter private final Map rowContextReferences = new HashMap<>(); public void addCTE(String cteName, String cteDefinition) { cteDefinitions.put(cteName, cteDefinition); @@ -44,6 +46,17 @@ public void addColumnMapping(String originalColumn, String cteReference) { columnMappings.put(originalColumn, cteReference); } + /** + * Adds a mapping between a row context column and the CTE name that it references. + * + * @param alias The alias of the row context column, for instance "EPEcjy3FWmI.lJTx9EZ1dk1" + * @param cteName The name of the CTE that the row context column references, for instance + * "ps_epecjy3fwmi_ljtx9ez1dk1" + */ + public void addRowContextColumnMapping(String alias, String cteName) { + rowContextReferences.put(alias, cteName); + } + public String getCTEDefinition() { if (cteDefinitions.isEmpty()) { return ""; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java index 08eb967609d..43511d86568 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java @@ -1,19 +1,46 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package org.hisp.dhis.analytics.common; import org.hisp.dhis.common.QueryItem; public class CTEUtils { - public static String createFilterName(QueryItem queryItem) { - return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); - } + public static String createFilterName(QueryItem queryItem) { + return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); + } - public static String createFilterNameByIdentifier(String identifier) { - return "filter_" + identifier.replace('.', '_').toLowerCase(); - } + public static String createFilterNameByIdentifier(String identifier) { + return "filter_" + identifier.replace('.', '_').toLowerCase(); + } - public static String getIdentifier(QueryItem queryItem) { - String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; - return stage + "." + queryItem.getItemId(); - } + public static String getIdentifier(QueryItem queryItem) { + String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; + return stage + "." + queryItem.getItemId(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java new file mode 100644 index 00000000000..d6341b17682 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.hisp.dhis.db.sql.SqlBuilder; + +public class RowContextUtils { + + public static List getRowContextColumns(CTEContext cteContext, SqlBuilder sqlBuilder) { + List columns = new ArrayList<>(); + Map rowCtxRefs = cteContext.getRowContextReferences(); + for (String aliases : rowCtxRefs.keySet()) { + columns.add(getStatusColumn(aliases, rowCtxRefs.get(aliases), sqlBuilder)); + columns.add(getExistsColumn(aliases, rowCtxRefs.get(aliases), sqlBuilder)); + } + + return columns; + } + + public static List getRowContextWhereClauses(CTEContext cteContext) { + List whereClauses = new ArrayList<>(); + Map rowCtxRefs = cteContext.getRowContextReferences(); + for (String alias : rowCtxRefs.values()) { + whereClauses.add("%s.value is null".formatted(alias)); + whereClauses.add("%s.exists_flag = true".formatted(alias)); + } + return whereClauses; + } + + private static String getExistsColumn( + String aliases, String cteReference, SqlBuilder sqlBuilder) { + return "coalesce(%s.exists_flag, false) as %s" + .formatted(cteReference, sqlBuilder.quote(aliases + ".status.exists")); + } + + private static String getStatusColumn(String alias, String cteReference, SqlBuilder sqlBuilder) { + return "%s_status.status as %s".formatted(cteReference, sqlBuilder.quote(alias)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 0b88484ca5b..f4ff6f8f2c4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -414,7 +414,7 @@ private void addItemSelectColumns( * @param queryItem * @return true when eligible for row context */ - private boolean rowContextAllowedAndNeeded(EventQueryParams params, QueryItem queryItem) { + protected boolean rowContextAllowedAndNeeded(EventQueryParams params, QueryItem queryItem) { return params.getEndpointItem() == ENROLLMENT && params.isRowContext() && queryItem.hasProgramStage() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 9453ba02cbf..41d686faa31 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -47,11 +47,9 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @@ -60,6 +58,7 @@ import org.hisp.dhis.analytics.common.CTEContext; import org.hisp.dhis.analytics.common.CTEUtils; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; +import org.hisp.dhis.analytics.common.RowContextUtils; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.table.AbstractJdbcTableManager; @@ -155,7 +154,7 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { if (params.isAggregatedEnrollments()) { sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); } else { - sql = buildEnrollmentQueryWithCte2(params); + sql = buildEnrollmentQueryWithCte(params); } System.out.println("SQL: " + sql); // FIXME: Remove debug line @@ -524,25 +523,31 @@ private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext ct String operator = getSqlOperator(filter); String value = getSqlFilterValue(filter, item); whereClause - .append(" AND ") - .append(cteName) - .append(".value ") - .append(operator) - .append(" ") - .append(value); + .append(" AND ") + .append(cteName) + .append(".value ") + .append(operator) + .append(" ") + .append(value); } } } else { // If the filter is not part of the CTE, apply it directly to the enrollment table // using the standard where clause method - String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); if (StringUtils.isNotBlank(filters) && filters.trim().startsWith("where")) { // remove the 'where' keyword - filters = filters.trim().substring(5); - whereClause.append("and ").append(filters); + filters = filters.trim().substring(5); + whereClause.append("and ").append(filters); } } } + // Handle the row context case + List rowContextColumns = RowContextUtils.getRowContextWhereClauses(cteContext); + if (!StringUtils.isEmpty(whereClause)) { + whereClause.append(" AND "); + } + whereClause.append(String.join(" AND ", rowContextColumns)); return whereClause.toString(); } @@ -611,7 +616,6 @@ SELECT DISTINCT ON (enrollment) enrollment, %s AS value .collect(Collectors.joining("\nUNION ALL\n")); } - @Override protected String getSelectClause(EventQueryParams params) { List selectCols = @@ -1035,42 +1039,54 @@ private CTEContext getCteDefinitions(EventQueryParams params) { String eventTableName = ANALYTICS_EVENT + queryItem.getProgram().getUid(); String cteSql = - String.format( - """ + """ -- Generate CTE for program stage items - SELECT DISTINCT ON (enrollment) enrollment, %s as value + SELECT DISTINCT ON (enrollment) enrollment, %s as value, %s FROM %s WHERE eventstatus != 'SCHEDULE' AND ps = '%s' - -- AND %s IS NOT NULL - ORDER BY enrollment, occurreddate DESC, created DESC""", - colName, eventTableName, queryItem.getProgramStage().getUid(), colName); + ORDER BY enrollment, occurreddate DESC, created DESC""" + .formatted( + colName, + rowContextAllowedAndNeeded(params, queryItem) ? "true as exists_flag" : "", + eventTableName, + queryItem.getProgramStage().getUid()); cteContext.addCTE(cteName, cteSql); - - String alias = - quote(queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid()); + String unquotedAlias = + queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(); cteContext.addColumnMapping( queryItem.getProgramStage().getUid() + "." + queryItem.getItem().getUid(), - cteName + ".value as " + alias); + cteName + ".value as " + quote(unquotedAlias)); + + if (rowContextAllowedAndNeeded( + params, queryItem)) { // TODO original condition && !isEmpty(columnAndAlias.alias)) { + + String statusCteSql = + """ + select + distinct on (enrollment) + enrollment, + eventstatus as status + from + %s + where + eventstatus != 'SCHEDULE' + and ps = '%s' + order by + enrollment, + occurreddate desc, + created desc + """ + .formatted(eventTableName, queryItem.getProgramStage().getUid()); + cteContext.addCTE(cteName + "_status", statusCteSql); + cteContext.addRowContextColumnMapping(unquotedAlias, cteName); + } } } return cteContext; } - private String buildFilterCteSql(QueryItem item, EventQueryParams params) { - String tableName = params.getTableName(); - String columnName = getColumn(item); - String filterCondition = - item.getFilters().stream() - .map(filter -> toSql(item, filter, params)) - .collect(Collectors.joining(" AND ")); - - return String.format( - "SELECT enrollment, %s AS value " + "FROM %s " + "WHERE %s", - columnName, tableName, filterCondition); - } - private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) { // Combine items and item filters List queryItems = @@ -1111,7 +1127,7 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) }); } - private String buildEnrollmentQueryWithCte2(EventQueryParams params) { + private String buildEnrollmentQueryWithCte(EventQueryParams params) { StringBuilder sql = new StringBuilder(); @@ -1131,7 +1147,8 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { List selectCols = ListUtils.distinctUnion( params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, - getSelectColumnsWithCTE(params, cteContext)); + getSelectColumnsWithCTE(params, cteContext), + getRowContextColumns(cteContext)); sql.append("SELECT ").append(String.join(",\n ", selectCols)); // 4. From clause @@ -1161,148 +1178,134 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) { return sql.toString(); } - private String getBasicSelectColumns() { - return String.join( - ",", - "ax.enrollment", - "trackedentity", - "enrollmentdate", - "occurreddate", - "storedby", - "createdbydisplayname", - "lastupdatedbydisplayname", - "lastupdated", - "ST_AsGeoJSON(enrollmentgeometry)", - "longitude", - "latitude", - "ouname", - "ounamehierarchy", - "oucode", - "enrollmentstatus", - "ax.\"ou\""); - } - - private String resolveOrderByOffset(int offset) { - - if (offset <= 0) { - return "desc"; - } - return "asc"; - } - - private String buildAllRankedEventsCTEs(List items) { - StringBuilder ctes = new StringBuilder(); - Set processedCombinations = new HashSet<>(); - - for (QueryItem item : items) { - if (!item.hasProgramStage()) { - continue; - } - - String stageUid = item.getProgramStage().getUid(); - int offset = createOffset2(item.getProgramStageOffset()); - String order = resolveOrderByOffset(item.getProgramStageOffset()); - - // Create unique key for this combination to avoid duplicate CTEs - String key = stageUid + "_" + offset + "_" + order; - if (processedCombinations.contains(key)) { - continue; - } - - if (!ctes.isEmpty()) { - ctes.append(",\n"); - } - - String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); - String columnName = quote(item.getItem().getUid()); - - ctes.append( - String.format( - """ - RankedEvents_%s as ( - select enrollment, %s as value, eventstatus, - row_number() over (partition by enrollment order by occurreddate %s, created %s) as rn - from %s - where eventstatus != 'SCHEDULE' - and ps = '%s' - ) - """, - key, columnName, order, order, eventTableName, stageUid)); - - processedCombinations.add(key); - } - - return !ctes.isEmpty() ? "with " + ctes + "\n" : ""; - } - - private String buildValueColumns(List items) { - StringBuilder columns = new StringBuilder(); - - for (QueryItem item : items) { - if (!item.hasProgramStage()) { - continue; - } - - String stageUid = item.getProgramStage().getUid(); - int offset = createOffset2(item.getProgramStageOffset()); - String key = - stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); - - String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; - String alias = "re_" + key; - - if (!columns.isEmpty()) { - columns.append(",\n"); - } - - // Add value column - columns.append( - String.format( - "%s.value as %s", - alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid()))); - - // Add exists column - columns.append( - String.format( - ",\n(%s.enrollment is not null) as %s", - alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".exists"))); - - // Add status column - columns.append( - String.format( - ",\n%s.eventstatus as %s", - alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + ".status"))); - } - - return columns.toString(); + private List getRowContextColumns(CTEContext cteContext) { + return RowContextUtils.getRowContextColumns(cteContext, sqlBuilder); } - private String buildFromClauseWithJoins(EventQueryParams params, List items) { - StringBuilder fromClause = new StringBuilder(); - - // Start with base table - fromClause.append("\nfrom ").append(params.getTableName()).append(" as ax"); - - // Add joins for each item - for (QueryItem item : items) { - if (!item.hasProgramStage()) { - continue; - } - - String stageUid = item.getProgramStage().getUid(); - int offset = createOffset2(item.getProgramStageOffset()); - String key = - stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); - String alias = "re_" + key; - - fromClause.append( - String.format( - "\nleft join RankedEvents_%s %s on ax.enrollment = %s.enrollment and %s.rn = %d", - key, alias, alias, alias, offset + 1)); - } - - return fromClause.toString(); - } + // private String resolveOrderByOffset(int offset) { + // + // if (offset <= 0) { + // return "desc"; + // } + // return "asc"; + // } + + // private String buildAllRankedEventsCTEs(List items) { + // StringBuilder ctes = new StringBuilder(); + // Set processedCombinations = new HashSet<>(); + // + // for (QueryItem item : items) { + // if (!item.hasProgramStage()) { + // continue; + // } + // + // String stageUid = item.getProgramStage().getUid(); + // int offset = createOffset2(item.getProgramStageOffset()); + // String order = resolveOrderByOffset(item.getProgramStageOffset()); + // + // // Create unique key for this combination to avoid duplicate CTEs + // String key = stageUid + "_" + offset + "_" + order; + // if (processedCombinations.contains(key)) { + // continue; + // } + // + // if (!ctes.isEmpty()) { + // ctes.append(",\n"); + // } + // + // String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid().toLowerCase(); + // String columnName = quote(item.getItem().getUid()); + // + // ctes.append( + // String.format( + // """ + // RankedEvents_%s as ( + // select enrollment, %s as value, eventstatus, + // row_number() over (partition by enrollment order by occurreddate %s, created + // %s) as rn + // from %s + // where eventstatus != 'SCHEDULE' + // and ps = '%s' + // ) + // """, + // key, columnName, order, order, eventTableName, stageUid)); + // + // processedCombinations.add(key); + // } + // + // return !ctes.isEmpty() ? "with " + ctes + "\n" : ""; + // } + // + // private String buildValueColumns(List items) { + // StringBuilder columns = new StringBuilder(); + // + // for (QueryItem item : items) { + // if (!item.hasProgramStage()) { + // continue; + // } + // + // String stageUid = item.getProgramStage().getUid(); + // int offset = createOffset2(item.getProgramStageOffset()); + // String key = + // stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + // + // String offsetLabel = offset == 0 ? "[0]" : "[-" + offset + "]"; + // String alias = "re_" + key; + // + // if (!columns.isEmpty()) { + // columns.append(",\n"); + // } + // + // // Add value column + // columns.append( + // String.format( + // "%s.value as %s", + // alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid()))); + // + // // Add exists column + // columns.append( + // String.format( + // ",\n(%s.enrollment is not null) as %s", + // alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + + // ".exists"))); + // + // // Add status column + // columns.append( + // String.format( + // ",\n%s.eventstatus as %s", + // alias, quote(stageUid + offsetLabel + "." + item.getItem().getUid() + + // ".status"))); + // } + // + // return columns.toString(); + // } + // + // private String buildFromClauseWithJoins(EventQueryParams params, List items) { + // StringBuilder fromClause = new StringBuilder(); + // + // // Start with base table + // fromClause.append("\nfrom ").append(params.getTableName()).append(" as ax"); + // + // // Add joins for each item + // for (QueryItem item : items) { + // if (!item.hasProgramStage()) { + // continue; + // } + // + // String stageUid = item.getProgramStage().getUid(); + // int offset = createOffset2(item.getProgramStageOffset()); + // String key = + // stageUid + "_" + offset + "_" + resolveOrderByOffset(item.getProgramStageOffset()); + // String alias = "re_" + key; + // + // fromClause.append( + // String.format( + // "\nleft join RankedEvents_%s %s on ax.enrollment = %s.enrollment and %s.rn = %d", + // key, alias, alias, alias, offset + 1)); + // } + // + // return fromClause.toString(); + // } protected String getSortClause(EventQueryParams params) { if (params.isSorting()) {