Skip to content

Commit

Permalink
WIP - Fix basic filters
Browse files Browse the repository at this point in the history
  • Loading branch information
@luciano-fiandesio
luciano-fiandesio committed Dec 18, 2024
1 parent dab977a commit f383bac
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 43 deletions.
4 changes: 4 additions & 0 deletions ...vices/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,4 +68,8 @@ public Set<String> getCTENames() {
public String getColumnMapping(String columnId) {
return columnMappings.getOrDefault(columnId, columnId);
}

public boolean containsCteFilter(String cteFilterName) {
return cteDefinitions.containsKey(cteFilterName);
}
}
19 changes: 19 additions & 0 deletions ...ervices/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.hisp.dhis.analytics.common;

import org.hisp.dhis.common.QueryItem;

public class CTEUtils {

public static String createFilterName(QueryItem queryItem) {
return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase();
}

public static String createFilterNameByIdentifier(String identifier) {
return "filter_" + identifier.replace('.', '_').toLowerCase();
}

public static String getIdentifier(QueryItem queryItem) {
String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default";
return stage + "." + queryItem.getItemId();
}
}
82 changes: 39 additions & 43 deletions ...tics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import static org.apache.commons.lang3.StringUtils.EMPTY;
import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS;
import static org.hisp.dhis.analytics.DataType.BOOLEAN;
import static org.hisp.dhis.analytics.common.CTEUtils.createFilterNameByIdentifier;
import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables;
import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling;
import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE;
Expand All @@ -57,6 +58,7 @@
import org.apache.commons.lang3.StringUtils;
import org.hisp.dhis.analytics.analyze.ExecutionPlanStore;
import org.hisp.dhis.analytics.common.CTEContext;
import org.hisp.dhis.analytics.common.CTEUtils;
import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder;
import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager;
import org.hisp.dhis.analytics.event.EventQueryParams;
Expand Down Expand Up @@ -503,7 +505,7 @@ private String addFiltersToWhereClause(EventQueryParams params) {
return getQueryItemsAndFiltersWhereClause(params, new SqlHelper());
}

private String addCteFiltersToWhereClause(EventQueryParams params) {
private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext cteContext) {
StringBuilder whereClause = new StringBuilder();

// Iterate over each filter and apply the correct condition
Expand All @@ -512,21 +514,32 @@ private String addCteFiltersToWhereClause(EventQueryParams params) {
.filter(QueryItem::hasFilter)
.toList()) {

String cteName = "filter_" + getIdentifier(item).replace('.', '_').toLowerCase();

for (QueryFilter filter : item.getFilters()) {
if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly
whereClause.append(" AND ").append(cteName).append(".value IS NULL");
} else {
String operator = getSqlOperator(filter);
String value = getSqlFilterValue(filter, item);
whereClause
.append(" AND ")
.append(cteName)
.append(".value ")
.append(operator)
.append(" ")
.append(value);
String cteName = CTEUtils.createFilterName(item);

if (cteContext.containsCteFilter(cteName)) {
for (QueryFilter filter : item.getFilters()) {
if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly
whereClause.append(" AND ").append(cteName).append(".value IS NULL");
} else {
String operator = getSqlOperator(filter);
String value = getSqlFilterValue(filter, item);
whereClause
.append(" AND ")
.append(cteName)
.append(".value ")
.append(operator)
.append(" ")
.append(value);
}
}
} else {
// If the filter is not part of the CTE, apply it directly to the enrollment table
// using the standard where clause method
String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper());
if (StringUtils.isNotBlank(filters) && filters.trim().startsWith("where")) {
// remove the 'where' keyword
filters = filters.trim().substring(5);
whereClause.append("and ").append(filters);
}
}
}
Expand Down Expand Up @@ -598,26 +611,6 @@ SELECT DISTINCT ON (enrollment) enrollment, %s AS value
.collect(Collectors.joining("\nUNION ALL\n"));
}

private Stream<IdentifiableSql> asSqlCollection(
List<QueryItem> queryItems, EventQueryParams params) {
return Optional.ofNullable(queryItems).orElse(List.of()).stream()
.map(
queryItem -> {
String sql =
queryItem.getFilters().stream()
.map(filter -> toSql(queryItem, filter, params))
.collect(Collectors.joining(" AND "));
return IdentifiableSql.builder()
.identifier(getIdentifier(queryItem))
.sql(sql)
.build();
});
}

private String getIdentifier(QueryItem queryItem) {
String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default";
return stage + "." + queryItem.getItemId();
}

@Override
protected String getSelectClause(EventQueryParams params) {
Expand Down Expand Up @@ -1009,6 +1002,7 @@ private String createOrderType(int offset) {

private CTEContext getCteDefinitions(EventQueryParams params) {
CTEContext cteContext = new CTEContext();

for (QueryItem queryItem : params.getItems()) {
if (queryItem.isProgramIndicator()) {
ProgramIndicator pi = (ProgramIndicator) queryItem.getItem();
Expand All @@ -1029,7 +1023,7 @@ private CTEContext getCteDefinitions(EventQueryParams params) {
params.getLatestEndDate(),
cteContext);
}
} else if (queryItem.hasProgramStage()) {
} else if (queryItem.hasProgramStage() && queryItem.hasProgram()) {
// Generate CTE for program stage items
String cteName =
String.format(
Expand Down Expand Up @@ -1096,10 +1090,10 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext)

// Process repeatable stage filters
itemsByRepeatableFlag.getOrDefault(true, List.of()).stream()
.collect(groupingBy(this::getIdentifier))
.collect(groupingBy(CTEUtils::getIdentifier))
.forEach(
(identifier, items) -> {
String cteName = "filter_" + identifier.replace('.', '_').toLowerCase();
String cteName = createFilterNameByIdentifier(identifier);
String cteSql = buildFilterCteSql(items, params);
cteContext.addCTE(cteName, cteSql);
});
Expand All @@ -1109,9 +1103,11 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext)
.getOrDefault(false, List.of())
.forEach(
queryItem -> {
String cteName = "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase();
String cteSql = buildFilterCteSql(List.of(queryItem), params);
cteContext.addCTE(cteName, cteSql);
if (queryItem.hasProgram() && queryItem.hasProgramStage()) {
String cteName = CTEUtils.createFilterName(queryItem);
String cteSql = buildFilterCteSql(List.of(queryItem), params);
cteContext.addCTE(cteName, cteSql);
}
});
}

Expand Down Expand Up @@ -1154,7 +1150,7 @@ private String buildEnrollmentQueryWithCte2(EventQueryParams params) {
sql.append(" ")
.append(getWhereClause(params))
.append(" ")
.append(addCteFiltersToWhereClause(params));
.append(addCteFiltersToWhereClause(params, cteContext));

// 7. Order by
sql.append(" ").append(getSortClause(params));
Expand Down

0 comments on commit f383bac

Please sign in to comment.