Skip to content

Commit

Permalink
Fix TableExportQuery conversion (#3509)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jnsrnhld
jnsrnhld authored Aug 1, 2024
1 parent 55af633 commit b1ad47a
Show file tree
Hide file tree
Showing 9 changed files with 117 additions and 89 deletions.
5 changes: 5 additions & 0 deletions backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.math.BigDecimal;
import java.util.Map;

import com.bakdata.conquery.models.datasets.Column;
import com.bakdata.conquery.models.events.MajorTypeId;

public class NumberMapUtil {
Expand All @@ -14,4 +15,8 @@ public class NumberMapUtil {
MajorTypeId.INTEGER, Integer.class
);

public static Class<? extends Number> getType(Column column) {
return NUMBER_MAP.get(column.getType());
}

}
9 changes: 5 additions & 4 deletions .../src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ public Condition convertForTableExport(SumFilter<RANGE> filter, FilterContext<RA
Column column = filter.getColumn();
String tableName = column.getTable().getName();
String columnName = column.getName();
Field<Number> field = DSL.field(DSL.name(tableName, columnName), Number.class);
Class<? extends Number> numberClass = NumberMapUtil.getType(column);
Field<? extends Number> field = DSL.field(DSL.name(tableName, columnName), numberClass);

Column subtractColumn = filter.getSubtractColumn();
if (subtractColumn == null) {
Expand All @@ -180,13 +181,13 @@ public Condition convertForTableExport(SumFilter<RANGE> filter, FilterContext<RA

String subtractColumnName = subtractColumn.getName();
String subtractTableName = subtractColumn.getTable().getName();
Field<Number> subtractField = DSL.field(DSL.name(subtractTableName, subtractColumnName), Number.class);
Field<? extends Number> subtractField = DSL.field(DSL.name(subtractTableName, subtractColumnName), numberClass);
return new SumCondition(field.minus(subtractField), filterContext.getValue()).condition();
}

private CommonAggregationSelect<BigDecimal> createSumAggregationSelect(Column sumColumn, Column subtractColumn, String alias, ConnectorSqlTables tables) {

Class<? extends Number> numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType());
Class<? extends Number> numberClass = NumberMapUtil.getType(sumColumn);
List<ExtractingSqlSelect<?>> preprocessingSelects = new ArrayList<>();

ExtractingSqlSelect<? extends Number> rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass);
Expand Down Expand Up @@ -228,7 +229,7 @@ private CommonAggregationSelect<BigDecimal> createDistinctSumAggregationSelect(
) {
List<ExtractingSqlSelect<?>> preprocessingSelects = new ArrayList<>();

Class<? extends Number> numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType());
Class<? extends Number> numberClass = NumberMapUtil.getType(sumColumn);
ExtractingSqlSelect<? extends Number> rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass);
preprocessingSelects.add(rootSelect);

Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public SqlFilters convertToSqlFilter(NumberFilter<RANGE> filter, FilterContext<R
Column column = filter.getColumn();
ConnectorSqlTables tables = filterContext.getTables();

Class<? extends Number> numberClass = NumberMapUtil.NUMBER_MAP.get(column.getType());
Class<? extends Number> numberClass = NumberMapUtil.getType(column);
ExtractingSqlSelect<? extends Number> rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass);

Field<? extends Number> eventFilterCtePredecessor = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.EVENT_FILTER)).select();
Expand Down
91 changes: 45 additions & 46 deletions ...nd/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package com.bakdata.conquery.sql.conversion.query;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import com.bakdata.conquery.apiv1.query.Query;
import com.bakdata.conquery.apiv1.query.TableExportQuery;
Expand All @@ -28,7 +27,6 @@
import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper;
import com.bakdata.conquery.util.TablePrimaryColumnUtil;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import lombok.RequiredArgsConstructor;
import org.jooq.Condition;
import org.jooq.Field;
Expand Down Expand Up @@ -113,24 +111,17 @@ private static QueryStep convertTable(
Field<Object> primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().getTable(), context.getConfig());
SqlIdColumns ids = new SqlIdColumns(primaryColumn);
String conceptConnectorName = context.getNameGenerator().conceptConnectorName(concept, cqTable.getConnector());
Optional<ColumnDateRange> validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, dateRestriction, context);

List<Field<?>> exportColumns = new ArrayList<>();
exportColumns.add(createSourceInfoSelect(cqTable));

positions.entrySet().stream()
.sorted(Comparator.comparingInt(Map.Entry::getValue))
.map(entry -> createColumnSelect(cqTable, entry))
.forEach(exportColumns::add);
Optional<ColumnDateRange> validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, context);

List<FieldWrapper<?>> exportColumns = initializeFields(cqTable, positions);
Selects selects = Selects.builder()
.ids(ids)
.validityDate(validityDate)
.sqlSelects(exportColumns.stream().map(FieldWrapper::new).collect(Collectors.toList()))
.sqlSelects(exportColumns)
.build();

List<Condition> filters = cqTable.getFilters().stream().map(filterValue -> filterValue.convertForTableExport(ids, context)).toList();
Table<Record> joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, context);
Table<Record> joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, dateRestriction, context);

return QueryStep.builder()
.cteName(conceptConnectorName)
Expand All @@ -140,68 +131,76 @@ private static QueryStep convertTable(
.build();
}

private static Optional<ColumnDateRange> convertTablesValidityDate(CQTable table, String alias, CDateRange dateRestriction, ConversionContext context) {
private static Optional<ColumnDateRange> convertTablesValidityDate(CQTable table, String alias, ConversionContext context) {
if (table.findValidityDate() == null) {
return Optional.of(ColumnDateRange.empty());
}
SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
ColumnDateRange validityDate = functionProvider.forValidityDate(table.findValidityDate(), dateRestriction);
ColumnDateRange validityDate = functionProvider.forValidityDate(table.findValidityDate());
// when exporting tables, we want the validity date as a single-column daterange string expression straightaway
Field<String> asStringExpression = functionProvider.encloseInCurlyBraces(functionProvider.daterangeStringExpression(validityDate));
return Optional.of(ColumnDateRange.of(asStringExpression).asValidityDateRange(alias));
}

private static Field<String> createSourceInfoSelect(CQTable cqTable) {
String tableName = cqTable.getConnector().getTable().getName();
return DSL.val(tableName).as(SharedAliases.SOURCE.getAlias());
}
private static List<FieldWrapper<?>> initializeFields(CQTable cqTable, Map<Column, Integer> positions) {

private static Field<?> createColumnSelect(CQTable table, Map.Entry<Column, Integer> entry) {

Column column = entry.getKey();
Integer columnPosition = entry.getValue();
String columnName = "%s-%s".formatted(column.getName(), columnPosition);

if (!isColumnOfTable(column, table)) {
return DSL.inline(null, Object.class).as(columnName);
Field<?>[] exportColumns = createPlaceholders(positions, cqTable);
for (Column column : cqTable.getConnector().getTable().getColumns()) {
// e.g. date column(s) are handled separately and not part of positions
if (!positions.containsKey(column)) {
continue;
}
int position = positions.get(column) - 1;
exportColumns[position] = createColumnSelect(column, position);
}
return DSL.field(DSL.name(column.getTable().getName(), column.getName()))
.as(columnName);
}

private static boolean isColumnOfTable(Column column, CQTable table) {
return columnIsConnectorColumn(column, table)
|| columnIsSecondaryIdOfConnectorTable(column, table)
|| columnIsConnectorTableColumn(column, table);
return Arrays.stream(exportColumns).map(FieldWrapper::new).collect(Collectors.toList());
}

private static boolean columnIsConnectorTableColumn(Column column, CQTable table) {
return matchesTableColumnOn(table, tableColumn -> tableColumn == column);
}
private static Field<?>[] createPlaceholders(Map<Column, Integer> positions, CQTable cqTable) {

Field<?>[] exportColumns = new Field[positions.size() + 1];
exportColumns[0] = createSourceInfoSelect(cqTable);

// if columns have the same computed position, they can share a common name because they will be unioned over multiple tables anyway
positions.forEach((column, position) -> {
int shifted = position - 1;
Field<?> columnSelect = DSL.inline(null, Object.class).as("%s-%d".formatted(column.getName(), shifted));
exportColumns[shifted] = columnSelect;
});

private static boolean columnIsSecondaryIdOfConnectorTable(Column column, CQTable table) {
return column.getSecondaryId() != null && matchesTableColumnOn(table, tableColumn -> tableColumn.getSecondaryId() == column.getSecondaryId());
return exportColumns;
}

private static boolean matchesTableColumnOn(CQTable table, Predicate<Column> condition) {
return Arrays.stream(table.getConnector().getTable().getColumns()).anyMatch(condition);
private static Field<String> createSourceInfoSelect(CQTable cqTable) {
String tableName = cqTable.getConnector().getTable().getName();
return DSL.val(tableName).as(SharedAliases.SOURCE.getAlias());
}

private static boolean columnIsConnectorColumn(Column column, CQTable table) {
return table.getConnector().getColumn() != null && table.getConnector().getColumn() == column;
private static Field<?> createColumnSelect(Column column, int position) {
String columnName = "%s-%s".formatted(column.getName(), position);
return DSL.field(DSL.name(column.getTable().getName(), column.getName()))
.as(columnName);
}

private static Table<Record> joinConnectorTableWithPrerequisite(
CQTable cqTable,
SqlIdColumns ids,
QueryStep convertedPrerequisite,
CDateRange dateRestriction,
ConversionContext context
) {
SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
Table<Record> connectorTable = DSL.table(DSL.name(cqTable.getConnector().getTable().getName()));
Table<Record> convertedPrerequisiteTable = DSL.table(DSL.name(convertedPrerequisite.getCteName()));
List<Condition> joinOnIds = ids.join(convertedPrerequisite.getQualifiedSelects().getIds());
return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinOnIds);

ColumnDateRange validityDate = functionProvider.forValidityDate(cqTable.findValidityDate());
List<Condition> joinConditions = Stream.concat(
ids.join(convertedPrerequisite.getQualifiedSelects().getIds()).stream(),
Stream.of(functionProvider.dateRestriction(functionProvider.forCDateRange(dateRestriction), validityDate))
).toList();

return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinConditions);
}

}
25 changes: 16 additions & 9 deletions backend/src/test/resources/shared/geschlecht.concept.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,23 @@
}
],
"table": "vers_stamm",
"filters": {
"type": "SELECT",
"name": "geschlecht",
"label": "Geschlecht",
"column": "vers_stamm.geschlecht",
"labels": {
"1": "Weiblich",
"2": "Männlich"
"filters": [
{
"type": "SELECT",
"name": "geschlecht",
"label": "Geschlecht",
"column": "vers_stamm.geschlecht",
"labels": {
"1": "Weiblich",
"2": "Männlich"
}
},
{
"type": "SUM",
"name": "sum",
"column": "vers_stamm.value"
}
},
],
"selects": [
{
"type": "FIRST",
Expand Down
46 changes: 23 additions & 23 deletions backend/src/test/resources/shared/vers_stamm.csv
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
pid,date_start,date_end,geburtsdatum,geschlecht
1,2012-01-01,2012-12-31,1957-01-01,1
2,2012-01-01,2012-12-31,1988-04-01,2
3,2012-03-01,2012-12-31,1989-01-01,2
4,2013-01-01,2013-12-31,1989-01-01,2
5,2012-07-01,2012-12-31,1972-07-01,2
6,2012-01-01,2012-12-31,1960-10-01,2
7,2012-01-01,2012-12-31,1983-01-01,1
8,2012-01-01,2012-12-31,1956-10-01,2
9,2012-01-01,2012-12-31,1951-07-01,2
10,2012-01-01,2012-12-31,1964-04-01,2
11,2012-01-01,2012-12-31,1966-04-01,1
12,2013-01-01,2013-12-31,1966-04-01,1
13,2012-01-01,2012-12-31,1995-01-01,1
14,2012-01-01,2012-12-31,1952-10-01,2
15,2012-01-01,2012-12-31,1983-01-01,1
16,2012-01-01,2012-12-31,1981-01-01,1
17,2012-01-01,2012-12-31,1979-01-01,2
18,2012-01-01,2012-12-31,1959-04-01,1
19,2012-01-01,2012-12-31,1991-10-01,1
20,2012-01-01,2012-12-31,1987-01-01,2
21,2012-01-01,2012-12-31,1952-01-01,1
22,2012-01-01,2012-10-01,1955-01-01,2
pid,date_start,date_end,geburtsdatum,geschlecht,value
1,2012-01-01,2012-12-31,1957-01-01,1,12
2,2012-01-01,2012-12-31,1988-04-01,2,34
3,2012-03-01,2012-12-31,1989-01-01,2,24
4,2013-01-01,2013-12-31,1989-01-01,2,66
5,2012-07-01,2012-12-31,1972-07-01,2,31
6,2012-01-01,2012-12-31,1960-10-01,2,41
7,2012-01-01,2012-12-31,1983-01-01,1,55
8,2012-01-01,2012-12-31,1956-10-01,2,7
9,2012-01-01,2012-12-31,1951-07-01,2,4
10,2012-01-01,2012-12-31,1964-04-01,2,1
11,2012-01-01,2012-12-31,1966-04-01,1,22
12,2013-01-01,2013-12-31,1966-04-01,1,4
13,2012-01-01,2012-12-31,1995-01-01,1,6
14,2012-01-01,2012-12-31,1952-10-01,2,788
15,2012-01-01,2012-12-31,1983-01-01,1,55
16,2012-01-01,2012-12-31,1981-01-01,1,2
17,2012-01-01,2012-12-31,1979-01-01,2,24
18,2012-01-01,2012-12-31,1959-04-01,1,68
19,2012-01-01,2012-12-31,1991-10-01,1,90
20,2012-01-01,2012-12-31,1987-01-01,2,765
21,2012-01-01,2012-12-31,1952-01-01,1,78
22,2012-01-01,2012-10-01,1955-01-01,2,5
4 changes: 4 additions & 0 deletions backend/src/test/resources/shared/vers_stamm.table.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@
{
"name": "geschlecht",
"type": "STRING"
},
{
"name": "value",
"type": "INTEGER"
}
]
}
15 changes: 14 additions & 1 deletion ...end/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/FULL_EXPORT_FORM_SECONDARY_ID.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
},
"form": {
"type": "FULL_EXPORT_FORM",
"dateRange": {
"min": "2012-01-01",
"max": "2012-03-03"
},
"tables": [
{
"type": "CONCEPT",
Expand All @@ -30,7 +34,16 @@
"id": "geschlecht.geschlecht",
"dateColumn": {
"value": "geschlecht.geschlecht.erster_tag"
}
},
"filters": [
{
"filter": "geschlecht.geschlecht.sum",
"type": "REAL_RANGE",
"value": {
"min": 10
}
}
]
}
}
],
Expand Down
9 changes: 4 additions & 5 deletions backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/expected.csv
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
result,dates,source,secondary,table column,vers_stamm date_end,vers_stamm geburtsdatum,vers_stamm geschlecht
1,{2012-03-01/2012-03-01},table,1,A,,,
1,{2012-03-02/2012-03-02},table,1,A,,,
1,{2012-03-04/2012-03-04},table,2,A,,,
1,{2012-01-01/2012-01-01},vers_stamm,,,2012-12-31,1957-01-01,1
result,dates,source,secondary,table column,vers_stamm date_end,vers_stamm geburtsdatum,vers_stamm geschlecht,vers_stamm value
1,{2012-03-01/2012-03-01},table,1,A,,,,
1,{2012-03-02/2012-03-02},table,1,A,,,,
1,{2012-01-01/2012-01-01},vers_stamm,,,2012-12-31,1957-01-01,1,12

0 comments on commit b1ad47a

Please sign in to comment.