From 86daceb1fe1fa8ea34987235f9c26383e7a3655c Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 16 Jul 2024 16:08:41 +0200 Subject: [PATCH] Implement DistinctSelect conversion (#3494) --- .../select/connector/DistinctSelect.java | 7 + .../conquery/models/types/ResultType.java | 40 ++++-- .../dialect/HanaSqlFunctionProvider.java | 12 -- .../dialect/PostgreSqlFunctionProvider.java | 12 -- .../dialect/SqlFunctionProvider.java | 13 +- .../sql/conversion/model/QueryStep.java | 4 + .../model/QueryStepTransformer.java | 14 +- .../model/select/DistinctSelectConverter.java | 135 ++++++++++++++++++ .../execution/DefaultResultSetProcessor.java | 39 ++++- .../sql/execution/ResultSetProcessor.java | 10 ++ .../tests/sql/combined/combined.json | 8 +- .../resources/tests/sql/combined/expected.csv | 12 +- .../tests/sql/selects/distinct/content.csv | 14 ++ .../tests/sql/selects/distinct/distinct.json | 65 +++++++++ .../tests/sql/selects/distinct/expected.csv | 6 + 15 files changed, 344 insertions(+), 47 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java create mode 100644 backend/src/test/resources/tests/sql/selects/distinct/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/distinct/distinct.json create mode 100644 backend/src/test/resources/tests/sql/selects/distinct/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index 4cd7c460a1..50a1316eff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -9,6 +9,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "DISTINCT", base = Select.class) @@ -29,4 +31,9 @@ public Aggregator createAggregator() { public ResultType getResultType() { return new ResultType.ListT(super.getResultType()); } + + @Override + public SelectConverter createConverter() { + return new DistinctSelectConverter(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java index f4ede6725c..02c82abfe1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java @@ -49,9 +49,7 @@ protected String print(PrintSettings cfg, @NonNull Object f) { public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; - protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { - throw new UnsupportedOperationException("ResultType list of type %s not supported for now.".formatted(getClass().getSimpleName())); - } + protected abstract List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; public static ResultType resolveResultType(MajorTypeId majorTypeId) { return switch (majorTypeId) { @@ -99,6 +97,11 @@ public String print(PrintSettings cfg, Object f) { public Boolean getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getBoolean(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getBooleanList(resultSet, columnIndex); + } } @@ -120,6 +123,11 @@ public String print(PrintSettings cfg, Object f) { public Integer getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getInteger(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getIntegerList(resultSet, columnIndex); + } } @CPSType(id = "NUMERIC", base = ResultType.class) @@ -140,6 +148,11 @@ public String print(PrintSettings cfg, Object f) { public Double getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDouble(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getDoubleList(resultSet, columnIndex); + } } @CPSType(id = "DATE", base = ResultType.class) @@ -162,6 +175,11 @@ public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetPr return resultSetProcessor.getDate(resultSet, columnIndex); } + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getDateList(resultSet, columnIndex); + } + public static String print(Number num, DateTimeFormatter formatter) { return formatter.format(LocalDate.ofEpochDay(num.intValue())); } @@ -276,6 +294,11 @@ public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultS public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) { return new BigDecimal(f.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits()); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getMoneyList(resultSet, columnIndex); + } } @CPSType(id = "LIST", base = ResultType.class) @@ -313,11 +336,12 @@ public String typeInfo() { @Override public List getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { - if (elementType.getClass() == DateRangeT.class || elementType.getClass() == StringT.class) { - return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor); - } - // TODO handle all other list types properly - throw new UnsupportedOperationException("Other result type lists not supported for now."); + return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor); + } + + @Override + protected List> getFromResultSetAsList(final ResultSet resultSet, final int columnIndex, final ResultSetProcessor resultSetProcessor) { + throw new UnsupportedOperationException("Nested lists not supported in SQL mode"); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 26a9c4eb57..a4c40c0d86 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -155,18 +155,6 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { return predecessor; } - @Override - public Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { - return DSL.field( - "{0}({1}, {2} {3})", - String.class, - DSL.keyword("STRING_AGG"), - stringField, - delimiter, - DSL.orderBy(orderByFields) - ); - } - @Override public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index 23128c0fc4..5d23751f56 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -164,18 +164,6 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { .build(); } - @Override - public Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { - return DSL.field( - "{0}({1}, {2} {3})", - String.class, - DSL.keyword("string_agg"), - stringField, - delimiter, - DSL.orderBy(orderByFields) - ); - } - @Override public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { Field asMultirange = rangeAgg(columnDateRange); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 1d7e09644e..9a0de8834b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -98,8 +98,6 @@ public interface SqlFunctionProvider { */ QueryStep unnestValidityDate(QueryStep predecessor, String cteName); - Field stringAggregation(Field stringField, Field delimiter, List> orderByFields); - /** * Aggregates the start and end columns of the validity date of entries into one compound string expression. *

@@ -140,6 +138,17 @@ public interface SqlFunctionProvider { */ Field yearQuarter(Field dateField); + default Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { + return DSL.field( + "{0}({1}, {2} {3})", + String.class, + DSL.keyword("string_agg"), + stringField, + delimiter, + DSL.orderBy(orderByFields) + ); + } + default Field concat(List> fields) { String concatenated = fields.stream() // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index f2f9b80cac..204bdd59e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -41,6 +41,10 @@ public class QueryStep { */ @Builder.Default boolean unionAll = true; + /** + * Determines if the select should be distinct. + */ + boolean selectDistinct; /** * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index 1c80762f63..04dcf0cd84 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -9,6 +9,7 @@ import org.jooq.Select; import org.jooq.SelectConditionStep; import org.jooq.SelectHavingStep; +import org.jooq.SelectSelectStep; import org.jooq.impl.DSL; /** @@ -70,10 +71,15 @@ private CommonTableExpression toCte(QueryStep queryStep) { private Select toSelectStep(QueryStep queryStep) { - Select selectStep = this.dslContext - .select(queryStep.getSelects().all()) - .from(queryStep.getFromTables()) - .where(queryStep.getConditions()); + SelectSelectStep selectClause; + if (queryStep.isSelectDistinct()) { + selectClause = dslContext.selectDistinct(queryStep.getSelects().all()); + } + else { + selectClause = dslContext.select(queryStep.getSelects().all()); + } + + Select selectStep = selectClause.from(queryStep.getFromTables()).where(queryStep.getConditions()); if (queryStep.isGroupBy()) { selectStep = ((SelectConditionStep) selectStep).groupBy(queryStep.getGroupBy()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java new file mode 100644 index 0000000000..ec2bf25a5f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -0,0 +1,135 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +/** + * 

+ *  The two additional CTEs this aggregator creates:
+ * 	
    
+ * 	    
  1. 
+ * 	        Select distinct values of a column.
+ *            {@code
+ * 	        	"distinct" as (
+ *     				select distinct "pid", "column"
+ *     				from "event_filter"
+ *  			)
+ *            }
+ * 	    
    2. 
+ * 	    
  2. 
+ * 	        String agg all distinct values of the column.
+ *            {@code
+ * 	        "aggregated" as (
+ *    			 select
+ *    			   "select-1-distinct"."pid",
+ *    			   string_agg(cast("column" as varchar), cast(' ' as varchar) ) as "select-1"
+ *    			 from "distinct"
+ *    			 group by "pid"
+ *   			)
+ *            }
+ * 	    
    3. 
+ * 	

+ *
+ */ +public class DistinctSelectConverter implements SelectConverter { + + @Getter + @RequiredArgsConstructor + private enum DistinctSelectCteStep implements CteStep { + + DISTINCT_SELECT("distinct", null), + STRING_AGG("aggregated", DISTINCT_SELECT); + + private final String suffix; + private final DistinctSelectCteStep predecessor; + } + + @Override + public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext selectContext) { + + String alias = selectContext.getNameGenerator().selectName(distinctSelect); + + ConnectorSqlTables tables = selectContext.getTables(); + FieldWrapper preprocessingSelect = new FieldWrapper<>(field(name(tables.getRootTable(), distinctSelect.getColumn().getName())).as(alias)); + + QueryStep distinctSelectCte = createDistinctSelectCte(preprocessingSelect, alias, selectContext); + QueryStep aggregatedCte = createAggregationCte(selectContext, preprocessingSelect, distinctSelectCte, alias); + + ExtractingSqlSelect finalSelect = preprocessingSelect.qualify(tables.cteName(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .preprocessingSelect(preprocessingSelect) + .additionalPredecessor(Optional.of(aggregatedCte)) + .finalSelect(finalSelect) + .build(); + } + + private static QueryStep createAggregationCte( + SelectContext selectContext, + FieldWrapper preprocessingSelect, + QueryStep distinctSelectCte, + String alias + ) { + SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); + Field castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR); + Field aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(castedColumn)) + .as(alias); + + SqlIdColumns ids = distinctSelectCte.getQualifiedSelects().getIds(); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(new FieldWrapper<>(aggregatedColumn)) + .build(); + + return QueryStep.builder() + .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias)) + .selects(selects) + .fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName())) + .groupBy(ids.toFields()) + .predecessor(distinctSelectCte) + .build(); + } + + private static QueryStep createDistinctSelectCte( + FieldWrapper preprocessingSelect, + String alias, + SelectContext selectContext + ) { + // values to aggregate must be event-filtered first + String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.EVENT_FILTER); + ExtractingSqlSelect qualified = preprocessingSelect.qualify(eventFilterTable); + SqlIdColumns ids = selectContext.getIds().qualify(eventFilterTable); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(qualified) + .build(); + + return QueryStep.builder() + .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias)) + .selectDistinct(true) + .selects(selects) + .fromTable(QueryStep.toTableLike(eventFilterTable)) + .build(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java index 62b430d9ed..415087b74b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java @@ -6,8 +6,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; -import java.util.Collections; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; import lombok.RequiredArgsConstructor; @@ -67,12 +67,47 @@ public List> getDateRangeList(ResultSet resultSet, int columnIndex @Override public List getStringList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, (string) -> string); + } + + @Override + public List getBooleanList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, Boolean::valueOf); + } + + @Override + public List getIntegerList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, Integer::valueOf); + } + + @Override + public List getDoubleList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, Double::valueOf); + } + + @Override + public List getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString( + resultSet, + columnIndex, + (string) -> BigDecimal.valueOf(Double.parseDouble(string)) + .setScale(2, RoundingMode.HALF_EVEN) + ); + } + + @Override + public List getDateList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, (string) -> Date.valueOf(string).toLocalDate().toEpochDay()); + } + + private List fromString(ResultSet resultSet, int columnIndex, Function parseFunction) throws SQLException { String arrayExpression = resultSet.getString(columnIndex); if (arrayExpression == null) { - return Collections.emptyList(); + return null; } return Arrays.stream(arrayExpression.split(String.valueOf(ResultSetProcessor.UNIT_SEPARATOR))) .filter(Predicate.not(String::isBlank)) + .map(parseFunction) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java index 9f63f58c36..6581ec8b21 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java @@ -26,4 +26,14 @@ public interface ResultSetProcessor { List> getDateRangeList(ResultSet resultSet, int columnIndex) throws SQLException; List getStringList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getBooleanList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getIntegerList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getDoubleList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getDateList(ResultSet resultSet, int columnIndex) throws SQLException; } diff --git a/backend/src/test/resources/tests/sql/combined/combined.json b/backend/src/test/resources/tests/sql/combined/combined.json index e7a04869d9..119b3c3f58 100644 --- a/backend/src/test/resources/tests/sql/combined/combined.json +++ b/backend/src/test/resources/tests/sql/combined/combined.json @@ -20,7 +20,8 @@ "concept.connector.event-date", "concept.connector.event_duration_sum", "concept.connector.first_value", - "concept.connector.sum_distinct" + "concept.connector.sum_distinct", + "concept.connector.distinct_select" ], "filters": [ { @@ -101,6 +102,11 @@ "column": "table.value", "type": "FIRST" }, + { + "label": "distinct_select", + "column": "table.value", + "type": "DISTINCT" + }, { "name": "sum_distinct", "type": "SUM", diff --git a/backend/src/test/resources/tests/sql/combined/expected.csv b/backend/src/test/resources/tests/sql/combined/expected.csv index 4902ce95be..f1c7b15856 100644 --- a/backend/src/test/resources/tests/sql/combined/expected.csv +++ b/backend/src/test/resources/tests/sql/combined/expected.csv @@ -1,6 +1,6 @@ -result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector exists,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept tree_label first_test_column -1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,A1 -2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,1,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,B2 -3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,1,{2012-01-01/2012-01-03},3,0.5,0.5,A1 -4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,1,{2012-01-01/2012-01-04},4,0.5,0.5,B2 -5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,1,{2012-01-01/2012-01-05},5,1.0,1.5, +result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector exists,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept connector distinct_select,concept tree_label first_test_column +1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,{1.0},A1 +2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,1,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,"{1.0,1.01}",B2 +3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,1,{2012-01-01/2012-01-03},3,0.5,0.5,{0.5},A1 +4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,1,{2012-01-01/2012-01-04},4,0.5,0.5,{0.5},B2 +5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,1,{2012-01-01/2012-01-05},5,1.0,1.5,"{0.5,1.0}", diff --git a/backend/src/test/resources/tests/sql/selects/distinct/content.csv b/backend/src/test/resources/tests/sql/selects/distinct/content.csv new file mode 100644 index 0000000000..03265056db --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/distinct/content.csv @@ -0,0 +1,14 @@ +pid,datum,value +1,2012-01-01,"f" +1,2012-01-02,"f" +2,2010-07-15, +3,2012-01-01,"f" +3,2012-01-02,"m" +4,2012-01-01,"f" +4,2012-01-02,"m" +4,2012-01-03,"f" +4,2012-01-04,"m" +5,2012-01-01,"f " +5,2012-01-02,"m " +5,2012-01-03,"f" +5,2012-01-04,"m" diff --git a/backend/src/test/resources/tests/sql/selects/distinct/distinct.json b/backend/src/test/resources/tests/sql/selects/distinct/distinct.json new file mode 100644 index 0000000000..4ff17cd780 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/distinct/distinct.json @@ -0,0 +1,65 @@ +{ + "type": "QUERY_TEST", + "label": "DISTINCT select", + "expectedCsv": "tests/sql/selects/distinct/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.select" + ] + } + ] + } + }, + "concepts": [ + { + "label": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "validityDates": { + "label": "datum", + "column": "table.datum" + }, + "selects": { + "name": "select", + "type": "DISTINCT", + "column": "table.value" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/distinct/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "value", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/distinct/expected.csv b/backend/src/test/resources/tests/sql/selects/distinct/expected.csv new file mode 100644 index 0000000000..6b7033299b --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/distinct/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept select +1,{2012-01-01/2012-01-02},{f} +2,{2010-07-15/2010-07-15}, +3,{2012-01-01/2012-01-02},"{f,m}" +4,{2012-01-01/2012-01-04},"{f,m}" +5,{2012-01-01/2012-01-04},"{f,f ,m,m }"