Add count filter and select converter (#3196)

Co-authored-by: Torben Meyer <[email protected]>

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/filter/CountFilterConverter.java

@@ -0,0 +1,56 @@
+package com.bakdata.conquery.sql.conversion.cqelement.concept.filter;
+
+import java.util.List;
+import java.util.Set;
+
+import com.bakdata.conquery.models.common.Range;
+import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.model.filter.ConceptFilter;
+import com.bakdata.conquery.sql.conversion.model.filter.CountCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.Filters;
+import com.bakdata.conquery.sql.conversion.model.select.CountSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
+import org.jooq.Field;
+
+public class CountFilterConverter implements FilterConverter<Range.LongRange, CountFilter> {
+
+	@Override
+	public ConceptFilter convert(CountFilter countFilter, FilterContext<Range.LongRange> context) {
+
+		SqlSelect rootSelect = new ExtractingSqlSelect<>(
+				context.getConceptTables().getPredecessorTableName(ConceptCteStep.PREPROCESSING),
+				countFilter.getColumn().getName(),
+				Object.class
+		);
+
+		Field<Object> qualifiedRootSelect = context.getConceptTables().qualifyOnPredecessorTableName(ConceptCteStep.AGGREGATION_SELECT, rootSelect.aliased());
+		CountSqlSelect countSqlSelect = new CountSqlSelect(qualifiedRootSelect, countFilter.getName(), CountSqlSelect.CountType.fromBoolean(countFilter.isDistinct()));
+
+		Field<Object> qualifiedCountGroupBy = context.getConceptTables().qualifyOnPredecessorTableName(ConceptCteStep.AGGREGATION_FILTER, countSqlSelect.aliased());
+		CountCondition countFilterCondition = new CountCondition(qualifiedCountGroupBy, context.getValue());
+
+		return new ConceptFilter(
+				SqlSelects.builder()
+						  .forPreprocessingStep(List.of(rootSelect))
+						  .forAggregationSelectStep(List.of(countSqlSelect))
+						  .build(),
+				Filters.builder()
+					   .group(List.of(countFilterCondition))
+					   .build()
+		);
+	}
+
+	@Override
+	public Set<ConceptCteStep> requiredSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.AGGREGATION_FILTER);
+	}
+
+	@Override
+	public Class<CountFilter> getConversionClass() {
+		return CountFilter.class;
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/select/CountSelectConverter.java

@@ -0,0 +1,45 @@
+package com.bakdata.conquery.sql.conversion.cqelement.concept.select;
+
+import java.util.List;
+
+import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.model.select.CountSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
+import org.jooq.Field;
+
+public class CountSelectConverter implements SelectConverter<CountSelect> {
+
+	@Override
+	public SqlSelects convert(CountSelect countSelect, SelectContext context) {
+
+		SqlSelect rootSelect = new ExtractingSqlSelect<>(
+				context.getConceptTables().getPredecessorTableName(ConceptCteStep.PREPROCESSING),
+				countSelect.getColumn().getName(),
+				Object.class
+		);
+
+		Field<Object> qualifiedRootSelect = context.getConceptTables().qualifyOnPredecessorTableName(ConceptCteStep.AGGREGATION_SELECT, rootSelect.aliased());
+		CountSqlSelect countSqlSelect = new CountSqlSelect(qualifiedRootSelect, countSelect.getName(), CountSqlSelect.CountType.fromBoolean(countSelect.isDistinct()));
+
+		ExtractingSqlSelect<Integer> finalSelect = new ExtractingSqlSelect<>(
+				context.getConceptTables().getPredecessorTableName(ConceptCteStep.FINAL),
+				countSqlSelect.aliased().getName(),
+				Integer.class
+		);
+
+		return SqlSelects.builder()
+						 .forPreprocessingStep(List.of(rootSelect))
+						 .forAggregationSelectStep(List.of(countSqlSelect))
+						 .forFinalStep(List.of(finalSelect))
+						 .build();
+	}
+
+	@Override
+	public Class<? extends CountSelect> getConversionClass() {
+		return CountSelect.class;
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java

@@ -15,13 +15,15 @@
 import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.BigMultiSelectFilterConverter;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.CountFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.DateDistanceFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.FilterConversions;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.FilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.MultiSelectFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.NumberFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.SingleSelectFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.SumFilterConverter;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.select.CountSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.DateDistanceSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.ExistsSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.FirstValueSelectConverter;
@@ -79,7 +81,8 @@ default List<SelectConverter<? extends Select>> customizeSelectConverters(List<S
 				new MultiSelectFilterConverter(),
 				new SingleSelectFilterConverter(),
 				new NumberFilterConverter(),
-				new SumFilterConverter()
+				new SumFilterConverter(),
+				new CountFilterConverter()
 		);
 	}
 
@@ -90,7 +93,8 @@ default List<SelectConverter<? extends Select>> getDefaultSelectConverters() {
 				new RandomValueSelectConverter(),
 				new DateDistanceSelectConverter(DEFAULT_DATE_NOW_SUPPLIER),
 				new ExistsSelectConverter(),
-				new SumSelectConverter()
+				new SumSelectConverter(),
+				new CountSelectConverter()
 		);
 	}
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java

@@ -0,0 +1,17 @@
+package com.bakdata.conquery.sql.conversion.model.filter;
+
+import com.bakdata.conquery.models.common.IRange;
+import org.jooq.Field;
+
+public class CountCondition extends RangeCondition {
+
+	public CountCondition(Field<?> column, IRange<? extends Number, ?> range) {
+		super(column, range);
+	}
+
+	@Override
+	public FilterType type() {
+		return FilterType.GROUP;
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java

@@ -1,19 +1,12 @@
 package com.bakdata.conquery.sql.conversion.model.filter;
 
 import com.bakdata.conquery.models.common.Range;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Condition;
 import org.jooq.Field;
 
-@RequiredArgsConstructor
-public class DateDistanceCondition implements FilterCondition {
+public class DateDistanceCondition extends RangeCondition {
 
-	private final Field<Integer> distanceColumn;
-	private final Range.LongRange range;
-
-	@Override
-	public Condition filterCondition() {
-		return ConditionUtil.rangeCondition(distanceColumn.coerce(Long.class), range);
+	public DateDistanceCondition(Field<Integer> column, Range.LongRange range) {
+		super(column, range);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java

@@ -1,19 +1,12 @@
 package com.bakdata.conquery.sql.conversion.model.filter;
 
 import com.bakdata.conquery.models.common.IRange;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Condition;
 import org.jooq.Field;
 
-@RequiredArgsConstructor
-public class NumberCondition implements FilterCondition {
+public class NumberCondition extends RangeCondition {
 
-	private final Field<? extends Number> numberColumn;
-	private final IRange<? extends Number, ?> range;
-
-	@Override
-	public Condition filterCondition() {
-		return ConditionUtil.rangeCondition(numberColumn, range);
+	public NumberCondition(Field<? extends Number> column, IRange<? extends Number, ?> range) {
+		super(column, range);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java

@@ -0,0 +1,19 @@
+package com.bakdata.conquery.sql.conversion.model.filter;
+
+import com.bakdata.conquery.models.common.IRange;
+import lombok.RequiredArgsConstructor;
+import org.jooq.Condition;
+import org.jooq.Field;
+
+@RequiredArgsConstructor
+abstract class RangeCondition implements FilterCondition {
+
+	private final Field<?> column;
+	private final IRange<?,?> range;
+
+	@Override
+	public Condition filterCondition() {
+		return ConditionUtil.rangeCondition(column, range);
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java

@@ -1,19 +1,12 @@
 package com.bakdata.conquery.sql.conversion.model.filter;
 
 import com.bakdata.conquery.models.common.IRange;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Condition;
 import org.jooq.Field;
 
-@RequiredArgsConstructor
-public class SumCondition implements FilterCondition {
+public class SumCondition extends RangeCondition {
 
-	private final Field<? extends Number> sumColumn;
-	private final IRange<? extends Number, ?> range;
-
-	@Override
-	public Condition filterCondition() {
-		return ConditionUtil.rangeCondition(sumColumn, range);
+	public SumCondition(Field<? extends Number> column, IRange<? extends Number, ?> range) {
+		super(column, range);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/CountSqlSelect.java

@@ -0,0 +1,43 @@
+package com.bakdata.conquery.sql.conversion.model.select;
+
+import java.math.BigDecimal;
+
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
+
+@RequiredArgsConstructor
+@EqualsAndHashCode
+public class CountSqlSelect implements SqlSelect {
+
+	private final Field<Object> columnToCount;
+	private final String alias;
+	private final CountType countType;
+
+	@Override
+	public Field<Integer> select() {
+		Field<Integer> countField = countType == CountType.DISTINCT ? DSL.countDistinct(columnToCount) : DSL.count(columnToCount);
+		return countField.as(alias);
+	}
+
+	@Override
+	public Field<BigDecimal> aliased() {
+		return DSL.field(alias, BigDecimal.class);
+	}
+
+	@Override
+	public String columnName() {
+		return columnToCount.getName();
+	}
+
+	public enum CountType {
+		DEFAULT,
+		DISTINCT;
+
+		public static CountType fromBoolean(boolean value) {
+			return value ? DISTINCT : DEFAULT;
+		}
+	}
+
+}

backend/src/test/resources/tests/sql/filter/count/content.csv

@@ -0,0 +1,9 @@
+pid,value
+1,0.9
+1,0.8
+1,0.7
+2,0.1
+2,0.1
+3,1.0
+3,0.5
+4,19.0

backend/src/test/resources/tests/sql/filter/count/count.json

@@ -0,0 +1,70 @@
+{
+  "label": "COUNT filter",
+  "type": "SQL_TEST",
+  "expectedCsv": "expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "type": "AND",
+      "children": [
+        {
+          "ids": [
+            "number"
+          ],
+          "type": "CONCEPT",
+          "label": "vs",
+          "tables": [
+            {
+              "id": "number.number_connector",
+              "filters": [
+                {
+                  "filter": "number.number_connector.value",
+                  "type": "REAL_RANGE",
+                  "value": {
+                    "min": 2
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "label": "number",
+      "type": "TREE",
+      "connectors": [
+        {
+          "label": "number_connector",
+          "table": "table1",
+          "filters": {
+            "label": "value",
+            "description": "xy",
+            "column": "table1.value",
+            "type": "COUNT"
+          }
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/filter/count/content.csv",
+        "name": "table1",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "value",
+            "type": "REAL"
+          }
+        ]
+      }
+    ]
+  }
+}

backend/src/test/resources/tests/sql/filter/count/expected.csv

@@ -0,0 +1,4 @@
+pid,dates
+1,{}
+2,{}
+3,{}

backend/src/test/resources/tests/sql/selects/count/content.csv

@@ -0,0 +1,8 @@
+pid,value
+1,0.9
+2,
+2,0.1
+2,0.3
+3,0.1
+3,1.0
+4,1.0