Merge pull request #3480 from ingef/fix/null-skip-null-searchCache

Fix performance issues with SearchFilter
ingef · Jul 2, 2024 · c0e8d68 · c0e8d68
2 parents 1cd9bf1 + 0fac81a
commit c0e8d68
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 13 deletions.
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java
@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -14,8 +15,6 @@
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter;
 import com.bakdata.conquery.util.search.TrieSearch;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import it.unimi.dsi.fastutil.objects.Object2LongMap;
-import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -35,7 +34,7 @@ public class FilterSearch {
 	 */
 	@JsonIgnore
 	private Map<Searchable, TrieSearch<FrontendValue>> searchCache = new HashMap<>();
-	private Object2LongMap<SelectFilter<?>> totals = new Object2LongOpenHashMap<>();
+	private Map<SelectFilter<?>, Integer> totals = new HashMap<>();
 
 	/**
 	 * From a given {@link FrontendValue} extract all relevant keywords.
@@ -69,12 +68,16 @@ public final List<TrieSearch<FrontendValue>> getSearchesFor(SelectFilter<?> sear
 						 .collect(Collectors.toList());
 	}
 
-	public long getTotal(SelectFilter<?> filter) {
-		return totals.computeIfAbsent(filter, (f) -> filter.getSearchReferences().stream()
-														   .map(searchCache::get)
-														   .flatMap(TrieSearch::stream)
-														   .distinct()
-														   .count());
+	public int getTotal(SelectFilter<?> filter) {
+		return totals.computeIfAbsent(filter, (f) -> {
+			HashSet<FrontendValue> count = new HashSet<>();
+
+			for (TrieSearch<FrontendValue> search : getSearchesFor(filter)) {
+				search.iterator().forEachRemaining(count::add);
+			}
+
+			return count.size();
+		});
 	}
 
 
@@ -116,7 +119,7 @@ public void shrinkSearch(Searchable searchable) {
 	}
 
 	public synchronized void clearSearch() {
-		totals = new Object2LongOpenHashMap<>();
+		totals = new HashMap<>();
 		searchCache = new HashMap<>();
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java
@@ -254,7 +254,7 @@ private Cursor<FrontendValue> listAllValues(SelectFilter<?> searchable) {
 		return new Cursor<>(Iterators.filter(iterators, seen::add));
 	}
 
-	private long countAllValues(SelectFilter<?> searchable) {
+	private int countAllValues(SelectFilter<?> searchable) {
 		final Namespace namespace = namespaces.get(searchable.getDataset().getId());
 
 		return namespace.getFilterSearch().getTotal(searchable);
@@ -313,10 +313,10 @@ public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List<S
 	/**
 	 * Container class to pair number of available values and Cursor for those values.
 	 */
-	private record CursorAndLength(Cursor<FrontendValue> values, long size) {
+	private record CursorAndLength(Cursor<FrontendValue> values, int size) {
 	}
 
-	public record AutoCompleteResult(List<FrontendValue> values, long total) {
+	public record AutoCompleteResult(List<FrontendValue> values, int total) {
 	}
 
 	public record ResolvedFilterResult(ConnectorId tableId, String filterId, Collection<FrontendValue> value) {