Merge pull request #2498 from opencb/TASK-6789

TASK-6789 - Variant query fails with large skip values
opencb · Aug 30, 2024 · 1b764f1 · 1b764f1
2 parents 90c4fe8 + 0c6551b
commit 1b764f1
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 1 deletion.
diff --git a/.../java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/.../java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java
@@ -262,6 +262,10 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) {
             intersect = true;
         } else {
             if (options.getBoolean(QueryOptions.COUNT)) {
+                // The SearchIndex is better than anyone in terms of counting
+                intersect = true;
+            } else if (options.getInt(QueryOptions.SKIP, 0) > 500) {
+                // Large "skip" queries should use SearchIndex when possible
                 intersect = true;
             } else {
                 // TODO: Improve this heuristic

diff --git a/...va/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java b/...va/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java
@@ -2,16 +2,22 @@
 
 import com.google.common.base.Throwables;
 import org.junit.ClassRule;
+import org.junit.Test;
 import org.opencb.biodata.models.variant.Variant;
 import org.opencb.commons.datastore.core.DataResult;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator;
+import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager;
 import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource;
 
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager.SEARCH_ENGINE_ID;
+
 /**
  * Created on 22/12/17.
  *
@@ -87,4 +93,32 @@ public DataResult rank(int limit, Query query, String field, boolean asc) {
             throw Throwables.propagate(e);
         }
     }
+
+    @Test
+    public void testQueryExecutor() {
+        assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions()
+                        .append(QueryOptions.LIMIT, 10)
+                        .append(QueryOptions.COUNT, false)
+                        .append(QueryOptions.SKIP, 0)).getSource(),
+                not(containsString(SEARCH_ENGINE_ID)));
+
+        assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions()
+                        .append(QueryOptions.LIMIT, 10)
+                        .append(QueryOptions.COUNT, true)
+                        .append(QueryOptions.SKIP, 0)).getSource(),
+                containsString(SEARCH_ENGINE_ID));
+
+        assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions()
+                        .append(QueryOptions.LIMIT, 10)
+                        .append(QueryOptions.COUNT, false)
+                        .append(QueryOptions.SKIP, 100)).getSource(),
+                not(containsString(SEARCH_ENGINE_ID)));
+
+        assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions()
+                        .append(QueryOptions.LIMIT, 10)
+                        .append(QueryOptions.COUNT, false)
+                        .append(QueryOptions.SKIP, 1000)).getSource(),
+                containsString(SEARCH_ENGINE_ID));
+
+    }
 }