TASK-6217 - Solr improvements: Stats and Update

opencga-analysis/pom.xml

@@ -43,6 +43,10 @@
             <groupId>org.opencb.commons</groupId>
             <artifactId>commons-datastore-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.opencb.commons</groupId>
+            <artifactId>commons-datastore-solr</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.opencb.opencga</groupId>
             <artifactId>opencga-core</artifactId>
@@ -248,6 +252,40 @@
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-test-framework</artifactId>
             <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-auth</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.htrace</groupId>
+                    <artifactId>htrace-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpcore</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpmime</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java

@@ -82,9 +82,11 @@
 import org.opencb.opencga.storage.core.StorageEngineFactory;
 import org.opencb.opencga.storage.core.StoragePipelineResult;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
+import org.opencb.opencga.storage.core.exceptions.VariantSearchException;
 import org.opencb.opencga.storage.core.metadata.VariantMetadataFactory;
 import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager;
 import org.opencb.opencga.storage.core.metadata.models.*;
+import org.opencb.opencga.storage.core.metadata.models.project.SearchIndexMetadata;
 import org.opencb.opencga.storage.core.utils.CellBaseUtils;
 import org.opencb.opencga.storage.core.variant.BeaconResponse;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
@@ -98,6 +100,7 @@
 import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser;
 import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor;
 import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchLoadResult;
+import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager;
 
 import java.io.IOException;
 import java.net.URI;
@@ -239,22 +242,6 @@ public List<StoragePipelineResult> index(String study, List<String> files, Strin
                         .index(study, files, UriUtils.createDirectoryUriSafe(outDir), params, token));
     }
 
-    public void secondaryIndexSamples(String study, List<String> samples, ObjectMap params, String token)
-            throws CatalogException, StorageEngineException {
-        secureOperation(VariantSecondaryIndexSamplesOperationTool.ID, study, params, token, engine -> {
-            engine.secondaryIndexSamples(study, samples);
-            return null;
-        });
-    }
-
-    public void removeSearchIndexSamples(String study, List<String> samples, ObjectMap params, String token)
-            throws CatalogException, StorageEngineException {
-        secureOperation("removeSecondaryIndexSamples", study, params, token, engine -> {
-            engine.removeSecondaryIndexSamples(study, samples);
-            return null;
-        });
-    }
-
     public VariantSearchLoadResult secondaryAnnotationIndex(String project, String region, boolean overwrite, ObjectMap params, String token)
             throws CatalogException, StorageEngineException {
         return secureOperationByProject(VariantSecondaryAnnotationIndexOperationTool.ID, project, params, token, engine -> {
@@ -1236,6 +1223,35 @@ public void variantPrune(String project, URI outdir, VariantPruneParams params,
         });
     }
 
+    public List<ObjectMap> getSearchStatus(String project, String token)
+            throws StorageEngineException, CatalogException, IOException, VariantSearchException {
+        List<ObjectMap> results = new ArrayList<>();
+        try (VariantStorageEngine engine = getVariantStorageEngineByProject(project, new ObjectMap(), token)) {
+            VariantSearchManager variantSearchManager = engine.getVariantSearchManager();
+            ProjectMetadata pm = engine.getMetadataManager().getProjectMetadata();
+            SearchIndexMetadata active = pm.getSecondaryAnnotationIndex().getActiveIndex();
+            if (active != null) {
+                String collection = variantSearchManager.buildCollectionName(active);
+                ObjectMap result = new ObjectMap();
+                result.put("metadata", active);
+                if (variantSearchManager.exists(active)) {
+                    result.put("collection", collection);
+                }
+                results.add(result);
+            }
+            for (SearchIndexMetadata stagingIndex : pm.getSecondaryAnnotationIndex().getStagingIndexes()) {
+                String collection = variantSearchManager.buildCollectionName(stagingIndex);
+                ObjectMap result = new ObjectMap();
+                result.put("metadata", stagingIndex);
+                if (variantSearchManager.exists(stagingIndex)) {
+                    result.put("collection", collection);
+                }
+                results.add(result);
+            }
+            return results;
+        }
+    }
+
     // Permission related methods
 
     private interface VariantReadOperation<R> {

opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java

@@ -123,8 +123,8 @@ public class VariantOperationsTest {
     public static Object[][] parameters() {
         return new Object[][]{
 //                {MongoDBVariantStorageEngine.STORAGE_ENGINE_ID},
-                {DummyVariantStorageEngine.STORAGE_ENGINE_ID},
-                {HadoopVariantStorageEngine.STORAGE_ENGINE_ID}
+                {HadoopVariantStorageEngine.STORAGE_ENGINE_ID},
+                {DummyVariantStorageEngine.STORAGE_ENGINE_ID}
         };
     }
 
@@ -174,7 +174,7 @@ public void tearDown() {
 
             try {
                 VariantStorageEngine engine = opencga.getStorageEngineFactory().getVariantStorageEngine(storageEngine, DB_NAME);
-                if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) {
+                if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID) && hadoopExternalResource.isReady()) {
                     VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants_AFTER")).toUri());
                 }
             } catch (Exception e) {
@@ -188,10 +188,6 @@ public void tearDown() {
 
     @BeforeClass
     public static void beforeClass() throws Exception {
-        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
-            solrExternalResource = new VariantSolrExternalResource();
-            solrExternalResource.before();
-        }
     }
 
     @AfterClass
@@ -201,8 +197,9 @@ public static void afterClass() {
             hadoopExternalResource.after();
             hadoopExternalResource = null;
         }
-        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+        if (solrExternalResource != null) {
             solrExternalResource.after();
+            solrExternalResource = null;
         }
     }
 
@@ -244,6 +241,14 @@ private void loadDataset() throws Throwable {
             DummyVariantStorageEngine.configure(opencga.getStorageEngineFactory(), true);
         }
 
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+            if (solrExternalResource == null) {
+                solrExternalResource = new VariantSolrExternalResource();
+                solrExternalResource.before();
+            } else {
+                solrExternalResource.clearCollections();
+            }
+        }
         catalogManager = opencga.getCatalogManager();
         if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
             variantStorageManager = opencga.getVariantStorageManager(solrExternalResource);
@@ -265,6 +270,12 @@ private void loadDataset() throws Throwable {
         }
 
         setUpCatalogManager();
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+            solrExternalResource.configure(variantStorageManager.getVariantStorageEngine(STUDY, token));
+            solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token));
+        }
+
+        dummyVariantSetup(variantStorageManager, STUDY, token);
 
         file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token);
 //            variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token);

opencga-app/app/misc/solr/INSTALL.md

@@ -11,6 +11,6 @@ After compiling and installing OpenCGA, the six Solr config sets are located at
 In order to upload all of them, you need to execute the following commands:
 
 ```
-$ ./bin/solr zk upconfig -n opencga-variant-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-variant-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983
-$ ./bin/solr zk upconfig -n opencga-rga-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-rga-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983
+$ ./bin/solr zk upconfig -n opencga-variant-configset-XYZ -d ~/opencga/build/misc/solr/opencga-variant-configset-XYZ -z localhost:9983
+$ ./bin/solr zk upconfig -n opencga-rga-configset-XYZ -d ~/opencga/build/misc/solr/opencga-rga-configset-XYZ -z localhost:9983
 ```