Merge branch 'release-2.8.x' into TASK-4872

opencb · Oct 2, 2023 · 1eb3fc4 · 1eb3fc4
2 parents c2bc6c7 + 8ba86bb
commit 1eb3fc4
Show file tree

Hide file tree

Showing 112 changed files with 592 additions and 328 deletions.
diff --git a/...pencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/...pencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java
@@ -100,6 +100,7 @@ public class VariantFileIndexerOperationManager extends OperationManager {
     private List<File> filesToIndex;
     private CatalogStorageMetadataSynchronizer synchronizer;
     private boolean fullSynchronize = false;
+    private boolean force;
 
     public VariantFileIndexerOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) {
         super(variantStorageManager, engine);
@@ -138,6 +139,7 @@ private void check(String study, ObjectMap params, String token) throws Exceptio
         }
         resume = params.getBoolean(VariantStorageOptions.RESUME.key());
         skipIndexedFiles = params.getBoolean(SKIP_INDEXED_FILES);
+        force = params.getBoolean(VariantStorageOptions.FORCE.key());
 
         // Obtain the type of analysis (transform, load or index)
         step = getType(load, transform);
@@ -589,6 +591,7 @@ private List<File> filterTransformFiles(List<File> fileList, boolean resume) thr
                         break;
                     case VariantIndexStatus.INDEXING:
                     case VariantIndexStatus.TRANSFORMING:
+                    case VariantIndexStatus.LOADING:
                         if (resume) {
                             filteredFiles.add(file);
                         } else {
@@ -603,14 +606,17 @@ private List<File> filterTransformFiles(List<File> fileList, boolean resume) thr
                         }
                         break;
                     case VariantIndexStatus.TRANSFORMED:
-                    case VariantIndexStatus.LOADING:
                     case VariantIndexStatus.READY:
                     default:
                         String msg = "We can only " + step + " VCF files not transformed, the status is " + indexStatus;
-                        if (skipIndexedFiles) {
-                            logger.warn(msg);
+                        if (force) {
+                            filteredFiles.add(file);
                         } else {
-                            throw new StorageEngineException(msg);
+                            if (skipIndexedFiles) {
+                                logger.warn(msg);
+                            } else {
+                                throw new StorageEngineException(msg);
+                            }
                         }
                         break;
                 }

diff --git a/...java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java b/...java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java
@@ -508,6 +508,16 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
             }
             fileSamplesMap.put(fileMetadata.getName(), samples);
             allSamples.addAll(fileMetadata.getSamples());
+            if (samples.size() > 100) {
+                // Try to reuse value.
+                // If the file holds more than 100 samples, it's most likely this same set of samples is already present
+                for (Set<String> value : fileSamplesMap.values()) {
+                    if (value.equals(samples)) {
+                        fileSamplesMap.put(fileMetadata.getName(), value);
+                        break;
+                    }
+                }
+            }
         }
 
         if (!indexedFilesFromStorage.isEmpty()) {

diff --git a/...pencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/...pencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java
@@ -592,7 +592,9 @@ private void computeSignatureFitting() throws IOException, ToolException, Catalo
         StringBuilder scriptParams = new StringBuilder("R CMD Rscript --vanilla ")
                 .append("/opt/opencga/signature.tools.lib/scripts/signatureFit")
                 .append(" --catalogues=/data/input/").append(cataloguesFile.getName())
-                .append(" --outdir=/data/output");
+                .append(" --outdir=/data/output")
+                .append(" --commonsigtier=T2");
+
         if (StringUtils.isNotEmpty(getFitMethod())) {
             scriptParams.append(" --fitmethod=").append(getFitMethod());
         }

diff --git a/...c/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/...c/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java
@@ -90,6 +90,7 @@ protected void check() throws Exception {
         params.put(VariantStorageOptions.ANNOTATE.key(), indexParams.isAnnotate());
         params.putIfNotEmpty(VariantStorageOptions.ANNOTATOR.key(), indexParams.getAnnotator());
         params.put(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), indexParams.isOverwriteAnnotations());
+        params.put(VariantStorageOptions.FORCE.key(), indexParams.isForceReload());
         params.put(VariantStorageOptions.RESUME.key(), indexParams.isResume());
         params.put(VariantStorageOptions.NORMALIZATION_SKIP.key(), indexParams.getNormalizationSkip());
         params.putIfNotEmpty(VariantStorageOptions.NORMALIZATION_REFERENCE_GENOME.key(), indexParams.getReferenceGenome());

diff --git a/...a-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/...a-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java
@@ -18,6 +18,7 @@
 
 import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
@@ -95,6 +96,7 @@ public AlignmentAnalysisTest(String storageEngine) {
     private CatalogManager catalogManager;
     private VariantStorageManager variantStorageManager;
 
+    @ClassRule
     public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource();
     public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource();
 

diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java
@@ -6,23 +6,28 @@
 import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
 import org.apache.solr.core.NodeConfig;
 import org.junit.rules.ExternalResource;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
 import org.opencb.commons.datastore.solr.SolrManager;
 import org.opencb.opencga.core.common.GitRepositoryState;
+import org.opencb.opencga.core.common.TimeUtils;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.*;
+import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri;
 
 public class RgaSolrExtenalResource extends ExternalResource {
 
     public String coreName = "opencga_rga_test";
 
     private SolrClient solrClient;
     protected boolean embeded = true;
+    private Class<?> testClass;
 
     public RgaSolrExtenalResource() {
         this(true);
@@ -32,16 +37,23 @@ public RgaSolrExtenalResource(boolean embeded) {
         this.embeded = embeded;
     }
 
+    @Override
+    public Statement apply(Statement base, Description description) {
+        testClass = description.getTestClass();
+        return super.apply(base, description);
+    }
+
     @Override
     protected void before() throws Throwable {
         super.before();
 
-        Path rootDir = getTmpRootDir();
+        Path rootDir = Paths.get("target/test-data", "junit-rga-solr-" + TimeUtils.getTimeMillis());
+        Files.createDirectories(rootDir);
 
         String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.getInstance().getBuildVersion();
         String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.getInstance().getBuildVersion();
-        copyConfigSetConfiguration(mainConfigSet, "managed-schema");
-        copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema");
+        copyConfigSetConfiguration(mainConfigSet, "managed-schema", rootDir);
+        copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema", rootDir);
 
         String solrHome = rootDir.resolve("solr").toString();
 
@@ -78,15 +90,15 @@ protected void after() {
         }
     }
 
-    private void copyConfigSetConfiguration(String configSet, String managedSchemaFile) throws IOException {
+    private void copyConfigSetConfiguration(String configSet, String managedSchemaFile, Path rootDir) throws IOException {
         // Copy configuration
-        getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml");
-        getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema");
-        getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json");
-        getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt");
-        getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt");
-        getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt");
-        getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt");
+        getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml", rootDir);
+        getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema", rootDir);
+        getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json", rootDir);
+        getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt", rootDir);
+        getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt", rootDir);
+        getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt", rootDir);
+        getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt", rootDir);
     }
 
     public RgaEngine configure(StorageConfiguration storageConfiguration) {

diff --git a/...ysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/...ysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java
@@ -38,6 +38,7 @@
 import org.opencb.opencga.core.common.YesNoAuto;
 import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.cohort.Cohort;
 import org.opencb.opencga.core.models.cohort.CohortCreateParams;
 import org.opencb.opencga.core.models.common.IndexStatus;
@@ -55,6 +56,7 @@
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.testclassification.duration.LongTests;
 import org.opencb.opencga.core.tools.result.ExecutionResult;
+import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
@@ -73,6 +75,7 @@
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 @RunWith(Parameterized.class)
 @Category(LongTests.class)
@@ -285,6 +288,29 @@ public void setUpCatalogManager() throws Exception {
 
     }
 
+    @Test
+    public void testVariantFileReload() throws Exception {
+
+        try {
+            toolRunner.execute(VariantIndexOperationTool.class, STUDY,
+                    new VariantIndexParams()
+                            .setForceReload(false)
+                            .setFile(file.getId()),
+                    Paths.get(opencga.createTmpOutdir()), "index_reload", token);
+            fail("Should have thrown an exception");
+        } catch (ToolException e) {
+            assertEquals(StorageEngineException.class, e.getCause().getClass());
+            assertEquals("We can only INDEX VCF files not transformed, the status is READY", e.getCause().getMessage());
+        }
+
+        toolRunner.execute(VariantIndexOperationTool.class, STUDY,
+                new VariantIndexParams()
+                        .setForceReload(true)
+                        .setFile(file.getId()),
+                Paths.get(opencga.createTmpOutdir()), "index_reload", token);
+
+    }
+
     @Test
     public void testVariantSecondaryAnnotationIndex() throws Exception {
 

diff --git a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile
@@ -17,14 +17,15 @@ RUN apt-get update -y && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt
     ## Installation dependencies using R install.packages() is slower than apt-get but final size is 400GB smaller.
     R -e "install.packages(c('BiocManager', 'RCircos', 'nnls', 'ggplot2', 'jsonlite', 'optparse', 'knitr', 'configr', 'dplyr', 'rmarkdown', 'tidyr', 'httr', 'kinship2', 'limSolve'))" && \
     R -e "BiocManager::install('BiocStyle')" && \
+    R -e "BiocManager::install('BSgenome.Hsapiens.UCSC.hg38')" && \
     ## signature.tools.lib installation \
-    R -e 'install.packages(c("devtools", "getopt"), repos="https://www.stats.bris.ac.uk/R/")' && \
+    R -e 'install.packages(c("devtools", "getopt"), repos=c("http://cran.rstudio.com/", "https://www.stats.bris.ac.uk/R/"))' && \
     git clone https://github.com/Nik-Zainal-Group/signature.tools.lib.git /opt/opencga/signature.tools.lib
 
 WORKDIR /opt/opencga/signature.tools.lib
 
 RUN git fetch origin --tags && \
-    git checkout tags/v2.4.1 && \
+    git checkout tags/v2.4.2 && \
     sed -i '/Mmusculus/d' DESCRIPTION && \
     sed -i '/Cfamiliaris/d' DESCRIPTION && \
     sed -i '/1000genomes/d' DESCRIPTION && \

diff --git a/...in/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/...in/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java
@@ -380,6 +380,7 @@ private void index() throws ToolException {
                 cliOptions.genericVariantIndexOptions.family,
                 cliOptions.genericVariantIndexOptions.somatic,
                 cliOptions.genericVariantIndexOptions.load,
+                cliOptions.genericVariantIndexOptions.forceReload,
                 cliOptions.genericVariantIndexOptions.loadSplitData,
                 cliOptions.genericVariantIndexOptions.loadMultiFileData,
                 cliOptions.genericVariantIndexOptions.loadSampleIndex,

diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java
@@ -1,5 +1,5 @@
 /*
-* Copyright 2015-2023-04-18 OpenCB
+* Copyright 2015-2023-07-28 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.

diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java
@@ -1,5 +1,5 @@
 /*
-* Copyright 2015-2023-04-18 OpenCB
+* Copyright 2015-2023-07-28 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.

diff --git a/...-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java b/...-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java
@@ -38,8 +38,8 @@ public CustomFilesCommandExecutor(ObjectMap options, String token, ClientConfigu
 
     public RestResponse<File> upload() throws Exception {
 //        ObjectMap params = new ObjectMap()
-        options.append("fileFormat", ParamUtils.defaultString(String.valueOf(options.get("fileFormat")), File.Format.UNKNOWN.toString()))
-                .append("bioformat", ParamUtils.defaultString(String.valueOf(options.get("bioformat")), File.Bioformat.UNKNOWN.toString()));
+        options.append("fileFormat", options.getString("fileFormat", File.Format.UNKNOWN.toString()))
+                .append("bioformat", options.getString("bioformat", File.Bioformat.UNKNOWN.toString()));
 //        //If the DEPRECATED parameter fileFormat has set we only override it if the new parameter format is also set
 //        params.append("fileFormat", ParamUtils.defaultString(commandOptions.format, params.getString("fileFormat")));
 //        params.putIfNotEmpty("study", commandOptions.study);

diff --git a/...c/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/...c/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java
@@ -862,6 +862,7 @@ private RestResponse<Job> runIndex() throws Exception {
             putNestedIfNotNull(beanParams, "family",commandOptions.family, true);
             putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true);
             putNestedIfNotNull(beanParams, "load",commandOptions.load, true);
+            putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true);
             putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true);
             putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true);
             putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true);

diff --git a/...va/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/...va/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java
@@ -530,6 +530,7 @@ private RestResponse<Job> indexVariant() throws Exception {
             putNestedIfNotNull(beanParams, "family",commandOptions.family, true);
             putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true);
             putNestedIfNotNull(beanParams, "load",commandOptions.load, true);
+            putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true);
             putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true);
             putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true);
             putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true);
@@ -600,6 +601,7 @@ private RestResponse<Job> launcherVariantIndex() throws Exception {
             putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true);
             putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true);
             putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true);
+            putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true);
             putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true);
             putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true);
             putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true);