TASK-6780 - Port Patch 3.2.1 -> 4.0.0 - Xetabase 2.2.1 -> 3.0.0

opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java

@@ -16,9 +16,13 @@
 
 package org.opencb.opencga.analysis;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.commons.utils.FileUtils;
+import org.opencb.opencga.core.config.AnalysisTool;
 import org.opencb.opencga.core.config.Configuration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,10 +32,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 public class ConfigurationUtils {
     private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class);
 
+    private ConfigurationUtils() {
+        throw new IllegalStateException("Utility class");
+    }
     /**
      * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml.
      *
@@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome)
                     .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml"));
         }
     }
+
+    public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException {
+        List<AnalysisTool> tools = new ArrayList<>();
+        for (AnalysisTool tool : configuration.getAnalysis().getTools()) {
+            if (tool.getId().equals(toolId)) {
+                tools.add(tool);
+            }
+        }
+        if (CollectionUtils.isEmpty(tools)) {
+            throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file");
+        }
+        if (tools.size() == 1) {
+            return tools.get(0).getVersion();
+        }
+        String defaultVersion = null;
+        for (AnalysisTool tool : tools) {
+            if (tool.isDefaultVersion()) {
+                if (!StringUtils.isEmpty(defaultVersion)) {
+                    throw new ToolException("More than one default version found for tool ID '" + toolId + "'");
+                } else {
+                    defaultVersion = tool.getVersion();
+                }
+            }
+        }
+        if (StringUtils.isEmpty(defaultVersion)) {
+            throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true");
+        }
+        return defaultVersion;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java

@@ -30,8 +30,10 @@
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.opencga.analysis.ConfigurationUtils;
 import org.opencb.opencga.analysis.clinical.InterpretationAnalysis;
 import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils;
+import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis;
 import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.utils.ParamUtils;
@@ -44,10 +46,10 @@
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.individual.Individual;
 import org.opencb.opencga.core.response.OpenCGAResult;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
+import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis {
     private String clinicalAnalysisId;
     private String sampleId;
     private ClinicalAnalysis.Type clinicalAnalysisType;
+    private String exomiserVersion;
 
     private ClinicalAnalysis clinicalAnalysis;
 
@@ -97,8 +100,7 @@ protected void check() throws Exception {
         try {
             clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(),
                     token);
-        } catch (
-                CatalogException e) {
+        } catch (CatalogException e) {
             throw new ToolException(e);
         }
         if (clinicalAnalysisQueryResult.getNumResults() != 1) {
@@ -117,6 +119,7 @@ protected void check() throws Exception {
         }
         sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId();
 
+        // Check clinical analysis type
         if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) {
             clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY;
         } else {
@@ -125,6 +128,13 @@ protected void check() throws Exception {
         logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(),
                 clinicalAnalysisType);
 
+        // Check exomiser version
+        if (StringUtils.isEmpty(exomiserVersion)) {
+            // Missing exomiser version use the default one
+            exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration);
+            logger.warn("Missing exomiser version, using the default {}", exomiserVersion);
+        }
+
         // Update executor params with OpenCGA home and session ID
         setUpStorageEngineExecutor(studyId);
     }
@@ -134,25 +144,28 @@ protected void run() throws ToolException {
         step(() -> {
 
             executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID);
-            getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
+            ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(studyId)
                     .setSampleId(sampleId)
                     .setClinicalAnalysisType(clinicalAnalysisType)
-                    .execute();
+                    .setExomiserVersion(exomiserVersion);
+
+            exomiserExecutor.execute();
 
-            saveInterpretation(studyId, clinicalAnalysis);
+            saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion());
         });
     }
 
-    protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException,
+    protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion)
+            throws ToolException, StorageEngineException,
             CatalogException, IOException {
         // Interpretation method
         InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(),
                 GitRepositoryState.getInstance().getCommitId(), Collections.singletonList(
                 new Software()
                         .setName("Exomiser")
-                        .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME)
-                        .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION)));
+                        .setRepository("Docker: " + dockerImage)
+                        .setVersion(dockerImageVersion)));
 
         // Analyst
         ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token);
@@ -274,8 +287,17 @@ private List<ClinicalVariant> getPrimaryFindings() throws IOException, StorageEn
                 // Convert variants to clinical variants
                 for (Variant variant : variantResults.getResults()) {
                     ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant);
-                    List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv
-                            .get(variant.toStringSimple())));
+                    List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>();
+                    if (normalizedToTsv.containsKey(variant.toStringSimple())) {
+                        if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) {
+                            exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple())));
+                        } else {
+                            logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(),
+                                    normalizedToTsv.get(variant.toStringSimple()));
+                        }
+                    } else {
+                        logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple());
+                    }
                     for (String[] fields : variantTsvMap.get(variant.toStringSimple())) {
                         ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]);
                         Map<String, Object> attributes = getAttributesFromTsv(fields);
@@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly
         this.clinicalAnalysisId = clinicalAnalysisId;
         return this;
     }
+
+    public String getExomiserVersion() {
+        return exomiserVersion;
+    }
+
+    public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) {
+        this.exomiserVersion = exomiserVersion;
+        return this;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java

@@ -552,7 +552,7 @@ protected final <T extends OpenCgaToolExecutor> T getToolExecutor(Class<T> clazz
                 toolExecutor.getSource(),
                 toolExecutor.getFramework()));
 
-        toolExecutor.setUp(erm, executorParams, outDir);
+        toolExecutor.setUp(erm, executorParams, outDir, configuration);
         return toolExecutor;
     }
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java

@@ -295,7 +295,7 @@ protected void run() throws ToolException {
         step("gwas", () -> {
             GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class);
 
-            gwasExecutor.setConfiguration(gwasConfiguration)
+            gwasExecutor.setGwasConfiguration(gwasConfiguration)
                     .setStudy(study)
                     .setSampleList1(caseCohortSamples)
                     .setSampleList2(controlCohortSamples)

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java

@@ -71,8 +71,9 @@
 import org.opencb.opencga.core.models.sample.SamplePermissions;
 import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.core.models.study.StudyPermissions;
+import org.opencb.opencga.core.models.study.VariantSetupResult;
+import org.opencb.opencga.core.models.variant.VariantSetupParams;
 import org.opencb.opencga.core.response.OpenCGAResult;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.core.tools.ToolParams;
 import org.opencb.opencga.storage.core.StorageEngineFactory;
 import org.opencb.opencga.storage.core.StoragePipelineResult;
@@ -88,6 +89,7 @@
 import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat;
 import org.opencb.opencga.storage.core.variant.query.ParsedQuery;
+import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils;
 import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser;
 import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor;
@@ -490,6 +492,18 @@ public void aggregate(String studyStr, VariantAggregateParams params, String tok
         });
     }
 
+    public VariantSetupResult variantSetup(String studyStr, VariantSetupParams params, String token)
+            throws CatalogException, StorageEngineException {
+        return secureOperation(VariantSetupOperationManager.ID, studyStr, params.toObjectMap(), token,
+                engine -> new VariantSetupOperationManager(this, engine).setup(getStudyFqn(studyStr, token), params, token));
+    }
+
+    public boolean hasVariantSetup(String studyStr, String token) throws CatalogException {
+        Study study = catalogManager.getStudyManager().get(studyStr,
+                new QueryOptions(INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token).first();
+        return VariantSetupOperationManager.hasVariantSetup(study);
+    }
+
     public ObjectMap configureProject(String projectStr, ObjectMap params, String token) throws CatalogException, StorageEngineException {
         return secureOperationByProject("configure", projectStr, params, token, engine -> {
             DataStore dataStore = getDataStoreByProjectId(projectStr, token);
@@ -1181,7 +1195,7 @@ private interface VariantOperationFunction<R> {
     private <R> R secureOperationByProject(String operationName, String project, ObjectMap params, String token, VariantOperationFunction<R> operation)
             throws CatalogException, StorageEngineException {
         try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineByProject(project, params, token)) {
-            return secureTool(operationName, true, params, token, variantStorageEngine, operation);
+            return secureTool(operationName, true, null, params, token, variantStorageEngine, operation);
         } catch (IOException e) {
             throw new StorageEngineException("Error closing the VariantStorageEngine", e);
         }
@@ -1190,7 +1204,7 @@ private <R> R secureOperationByProject(String operationName, String project, Obj
     private <R> R secureOperation(String operationName, String study, ObjectMap params, String token, VariantOperationFunction<R> operation)
             throws CatalogException, StorageEngineException {
         try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) {
-            return secureTool(operationName, true, params, token, variantStorageEngine, operation);
+            return secureTool(operationName, true, study, params, token, variantStorageEngine, operation);
         } catch (IOException e) {
             throw new StorageEngineException("Error closing the VariantStorageEngine", e);
         }
@@ -1199,7 +1213,7 @@ private <R> R secureOperation(String operationName, String study, ObjectMap para
     private <R> R secureAnalysis(String operationName, String study, ObjectMap params, String token, VariantOperationFunction<R> operation)
             throws CatalogException, StorageEngineException {
         try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) {
-            return secureTool(operationName, false, params, token, variantStorageEngine, operation);
+            return secureTool(operationName, false, study, params, token, variantStorageEngine, operation);
         } catch (IOException e) {
             throw new StorageEngineException("Error closing the VariantStorageEngine", e);
         }
@@ -1221,7 +1235,7 @@ private <R> R secureOperationByProject(String operationName, String projectStr,
         return secureOperationByProject(operationName, projectStr, params, token, operation);
     }
 
-    private <R> R secureTool(String toolId, boolean isOperation, ObjectMap params, String token,
+    private <R> R secureTool(String toolId, boolean isOperation, String study, ObjectMap params, String token,
                              VariantStorageEngine variantStorageEngine, VariantOperationFunction<R> operation)
             throws CatalogException, StorageEngineException {
 
@@ -1241,6 +1255,15 @@ private <R> R secureTool(String toolId, boolean isOperation, ObjectMap params, S
                 throw new StorageEngineException("Unable to execute operation '" + toolId + "'. "
                         + "The storage engine is in mode=" + storageConfiguration.getMode());
             }
+            if (isOperation && study != null && !VariantSetupOperationManager.ID.equals(toolId)) {
+                // Ensure that the variant setup has been executed
+                //   do not check for the setup operation itself
+                // Project level operations can not be checked for setup.
+                if (!hasVariantSetup(study, token)) {
+                    throw new StorageEngineException("Unable to execute operation '" + toolId + "'. "
+                            + "The variant storage has not been setup for study '" + study + "'");
+                }
+            }
             result = operation.apply(variantStorageEngine);
             return result;
         } catch (CatalogException | StorageEngineException e) {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java

@@ -27,13 +27,17 @@
 import org.opencb.opencga.storage.core.metadata.models.TaskMetadata;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 import org.opencb.opencga.storage.core.variant.VariantStorageOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 
 public class VariantDeleteOperationManager extends OperationManager {
 
+    private final Logger logger = LoggerFactory.getLogger(VariantDeleteOperationManager.class);
+
     public VariantDeleteOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) {
         super(variantStorageManager, engine);
     }
@@ -63,7 +67,14 @@ public void removeFile(String study, List<String> inputFiles, URI outdir, String
                 String catalogIndexStatus = file.getInternal().getVariant().getIndex().getStatus().getId();
                 if (!catalogIndexStatus.equals(VariantIndexStatus.READY)) {
                     // Might be partially loaded in VariantStorage. Check FileMetadata
-                    FileMetadata fileMetadata = variantStorageEngine.getMetadataManager().getFileMetadata(studyMetadata.getId(), fileStr);
+                    FileMetadata fileMetadata = variantStorageEngine.getMetadataManager()
+                            .getFileMetadata(studyMetadata.getId(), file.getName());
+                    if (fileMetadata != null && !fileMetadata.getPath().equals(file.getUri().getPath())) {
+                        // FileMetadata path does not match the catalog path. This file is not registered in the storage.
+                        throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. "
+                                + "File is not registered in the storage. "
+                                + "Instead, found file with same name but different path '" + fileMetadata.getPath() + "'");
+                    }
                     boolean canBeRemoved;
                     if (force) {
                         // When forcing remove, just require the file to be registered in the storage
@@ -73,17 +84,18 @@ public void removeFile(String study, List<String> inputFiles, URI outdir, String
                         canBeRemoved = fileMetadata != null && fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE;
                     }
                     if (!canBeRemoved) {
-                        throw new CatalogException("Unable to remove variants from file " + file.getName() + ". "
-                                + "IndexStatus = " + catalogIndexStatus);
+                        throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. "
+                                + "IndexStatus = " + catalogIndexStatus + "."
+                                + (fileMetadata == null ? " File not found in storage." : ""));
                     }
                 }
                 fileNames.add(file.getName());
 //                        filePaths.add(file.getPath());
             }
 
-            if (fileNames.isEmpty()) {
-                throw new CatalogException("Nothing to do!");
-            }
+        }
+        if (fileNames.isEmpty()) {
+            throw new CatalogException("Nothing to do!");
         }
 
         variantStorageEngine.removeFiles(study, fileNames, outdir);