Merge branch 'release-3.2.x' into TASK-6494

opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java

@@ -16,9 +16,13 @@
 
 package org.opencb.opencga.analysis;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.commons.utils.FileUtils;
+import org.opencb.opencga.core.config.AnalysisTool;
 import org.opencb.opencga.core.config.Configuration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,10 +32,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 public class ConfigurationUtils {
     private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class);
 
+    private ConfigurationUtils() {
+        throw new IllegalStateException("Utility class");
+    }
     /**
      * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml.
      *
@@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome)
                     .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml"));
         }
     }
+
+    public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException {
+        List<AnalysisTool> tools = new ArrayList<>();
+        for (AnalysisTool tool : configuration.getAnalysis().getTools()) {
+            if (tool.getId().equals(toolId)) {
+                tools.add(tool);
+            }
+        }
+        if (CollectionUtils.isEmpty(tools)) {
+            throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file");
+        }
+        if (tools.size() == 1) {
+            return tools.get(0).getVersion();
+        }
+        String defaultVersion = null;
+        for (AnalysisTool tool : tools) {
+            if (tool.isDefaultVersion()) {
+                if (!StringUtils.isEmpty(defaultVersion)) {
+                    throw new ToolException("More than one default version found for tool ID '" + toolId + "'");
+                } else {
+                    defaultVersion = tool.getVersion();
+                }
+            }
+        }
+        if (StringUtils.isEmpty(defaultVersion)) {
+            throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true");
+        }
+        return defaultVersion;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java

@@ -30,8 +30,10 @@
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.opencga.analysis.ConfigurationUtils;
 import org.opencb.opencga.analysis.clinical.InterpretationAnalysis;
 import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils;
+import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis;
 import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.utils.ParamUtils;
@@ -44,10 +46,10 @@
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.individual.Individual;
 import org.opencb.opencga.core.response.OpenCGAResult;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
+import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis {
     private String clinicalAnalysisId;
     private String sampleId;
     private ClinicalAnalysis.Type clinicalAnalysisType;
+    private String exomiserVersion;
 
     private ClinicalAnalysis clinicalAnalysis;
 
@@ -97,8 +100,7 @@ protected void check() throws Exception {
         try {
             clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(),
                     token);
-        } catch (
-                CatalogException e) {
+        } catch (CatalogException e) {
             throw new ToolException(e);
         }
         if (clinicalAnalysisQueryResult.getNumResults() != 1) {
@@ -117,6 +119,7 @@ protected void check() throws Exception {
         }
         sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId();
 
+        // Check clinical analysis type
         if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) {
             clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY;
         } else {
@@ -125,6 +128,13 @@ protected void check() throws Exception {
         logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(),
                 clinicalAnalysisType);
 
+        // Check exomiser version
+        if (StringUtils.isEmpty(exomiserVersion)) {
+            // Missing exomiser version use the default one
+            exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration);
+            logger.warn("Missing exomiser version, using the default {}", exomiserVersion);
+        }
+
         // Update executor params with OpenCGA home and session ID
         setUpStorageEngineExecutor(studyId);
     }
@@ -134,25 +144,28 @@ protected void run() throws ToolException {
         step(() -> {
 
             executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID);
-            getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
+            ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(studyId)
                     .setSampleId(sampleId)
                     .setClinicalAnalysisType(clinicalAnalysisType)
-                    .execute();
+                    .setExomiserVersion(exomiserVersion);
+
+            exomiserExecutor.execute();
 
-            saveInterpretation(studyId, clinicalAnalysis);
+            saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion());
         });
     }
 
-    protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException,
+    protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion)
+            throws ToolException, StorageEngineException,
             CatalogException, IOException {
         // Interpretation method
         InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(),
                 GitRepositoryState.getInstance().getCommitId(), Collections.singletonList(
                 new Software()
                         .setName("Exomiser")
-                        .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME)
-                        .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION)));
+                        .setRepository("Docker: " + dockerImage)
+                        .setVersion(dockerImageVersion)));
 
         // Analyst
         ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token);
@@ -274,8 +287,17 @@ private List<ClinicalVariant> getPrimaryFindings() throws IOException, StorageEn
                 // Convert variants to clinical variants
                 for (Variant variant : variantResults.getResults()) {
                     ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant);
-                    List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv
-                            .get(variant.toStringSimple())));
+                    List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>();
+                    if (normalizedToTsv.containsKey(variant.toStringSimple())) {
+                        if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) {
+                            exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple())));
+                        } else {
+                            logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(),
+                                    normalizedToTsv.get(variant.toStringSimple()));
+                        }
+                    } else {
+                        logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple());
+                    }
                     for (String[] fields : variantTsvMap.get(variant.toStringSimple())) {
                         ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]);
                         Map<String, Object> attributes = getAttributesFromTsv(fields);
@@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly
         this.clinicalAnalysisId = clinicalAnalysisId;
         return this;
     }
+
+    public String getExomiserVersion() {
+        return exomiserVersion;
+    }
+
+    public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) {
+        this.exomiserVersion = exomiserVersion;
+        return this;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java

@@ -552,7 +552,7 @@ protected final <T extends OpenCgaToolExecutor> T getToolExecutor(Class<T> clazz
                 toolExecutor.getSource(),
                 toolExecutor.getFramework()));
 
-        toolExecutor.setUp(erm, executorParams, outDir);
+        toolExecutor.setUp(erm, executorParams, outDir, configuration);
         return toolExecutor;
     }
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java

@@ -295,7 +295,7 @@ protected void run() throws ToolException {
         step("gwas", () -> {
             GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class);
 
-            gwasExecutor.setConfiguration(gwasConfiguration)
+            gwasExecutor.setGwasConfiguration(gwasConfiguration)
                     .setStudy(study)
                     .setSampleList1(caseCohortSamples)
                     .setSampleList2(controlCohortSamples)

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java

@@ -134,7 +134,6 @@ private void rebuildSampleFileIds(VariantStorageMetadataManager metadataManager,
             for (Map.Entry<Integer, List<Integer>> entry : batch.entrySet()) {
                 Integer sampleId = entry.getKey();
                 List<Integer> fileIds = entry.getValue();
-
                 List<Integer> actualFiles = metadataManager.getSampleMetadata(studyId, sampleId).getFiles();
                 if (actualFiles.size() != fileIds.size() || !actualFiles.containsAll(fileIds)) {
                     fixedSamples++;

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java

@@ -7,8 +7,8 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.exec.Command;
-import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis;
 import org.opencb.opencga.core.common.GitRepositoryState;
+import org.opencb.opencga.core.config.AnalysisTool;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.tools.OpenCgaToolExecutor;
 import org.slf4j.Logger;
@@ -18,27 +18,61 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.*;
 
-public abstract class DockerWrapperAnalysisExecutor  extends OpenCgaToolExecutor {
+public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor {
 
     public final static String DOCKER_INPUT_PATH = "/data/input";
     public final static String DOCKER_OUTPUT_PATH = "/data/output";
 
     public static final String STDOUT_FILENAME = "stdout.txt";
     public static final String STDERR_FILENAME = "stderr.txt";
 
+    public String getDockerImageName() throws ToolException {
+        return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0];
+    }
+
     public static final String DOCKER_CLI_MSG = "Docker CLI: ";
 
-    public String getDockerImageName() {
-        return "opencb/opencga-ext-tools";
+    public String getDockerImageVersion() throws ToolException {
+        if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) {
+            return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1];
+        } else {
+            return GitRepositoryState.getInstance().getBuildVersion();
+        }
+    }
+
+    protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException {
+        for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) {
+            if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) {
+                return tool;
+            }
+        }
+        throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file");
+    }
+
+    public String getDockerImageName(String toolId, String version) throws ToolException {
+        AnalysisTool tool = getAnalysisTool(toolId, version);
+        return tool.getDockerId().split(":")[0];
+    }
+
+    public String getDockerImageVersion(String toolId, String version) throws ToolException {
+        AnalysisTool tool = getAnalysisTool(toolId, version);
+        if (tool.getDockerId().contains(":")) {
+            return tool.getDockerId().split(":")[1];
+        } else {
+            return null;
+        }
     }
 
-    public String getDockerImageVersion() {
-        return GitRepositoryState.getInstance().getBuildVersion();
+    protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException {
+        // Get resources from the configuration file
+        AnalysisTool tool = getAnalysisTool(toolId, version);
+        if (!tool.getResources().containsKey(resourceKey)) {
+            throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version =  "
+                    + version + "): it does not exist in the configuration file");
+        }
+        return tool.getResources().get(resourceKey);
     }
 
     private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class);
@@ -59,6 +93,14 @@ protected StringBuilder initCommandLine() {
         return new StringBuilder("docker run --log-driver=none -a stdin -a stdout -a stderr ");
     }
 
+    protected StringBuilder initCommandLine(String user) {
+        StringBuilder sb = initCommandLine();
+        if (StringUtils.isNotEmpty(user)) {
+            sb.append("--user ").append(user);
+        }
+        return sb;
+    }
+
     protected Map<String, String> appendMounts(List<Pair<String, String>> inputFilenames, StringBuilder sb) {
         Map<String, String> mountMap = new HashMap<>();
 
@@ -84,7 +126,7 @@ protected Map<String, String> appendMounts(List<Pair<String, String>> inputFilen
         return mountMap;
     }
 
-    protected void appendCommand(String command, StringBuilder sb) {
+    protected void appendCommand(String command, StringBuilder sb) throws ToolException {
         // Docker image and version
         sb.append(getDockerImageName());
         if (StringUtils.isNotEmpty(getDockerImageVersion())) {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java

@@ -17,6 +17,7 @@
 package org.opencb.opencga.analysis.wrappers.exomiser;
 
 import org.apache.commons.lang3.StringUtils;
+import org.opencb.opencga.analysis.ConfigurationUtils;
 import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
@@ -34,6 +35,14 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy {
     public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants"
             + " from whole-exome or whole-genome sequencing data.";
 
+    // It must match the tool prefix in the tool keys for exomiser in the configuration file
+    public final static String EXOMISER_PREFIX = "exomiser-";
+
+    // It must match the resources key in the exomiser/tool section in the configuration file
+    public final static String HG19_RESOURCE_KEY = "HG19";
+    public final static String HG38_RESOURCE_KEY = "HG38";
+    public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE";
+
     @ToolParams
     protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams();
 
@@ -43,6 +52,14 @@ protected void check() throws Exception {
         if (StringUtils.isEmpty(getStudy())) {
             throw new ToolException("Missing study");
         }
+
+        // Check exomiser version
+        if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) {
+            // Missing exomiser version use the default one
+            String exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration);
+            logger.warn("Missing exomiser version, using the default {}", exomiserVersion);
+            analysisParams.setExomiserVersion(exomiserVersion);
+        }
     }
 
     @Override
@@ -53,6 +70,7 @@ protected void run() throws Exception {
             getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(study)
                     .setSampleId(analysisParams.getSample())
+                    .setExomiserVersion(analysisParams.getExomiserVersion())
                     .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType()))
                     .execute();
         });