Skip to content

Commit

Permalink
Merge branch 'release-3.2.x' into TASK-5964
Browse files Browse the repository at this point in the history
  • Loading branch information
pfurio authored Aug 22, 2024
2 parents 831c2cb + 35074a6 commit 40651f8
Show file tree
Hide file tree
Showing 174 changed files with 3,271 additions and 1,523 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,13 @@

package org.opencb.opencga.analysis;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.opencb.commons.utils.FileUtils;
import org.opencb.opencga.core.config.AnalysisTool;
import org.opencb.opencga.core.config.Configuration;
import org.opencb.opencga.core.config.storage.StorageConfiguration;
import org.opencb.opencga.core.exceptions.ToolException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -28,10 +32,15 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

public class ConfigurationUtils {
private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class);

private ConfigurationUtils() {
throw new IllegalStateException("Utility class");
}
/**
* This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml.
*
Expand Down Expand Up @@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome)
.load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml"));
}
}

public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException {
List<AnalysisTool> tools = new ArrayList<>();
for (AnalysisTool tool : configuration.getAnalysis().getTools()) {
if (tool.getId().equals(toolId)) {
tools.add(tool);
}
}
if (CollectionUtils.isEmpty(tools)) {
throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file");
}
if (tools.size() == 1) {
return tools.get(0).getVersion();
}
String defaultVersion = null;
for (AnalysisTool tool : tools) {
if (tool.isDefaultVersion()) {
if (!StringUtils.isEmpty(defaultVersion)) {
throw new ToolException("More than one default version found for tool ID '" + toolId + "'");
} else {
defaultVersion = tool.getVersion();
}
}
}
if (StringUtils.isEmpty(defaultVersion)) {
throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true");
}
return defaultVersion;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.ConfigurationUtils;
import org.opencb.opencga.analysis.clinical.InterpretationAnalysis;
import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils;
import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis;
import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.utils.ParamUtils;
Expand All @@ -44,10 +46,10 @@
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.individual.Individual;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
import org.opencb.opencga.core.tools.annotations.Tool;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;

import java.io.BufferedReader;
import java.io.File;
Expand All @@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis {
private String clinicalAnalysisId;
private String sampleId;
private ClinicalAnalysis.Type clinicalAnalysisType;
private String exomiserVersion;

private ClinicalAnalysis clinicalAnalysis;

Expand Down Expand Up @@ -97,8 +100,7 @@ protected void check() throws Exception {
try {
clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(),
token);
} catch (
CatalogException e) {
} catch (CatalogException e) {
throw new ToolException(e);
}
if (clinicalAnalysisQueryResult.getNumResults() != 1) {
Expand All @@ -117,6 +119,7 @@ protected void check() throws Exception {
}
sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId();

// Check clinical analysis type
if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) {
clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY;
} else {
Expand All @@ -125,6 +128,13 @@ protected void check() throws Exception {
logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(),
clinicalAnalysisType);

// Check exomiser version
if (StringUtils.isEmpty(exomiserVersion)) {
// Missing exomiser version use the default one
exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration);
logger.warn("Missing exomiser version, using the default {}", exomiserVersion);
}

// Update executor params with OpenCGA home and session ID
setUpStorageEngineExecutor(studyId);
}
Expand All @@ -134,25 +144,28 @@ protected void run() throws ToolException {
step(() -> {

executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID);
getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
.setStudyId(studyId)
.setSampleId(sampleId)
.setClinicalAnalysisType(clinicalAnalysisType)
.execute();
.setExomiserVersion(exomiserVersion);

exomiserExecutor.execute();

saveInterpretation(studyId, clinicalAnalysis);
saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion());
});
}

protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException,
protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion)
throws ToolException, StorageEngineException,
CatalogException, IOException {
// Interpretation method
InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(),
GitRepositoryState.getInstance().getCommitId(), Collections.singletonList(
new Software()
.setName("Exomiser")
.setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME)
.setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION)));
.setRepository("Docker: " + dockerImage)
.setVersion(dockerImageVersion)));

// Analyst
ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token);
Expand Down Expand Up @@ -274,8 +287,17 @@ private List<ClinicalVariant> getPrimaryFindings() throws IOException, StorageEn
// Convert variants to clinical variants
for (Variant variant : variantResults.getResults()) {
ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant);
List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv
.get(variant.toStringSimple())));
List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>();
if (normalizedToTsv.containsKey(variant.toStringSimple())) {
if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) {
exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple())));
} else {
logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(),
normalizedToTsv.get(variant.toStringSimple()));
}
} else {
logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple());
}
for (String[] fields : variantTsvMap.get(variant.toStringSimple())) {
ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]);
Map<String, Object> attributes = getAttributesFromTsv(fields);
Expand Down Expand Up @@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly
this.clinicalAnalysisId = clinicalAnalysisId;
return this;
}

public String getExomiserVersion() {
return exomiserVersion;
}

public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) {
this.exomiserVersion = exomiserVersion;
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ protected final <T extends OpenCgaToolExecutor> T getToolExecutor(Class<T> clazz
toolExecutor.getSource(),
toolExecutor.getFramework()));

toolExecutor.setUp(erm, executorParams, outDir);
toolExecutor.setUp(erm, executorParams, outDir, configuration);
return toolExecutor;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ protected void run() throws ToolException {
step("gwas", () -> {
GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class);

gwasExecutor.setConfiguration(gwasConfiguration)
gwasExecutor.setGwasConfiguration(gwasConfiguration)
.setStudy(study)
.setSampleList1(caseCohortSamples)
.setSampleList2(controlCohortSamples)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ private void rebuildSampleFileIds(VariantStorageMetadataManager metadataManager,
for (Map.Entry<Integer, List<Integer>> entry : batch.entrySet()) {
Integer sampleId = entry.getKey();
List<Integer> fileIds = entry.getValue();

List<Integer> actualFiles = metadataManager.getSampleMetadata(studyId, sampleId).getFiles();
if (actualFiles.size() != fileIds.size() || !actualFiles.containsAll(fileIds)) {
fixedSamples++;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
import org.apache.commons.lang3.tuple.Pair;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.exec.Command;
import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis;
import org.opencb.opencga.core.common.GitRepositoryState;
import org.opencb.opencga.core.config.AnalysisTool;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.tools.OpenCgaToolExecutor;
import org.slf4j.Logger;
Expand All @@ -18,27 +18,61 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor {
public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor {

public final static String DOCKER_INPUT_PATH = "/data/input";
public final static String DOCKER_OUTPUT_PATH = "/data/output";

public static final String STDOUT_FILENAME = "stdout.txt";
public static final String STDERR_FILENAME = "stderr.txt";

public String getDockerImageName() throws ToolException {
return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0];
}

public static final String DOCKER_CLI_MSG = "Docker CLI: ";

public String getDockerImageName() {
return "opencb/opencga-ext-tools";
public String getDockerImageVersion() throws ToolException {
if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) {
return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1];
} else {
return GitRepositoryState.getInstance().getBuildVersion();
}
}

protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException {
for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) {
if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) {
return tool;
}
}
throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file");
}

public String getDockerImageName(String toolId, String version) throws ToolException {
AnalysisTool tool = getAnalysisTool(toolId, version);
return tool.getDockerId().split(":")[0];
}

public String getDockerImageVersion(String toolId, String version) throws ToolException {
AnalysisTool tool = getAnalysisTool(toolId, version);
if (tool.getDockerId().contains(":")) {
return tool.getDockerId().split(":")[1];
} else {
return null;
}
}

public String getDockerImageVersion() {
return GitRepositoryState.getInstance().getBuildVersion();
protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException {
// Get resources from the configuration file
AnalysisTool tool = getAnalysisTool(toolId, version);
if (!tool.getResources().containsKey(resourceKey)) {
throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version = "
+ version + "): it does not exist in the configuration file");
}
return tool.getResources().get(resourceKey);
}

private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class);
Expand All @@ -59,6 +93,14 @@ protected StringBuilder initCommandLine() {
return new StringBuilder("docker run --log-driver=none -a stdin -a stdout -a stderr ");
}

protected StringBuilder initCommandLine(String user) {
StringBuilder sb = initCommandLine();
if (StringUtils.isNotEmpty(user)) {
sb.append("--user ").append(user);
}
return sb;
}

protected Map<String, String> appendMounts(List<Pair<String, String>> inputFilenames, StringBuilder sb) {
Map<String, String> mountMap = new HashMap<>();

Expand All @@ -84,7 +126,7 @@ protected Map<String, String> appendMounts(List<Pair<String, String>> inputFilen
return mountMap;
}

protected void appendCommand(String command, StringBuilder sb) {
protected void appendCommand(String command, StringBuilder sb) throws ToolException {
// Docker image and version
sb.append(getDockerImageName());
if (StringUtils.isNotEmpty(getDockerImageVersion())) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.opencb.opencga.analysis.wrappers.exomiser;

import org.apache.commons.lang3.StringUtils;
import org.opencb.opencga.analysis.ConfigurationUtils;
import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
Expand All @@ -34,6 +35,14 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy {
public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants"
+ " from whole-exome or whole-genome sequencing data.";

// It must match the tool prefix in the tool keys for exomiser in the configuration file
public final static String EXOMISER_PREFIX = "exomiser-";

// It must match the resources key in the exomiser/tool section in the configuration file
public final static String HG19_RESOURCE_KEY = "HG19";
public final static String HG38_RESOURCE_KEY = "HG38";
public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE";

@ToolParams
protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams();

Expand All @@ -43,6 +52,14 @@ protected void check() throws Exception {
if (StringUtils.isEmpty(getStudy())) {
throw new ToolException("Missing study");
}

// Check exomiser version
if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) {
// Missing exomiser version use the default one
String exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration);
logger.warn("Missing exomiser version, using the default {}", exomiserVersion);
analysisParams.setExomiserVersion(exomiserVersion);
}
}

@Override
Expand All @@ -53,6 +70,7 @@ protected void run() throws Exception {
getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
.setStudyId(study)
.setSampleId(analysisParams.getSample())
.setExomiserVersion(analysisParams.getExomiserVersion())
.setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType()))
.execute();
});
Expand Down
Loading

0 comments on commit 40651f8

Please sign in to comment.