Skip to content

Commit

Permalink
Merge pull request #2368 from opencb/TASK-5425
Browse files Browse the repository at this point in the history
TASK-5425 - PortPatch 1.10.1 -> 2.0.0
  • Loading branch information
juanfeSanahuja authored Dec 22, 2023
2 parents 02cdabc + 8dd7bb2 commit 1c090de
Show file tree
Hide file tree
Showing 149 changed files with 2,833 additions and 702 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/manual-deploy-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jobs:
./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
fi
- name: Maven Build (skip tests)
run: mvn -T 2 clean install -DskipTests
run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
- uses: actions/upload-artifact@v3
with:
name: build-folder
Expand Down
3 changes: 2 additions & 1 deletion opencga-analysis/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@
<dependency>
<groupId>org.opencb.opencga</groupId>
<artifactId>${opencga-storage-hadoop-deps.artifactId}</artifactId>
<version>${project.parent.version}</version>
<classifier>shaded</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.opencb.opencga</groupId>
Expand Down Expand Up @@ -274,6 +274,7 @@
com.google.code.findbugs:jsr305:jar:3.0.2
</ignoredUsedUndeclaredDependency>
</ignoredUsedUndeclaredDependencies>
<ignoredNonTestScopedDependencies><ignoredNonTestScopedDependency>*</ignoredNonTestScopedDependency></ignoredNonTestScopedDependencies>
<ignoredUnusedDeclaredDependencies>
<ignoredUnusedDeclaredDependency>*</ignoredUnusedDeclaredDependency>
</ignoredUnusedDeclaredDependencies>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -824,7 +824,7 @@ public ClinicalAnalyst getAnalyst(String token) throws ToolException {
OpenCGAResult<User> userQueryResult = catalogManager.getUserManager().get(userId, new QueryOptions(QueryOptions.INCLUDE,
Arrays.asList(UserDBAdaptor.QueryParams.EMAIL.key(), UserDBAdaptor.QueryParams.ORGANIZATION.key())), token);
User user = userQueryResult.first();
return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", "");
return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", Collections.emptyMap());
} catch (CatalogException e) {
throw new ToolException(e);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Copyright 2015-2020 OpenCB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.opencb.opencga.analysis.clinical;

import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.annotations.TsvAnnotationLoader;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.AnnotationSetManager;
import org.opencb.opencga.catalog.utils.Constants;
import org.opencb.opencga.catalog.utils.ParamUtils;
import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams;
import org.opencb.opencga.core.models.common.AnnotationSet;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.tools.annotations.Tool;

import java.util.Collections;

@Tool(id = ClinicalTsvAnnotationLoader.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, type = Tool.Type.OPERATION,
description = "Load annotations from TSV file.")
public class ClinicalTsvAnnotationLoader extends TsvAnnotationLoader {
public final static String ID = "clinical-tsv-load";

@Override
public int count(Query query) throws CatalogException {
return catalogManager.getClinicalAnalysisManager().count(study, query, token).getNumResults();
}

@Override
public void addAnnotationSet(String entryId, AnnotationSet annotationSet, QueryOptions options) throws CatalogException {
ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams()
.setAnnotationSets(Collections.singletonList(annotationSet));
QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions();
queryOptions.put(Constants.ACTIONS, new ObjectMap(AnnotationSetManager.ANNOTATION_SETS, ParamUtils.BasicUpdateAction.ADD));

catalogManager.getClinicalAnalysisManager().update(study, entryId, updateParams, queryOptions, token);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package org.opencb.opencga.analysis.variant.circos;

import org.opencb.opencga.analysis.ResourceUtils;
import org.opencb.opencga.analysis.tools.OpenCgaTool;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.common.Enums;
Expand All @@ -34,6 +35,8 @@ public class CircosAnalysis extends OpenCgaTool {
private String study;
private CircosAnalysisParams circosParams;

private String assembly;

@Override
protected void check() throws Exception {
super.check();
Expand All @@ -58,6 +61,8 @@ protected void check() throws Exception {
// }
//
// addAttribute("sampleName", sampleName);

assembly = ResourceUtils.getAssembly(catalogManager, study, token);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.utils.DockerUtils;
import org.opencb.opencga.analysis.ResourceUtils;
import org.opencb.opencga.analysis.StorageToolExecutor;
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.common.GitRepositoryState;
import org.opencb.opencga.core.common.TimeUtils;
import org.opencb.opencga.core.exceptions.ToolException;
Expand Down Expand Up @@ -93,7 +95,7 @@ public VariantStorageManager getVariantStorageManager() throws ToolExecutorExcep
}

@Override
public void run() throws ToolException, IOException {
public void run() throws ToolException, IOException, CatalogException {

// Create query
Query query = new Query();
Expand Down Expand Up @@ -128,7 +130,6 @@ public void run() throws ToolException, IOException {
throw new ToolException("Error launching threads when executing the Circos analysis", e);
}


if (MapUtils.isEmpty(errors)) {
// Execute R script
// circos.R ./snvs.tsv ./indels.tsv ./cnvs.tsv ./rearrs.tsv SampleId
Expand All @@ -138,10 +139,19 @@ public void run() throws ToolException, IOException {
AbstractMap.SimpleEntry<String, String> outputBinding = new AbstractMap.SimpleEntry<>(getOutDir()
.toAbsolutePath().toString(),
DOCKER_OUTPUT_PATH);

// Get genome version
String genomeVersion = "hg38";
String assembly = ResourceUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken());
if (StringUtils.isNotEmpty(assembly) && assembly.toUpperCase(Locale.ROOT).equals("GRCH37")) {
genomeVersion = "hg19";
}

String scriptParams = "R CMD Rscript --vanilla " + DOCKER_INPUT_PATH + "/circos.R"
+ (plotCopynumber ? "" : " --no_copynumber")
+ (plotIndels ? "" : " --no_indels")
+ (plotRearrangements ? "" : " --no_rearrangements")
+ " --genome_version " + genomeVersion
+ " --out_path " + DOCKER_OUTPUT_PATH
+ " " + DOCKER_OUTPUT_PATH + "/" + snvsFile.getName()
+ " " + DOCKER_OUTPUT_PATH + "/" + indelsFile.getName()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.opencb.biodata.models.clinical.qc.GenomePlotConfig;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.AnalysisUtils;
import org.opencb.opencga.analysis.ResourceUtils;
import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
import org.opencb.opencga.core.common.JacksonUtils;
import org.opencb.opencga.core.exceptions.ToolException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.utils.DockerUtils;
import org.opencb.opencga.analysis.ResourceUtils;
import org.opencb.opencga.analysis.StorageToolExecutor;
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.common.GitRepositoryState;
import org.opencb.opencga.core.common.JacksonUtils;
import org.opencb.opencga.core.common.TimeUtils;
Expand All @@ -46,10 +48,7 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.*;

import static org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor.DOCKER_INPUT_PATH;
Expand Down Expand Up @@ -79,7 +78,7 @@ public class GenomePlotLocalAnalysisExecutor extends GenomePlotAnalysisExecutor
private Logger logger = LoggerFactory.getLogger(this.getClass());

@Override
public void run() throws ToolException, IOException {
public void run() throws ToolException, IOException, CatalogException {

plotConfig = JacksonUtils.getDefaultObjectMapper().readerFor(GenomePlotConfig.class).readValue(getConfigFile());

Expand Down Expand Up @@ -125,10 +124,19 @@ public void run() throws ToolException, IOException {
inputBindings.add(new AbstractMap.SimpleEntry<>(rScriptPath, DOCKER_INPUT_PATH));
AbstractMap.SimpleEntry<String, String> outputBinding = new AbstractMap.SimpleEntry<>(getOutDir()
.toAbsolutePath().toString(), DOCKER_OUTPUT_PATH);

// Get genome version
String genomeVersion = "hg38";
String assembly = ResourceUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken());
if (StringUtils.isNotEmpty(assembly) && assembly.toUpperCase(Locale.ROOT).equals("GRCH37")) {
genomeVersion = "hg19";
}

String scriptParams = "R CMD Rscript --vanilla " + DOCKER_INPUT_PATH + "/circos.R"
+ (plotCopynumber ? "" : " --no_copynumber")
+ (plotIndels ? "" : " --no_indels")
+ (plotRearrangements ? "" : " --no_rearrangements")
+ " --genome_version " + genomeVersion
+ " --out_path " + DOCKER_OUTPUT_PATH
+ " " + DOCKER_OUTPUT_PATH + "/" + snvsFile.getName()
+ " " + DOCKER_OUTPUT_PATH + "/" + indelsFile.getName()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,15 +147,23 @@ public void run() throws ToolException {
Path openCgaHome = getOpencgaHomePath();
Path exomiserDataPath = getAnalysisDataPath(ExomiserWrapperAnalysis.ID);

// And copy the application.properties
// Copy the analysis
try {
FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(),
getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile());
} catch (IOException e) {
throw new ToolException("Error copying Exomiser analysis file", e);
}

// Copy the application.properties
try {
FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(),
getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile());
} catch (IOException e) {
throw new ToolException("Error copying Exomiser properties file", e);
}

// And copy the output options
// Copy the output options
try {
FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(),
getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile());
Expand All @@ -174,7 +182,7 @@ public void run() throws ToolException {
appendCommand("", sb);

// Append input file params
// sb.append(" --analysis /jobdir/").append(EXOMISER_ANALYSIS_TEMPLATE_FILENAME)
sb.append(" --analysis /jobdir/").append(EXOMISER_ANALYSIS_TEMPLATE_FILENAME);
sb.append(" --sample /jobdir/").append(sampleFile.getName());
if (pedigreeFile != null && pedigreeFile.exists()) {
sb.append(" --ped /jobdir/").append(pedigreeFile.getName());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeThat;

@Category(MediumTests.class)
Expand Down Expand Up @@ -93,7 +94,7 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE
clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
.get(clinicalTest.studyFqn, clinicalTest.CA_ID2, QueryOptions.empty(), clinicalTest.token).first();
assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
assertEquals(22, clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size());
assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
}

@Test
Expand All @@ -119,7 +120,7 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE
clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
.get(clinicalTest.studyFqn, clinicalTest.CA_ID3, QueryOptions.empty(), clinicalTest.token).first();
assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
assertEquals(2, clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size());
assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
System.out.println("results at out dir = " + outDir.toAbsolutePath());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;

/**
Expand Down Expand Up @@ -260,6 +262,14 @@ public Path isolateOpenCGA() throws IOException {
inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R");
Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING);

// Exomiser analysis files
analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser")).toAbsolutePath();
List<String> exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml");
for (String exomiserFile : exomiserFiles) {
inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserFile);
Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING);
}

return opencgaHome;
}

Expand Down
70 changes: 70 additions & 0 deletions opencga-app/app/analysis/exomiser/exomiser-analysis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
---
analysisMode: PASS_ONLY
inheritanceModes: {
AUTOSOMAL_DOMINANT: 0.1,
AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1,
AUTOSOMAL_RECESSIVE_COMP_HET: 2.0,
X_DOMINANT: 0.1,
X_RECESSIVE_HOM_ALT: 0.1,
X_RECESSIVE_COMP_HET: 2.0,
MITOCHONDRIAL: 0.2
}
frequencySources: [
THOUSAND_GENOMES,
TOPMED,
UK10K,

ESP_AFRICAN_AMERICAN, ESP_EUROPEAN_AMERICAN, ESP_ALL,

EXAC_AFRICAN_INC_AFRICAN_AMERICAN, EXAC_AMERICAN,
EXAC_SOUTH_ASIAN, EXAC_EAST_ASIAN,
EXAC_FINNISH, EXAC_NON_FINNISH_EUROPEAN,
EXAC_OTHER,

GNOMAD_E_AFR,
GNOMAD_E_AMR,
# GNOMAD_E_ASJ,
GNOMAD_E_EAS,
GNOMAD_E_FIN,
GNOMAD_E_NFE,
GNOMAD_E_OTH,
GNOMAD_E_SAS,

GNOMAD_G_AFR,
GNOMAD_G_AMR,
# GNOMAD_G_ASJ,
GNOMAD_G_EAS,
GNOMAD_G_FIN,
GNOMAD_G_NFE,
GNOMAD_G_OTH,
GNOMAD_G_SAS
]
# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM
# REMM is trained on non-coding regulatory regions
# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files
# and updated their location in the application.properties. Exomiser will not run without this.
pathogenicitySources: [ REVEL, MVP ]
#this is the standard exomiser order.
steps: [
failedVariantFilter: { },
variantEffectFilter: {
remove: [
FIVE_PRIME_UTR_EXON_VARIANT,
FIVE_PRIME_UTR_INTRON_VARIANT,
THREE_PRIME_UTR_EXON_VARIANT,
THREE_PRIME_UTR_INTRON_VARIANT,
NON_CODING_TRANSCRIPT_EXON_VARIANT,
NON_CODING_TRANSCRIPT_INTRON_VARIANT,
CODING_TRANSCRIPT_INTRON_VARIANT,
UPSTREAM_GENE_VARIANT,
DOWNSTREAM_GENE_VARIANT,
INTERGENIC_VARIANT,
REGULATORY_REGION_VARIANT
]
},
frequencyFilter: { maxFrequency: 2.0 },
pathogenicityFilter: { keepNonPathogenic: true },
inheritanceFilter: { },
omimPrioritiser: { },
hiPhivePrioritiser: { }
]
2 changes: 1 addition & 1 deletion opencga-app/app/cloud/docker/opencga-base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ ARG BUILD_PATH="."
ENV OPENCGA_HOME=/opt/opencga
ENV OPENCGA_CONFIG_DIR=${OPENCGA_HOME}/conf

RUN apt-get update && apt-get -y upgrade && apt-get install -y lsb-release sshpass ca-certificates curl gnupg jq ncurses-bin && \
RUN apt-get update && apt-get -y upgrade && apt-get install -y lsb-release sshpass ca-certificates curl gnupg jq ncurses-bin unzip vim && \
## Install Docker repository
install -m 0755 -d /etc/apt/keyrings && \
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ public InternalCliOptionsParser() {
clinicalSubCommands.addCommand(RGA_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaSecondaryIndexCommandOptions);
clinicalSubCommands.addCommand(RGA_AUX_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaAuxiliarSecondaryIndexCommandOptions);
clinicalSubCommands.addCommand(EXOMISER_INTERPRETATION_RUN_COMMAND, clinicalCommandOptions.exomiserInterpretationCommandOptions);
clinicalSubCommands.addCommand("tsv-load", clinicalCommandOptions.tsvLoad);

fileCommandOptions = new FileCommandOptions(commonCommandOptions, jCommander);
jCommander.addCommand("files", fileCommandOptions);
Expand Down
Loading

0 comments on commit 1c090de

Please sign in to comment.