TASK-5425 - Port Patch 2.12.1 -> 3.0.0 (XB 1.10.1 -> 2.0.0)

.github/workflows/manual-deploy-docker.yml

@@ -35,7 +35,7 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests
+        run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
       - uses: actions/upload-artifact@v3
         with:
           name: build-folder

opencga-analysis/pom.xml

@@ -72,8 +72,8 @@
         <dependency>
             <groupId>org.opencb.opencga</groupId>
             <artifactId>${opencga-storage-hadoop-deps.artifactId}</artifactId>
+            <version>${project.parent.version}</version>
             <classifier>shaded</classifier>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.opencb.opencga</groupId>
@@ -274,6 +274,7 @@
                                     com.google.code.findbugs:jsr305:jar:3.0.2
                                 </ignoredUsedUndeclaredDependency>
                             </ignoredUsedUndeclaredDependencies>
+                            <ignoredNonTestScopedDependencies><ignoredNonTestScopedDependency>*</ignoredNonTestScopedDependency></ignoredNonTestScopedDependencies>
                             <ignoredUnusedDeclaredDependencies>
                                 <ignoredUnusedDeclaredDependency>*</ignoredUnusedDeclaredDependency>
                             </ignoredUnusedDeclaredDependencies>

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java

@@ -824,7 +824,7 @@ public ClinicalAnalyst getAnalyst(String token) throws ToolException {
             OpenCGAResult<User> userQueryResult = catalogManager.getUserManager().get(userId, new QueryOptions(QueryOptions.INCLUDE,
                     Arrays.asList(UserDBAdaptor.QueryParams.EMAIL.key(), UserDBAdaptor.QueryParams.ORGANIZATION.key())), token);
             User user = userQueryResult.first();
-            return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", "");
+            return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", Collections.emptyMap());
         } catch (CatalogException e) {
             throw new ToolException(e);
         }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015-2020 OpenCB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opencb.opencga.analysis.clinical;
+
+import org.opencb.commons.datastore.core.ObjectMap;
+import org.opencb.commons.datastore.core.Query;
+import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.opencga.analysis.annotations.TsvAnnotationLoader;
+import org.opencb.opencga.catalog.exceptions.CatalogException;
+import org.opencb.opencga.catalog.managers.AnnotationSetManager;
+import org.opencb.opencga.catalog.utils.Constants;
+import org.opencb.opencga.catalog.utils.ParamUtils;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams;
+import org.opencb.opencga.core.models.common.AnnotationSet;
+import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.tools.annotations.Tool;
+
+import java.util.Collections;
+
+@Tool(id = ClinicalTsvAnnotationLoader.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, type = Tool.Type.OPERATION,
+        description = "Load annotations from TSV file.")
+public class ClinicalTsvAnnotationLoader extends TsvAnnotationLoader {
+    public final static String ID = "clinical-tsv-load";
+
+    @Override
+    public int count(Query query) throws CatalogException {
+        return catalogManager.getClinicalAnalysisManager().count(study, query, token).getNumResults();
+    }
+
+    @Override
+    public void addAnnotationSet(String entryId, AnnotationSet annotationSet, QueryOptions options) throws CatalogException {
+        ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams()
+                .setAnnotationSets(Collections.singletonList(annotationSet));
+        QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions();
+        queryOptions.put(Constants.ACTIONS, new ObjectMap(AnnotationSetManager.ANNOTATION_SETS, ParamUtils.BasicUpdateAction.ADD));
+
+        catalogManager.getClinicalAnalysisManager().update(study, entryId, updateParams, queryOptions, token);
+    }
+}

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosAnalysis.java

@@ -16,6 +16,7 @@
 
 package org.opencb.opencga.analysis.variant.circos;
 
+import org.opencb.opencga.analysis.ResourceUtils;
 import org.opencb.opencga.analysis.tools.OpenCgaTool;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.common.Enums;
@@ -34,6 +35,8 @@ public class CircosAnalysis extends OpenCgaTool {
     private String study;
     private CircosAnalysisParams circosParams;
 
+    private String assembly;
+
     @Override
     protected void check() throws Exception {
         super.check();
@@ -58,6 +61,8 @@ protected void check() throws Exception {
 //        }
 //
 //        addAttribute("sampleName", sampleName);
+
+        assembly = ResourceUtils.getAssembly(catalogManager, study, token);
     }
 
     @Override

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java

@@ -28,8 +28,10 @@
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.commons.utils.DockerUtils;
+import org.opencb.opencga.analysis.ResourceUtils;
 import org.opencb.opencga.analysis.StorageToolExecutor;
 import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
+import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.common.GitRepositoryState;
 import org.opencb.opencga.core.common.TimeUtils;
 import org.opencb.opencga.core.exceptions.ToolException;
@@ -93,7 +95,7 @@ public VariantStorageManager getVariantStorageManager() throws ToolExecutorExcep
     }
 
     @Override
-    public void run() throws ToolException, IOException {
+    public void run() throws ToolException, IOException, CatalogException {
 
         // Create query
         Query query = new Query();
@@ -128,7 +130,6 @@ public void run() throws ToolException, IOException {
             throw new ToolException("Error launching threads when executing the Circos analysis", e);
         }
 
-
         if (MapUtils.isEmpty(errors)) {
             // Execute R script
             // circos.R ./snvs.tsv ./indels.tsv ./cnvs.tsv ./rearrs.tsv SampleId
@@ -138,10 +139,19 @@ public void run() throws ToolException, IOException {
             AbstractMap.SimpleEntry<String, String> outputBinding = new AbstractMap.SimpleEntry<>(getOutDir()
                     .toAbsolutePath().toString(),
                     DOCKER_OUTPUT_PATH);
+
+            // Get genome version
+            String genomeVersion = "hg38";
+            String assembly = ResourceUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken());
+            if (StringUtils.isNotEmpty(assembly) && assembly.toUpperCase(Locale.ROOT).equals("GRCH37")) {
+                genomeVersion = "hg19";
+            }
+
             String scriptParams = "R CMD Rscript --vanilla " + DOCKER_INPUT_PATH + "/circos.R"
                     + (plotCopynumber ? "" : " --no_copynumber")
                     + (plotIndels ? "" : " --no_indels")
                     + (plotRearrangements ? "" : " --no_rearrangements")
+                    + " --genome_version " + genomeVersion
                     + " --out_path " + DOCKER_OUTPUT_PATH
                     + " " + DOCKER_OUTPUT_PATH + "/" + snvsFile.getName()
                     + " " + DOCKER_OUTPUT_PATH + "/" + indelsFile.getName()

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java

@@ -21,6 +21,7 @@
 import org.opencb.biodata.models.clinical.qc.GenomePlotConfig;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.AnalysisUtils;
+import org.opencb.opencga.analysis.ResourceUtils;
 import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.core.common.JacksonUtils;
 import org.opencb.opencga.core.exceptions.ToolException;

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java

@@ -30,8 +30,10 @@
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.commons.utils.DockerUtils;
+import org.opencb.opencga.analysis.ResourceUtils;
 import org.opencb.opencga.analysis.StorageToolExecutor;
 import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
+import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.common.GitRepositoryState;
 import org.opencb.opencga.core.common.JacksonUtils;
 import org.opencb.opencga.core.common.TimeUtils;
@@ -46,10 +48,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.*;
 
 import static org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor.DOCKER_INPUT_PATH;
@@ -79,7 +78,7 @@ public class GenomePlotLocalAnalysisExecutor extends GenomePlotAnalysisExecutor
     private Logger logger = LoggerFactory.getLogger(this.getClass());
 
     @Override
-    public void run() throws ToolException, IOException {
+    public void run() throws ToolException, IOException, CatalogException {
 
         plotConfig = JacksonUtils.getDefaultObjectMapper().readerFor(GenomePlotConfig.class).readValue(getConfigFile());
 
@@ -125,10 +124,19 @@ public void run() throws ToolException, IOException {
             inputBindings.add(new AbstractMap.SimpleEntry<>(rScriptPath, DOCKER_INPUT_PATH));
             AbstractMap.SimpleEntry<String, String> outputBinding = new AbstractMap.SimpleEntry<>(getOutDir()
                     .toAbsolutePath().toString(), DOCKER_OUTPUT_PATH);
+
+            // Get genome version
+            String genomeVersion = "hg38";
+            String assembly = ResourceUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken());
+            if (StringUtils.isNotEmpty(assembly) && assembly.toUpperCase(Locale.ROOT).equals("GRCH37")) {
+                genomeVersion = "hg19";
+            }
+
             String scriptParams = "R CMD Rscript --vanilla " + DOCKER_INPUT_PATH + "/circos.R"
                     + (plotCopynumber ? "" : " --no_copynumber")
                     + (plotIndels ? "" : " --no_indels")
                     + (plotRearrangements ? "" : " --no_rearrangements")
+                    + " --genome_version " + genomeVersion
                     + " --out_path " + DOCKER_OUTPUT_PATH
                     + " " + DOCKER_OUTPUT_PATH + "/" + snvsFile.getName()
                     + " " + DOCKER_OUTPUT_PATH + "/" + indelsFile.getName()

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java

@@ -147,15 +147,23 @@ public void run() throws ToolException {
         Path openCgaHome = getOpencgaHomePath();
         Path exomiserDataPath = getAnalysisDataPath(ExomiserWrapperAnalysis.ID);
 
-        // And copy the application.properties
+        // Copy the analysis
+        try {
+            FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(),
+                    getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile());
+        } catch (IOException e) {
+            throw new ToolException("Error copying Exomiser analysis file", e);
+        }
+
+        // Copy the application.properties
         try {
             FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(),
                     getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile());
         } catch (IOException e) {
             throw new ToolException("Error copying Exomiser properties file", e);
         }
 
-        // And copy the output options
+        // Copy the output options
         try {
             FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(),
                     getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile());
@@ -174,7 +182,7 @@ public void run() throws ToolException {
         appendCommand("", sb);
 
         // Append input file params
-//        sb.append(" --analysis /jobdir/").append(EXOMISER_ANALYSIS_TEMPLATE_FILENAME)
+        sb.append(" --analysis /jobdir/").append(EXOMISER_ANALYSIS_TEMPLATE_FILENAME);
         sb.append(" --sample /jobdir/").append(sampleFile.getName());
         if (pedigreeFile != null && pedigreeFile.exists()) {
             sb.append(" --ped /jobdir/").append(pedigreeFile.getName());

opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java

@@ -31,6 +31,7 @@
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeThat;
 
 @Category(MediumTests.class)
@@ -93,7 +94,7 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE
         clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
                 .get(clinicalTest.studyFqn, clinicalTest.CA_ID2, QueryOptions.empty(), clinicalTest.token).first();
         assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
-        assertEquals(22, clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size());
+        assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
     }
 
     @Test
@@ -119,7 +120,7 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE
         clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
                 .get(clinicalTest.studyFqn, clinicalTest.CA_ID3, QueryOptions.empty(), clinicalTest.token).first();
         assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
-        assertEquals(2, clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size());
+        assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
         System.out.println("results at out dir = " + outDir.toAbsolutePath());
     }
 

opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java

@@ -49,7 +49,9 @@
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.text.SimpleDateFormat;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -260,6 +262,14 @@ public Path isolateOpenCGA() throws IOException {
         inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R");
         Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING);
 
+        // Exomiser analysis files
+        analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser")).toAbsolutePath();
+        List<String> exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml");
+        for (String exomiserFile : exomiserFiles) {
+            inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserFile);
+            Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING);
+        }
+
         return opencgaHome;
     }
 

opencga-app/app/analysis/exomiser/exomiser-analysis.yml

@@ -0,0 +1,70 @@
+---
+analysisMode: PASS_ONLY
+inheritanceModes: {
+  AUTOSOMAL_DOMINANT: 0.1,
+  AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1,
+  AUTOSOMAL_RECESSIVE_COMP_HET: 2.0,
+  X_DOMINANT: 0.1,
+  X_RECESSIVE_HOM_ALT: 0.1,
+  X_RECESSIVE_COMP_HET: 2.0,
+  MITOCHONDRIAL: 0.2
+}
+frequencySources: [
+    THOUSAND_GENOMES,
+    TOPMED,
+    UK10K,
+
+    ESP_AFRICAN_AMERICAN, ESP_EUROPEAN_AMERICAN, ESP_ALL,
+
+    EXAC_AFRICAN_INC_AFRICAN_AMERICAN, EXAC_AMERICAN,
+    EXAC_SOUTH_ASIAN, EXAC_EAST_ASIAN,
+    EXAC_FINNISH, EXAC_NON_FINNISH_EUROPEAN,
+    EXAC_OTHER,
+
+    GNOMAD_E_AFR,
+    GNOMAD_E_AMR,
+  #        GNOMAD_E_ASJ,
+    GNOMAD_E_EAS,
+    GNOMAD_E_FIN,
+    GNOMAD_E_NFE,
+    GNOMAD_E_OTH,
+    GNOMAD_E_SAS,
+
+    GNOMAD_G_AFR,
+    GNOMAD_G_AMR,
+  #        GNOMAD_G_ASJ,
+    GNOMAD_G_EAS,
+    GNOMAD_G_FIN,
+    GNOMAD_G_NFE,
+    GNOMAD_G_OTH,
+    GNOMAD_G_SAS
+]
+# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM
+# REMM is trained on non-coding regulatory regions
+# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files
+# and updated their location in the application.properties. Exomiser will not run without this.
+pathogenicitySources: [ REVEL, MVP ]
+#this is the standard exomiser order.
+steps: [
+    failedVariantFilter: { },
+    variantEffectFilter: {
+      remove: [
+          FIVE_PRIME_UTR_EXON_VARIANT,
+          FIVE_PRIME_UTR_INTRON_VARIANT,
+          THREE_PRIME_UTR_EXON_VARIANT,
+          THREE_PRIME_UTR_INTRON_VARIANT,
+          NON_CODING_TRANSCRIPT_EXON_VARIANT,
+          NON_CODING_TRANSCRIPT_INTRON_VARIANT,
+          CODING_TRANSCRIPT_INTRON_VARIANT,
+          UPSTREAM_GENE_VARIANT,
+          DOWNSTREAM_GENE_VARIANT,
+          INTERGENIC_VARIANT,
+          REGULATORY_REGION_VARIANT
+      ]
+    },
+    frequencyFilter: { maxFrequency: 2.0 },
+    pathogenicityFilter: { keepNonPathogenic: true },
+    inheritanceFilter: { },
+    omimPrioritiser: { },
+    hiPhivePrioritiser: { }
+]

opencga-app/app/cloud/docker/opencga-base/Dockerfile

@@ -7,7 +7,7 @@ ARG BUILD_PATH="."
 ENV OPENCGA_HOME=/opt/opencga
 ENV OPENCGA_CONFIG_DIR=${OPENCGA_HOME}/conf
 
-RUN apt-get update && apt-get -y upgrade && apt-get install -y lsb-release sshpass ca-certificates curl gnupg jq ncurses-bin && \
+RUN apt-get update && apt-get -y upgrade && apt-get install -y lsb-release sshpass ca-certificates curl gnupg jq ncurses-bin unzip vim && \
     ## Install Docker repository
     install -m 0755 -d /etc/apt/keyrings && \
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java

@@ -226,6 +226,7 @@ public InternalCliOptionsParser() {
         clinicalSubCommands.addCommand(RGA_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaSecondaryIndexCommandOptions);
         clinicalSubCommands.addCommand(RGA_AUX_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaAuxiliarSecondaryIndexCommandOptions);
         clinicalSubCommands.addCommand(EXOMISER_INTERPRETATION_RUN_COMMAND, clinicalCommandOptions.exomiserInterpretationCommandOptions);
+        clinicalSubCommands.addCommand("tsv-load", clinicalCommandOptions.tsvLoad);
 
         fileCommandOptions = new FileCommandOptions(commonCommandOptions, jCommander);
         jCommander.addCommand("files", fileCommandOptions);