Merge branch 'release-2.12.x' into TASK-5858

Resolve conflicts by generating clients

opencga-analysis/pom.xml

@@ -243,6 +243,14 @@
     </dependencies>
 
     <build>
+        <testResources>
+            <testResource>
+                <directory>../opencga-core/src/test/resources</directory>
+                <includes>
+                    <include>log4j2-test.xml</include>
+                </includes>
+            </testResource>
+        </testResources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java

@@ -68,6 +68,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis {
     private String studyId;
     private String clinicalAnalysisId;
     private String sampleId;
+    private ClinicalAnalysis.Type clinicalAnalysisType;
 
     private ClinicalAnalysis clinicalAnalysis;
 
@@ -116,6 +117,14 @@ protected void check() throws Exception {
         }
         sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId();
 
+        if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) {
+            clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY;
+        } else {
+            clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE;
+        }
+        logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(),
+                clinicalAnalysisType);
+
         // Update executor params with OpenCGA home and session ID
         setUpStorageEngineExecutor(studyId);
     }
@@ -128,6 +137,7 @@ protected void run() throws ToolException {
             getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(studyId)
                     .setSampleId(sampleId)
+                    .setClinicalAnalysisType(clinicalAnalysisType)
                     .execute();
 
             saveInterpretation(studyId, clinicalAnalysis);
@@ -181,7 +191,8 @@ private List<ClinicalVariant> getPrimaryFindings() throws IOException, StorageEn
 
         // Prepare variant query
         List<String> sampleIds = new ArrayList<>();
-        if (clinicalAnalysis.getFamily() != null && CollectionUtils.isNotEmpty(clinicalAnalysis.getFamily().getMembers())) {
+        if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY && clinicalAnalysis.getFamily() != null
+                && CollectionUtils.isNotEmpty(clinicalAnalysis.getFamily().getMembers())) {
             for (Individual member : clinicalAnalysis.getFamily().getMembers()) {
                 Individual individual = IndividualQcUtils.getIndividualById(studyId, member.getId(), getCatalogManager(), getToken());
                 if (CollectionUtils.isNotEmpty(individual.getSamples())) {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java

@@ -31,6 +31,8 @@ public abstract class DockerWrapperAnalysisExecutor  extends OpenCgaToolExecutor
     public static final String STDOUT_FILENAME = "stdout.txt";
     public static final String STDERR_FILENAME = "stderr.txt";
 
+    public static final String DOCKER_CLI_MSG = "Docker CLI: ";
+
     public String getDockerImageName() {
         return "opencb/opencga-ext-tools";
     }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java

@@ -19,6 +19,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
 import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.tools.annotations.Tool;
@@ -52,6 +53,7 @@ protected void run() throws Exception {
             getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(study)
                     .setSampleId(analysisParams.getSample())
+                    .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType()))
                     .execute();
         });
     }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java

@@ -18,6 +18,7 @@
 import org.opencb.opencga.catalog.managers.FamilyManager;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.exceptions.ToolExecutorException;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
 import org.opencb.opencga.core.models.family.Family;
 import org.opencb.opencga.core.models.individual.Individual;
 import org.opencb.opencga.core.tools.annotations.ToolExecutor;
@@ -50,6 +51,7 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut
 
     private String studyId;
     private String sampleId;
+    private ClinicalAnalysis.Type clinicalAnalysisType;
 
     private Logger logger = LoggerFactory.getLogger(this.getClass());
 
@@ -100,22 +102,24 @@ public void run() throws ToolException {
         // Check multi-sample (family) analysis
         File pedigreeFile = null;
         Pedigree pedigree = null;
-        if (individual.getMother() != null && individual.getMother().getId() != null
-                && individual.getFather() != null && individual.getFather().getId() != null) {
-            Family family = IndividualQcUtils.getFamilyByIndividualId(getStudyId(), individual.getId(),
-                    getVariantStorageManager().getCatalogManager(), getToken());
-            if (family != null) {
-                pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId());
-            }
-
-            if (pedigree != null) {
-                if (individual.getFather() != null) {
-                    samples.add(individual.getFather().getSamples().get(0).getId());
+        if (clinicalAnalysisType == ClinicalAnalysis.Type.FAMILY) {
+            if (individual.getMother() != null && individual.getMother().getId() != null
+                    && individual.getFather() != null && individual.getFather().getId() != null) {
+                Family family = IndividualQcUtils.getFamilyByIndividualId(getStudyId(), individual.getId(),
+                        getVariantStorageManager().getCatalogManager(), getToken());
+                if (family != null) {
+                    pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId());
                 }
-                if (individual.getMother() != null) {
-                    samples.add(individual.getMother().getSamples().get(0).getId());
+
+                if (pedigree != null) {
+                    if (individual.getFather() != null) {
+                        samples.add(individual.getFather().getSamples().get(0).getId());
+                    }
+                    if (individual.getMother() != null) {
+                        samples.add(individual.getMother().getSamples().get(0).getId());
+                    }
+                    pedigreeFile = createPedigreeFile(family, pedigree);
                 }
-                pedigreeFile = createPedigreeFile(family, pedigree);
             }
         }
         File sampleFile = createSampleFile(individual, hpos, pedigree);
@@ -192,7 +196,9 @@ public void run() throws ToolException {
                 .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME);
 
         // Execute command and redirect stdout and stderr to the files
-        logger.info("{}: Docker command line: {}", ID, sb);
+        String msg = DOCKER_CLI_MSG + sb;
+        logger.info(msg);
+        addWarning(msg);
         runCommandLine(sb.toString());
     }
 
@@ -483,4 +489,13 @@ public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) {
         this.sampleId = sampleId;
         return this;
     }
+
+    public ClinicalAnalysis.Type getClinicalAnalysisType() {
+        return clinicalAnalysisType;
+    }
+
+    public ExomiserWrapperAnalysisExecutor setClinicalAnalysisType(ClinicalAnalysis.Type clinicalAnalysisType) {
+        this.clinicalAnalysisType = clinicalAnalysisType;
+        return this;
+    }
 }

opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java

@@ -76,6 +76,10 @@ public static AbstractClinicalManagerTest getClinicalTest(OpenCGATestExternalRes
         variantStorageManager.index(clinicalTest.studyFqn, "HG005.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
         variantStorageManager.index(clinicalTest.studyFqn, "HG006.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
         variantStorageManager.index(clinicalTest.studyFqn, "HG007.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
+        variantStorageManager.index(clinicalTest.studyFqn, "HG104.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
+        variantStorageManager.index(clinicalTest.studyFqn, "HG105.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
+        variantStorageManager.index(clinicalTest.studyFqn, "HG106.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
+        variantStorageManager.index(clinicalTest.studyFqn, "HG107.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token);
 
         return clinicalTest;
     }

opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java

@@ -1,19 +1,23 @@
 package org.opencb.opencga.analysis.clinical.exomiser;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
+import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant;
 import org.opencb.biodata.models.variant.Variant;
 import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField;
 import org.opencb.biodata.tools.variant.VariantNormalizer;
 import org.junit.*;
 import org.eclipse.jetty.util.Scanner;
 import org.junit.experimental.categories.Category;
+import org.opencb.commons.datastore.core.Event;
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.clinical.ClinicalAnalysisUtilsTest;
 import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource;
+import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest;
 import org.opencb.opencga.core.exceptions.ToolException;
@@ -26,9 +30,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 
+import static com.mongodb.assertions.Assertions.assertFalse;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -50,8 +58,8 @@ public static void setUp() throws Exception {
         clinicalTest = ClinicalAnalysisUtilsTest.getClinicalTest(opencga);
     }
 
-    @After
-    public void tearDown() throws Exception {
+    @AfterClass
+    public static void tearDown() throws Exception {
         opencga.clear();
     }
 
@@ -71,7 +79,7 @@ public void testNormalization() throws NonStandardCompliantSampleField {
     }
 
     @Test
-    public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException {
+    public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ToolException {
         assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true));
 
         prepareExomiserData();
@@ -95,14 +103,30 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE
                 .get(clinicalTest.studyFqn, clinicalTest.CA_ID2, QueryOptions.empty(), clinicalTest.token).first();
         assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
         assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
+
+        // Check Exomiser docker CLI
+        boolean pedFound = false;
+        for (Event event : result.getEvents()) {
+            if (event.getType() == Event.Type.WARNING && StringUtils.isNotEmpty(event.getMessage())
+                    && event.getMessage().startsWith(DockerWrapperAnalysisExecutor.DOCKER_CLI_MSG)) {
+                List<String> splits = Arrays.asList(event.getMessage().split(" "));
+                pedFound = splits.contains("--ped") && splits.contains("/jobdir/" + clinicalTest.PROBAND_ID2 + ".ped");
+            }
+        }
+        assertFalse(pedFound);
+
+        // Only proband sample is returned in primary findings
+        for (ClinicalVariant cv : clinicalAnalysis.getInterpretation().getPrimaryFindings()) {
+            assertEquals(1, cv.getStudies().get(0).getSamples().size());
+        }
     }
 
     @Test
-    public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException {
+    public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, ToolException {
         assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true));
 
         prepareExomiserData();
-        outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_family"));
+        outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_family"));
 
         ClinicalAnalysis clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
                 .get(clinicalTest.studyFqn, clinicalTest.CA_ID3, QueryOptions.empty(), clinicalTest.token).first();
@@ -121,6 +145,67 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE
                 .get(clinicalTest.studyFqn, clinicalTest.CA_ID3, QueryOptions.empty(), clinicalTest.token).first();
         assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
         assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
+
+        // Check Exomiser docker CLI
+        boolean pedFound = false;
+        for (Event event : result.getEvents()) {
+            if (event.getType() == Event.Type.WARNING && StringUtils.isNotEmpty(event.getMessage())
+                    && event.getMessage().startsWith(DockerWrapperAnalysisExecutor.DOCKER_CLI_MSG)) {
+                List<String> splits = Arrays.asList(event.getMessage().split(" "));
+                pedFound = splits.contains("--ped") && splits.contains("/jobdir/" + clinicalTest.PROBAND_ID3 + ".ped");
+            }
+        }
+        assertTrue(pedFound);
+
+        // All family samples are returned in primary findings
+        for (ClinicalVariant cv : clinicalAnalysis.getInterpretation().getPrimaryFindings()) {
+            assertEquals(4, cv.getStudies().get(0).getSamples().size());
+        }
+
+        System.out.println("results at out dir = " + outDir.toAbsolutePath());
+    }
+
+    @Test
+    public void trioSingleExomiserAnalysis() throws IOException, CatalogException, ToolException {
+        assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true));
+
+        prepareExomiserData();
+        outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_single"));
+
+        ClinicalAnalysis clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
+                .get(clinicalTest.studyFqn, clinicalTest.CA_ID4, QueryOptions.empty(), clinicalTest.token).first();
+        assertEquals(0, clinicalAnalysis.getSecondaryInterpretations().size());
+
+        ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis();
+
+        exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token);
+        exomiser.setStudyId(clinicalTest.studyFqn)
+                .setClinicalAnalysisId(clinicalTest.CA_ID4);
+
+        ExecutionResult result = exomiser.start();
+
+        // Refresh clinical analysis
+        clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager()
+                .get(clinicalTest.studyFqn, clinicalTest.CA_ID4, QueryOptions.empty(), clinicalTest.token).first();
+        assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size());
+        assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0);
+
+        // Check Exomiser docker CLI
+        boolean pedFound = false;
+        for (Event event : result.getEvents()) {
+            if (event.getType() == Event.Type.WARNING && StringUtils.isNotEmpty(event.getMessage())
+                    && event.getMessage().startsWith(DockerWrapperAnalysisExecutor.DOCKER_CLI_MSG)) {
+                List<String> splits = Arrays.asList(event.getMessage().split(" "));
+                pedFound = splits.contains("--ped") && splits.contains("/jobdir/" + clinicalTest.PROBAND_ID4 + ".ped");
+            }
+        }
+        assertFalse(pedFound);
+
+        // Only proband sample is returned in primary findings
+        for (ClinicalVariant cv : clinicalAnalysis.getInterpretation().getPrimaryFindings()) {
+            assertEquals(1, cv.getStudies().get(0).getSamples().size());
+        }
+
         System.out.println("results at out dir = " + outDir.toAbsolutePath());
     }
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java

@@ -62,6 +62,7 @@
 import org.opencb.opencga.core.common.YesNoAuto;
 import org.opencb.opencga.core.exceptions.AnalysisExecutionException;
 import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
 import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams;
 import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin;
 import org.opencb.opencga.core.models.operations.variant.*;
@@ -1017,6 +1018,7 @@ private void exomiser() throws Exception {
 
         ObjectMap params = new ExomiserWrapperParams(
                 cliOptions.sample,
+                cliOptions.clinicalAnalysisType,
                 cliOptions.outdir)
                 .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study);
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java

@@ -48,6 +48,7 @@
 import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser;
 import org.opencb.opencga.core.api.FieldConstants;
 import org.opencb.opencga.core.api.ParamConstants;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
 import org.opencb.opencga.core.models.variant.AnnotationVariantQueryParams;
 import org.opencb.opencga.core.models.variant.SampleVariantFilterParams;
 import org.opencb.opencga.core.tools.variant.IndividualQcAnalysisExecutor;
@@ -1841,10 +1842,13 @@ public class ExomiserAnalysisCommandOptions {
         @Parameter(names = {"--study"}, description = "Study where all the samples belong to.")
         public String study;
 
-        @Parameter(names = {"--sample"}, description = "Sample ID.", required = true)
+        @Parameter(names = {"--sample"}, description = FieldConstants.SAMPLE_ID_DESCRIPTION, required = true)
         public String sample;
 
-        @Parameter(names = {"-o", "--outdir"}, description = "Output directory.")
+        @Parameter(names = {"--clinical-analysis-type"}, description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION)
+        public String clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE.name();
+
+        @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION)
         public String outdir;
     }
 }

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java

@@ -1,5 +1,5 @@
 /*
-* Copyright 2015-2024-03-25 OpenCB
+* Copyright 2015-2024-04-17 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -110,7 +110,7 @@ public abstract class OpenCgaCompleter implements Completer {
             .map(Candidate::new)
             .collect(toList());
 
-    private List<Candidate> filesList = asList( "acl-update","aggregationstats","annotation-sets-load","bioformats","create","distinct","fetch","formats","link","link-run","postlink-run","search","upload","acl","delete","info","unlink","update","annotation-sets-annotations-update","download","grep","head","image","refresh","tail","list","tree")
+    private List<Candidate> filesList = asList( "acl-update","aggregationstats","annotation-sets-load","bioformats","create","distinct","fetch","formats","link","link-run","postlink-run","search","upload","acl","delete","info","unlink","update","annotation-sets-annotations-update","download","grep","head","image","move","refresh","tail","list","tree")
             .stream()
             .map(Candidate::new)
             .collect(toList());