Merge branch 'release-2.12.x' into TASK-5877

opencga-analysis/pom.xml

@@ -243,6 +243,17 @@
     </dependencies>
 
     <build>
+        <testResources>
+            <testResource>
+                <directory>../opencga-core/src/test/resources</directory>
+                <includes>
+                    <include>log4j2-test.xml</include>
+                </includes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
+        </testResources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java

@@ -54,6 +54,10 @@ protected void check() throws Exception {
         super.check();
 
         // Sanity check
+        if (StringUtils.isEmpty(getJobId())) {
+            throw new ToolException("Missing job ID");
+        }
+
         if (StringUtils.isEmpty(getStudy())) {
             throw new ToolException("Missing study when computing alignment coverage");
         }
@@ -168,20 +172,22 @@ protected void run() throws Exception {
                         + ") was not create, please, check log files.");
             }
 
-            // Try to copy the BW file into the BAM file directory
+            // Try to move the BW file into the BAM file directory
+            boolean moveSuccessful = false;
             Path targetPath = Paths.get(bamCatalogFile.getUri()).getParent().resolve(bwPath.getFileName());
             try {
-                Files.move(bwPath, targetPath);
+                Path movedPath = Files.move(bwPath, targetPath);
+                moveSuccessful = targetPath.equals(movedPath);
             } catch (Exception e) {
-                // Do nothing
-                logger.info("Moving from {} to {}: {}", bwPath, targetPath, e.getMessage());
+                // Log message
+                logger.info("Error moving the coverage file into the BAM folder {} to {}", bwPath, targetPath, e);
             }
 
-            if (targetPath.toFile().exists()) {
+            if (moveSuccessful) {
                 bwPath = targetPath;
-                logger.info("Coverage file was copied into the BAM folder: {}", bwPath);
+                logger.info("Coverage file was moved into the BAM folder: {}", bwPath);
             } else {
-                logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
+                logger.info("Couldn't move the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
                         bwPath);
             }
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java

@@ -16,29 +16,34 @@
 
 package org.opencb.opencga.analysis.alignment;
 
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.tools.alignment.BamManager;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.tools.OpenCgaTool;
+import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
+import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.InternalStatus;
 import org.opencb.opencga.core.models.file.*;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
+import org.opencb.opencga.core.tools.annotations.ToolParams;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 @Tool(id = AlignmentIndexOperation.ID, resource = Enums.Resource.ALIGNMENT, description = "Index alignment.")
-public class AlignmentIndexOperation extends OpenCgaTool {
+public class AlignmentIndexOperation extends OpenCgaToolScopeStudy {
 
     public static final String ID = "alignment-index-run";
     public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";
 
-    private String study;
-    private String inputFile;
+    @ToolParams
+    protected final AlignmentIndexParams indexParams = new AlignmentIndexParams();
 
     private File inputCatalogFile;
     private Path inputPath;
@@ -48,14 +53,23 @@ public class AlignmentIndexOperation extends OpenCgaTool {
     protected void check() throws Exception {
         super.check();
 
+        // Sanity check
+        if (StringUtils.isEmpty(getJobId())) {
+            throw new ToolException("Missing job ID");
+        }
+
+        if (StringUtils.isEmpty(getStudy())) {
+            throw new ToolException("Missing study when computing alignment index");
+        }
+
         OpenCGAResult<File> fileResult;
         try {
-            fileResult = catalogManager.getFileManager().get(getStudy(), inputFile, QueryOptions.empty(), token);
+            fileResult = catalogManager.getFileManager().get(getStudy(), indexParams.getFileId(), QueryOptions.empty(), token);
         } catch (CatalogException e) {
-            throw new ToolException("Error accessing file '" + inputFile + "' of the study " + study + "'", e);
+            throw new ToolException("Error accessing file '" + indexParams.getFileId() + "' of the study " + study + "'", e);
         }
         if (fileResult.getNumResults() <= 0) {
-            throw new ToolException("File '" + inputFile + "' not found in study '" + study + "'");
+            throw new ToolException("File '" + indexParams.getFileId() + "' not found in study '" + study + "'");
         }
 
         inputCatalogFile = fileResult.getResults().get(0);
@@ -64,7 +78,7 @@ protected void check() throws Exception {
 
         // Check if the input file is .bam or .cram
         if (!filename.endsWith(AlignmentConstants.BAM_EXTENSION) && !filename.endsWith(AlignmentConstants.CRAM_EXTENSION)) {
-            throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format");
+            throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format");
         }
 
         outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
@@ -73,6 +87,9 @@ protected void check() throws Exception {
 
     @Override
     protected void run() throws Exception {
+        setUpStorageEngineExecutor(study);
+
+        logger.info("Running with parameters {}", indexParams);
 
         step(ID, () -> {
             // Compute index if necessary
@@ -82,51 +99,36 @@ protected void run() throws Exception {
             bamManager.close();
 
             if (!outputPath.toFile().exists()) {
-                throw new ToolException("Something wrong happened when computing index file for '" + inputFile + "'");
+                throw new ToolException("Something wrong happened when computing index file for '" + indexParams.getFileId() + "'");
             }
 
-            // Try to copy the BAI file into the BAM file directory
+            // Try to move the BAI file into the BAM file directory
+            boolean moveSuccessful = false;
             Path targetPath = inputPath.getParent().resolve(outputPath.getFileName());
             try {
-                Files.move(outputPath, targetPath);
+                Path movedPath = Files.move(outputPath, targetPath);
+                moveSuccessful = targetPath.equals(movedPath);
             } catch (Exception e) {
-                // Do nothing
-                logger.info("Moving from {} to {}: {}", outputPath, targetPath, e.getMessage());
+                // Log message
+                logger.info("Error moving from {} to {}", outputPath, targetPath, e);
             }
 
-            if (targetPath.toFile().exists()) {
+            if (moveSuccessful) {
                 outputPath = targetPath;
-                logger.info("Alignment index file was copied into the BAM folder: {}", outputPath);
+                logger.info("Alignment index file was moved into the BAM folder: {}", outputPath);
             } else {
-                logger.info("Couldn't copy the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
+                logger.info("Couldn't move the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
                         outputPath);
             }
 
             // Link generated BAI file and update samples info, related file
-            File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token);
+            File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager,
+                    token);
 
             // Update BAM file internal in order to set the alignment index (BAI)
             FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY),
                     baiCatalogFile.getId(), "HTSJDK library");
             catalogManager.getFileManager().updateFileInternalAlignmentIndex(inputCatalogFile, fileAlignmentIndex, token);
         });
     }
-
-    public String getStudy() {
-        return study;
-    }
-
-    public AlignmentIndexOperation setStudy(String study) {
-        this.study = study;
-        return this;
-    }
-
-    public String getInputFile() {
-        return inputFile;
-    }
-
-    public AlignmentIndexOperation setInputFile(String inputFile) {
-        this.inputFile = inputFile;
-        return this;
-    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java

@@ -34,12 +34,16 @@
 import org.opencb.opencga.analysis.StorageManager;
 import org.opencb.opencga.analysis.models.FileInfo;
 import org.opencb.opencga.analysis.models.StudyInfo;
+import org.opencb.opencga.analysis.tools.ToolRunner;
 import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
 import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.managers.CatalogManager;
 import org.opencb.opencga.catalog.utils.ParamUtils;
+import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
+import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.file.File;
 import org.opencb.opencga.core.models.project.Project;
 import org.opencb.opencga.core.models.study.Study;
@@ -87,21 +91,18 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto
         initStatsMap();
     }
 
-    //-------------------------------------------------------------------------
-    // INDEX
-    //-------------------------------------------------------------------------
-
-    public void index(String study, String inputFile, String outdir, String token) throws ToolException {
-        ObjectMap params = new ObjectMap();
-
-        AlignmentIndexOperation indexOperation = new AlignmentIndexOperation();
-        indexOperation.setUp(null, catalogManager, storageEngineFactory, params, Paths.get(outdir), jobId, token);
-
-        indexOperation.setStudy(study);
-        indexOperation.setInputFile(inputFile);
-
-        indexOperation.start();
-    }
+//    //-------------------------------------------------------------------------
+//    // INDEX
+//    //-------------------------------------------------------------------------
+//
+//    public void index(String study, String inputFile, String outdir, String token) throws ToolException {
+//        ToolRunner toolRunner = new ToolRunner("", catalogManager, storageEngineFactory);
+//
+//        AlignmentIndexParams params = new AlignmentIndexParams();
+//        params.setFileId(inputFile);
+//        toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, study), Paths.get(outdir),
+//                jobId, token);
+//    }
 
     //-------------------------------------------------------------------------
     // QUERY

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java

@@ -68,6 +68,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis {
     private String studyId;
     private String clinicalAnalysisId;
     private String sampleId;
+    private ClinicalAnalysis.Type clinicalAnalysisType;
 
     private ClinicalAnalysis clinicalAnalysis;
 
@@ -116,6 +117,14 @@ protected void check() throws Exception {
         }
         sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId();
 
+        if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) {
+            clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY;
+        } else {
+            clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE;
+        }
+        logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(),
+                clinicalAnalysisType);
+
         // Update executor params with OpenCGA home and session ID
         setUpStorageEngineExecutor(studyId);
     }
@@ -128,6 +137,7 @@ protected void run() throws ToolException {
             getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(studyId)
                     .setSampleId(sampleId)
+                    .setClinicalAnalysisType(clinicalAnalysisType)
                     .execute();
 
             saveInterpretation(studyId, clinicalAnalysis);
@@ -181,7 +191,8 @@ private List<ClinicalVariant> getPrimaryFindings() throws IOException, StorageEn
 
         // Prepare variant query
         List<String> sampleIds = new ArrayList<>();
-        if (clinicalAnalysis.getFamily() != null && CollectionUtils.isNotEmpty(clinicalAnalysis.getFamily().getMembers())) {
+        if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY && clinicalAnalysis.getFamily() != null
+                && CollectionUtils.isNotEmpty(clinicalAnalysis.getFamily().getMembers())) {
             for (Individual member : clinicalAnalysis.getFamily().getMembers()) {
                 Individual individual = IndividualQcUtils.getIndividualById(studyId, member.getId(), getCatalogManager(), getToken());
                 if (CollectionUtils.isNotEmpty(individual.getSamples())) {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java

@@ -460,7 +460,7 @@ protected final void step(String stepId, StepRunnable step) throws ToolException
             } catch (ToolException e) {
                 throw e;
             } catch (Exception e) {
-                throw new ToolException("Exception from step " + stepId, e);
+                throw new ToolException("Exception from step '" + stepId + "'", e);
             }
         } else {
             privateLogger.info("------- Skip step " + stepId + " -------");

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java

@@ -68,7 +68,6 @@
 import org.opencb.opencga.core.models.sample.SamplePermissions;
 import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.core.models.study.StudyPermissions;
-import org.opencb.opencga.core.models.variant.VariantPruneParams;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.response.VariantQueryResult;
 import org.opencb.opencga.core.tools.ToolParams;
@@ -408,7 +407,6 @@ public DataResult<Trio> familyIndex(String study, List<String> familiesStr, bool
             throws CatalogException, StorageEngineException {
         return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> {
             List<Trio> trios = new LinkedList<>();
-            List<Event> events = new LinkedList<>();
             VariantStorageMetadataManager metadataManager = engine.getMetadataManager();
             VariantCatalogQueryUtils catalogUtils = new VariantCatalogQueryUtils(catalogManager);
             if (familiesStr.size() == 1 && familiesStr.get(0).equals(VariantQueryUtils.ALL)) {
@@ -425,7 +423,7 @@ public DataResult<Trio> familyIndex(String study, List<String> familiesStr, bool
             }
             DataResult<Trio> dataResult = engine.familyIndex(study, trios, params);
             getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream()
-                    .flatMap(t->t.toList().stream())
+                    .flatMap(t -> t.toList().stream())
                     .collect(Collectors.toList()), token);
             return dataResult;
         });
@@ -441,11 +439,29 @@ public DataResult<Trio> familyIndexBySamples(String study, Collection<String> sa
             throws CatalogException, StorageEngineException {
         return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> {
             Collection<String> thisSamples = samples;
+            boolean allSamples;
             if (CollectionUtils.size(thisSamples) == 1 && thisSamples.iterator().next().equals(ParamConstants.ALL)) {
                 thisSamples = getIndexedSamples(study, token);
+                allSamples = true;
+            } else {
+                allSamples = false;
             }
 
             List<Trio> trios = catalogUtils.getTriosFromSamples(study, engine.getMetadataManager(), thisSamples, token);
+            if (trios.isEmpty()) {
+                String msg;
+                if (thisSamples.size() > 6) {
+                    msg = "No trios found for " + thisSamples.size() + " samples";
+                } else {
+                    msg = "No trios found for samples " + thisSamples;
+                }
+                if (allSamples) {
+                    logger.info(msg);
+                    return new DataResult<>(0, Collections.singletonList(new Event(Event.Type.INFO, msg)), 0, Collections.emptyList(), 0);
+                } else {
+                    throw new StorageEngineException(msg);
+                }
+            }
             DataResult<Trio> dataResult = engine.familyIndex(study, trios, params);
             getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream()
                     .flatMap(t -> t.toList().stream())

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java

@@ -20,7 +20,7 @@
 import org.apache.solr.common.StringUtils;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.common.Enums;
-import org.opencb.opencga.core.models.variant.VariantFileDeleteParams;
+import org.opencb.opencga.core.models.operations.variant.VariantFileDeleteParams;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 import org.opencb.opencga.storage.core.variant.VariantStorageOptions;

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java

@@ -16,8 +16,8 @@
 import org.opencb.opencga.core.models.file.FileInternal;
 import org.opencb.opencga.core.models.file.VariantIndexStatus;
 import org.opencb.opencga.core.models.job.Job;
-import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams;
-import org.opencb.opencga.core.models.variant.VariantIndexParams;
+import org.opencb.opencga.core.models.operations.variant.VariantFileIndexJobLauncherParams;
+import org.opencb.opencga.core.models.operations.variant.VariantIndexParams;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;