Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TASK-5858 - Fix alignment and coverage index analyses in blob storage #2416

Merged
merged 7 commits into from
Apr 19, 2024
Merged

TASK-5858 - Fix alignment and coverage index analyses in blob storage #2416

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
060c26f
analysis: improve alignment and coverage index by taking into account…
jtarraga Mar 14, 2024
57fbbe1
tests: add Junit tests, #TASK-5858
jtarraga Mar 25, 2024
c550fd3
client: generate clients, #TASK-5858
jtarraga Mar 25, 2024
f26c9e8
core: fix Integer conversion, #TASK-5858
jtarraga Mar 25, 2024
1e5f15b
analysis: remove jobID check from OpenCgaTool, #TASK-5858
jtarraga Mar 25, 2024
80a2e9a
Merge branch 'release-2.12.x' into TASK-5858
jtarraga Apr 16, 2024
313ae7f
Merge branch 'release-2.12.x' into TASK-5858
jtarraga Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions ...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ protected void check() throws Exception {
super.check();

// Sanity check
if (StringUtils.isEmpty(getJobId())) {
throw new ToolException("Missing job ID");
}

if (StringUtils.isEmpty(getStudy())) {
throw new ToolException("Missing study when computing alignment coverage");
}
Expand Down Expand Up @@ -168,20 +172,22 @@ protected void run() throws Exception {
+ ") was not create, please, check log files.");
}

// Try to copy the BW file into the BAM file directory
// Try to move the BW file into the BAM file directory
boolean moveSuccessful = false;
Path targetPath = Paths.get(bamCatalogFile.getUri()).getParent().resolve(bwPath.getFileName());
try {
Files.move(bwPath, targetPath);
Path movedPath = Files.move(bwPath, targetPath);
moveSuccessful = targetPath.equals(movedPath);
} catch (Exception e) {
// Do nothing
logger.info("Moving from {} to {}: {}", bwPath, targetPath, e.getMessage());
// Log message
logger.info("Error moving the coverage file into the BAM folder {} to {}", bwPath, targetPath, e);
}

if (targetPath.toFile().exists()) {
if (moveSuccessful) {
bwPath = targetPath;
logger.info("Coverage file was copied into the BAM folder: {}", bwPath);
logger.info("Coverage file was moved into the BAM folder: {}", bwPath);
} else {
logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
logger.info("Couldn't move the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
bwPath);
}

Expand Down
70 changes: 36 additions & 34 deletions ...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,34 @@

package org.opencb.opencga.analysis.alignment;

import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.tools.alignment.BamManager;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.tools.OpenCgaTool;
import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.common.InternalStatus;
import org.opencb.opencga.core.models.file.*;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.tools.annotations.Tool;
import org.opencb.opencga.core.tools.annotations.ToolParams;

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

@Tool(id = AlignmentIndexOperation.ID, resource = Enums.Resource.ALIGNMENT, description = "Index alignment.")
public class AlignmentIndexOperation extends OpenCgaTool {
public class AlignmentIndexOperation extends OpenCgaToolScopeStudy {

public static final String ID = "alignment-index-run";
public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";

private String study;
private String inputFile;
@ToolParams
protected final AlignmentIndexParams indexParams = new AlignmentIndexParams();

private File inputCatalogFile;
private Path inputPath;
Expand All @@ -48,14 +53,23 @@ public class AlignmentIndexOperation extends OpenCgaTool {
protected void check() throws Exception {
super.check();

// Sanity check
if (StringUtils.isEmpty(getJobId())) {
throw new ToolException("Missing job ID");
}

if (StringUtils.isEmpty(getStudy())) {
throw new ToolException("Missing study when computing alignment index");
}

OpenCGAResult<File> fileResult;
try {
fileResult = catalogManager.getFileManager().get(getStudy(), inputFile, QueryOptions.empty(), token);
fileResult = catalogManager.getFileManager().get(getStudy(), indexParams.getFileId(), QueryOptions.empty(), token);
} catch (CatalogException e) {
throw new ToolException("Error accessing file '" + inputFile + "' of the study " + study + "'", e);
throw new ToolException("Error accessing file '" + indexParams.getFileId() + "' of the study " + study + "'", e);
}
if (fileResult.getNumResults() <= 0) {
throw new ToolException("File '" + inputFile + "' not found in study '" + study + "'");
throw new ToolException("File '" + indexParams.getFileId() + "' not found in study '" + study + "'");
}

inputCatalogFile = fileResult.getResults().get(0);
Expand All @@ -64,7 +78,7 @@ protected void check() throws Exception {

// Check if the input file is .bam or .cram
if (!filename.endsWith(AlignmentConstants.BAM_EXTENSION) && !filename.endsWith(AlignmentConstants.CRAM_EXTENSION)) {
throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format");
throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format");
}

outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
Expand All @@ -73,6 +87,9 @@ protected void check() throws Exception {

@Override
protected void run() throws Exception {
setUpStorageEngineExecutor(study);

logger.info("Running with parameters {}", indexParams);

step(ID, () -> {
// Compute index if necessary
Expand All @@ -82,51 +99,36 @@ protected void run() throws Exception {
bamManager.close();

if (!outputPath.toFile().exists()) {
throw new ToolException("Something wrong happened when computing index file for '" + inputFile + "'");
throw new ToolException("Something wrong happened when computing index file for '" + indexParams.getFileId() + "'");
}

// Try to copy the BAI file into the BAM file directory
// Try to move the BAI file into the BAM file directory
boolean moveSuccessful = false;
Path targetPath = inputPath.getParent().resolve(outputPath.getFileName());
try {
Files.move(outputPath, targetPath);
Path movedPath = Files.move(outputPath, targetPath);
moveSuccessful = targetPath.equals(movedPath);
} catch (Exception e) {
// Do nothing
logger.info("Moving from {} to {}: {}", outputPath, targetPath, e.getMessage());
// Log message
logger.info("Error moving from {} to {}", outputPath, targetPath, e);
}

if (targetPath.toFile().exists()) {
if (moveSuccessful) {
outputPath = targetPath;
logger.info("Alignment index file was copied into the BAM folder: {}", outputPath);
logger.info("Alignment index file was moved into the BAM folder: {}", outputPath);
} else {
logger.info("Couldn't copy the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
logger.info("Couldn't move the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
outputPath);
}

// Link generated BAI file and update samples info, related file
File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token);
File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager,
token);

// Update BAM file internal in order to set the alignment index (BAI)
FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY),
baiCatalogFile.getId(), "HTSJDK library");
catalogManager.getFileManager().updateFileInternalAlignmentIndex(inputCatalogFile, fileAlignmentIndex, token);
});
}

public String getStudy() {
return study;
}

public AlignmentIndexOperation setStudy(String study) {
this.study = study;
return this;
}

public String getInputFile() {
return inputFile;
}

public AlignmentIndexOperation setInputFile(String inputFile) {
this.inputFile = inputFile;
return this;
}
}
31 changes: 16 additions & 15 deletions ...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,16 @@
import org.opencb.opencga.analysis.StorageManager;
import org.opencb.opencga.analysis.models.FileInfo;
import org.opencb.opencga.analysis.models.StudyInfo;
import org.opencb.opencga.analysis.tools.ToolRunner;
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.catalog.utils.ParamUtils;
import org.opencb.opencga.core.api.ParamConstants;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.project.Project;
import org.opencb.opencga.core.models.study.Study;
Expand Down Expand Up @@ -87,21 +91,18 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto
initStatsMap();
}

//-------------------------------------------------------------------------
// INDEX
//-------------------------------------------------------------------------

public void index(String study, String inputFile, String outdir, String token) throws ToolException {
ObjectMap params = new ObjectMap();

AlignmentIndexOperation indexOperation = new AlignmentIndexOperation();
indexOperation.setUp(null, catalogManager, storageEngineFactory, params, Paths.get(outdir), jobId, token);

indexOperation.setStudy(study);
indexOperation.setInputFile(inputFile);

indexOperation.start();
}
// //-------------------------------------------------------------------------
// // INDEX
// //-------------------------------------------------------------------------
//
// public void index(String study, String inputFile, String outdir, String token) throws ToolException {
// ToolRunner toolRunner = new ToolRunner("", catalogManager, storageEngineFactory);
//
// AlignmentIndexParams params = new AlignmentIndexParams();
// params.setFileId(inputFile);
// toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, study), Paths.get(outdir),
// jobId, token);
// }

//-------------------------------------------------------------------------
// QUERY
Expand Down
Loading
Loading