Skip to content

Commit

Permalink
analysis: implement overwrite for alignment index and coverage; and g…
Browse files Browse the repository at this point in the history 
…enerate clients, #TASK-5679, #TASK-5662
  • Loading branch information
@jtarraga
jtarraga committed Feb 19, 2024
1 parent aff4ce5 commit f9da105
Show file tree
Hide file tree
Showing 84 changed files with 246 additions and 315 deletions.
11 changes: 5 additions & 6 deletions ...-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,13 @@

public class AlignmentAnalysisUtils {

public static File linkAndUpdate(File bamCatalogFile, Path outPath, String study, CatalogManager catalogManager, String token)
public static File linkAndUpdate(File bamCatalogFile, Path outPath, String jobId, String study, CatalogManager catalogManager, String token)
throws CatalogException, ToolException {
// Link BW file and update sample info
FileLinkParams fileLinkParams = new FileLinkParams().setUri(outPath.toString());
if (Paths.get(bamCatalogFile.getPath()).getParent() != null) {
fileLinkParams.setPath(Paths.get(bamCatalogFile.getPath()).getParent().resolve(outPath.getFileName()).toString());
}
OpenCGAResult<File> fileResult = catalogManager.getFileManager().link(study, fileLinkParams, false, token);
FileLinkParams fileLinkParams = new FileLinkParams()
.setUri(outPath.toString())
.setPath(Paths.get(jobId).resolve(outPath.getFileName()).toString());
OpenCGAResult<File> fileResult = catalogManager.getFileManager().link(study, fileLinkParams, true, token);
if (fileResult.getNumResults() != 1) {
throw new ToolException("It could not link OpenCGA file catalog file for '" + outPath + "'");
}
Expand Down
51 changes: 28 additions & 23 deletions ...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.opencb.opencga.core.models.common.InternalStatus;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.file.FileInternalCoverageIndex;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.tools.annotations.Tool;
import org.opencb.opencga.core.tools.annotations.ToolParams;

Expand All @@ -43,7 +42,7 @@
public class AlignmentCoverageAnalysis extends OpenCgaToolScopeStudy {

public static final String ID = "coverage-index-run";
public static final String DESCRIPTION = "Compute the coverage from a given alignment file, e.g., create a "
public static final String DESCRIPTION = "Compute the coverage from a given BAM alignment file, e.g., create a "
+ AlignmentConstants.BIGWIG_EXTENSION + " file from a " + AlignmentConstants.BAM_EXTENSION + " file";

@ToolParams
Expand All @@ -63,37 +62,36 @@ protected void check() throws Exception {

// Checking BAM file ID
try {
bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getBamFileId(), QueryOptions.empty(),
bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getFileId(), QueryOptions.empty(),
getToken()).first();
if (bamCatalogFile == null) {
throw new ToolException("Could not find BAM file from ID '" + coverageParams.getBamFileId() + "'");
throw new ToolException("Could not find BAM file from ID '" + coverageParams.getFileId() + "'");
}
} catch (Exception e) {
throw new ToolException("Could not get BAM file from ID " + coverageParams.getBamFileId());
throw new ToolException("Could not get BAM file from ID " + coverageParams.getFileId());
}

// Check if the input file is .bam
if (!bamCatalogFile.getName().endsWith(AlignmentConstants.BAM_EXTENSION)) {
throw new ToolException("Invalid input alignment file '" + coverageParams.getBamFileId() + "' (" + bamCatalogFile.getName()
throw new ToolException("Invalid input alignment file '" + coverageParams.getFileId() + "' (" + bamCatalogFile.getName()
+ "): it must be in BAM format");
}

// Getting BAI file
String baiFileId = coverageParams.getBaiFileId();
String baiFileId;
try {
baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
} catch (Exception e) {
throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId() + "'");
}
if (StringUtils.isEmpty(baiFileId)) {
// BAI file ID was not provided, looking for it
logger.info("BAI file ID was not provided, getting it from the internal alignment index of the BAM file ID {}",
bamCatalogFile.getId());
try {
baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
} catch (Exception e) {
throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId());
}
throw new ToolException("Could not find the alignment index file for the BAM file ID '" + bamCatalogFile.getId() + "'. Please,"
+ " create the alignment index file before computing the coverage");
}
try {
baiCatalogFile = catalogManager.getFileManager().get(getStudy(), baiFileId, QueryOptions.empty(), getToken()).first();
if (baiCatalogFile == null) {
throw new ToolException("Could not find BAI file from ID '" + coverageParams.getBaiFileId() + "'");
throw new ToolException("Could not find BAI file from ID '" + baiFileId + "'");
}
} catch (Exception e) {
throw new ToolException("Could not get BAI file from file ID " + baiFileId);
Expand All @@ -114,6 +112,18 @@ protected void check() throws Exception {
coverageParams.setWindowSize(Integer.parseInt(COVERAGE_WINDOW_SIZE_DEFAULT));
logger.info("Window size is set to {}", coverageParams.getWindowSize());
}

// Check overwrite
String bwFileId;
try {
bwFileId = bamCatalogFile.getInternal().getAlignment().getCoverage().getFileId();
} catch (Exception e) {
bwFileId = null;
}
if (StringUtils.isNotEmpty(bwFileId) && !coverageParams.isOverwrite()) {
throw new ToolException("Coverage file ID '" + bwFileId + "' already exists for file ID '" + bamCatalogFile.getId()
+ "'. To overwrite the BIGWIG file use the flag --overwrite");
}
}

@Override
Expand Down Expand Up @@ -189,17 +199,12 @@ protected void run() throws Exception {
}

// Link generated BIGWIG file and update samples info
File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, study, catalogManager, token);
File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);

// Update BAM file internal in order to set the coverage index (BIGWIG)
FileInternalCoverageIndex fileCoverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY),
bwCatalogFile.getId(), "deeptools bamCoverage", coverageParams.getWindowSize());
OpenCGAResult<?> openCGAResult = catalogManager.getFileManager().updateFileInternalCoverageIndex(bamCatalogFile,
fileCoverageIndex, token);
if (openCGAResult.getNumUpdated() != 1) {
throw new ToolException("It could not update OpenCGA file catalog (alignment index " + bamCatalogFile.getId() + ") with"
+ " the coverage index (BIGWIG) file '" + bwCatalogFile.getId() + "'");
}
catalogManager.getFileManager().updateFileInternalCoverageIndex(bamCatalogFile, fileCoverageIndex, token);
});
}
}
31 changes: 28 additions & 3 deletions ...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,16 @@

package org.opencb.opencga.analysis.alignment;

import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.tools.alignment.BamManager;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.tools.OpenCgaTool;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.common.InternalStatus;
import org.opencb.opencga.core.models.file.*;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.file.FileInternalAlignmentIndex;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.tools.annotations.Tool;

Expand All @@ -35,10 +37,12 @@
public class AlignmentIndexOperation extends OpenCgaTool {

public static final String ID = "alignment-index-run";
public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";
public static final String DESCRIPTION = "Index a given alignment file BAM/CRAM, e.g., create a " + AlignmentConstants.BAI_EXTENSION
+ " file from a " + AlignmentConstants.BAM_EXTENSION + " file";

private String study;
private String inputFile;
private boolean overwrite = false;

private File inputCatalogFile;
private Path inputPath;
Expand Down Expand Up @@ -67,6 +71,18 @@ protected void check() throws Exception {
throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format");
}

// Check overwrite
String baiFileId;
try {
baiFileId = inputCatalogFile.getInternal().getAlignment().getIndex().getFileId();
} catch (Exception e) {
baiFileId = null;
}
if (StringUtils.isNotEmpty(baiFileId) && !overwrite) {
throw new ToolException("Alignment index file ID '" + baiFileId + "' already exists for file ID '" + inputCatalogFile.getId()
+ "'. To overwrite the alignment index file use the flag --overwrite");
}

outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
? AlignmentConstants.BAI_EXTENSION : AlignmentConstants.CRAI_EXTENSION));
}
Expand Down Expand Up @@ -103,7 +119,7 @@ protected void run() throws Exception {
}

// Link generated BAI file and update samples info, related file
File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, study, catalogManager, token);
File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token);

// Update BAM file internal in order to set the alignment index (BAI)
FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY),
Expand All @@ -129,4 +145,13 @@ public AlignmentIndexOperation setInputFile(String inputFile) {
this.inputFile = inputFile;
return this;
}

public boolean isOverwrite() {
return overwrite;
}

public AlignmentIndexOperation setOverwrite(boolean overwrite) {
this.overwrite = overwrite;
return this;
}
}
3 changes: 2 additions & 1 deletion ...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,15 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto
// INDEX
//-------------------------------------------------------------------------

public void index(String study, String inputFile, String outdir, String token) throws ToolException {
public void index(String study, String inputFile, boolean overwrite, String outdir, String token) throws ToolException {
ObjectMap params = new ObjectMap();

AlignmentIndexOperation indexOperation = new AlignmentIndexOperation();
indexOperation.setUp(null, catalogManager, storageEngineFactory, params, Paths.get(outdir), jobId, token);

indexOperation.setStudy(study);
indexOperation.setInputFile(inputFile);
indexOperation.setOverwrite(overwrite);

indexOperation.start();
}
Expand Down
123 changes: 5 additions & 118 deletions ...src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ public class AlignmentCommandExecutor extends InternalCommandExecutor {

private AlignmentCommandOptions alignmentCommandOptions;
private String jobId;
// private AlignmentStorageEngine alignmentStorageManager;

public AlignmentCommandExecutor(AlignmentCommandOptions options) {
super(options.analysisCommonOptions);
Expand All @@ -76,18 +75,6 @@ public void execute() throws Exception {
case "gene-coveratge-stats-run":
geneCoverageStatsRun();
break;
// case "stats-run":
// statsRun();
// break;
// case "flagstats-run":
// flagStatsRun();
// break;
// case "fastqcmetrics-run":
// fastQcMetricsRun();
// break;
// case "hsmetrics-run":
// hsMetricsRun();
// break;
case "coverage-index-run":
coverageRun();
break;
Expand Down Expand Up @@ -119,38 +106,12 @@ public void execute() throws Exception {
private void indexRun() throws Exception {
AlignmentCommandOptions.IndexAlignmentCommandOptions cliOptions = alignmentCommandOptions.indexAlignmentCommandOptions;

AlignmentStorageManager alignmentManager = new AlignmentStorageManager(catalogManager, storageEngineFactory, alignmentCommandOptions.internalJobOptions.jobId);
AlignmentStorageManager alignmentManager = new AlignmentStorageManager(catalogManager, storageEngineFactory,
alignmentCommandOptions.internalJobOptions.jobId);

alignmentManager.index(cliOptions.study, cliOptions.file, cliOptions.outdir, cliOptions.commonOptions.token);
alignmentManager.index(cliOptions.study, cliOptions.file, cliOptions.overwrite, cliOptions.outdir, cliOptions.commonOptions.token);
}


// private void query() throws InterruptedException, CatalogException, IOException {
// ObjectMap objectMap = new ObjectMap();
// objectMap.putIfNotNull("sid", alignmentCommandOptions.queryAlignmentCommandOptions.commonOptions.token);
// objectMap.putIfNotNull("study", alignmentCommandOptions.queryAlignmentCommandOptions.study);
// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.REGION.key(), alignmentCommandOptions.queryAlignmentCommandOptions.region);
// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.MIN_MAPQ.key(),
// alignmentCommandOptions.queryAlignmentCommandOptions.minMappingQuality);
// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.CONTAINED.key(),
// alignmentCommandOptions.queryAlignmentCommandOptions.contained);
// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.MD_FIELD.key(),
// alignmentCommandOptions.queryAlignmentCommandOptions.mdField);
// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.BIN_QUALITIES.key(),
// alignmentCommandOptions.queryAlignmentCommandOptions.binQualities);
// objectMap.putIfNotNull(QueryOptions.LIMIT, alignmentCommandOptions.queryAlignmentCommandOptions.limit);
// objectMap.putIfNotNull(QueryOptions.SKIP, alignmentCommandOptions.queryAlignmentCommandOptions.skip);
// objectMap.putIfNotNull(QueryOptions.COUNT, alignmentCommandOptions.queryAlignmentCommandOptions.count);
//
// OpenCGAClient openCGAClient = new OpenCGAClient(clientConfiguration);
// RestResponse<ReadAlignment> alignments = openCGAClient.getAlignmentClient()
// .query(alignmentCommandOptions.queryAlignmentCommandOptions.fileId, objectMap);
//
// for (ReadAlignment readAlignment : alignments.allResults()) {
// System.out.println(readAlignment);
// }
// }

private void qcRun() throws ToolException {
AlignmentCommandOptions.QcAlignmentCommandOptions cliOptions = alignmentCommandOptions.qcAlignmentCommandOptions;

Expand Down Expand Up @@ -179,64 +140,13 @@ private void geneCoverageStatsRun() throws ToolException {
toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
}

// private void statsRun() throws ToolException {
// AlignmentCommandOptions.StatsAlignmentCommandOptions cliOptions = alignmentCommandOptions.statsAlignmentCommandOptions;
//
// ObjectMap params = new AlignmentStatsParams(
// cliOptions.file,
// cliOptions.outdir
// ).toObjectMap(cliOptions.commonOptions.params)
// .append(ParamConstants.STUDY_PARAM, cliOptions.study);
//
// toolRunner.execute(AlignmentStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
// }
//
// private void flagStatsRun() throws ToolException {
// AlignmentCommandOptions.FlagStatsAlignmentCommandOptions cliOptions = alignmentCommandOptions.flagStatsAlignmentCommandOptions;
//
// ObjectMap params = new AlignmentFlagStatsParams(
// cliOptions.file,
// cliOptions.outdir
// ).toObjectMap(cliOptions.commonOptions.params)
// .append(ParamConstants.STUDY_PARAM, cliOptions.study);
//
// toolRunner.execute(AlignmentFlagStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
// }
//
// private void fastQcMetricsRun() throws ToolException {
// AlignmentCommandOptions.FastQcMetricsAlignmentCommandOptions cliOptions = alignmentCommandOptions
// .fastQcMetricsAlignmentCommandOptions;
//
// ObjectMap params = new AlignmentFastQcMetricsParams(
// cliOptions.file,
// cliOptions.outdir
// ).toObjectMap(cliOptions.commonOptions.params)
// .append(ParamConstants.STUDY_PARAM, cliOptions.study);
//
// toolRunner.execute(AlignmentFastQcMetricsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
// }
//
// private void hsMetricsRun() throws ToolException {
// AlignmentCommandOptions.HsMetricsAlignmentCommandOptions cliOptions = alignmentCommandOptions.hsMetricsAlignmentCommandOptions;
//
// ObjectMap params = new AlignmentHsMetricsParams(
// cliOptions.bamFile,
// cliOptions.bedFile,
// cliOptions.dictFile,
// cliOptions.outdir
// ).toObjectMap(cliOptions.commonOptions.params)
// .append(ParamConstants.STUDY_PARAM, cliOptions.study);
//
// toolRunner.execute(AlignmentHsMetricsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
// }

private void coverageRun() throws ToolException {
AlignmentCommandOptions.CoverageAlignmentCommandOptions cliOptions = alignmentCommandOptions.coverageAlignmentCommandOptions;

ObjectMap params = new CoverageIndexParams(
cliOptions.bamFileId,
cliOptions.baiFileId,
cliOptions.windowSize
cliOptions.windowSize,
cliOptions.overwrite
).toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study);

toolRunner.execute(AlignmentCoverageAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
Expand Down Expand Up @@ -323,27 +233,4 @@ private void picard() throws Exception {

toolRunner.execute(PicardWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token);
}

//-------------------------------------------------------------------------
// M I S C E L A N E O U S M E T H O D S
//-------------------------------------------------------------------------

private void addParam(Map<String, String> map, String key, Object value) {
if (value == null) {
return;
}

if (value instanceof String) {
if (!((String) value).isEmpty()) {
map.put(key, (String) value);
}
} else if (value instanceof Integer) {
map.put(key, Integer.toString((int) value));
} else if (value instanceof Boolean) {
map.put(key, Boolean.toString((boolean) value));
} else {
throw new UnsupportedOperationException();
}
}

}
Loading

0 comments on commit f9da105

Please sign in to comment.