Skip to content

Commit

Permalink
Removed multimapping support and associated dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-cameron
d-cameron committed Nov 21, 2017
1 parent 5a8e7a6 commit 148c8bb
Show file tree
Hide file tree
Showing 21 changed files with 68 additions and 1,129 deletions.
67 changes: 16 additions & 51 deletions .classpath
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry including="**/*.java" kind="src" output="target/test-classes" path="src/test/java"/>
<classpathentry excluding="**/*.java" kind="src" output="target/test-classes" path="src/test/resources"/>
<classpathentry including="**/*.java" kind="src" path="src/main/java"/>
<classpathentry excluding="**/*.java" kind="src" path="src/main/resources"/>
<classpathentry kind="var" path="M2_REPO/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
<classpathentry kind="var" path="M2_REPO/jaligner/jaligner/1.0/jaligner-1.0.jar"/>
<classpathentry kind="var" path="M2_REPO/ssw/ssw/1.0/ssw-1.0.jar"/>
<classpathentry kind="var" path="M2_REPO/junit/junit/4.12/junit-4.12.jar"/>
<classpathentry kind="var" path="M2_REPO/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/guava/guava/23.2-jre/guava-23.2-jre.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/guava/guava/23.4-jre/guava-23.4-jre.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/errorprone/error_prone_annotations/2.0.18/error_prone_annotations-2.0.18.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar"/>
Expand All @@ -48,45 +26,32 @@
<classpathentry kind="var" path="M2_REPO/org/codehaus/woodstox/stax2-api/3.1.4/stax2-api-3.1.4.jar"/>
<classpathentry kind="var" path="M2_REPO/it/unimi/dsi/fastutil/8.1.0/fastutil-8.1.0.jar"/>
<classpathentry kind="var" path="M2_REPO/net/sf/trove4j/trove4j/3.0.3/trove4j-3.0.3.jar"/>
<classpathentry kind="var" path="M2_REPO/com/github/samtools/htsjdk/2.12.0/htsjdk-2.12.0.jar"/>
<classpathentry kind="var" path="M2_REPO/com/github/samtools/htsjdk/2.13.1/htsjdk-2.13.1.jar" sourcepath="/HTSJDK_ROOT"/>
<classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-jexl/2.1.1/commons-jexl-2.1.1.jar"/>
<classpathentry kind="var" path="M2_REPO/org/xerial/snappy/snappy-java/1.1.4/snappy-java-1.1.4.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar"/>
<classpathentry kind="var" path="M2_REPO/org/tukaani/xz/1.5/xz-1.5.jar"/>
<classpathentry kind="var" path="M2_REPO/gov/nih/nlm/ncbi/ngs-java/1.2.4/ngs-java-1.2.4.jar"/>
<classpathentry kind="var" path="M2_REPO/com/github/broadinstitute/picard/2.13.2/picard-2.13.2.jar" sourcepath="/PICARD_ROOT"/>
<classpathentry kind="var" path="M2_REPO/com/github/broadinstitute/picard/2.15.0/picard-2.15.0.jar"/>
<classpathentry kind="var" path="M2_REPO/com/intel/gkl/gkl/0.5.3/gkl-0.5.3.jar"/>
<classpathentry kind="var" path="M2_REPO/org/broadinstitute/gatk-native-bindings/0.0.3/gatk-native-bindings-0.0.3.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/logging/log4j/log4j-api/2.5/log4j-api-2.5.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/logging/log4j/log4j-core/2.5/log4j-core-2.5.jar"/>
<classpathentry kind="var" path="M2_REPO/org/broadinstitute/barclay/1.1.0/barclay-1.1.0.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/logging/log4j/log4j-api/2.3/log4j-api-2.3.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/logging/log4j/log4j-core/2.3/log4j-core-2.3.jar"/>
<classpathentry kind="var" path="M2_REPO/org/broadinstitute/barclay/1.2.2/barclay-1.2.2.jar"/>
<classpathentry kind="var" path="M2_REPO/net/sf/jopt-simple/jopt-simple/5.0.3/jopt-simple-5.0.3.jar"/>
<classpathentry kind="var" path="M2_REPO/org/freemarker/freemarker/2.3.18/freemarker-2.3.18.jar"/>
<classpathentry kind="var" path="M2_REPO/org/freemarker/freemarker/2.3.23/freemarker-2.3.23.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/code/gson/gson/2.2.2/gson-2.2.2.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-map/3.13.0/chronicle-map-3.13.0.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-core/1.7.15/chronicle-core-1.7.15.jar"/>
<classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.7.14/slf4j-api-1.7.14.jar"/>
<classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar"/>
<classpathentry kind="var" path="M2_REPO/com/intellij/annotations/12.0/annotations-12.0.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-values/1.5.5/chronicle-values-1.5.5.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-bytes/1.7.34/chronicle-bytes-1.7.34.jar"/>
<classpathentry kind="var" path="M2_REPO/com/squareup/javapoet/1.5.1/javapoet-1.5.1.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-threads/1.7.7/chronicle-threads-1.7.7.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/affinity/3.0.5/affinity-3.0.5.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna/4.2.1/jna-4.2.1.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna-platform/4.2.1/jna-platform-4.2.1.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-wire/1.7.30/chronicle-wire-1.7.30.jar"/>
<classpathentry kind="var" path="M2_REPO/net/openhft/chronicle-algorithms/1.1.8/chronicle-algorithms-1.1.8.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna/4.4.0/jna-4.4.0.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna-platform/4.4.0/jna-platform-4.4.0.jar"/>
<classpathentry kind="var" path="M2_REPO/com/thoughtworks/xstream/xstream/1.4.9/xstream-1.4.9.jar"/>
<classpathentry kind="var" path="M2_REPO/xmlpull/xmlpull/1.1.3.1/xmlpull-1.1.3.1.jar"/>
<classpathentry kind="var" path="M2_REPO/xpp3/xpp3_min/1.1.4c/xpp3_min-1.1.4c.jar"/>
<classpathentry kind="var" path="M2_REPO/org/codehaus/jettison/jettison/1.3.7/jettison-1.3.7.jar"/>
<classpathentry kind="var" path="M2_REPO/org/codehaus/jettison/jettison/1.3.8/jettison-1.3.8.jar"/>
<classpathentry kind="var" path="M2_REPO/stax/stax-api/1.0.1/stax-api-1.0.1.jar"/>
<classpathentry kind="var" path="M2_REPO/org/ops4j/pax/url/pax-url-aether/2.4.5/pax-url-aether-2.4.5.jar"/>
<classpathentry kind="var" path="M2_REPO/org/slf4j/jcl-over-slf4j/1.6.6/jcl-over-slf4j-1.6.6.jar"/>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
37 changes: 16 additions & 21 deletions .project
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>gridss</name>
<comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
<name>gridss</name>
<comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
<projects/>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
47 changes: 21 additions & 26 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,24 +78,24 @@
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.7.7.201606060606</version>
<configuration>
<destfile>${basedir}/target/coverage-reports/jacoco-unit.exec</destfile>
<datafile>${basedir}/target/coverage-reports/jacoco-unit.exec</datafile>
</configuration>
<destfile>${basedir}/target/coverage-reports/jacoco-unit.exec</destfile>
<datafile>${basedir}/target/coverage-reports/jacoco-unit.exec</datafile>
</configuration>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.eluder.coveralls</groupId>
Expand Down Expand Up @@ -131,7 +131,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.2-jre</version>
<version>23.4-jre</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down Expand Up @@ -171,22 +171,17 @@
<dependency>
<groupId>com.github.samtools</groupId>
<artifactId>htsjdk</artifactId>
<version>2.12.0</version>
<version>2.13.1</version>
</dependency>
<dependency>
<groupId>com.github.broadinstitute</groupId>
<artifactId>picard</artifactId>
<version>2.13.2</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>
<artifactId>chronicle-map</artifactId>
<version>3.13.0</version>
<version>2.15.0</version>
</dependency>
<dependency>
<groupId>org.broadinstitute</groupId>
<artifactId>barclay</artifactId>
<version>1.2.1</version>
<version>1.2.2</version>
</dependency>
</dependencies>
<scm>
Expand Down
116 changes: 3 additions & 113 deletions src/main/java/au/edu/wehi/idsv/AssemblyEvidenceSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,12 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.MoreExecutors;

import au.edu.wehi.idsv.SequentialEvidenceAssemblyAllocator.BreakendAssemblyEvidenceSupport;
import au.edu.wehi.idsv.bed.IntervalBed;
import au.edu.wehi.idsv.configuration.AssemblyConfiguration;
import au.edu.wehi.idsv.debruijn.positional.PositionalAssembler;
Expand All @@ -33,8 +30,6 @@
import htsjdk.samtools.SAMFileWriter;
import htsjdk.samtools.SAMFileWriterFactory;
import htsjdk.samtools.SAMRecord;
import htsjdk.samtools.SAMRecordIterator;
import htsjdk.samtools.SamReader;
import htsjdk.samtools.util.CloseableIterator;
import htsjdk.samtools.util.Log;
import htsjdk.samtools.util.ProgressLogger;
Expand Down Expand Up @@ -102,48 +97,9 @@ public void assembleBreakends(ExecutorService threadpool) throws IOException {
}
log.info("Breakend assembly complete.");
List<File> deduplicatedChunks = assembledChunk;
if (Iterables.any(source, ses -> ses.getMetrics().getIdsvMetrics().SECONDARY_NOT_SPLIT > 0)) {
if (!getContext().getConfig().multimapping) {
log.info("Not performing unique assembly allocation of multimapping reads as the configuration setting multimapping is set to false.");
} else if (!getContext().getConfig().multimappingUniqueAssemblyAllocation) {
log.info("Not performing unique assembly allocation of multimapping reads as the configuration setting multimappingUniqueAssemblyAllocation is set to false.");
} else {
log.info("Multimapping mode invoked due to existiance of at least one BAM file with a non-split secondary alignment.");
log.info("Uniquely assigning multi-mapping reads to alignment location supported by the best assembly.");
log.info("Loading assembly evidence allocations");
deduplicatedChunks = assembledChunk.stream()
.map(f -> FileSystemContext.getWorkingFileFor(f, "deduplicated."))
.collect(Collectors.toList());
// technically this is an overestimation as a single alignment can
// a DP or OEA and an Indel/SC/SR.
long svReadAlignmentCount = source.stream()
.map(ses -> ses.getSVMetrics())
.mapToLong(m -> m.DISCORDANT_READ_PAIR_ALIGNMENTS +
m.UNMAPPED_MATE_READ_ALIGNMENTS +
m.STRUCTURAL_VARIANT_READ_ALIGNMENTS)
.sum();
try (GreedyAssemblyAllocationCache cache = new GreedyAssemblyAllocationCache(svReadAlignmentCount)) {
tasks = new ArrayList<>();
for (int i = 0; i < chunks.size(); i++) {
QueryInterval[] chunk = chunks.get(i);
File in = assembledChunk.get(i);
tasks.add(threadpool.submit(() -> { loadAssemblyEvidenceAllocation(cache, in, chunk); return null; }));
}
runTasks(tasks);

log.info("Allocating multi-mapping reads to assemblies");
tasks = new ArrayList<>();
for (int i = 0; i < chunks.size(); i++) {
QueryInterval[] chunk = chunks.get(i);
File in = assembledChunk.get(i);
File out = deduplicatedChunks.get(i);
if (!out.exists()) {
tasks.add(threadpool.submit(() -> { deduplicateChunk(in, out, chunk, cache); return null; }));
}
}
runTasks(tasks);
}
}
long secondaryNotSplit = source.stream().mapToLong(ses -> ses.getMetrics().getIdsvMetrics().SECONDARY_NOT_SPLIT).sum();
if (secondaryNotSplit > 0) {
log.warn(String.format("Found %d secondary alignments that were not split read alignments. GRIDSS no longer supports multi-mapping alignment. These reads will be ignored.", secondaryNotSplit));
}
log.info("Merging assembly files");
// Merge chunk files
Expand Down Expand Up @@ -285,72 +241,6 @@ private void assembleChunk(SAMFileWriter writer, SAMFileWriter filteredWriter, i
}
}
}
private void loadAssemblyEvidenceAllocation(GreedyAssemblyAllocationCache cache, File in, QueryInterval intervals[]) throws IOException {
log.debug(String.format("Caching assembly evidence allocation in interval %s:%d-%s:%d",
getContext().getDictionary().getSequence(intervals[0].referenceIndex).getSequenceName(), intervals[0].start,
getContext().getDictionary().getSequence(intervals[intervals.length-1].referenceIndex).getSequenceName(), intervals[intervals.length-1].end));
QueryInterval[] expanded = getExpanded(intervals);
try (CloseableIterator<DirectedEvidence> reads = mergedIterator(source, expanded)) {
try (SamReader reader = factory.open(in)) {
try (SAMRecordIterator assemblies = reader.iterator()) {
SequentialEvidenceAssemblyAllocator it = new SequentialEvidenceAssemblyAllocator(
getContext().getLinear(),
reads,
assemblies, 0);
while (it.hasNext()) {
BreakendAssemblyEvidenceSupport allocation = it.next();
float score = 0;
for (DirectedEvidence assemblyEvidence : SingleReadEvidence.createEvidence(this, 0, allocation.assemblyRecord)) {
score = Math.max(score, assemblyEvidence.getBreakendQual());
}
for (DirectedEvidence support : allocation.support) {
if (support.isFromMultimappingFragment()) {
cache.addBreakendAssemblyAllocation(score, support);
}
}
}
}
}
}
}
private void deduplicateChunk(File in, File out, QueryInterval[] intervals, GreedyAssemblyAllocationCache cache) throws IOException {
log.debug(String.format("Uniquely assigning assembly evidence allocation in interval %s:%d-%s:%d",
getContext().getDictionary().getSequence(intervals[0].referenceIndex).getSequenceName(), intervals[0].start,
getContext().getDictionary().getSequence(intervals[intervals.length-1].referenceIndex).getSequenceName(), intervals[intervals.length-1].end));
File tmpout = gridss.Defaults.OUTPUT_TO_TEMP_FILE ? FileSystemContext.getWorkingFileFor(out) : out;
QueryInterval[] expanded = getExpanded(intervals);
try (CloseableIterator<DirectedEvidence> reads = mergedIterator(source, expanded)) {
try (SamReader reader = factory.open(in)) {
try (SAMRecordIterator assemblies = reader.iterator()) {
try (SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(reader.getFileHeader(), true, tmpout)) {
SequentialEvidenceAssemblyAllocator it = new SequentialEvidenceAssemblyAllocator(
getContext().getLinear(),
reads,
assemblies, 0);
while (it.hasNext()) {
BreakendAssemblyEvidenceSupport allocation = it.next();
SAMRecord asm = allocation.assemblyRecord;
List<DirectedEvidence> newSupport = new ArrayList<>(allocation.support.size());
for (DirectedEvidence support : allocation.support) {
if (!support.isFromMultimappingFragment() || cache.isBestBreakendAssemblyAllocation(support)) {
newSupport.add(support);
}
}
AssemblyAttributes.annotateAssembly(getContext(), asm, newSupport);
if (shouldFilterAssembly(asm)) {
// TODO: append to filtered bam
} else {
writer.addAlignment(asm);
}
}
}
}
}
}
if (tmpout != out) {
FileHelper.move(tmpout, out, true);
}
}
@Override
public void ensureExtracted() throws IOException {
ensureMetrics();
Expand Down
Loading

0 comments on commit 148c8bb

Please sign in to comment.