Skip to content

Commit

Permalink
[ALS-4947] Add anyRecordOfMulti field to the query object (#77)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ramari16 @Luke-Sikina
ramari16 authored and Luke-Sikina committed Sep 11, 2023
1 parent 73d1759 commit 04421cd
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 26 deletions.
16 changes: 14 additions & 2 deletions client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public Query(Query query) {
private List<String> fields = new ArrayList<>();
private List<String> requiredFields = new ArrayList<>();
private List<String> anyRecordOf = new ArrayList<>();
private List<List<String>> anyRecordOfMulti = new ArrayList<>();
private Map<String, DoubleFilter> numericFilters = new HashMap<>();
private Map<String, String[]> categoryFilters = new HashMap<>();
private List<VariantInfoFilter> variantInfoFilters = new ArrayList<>();
Expand All @@ -62,6 +63,14 @@ public List<String> getRequiredFields() {
public List<String> getAnyRecordOf() {
return anyRecordOf;
}
public List<List<String>> getAnyRecordOfMulti() {
return anyRecordOfMulti;
}
public List<List<String>> getAllAnyRecordOf() {
List<List<String>> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
anyRecordOfMultiCopy.add(anyRecordOf);
return anyRecordOfMultiCopy;
}

public Map<String, DoubleFilter> getNumericFilters() {
return numericFilters;
Expand Down Expand Up @@ -98,6 +107,9 @@ public void setRequiredFields(Collection<String> requiredFields) {
public void setAnyRecordOf(Collection<String> anyRecordOf) {
this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
}
public void setAnyRecordOfMulti(Collection<List<String>> anyRecordOfMulti) {
this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
}

public void setNumericFilters(Map<String, DoubleFilter> numericFilters) {
this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
Expand Down Expand Up @@ -191,7 +203,7 @@ public String toString() {
writePartFormat("Numeric filters", numericFilters, builder);
writePartFormat("Category filters", categoryFilters, builder);
writePartFormat("Variant Info filters", variantInfoFilters, builder, false);
writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true);
writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true);

return builder.toString();
}
Expand Down Expand Up @@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder

Integer count = countMap.get(firstLevel);
if(count == null) {
count = new Integer(1);
count = 1;
} else {
count = count + 1;
}
Expand Down
49 changes: 25 additions & 24 deletions ...essing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.util.Map.Entry;
import java.util.concurrent.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;

import com.google.common.util.concurrent.UncheckedExecutionException;
Expand Down Expand Up @@ -195,24 +196,27 @@ protected Set<Integer> applyBooleanLogic(List<Set<Integer>> filteredIdSets) {
* @return
*/
protected List<Set<Integer>> idSetsForEachFilter(Query query) {
ArrayList<Set<Integer>> filteredIdSets = new ArrayList<Set<Integer>>();
final ArrayList<Set<Integer>> filteredIdSets = new ArrayList<>();

try {
addIdSetsForAnyRecordOf(query, filteredIdSets);
query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> {
addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets);
});
addIdSetsForRequiredFields(query, filteredIdSets);
addIdSetsForNumericFilters(query, filteredIdSets);
addIdSetsForCategoryFilters(query, filteredIdSets);
} catch (InvalidCacheLoadException e) {
log.warn("Invalid query supplied: " + e.getLocalizedMessage());
filteredIdSets.add(new HashSet<Integer>()); // if an invalid path is supplied, no patients should match.
filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match.
}

//AND logic to make sure all patients match each filter
if(filteredIdSets.size()>1) {
filteredIdSets = new ArrayList<Set<Integer>>(List.of(applyBooleanLogic(filteredIdSets)));
List<Set<Integer>> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets)));
return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets);
} else {
return addIdSetsForVariantInfoFilters(query, filteredIdSets);
}

return addIdSetsForVariantInfoFilters(query, filteredIdSets);
}

/**
Expand Down Expand Up @@ -260,22 +264,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList<Set<Integer>> fil
}
}

private void addIdSetsForAnyRecordOf(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getAnyRecordOf().isEmpty()) {
Set<Integer> patientsInScope = new ConcurrentSkipListSet<Integer>();
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
query.getAnyRecordOf().parallelStream().forEach(path->{
if(patientsInScope.size()<Math.max(
phenotypeMetaStore.getPatientIds().size(),
variantService.getPatientIds().length)) {
if(VariantUtils.pathIsVariantSpec(path)) {
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
} else {
patientsInScope.addAll(getCube(path).keyBasedIndex());
}
private void addIdSetsForAnyRecordOf(List<String> anyRecordOfFilters, ArrayList<Set<Integer>> filteredIdSets) {
if(!anyRecordOfFilters.isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
Set<Integer> anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> {
if (VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache);
return patientsInScope.stream();
} else {
return (Stream<Integer>) getCube(path).keyBasedIndex().stream();
}
});
filteredIdSets.add(patientsInScope);
}).collect(Collectors.toSet());
filteredIdSets.add(anyRecordOfPatientSet);
}
}

Expand All @@ -289,9 +290,9 @@ private void addIdSetsForNumericFilters(Query query, ArrayList<Set<Integer>> fil

private void addIdSetsForCategoryFilters(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getCategoryFilters().isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
Set<Set<Integer>> idsThatMatchFilters = (Set<Set<Integer>>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{
Set<Integer> ids = new TreeSet<Integer>();
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
Set<Set<Integer>> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{
Set<Integer> ids = new TreeSet<>();
if(VariantUtils.pathIsVariantSpec(entry.getKey())) {
addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache);
} else {
Expand Down

0 comments on commit 04421cd

Please sign in to comment.