Skip to content

Commit

Permalink
clean up
Browse files Browse the repository at this point in the history
  • Loading branch information
Luke Sikina committed Jan 4, 2024
1 parent 152918b commit d33bd3c
Showing 1 changed file with 66 additions and 67 deletions.
133 changes: 66 additions & 67 deletions ...essing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;


@Component
public class AbstractProcessor {
Expand Down Expand Up @@ -196,27 +199,24 @@ protected Set<Integer> applyBooleanLogic(List<Set<Integer>> filteredIdSets) {
* @return
*/
protected List<Set<Integer>> idSetsForEachFilter(Query query) {
final ArrayList<Set<Integer>> filteredIdSets = new ArrayList<>();
Set<Integer> intersectedIdSets = null;

try {
query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> {
addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets);
});
addIdSetsForRequiredFields(query, filteredIdSets);
addIdSetsForNumericFilters(query, filteredIdSets);
addIdSetsForCategoryFilters(query, filteredIdSets);
for (List<String> anyRecordOfFilterList : query.getAllAnyRecordOf()) {
intersectedIdSets = addIdSetsForAnyRecordOf(anyRecordOfFilterList, intersectedIdSets);
}
intersectedIdSets = addIdSetsForRequiredFields(query, intersectedIdSets);
intersectedIdSets = addIdSetsForNumericFilters(query, intersectedIdSets);
intersectedIdSets = addIdSetsForCategoryFilters(query, intersectedIdSets);
} catch (InvalidCacheLoadException e) {
log.warn("Invalid query supplied: " + e.getLocalizedMessage());
filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match.
return List.of(new HashSet<>());
}



//AND logic to make sure all patients match each filter
if(filteredIdSets.size()>1) {
List<Set<Integer>> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets)));
return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets);
} else {
return addIdSetsForVariantInfoFilters(query, filteredIdSets);
}
return addIdSetsForVariantInfoFilters(query, new ArrayList<>(List.of(intersectedIdSets)));
}

/**
Expand Down Expand Up @@ -249,67 +249,58 @@ public TreeSet<Integer> getPatientSubsetForQuery(Query query) {
return idList;
}

private void addIdSetsForRequiredFields(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getRequiredFields().isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
filteredIdSets.addAll(query.getRequiredFields().parallelStream().map(path->{
if(VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
return patientsInScope;
} else {
return new TreeSet<Integer>(getCube(path).keyBasedIndex());
}
}).collect(Collectors.toSet()));
}
private Set<Integer> addIdSetsForRequiredFields(Query query, Set<Integer> filteredIdSets) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
return query.getRequiredFields().parallelStream().map(path->{
if(VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
return patientsInScope;
} else {
return (Set<Integer>) new TreeSet<Integer>(getCube(path).keyBasedIndex());
}
}).reduce(filteredIdSets, this::nullSafeIntersect);
}

private void addIdSetsForAnyRecordOf(List<String> anyRecordOfFilters, ArrayList<Set<Integer>> filteredIdSets) {
if(!anyRecordOfFilters.isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
Set<Integer> anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> {
if (VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache);
return patientsInScope.stream();
} else {
try {
return (Stream<Integer>) getCube(path).keyBasedIndex().stream();
} catch (InvalidCacheLoadException e) {
// return an empty stream if this concept doesn't exist
return Stream.empty();
}
private Set<Integer> addIdSetsForAnyRecordOf(List<String> anyRecordOfFilters, @Nullable Set<Integer> filteredIdSets) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
Set<Integer> anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> {
if (VariantUtils.pathIsVariantSpec(path)) {
TreeSet<Integer> patientsInScope = new TreeSet<>();
addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache);
return patientsInScope.stream();
} else {
try {
return (Stream<Integer>) getCube(path).keyBasedIndex().stream();
} catch (InvalidCacheLoadException e) {
// return an empty stream if this concept doesn't exist
return Stream.empty();
}
}).collect(Collectors.toSet());
filteredIdSets.add(anyRecordOfPatientSet);
}
}
}).collect(Collectors.toSet());
return filteredIdSets != null ? Sets.intersection(filteredIdSets, anyRecordOfPatientSet) : anyRecordOfPatientSet;
}

private void addIdSetsForNumericFilters(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getNumericFilters().isEmpty()) {
filteredIdSets.addAll((Set<TreeSet<Integer>>)(query.getNumericFilters().entrySet().parallelStream().map(entry->{
return (TreeSet<Integer>)(getCube(entry.getKey()).getKeysForRange(entry.getValue().getMin(), entry.getValue().getMax()));
}).collect(Collectors.toSet())));
}
private Set<Integer> addIdSetsForNumericFilters(Query query, Set<Integer> filteredIdSets) {
return query.getNumericFilters().entrySet().parallelStream().map(entry-> {
return (Set<Integer>)(getCube(entry.getKey()).getKeysForRange(entry.getValue().getMin(), entry.getValue().getMax()));
}).reduce(filteredIdSets, this::nullSafeIntersect);
}

private void addIdSetsForCategoryFilters(Query query, ArrayList<Set<Integer>> filteredIdSets) {
if(!query.getCategoryFilters().isEmpty()) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
Set<Set<Integer>> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{
Set<Integer> ids = new TreeSet<>();
if(VariantUtils.pathIsVariantSpec(entry.getKey())) {
addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache);
} else {
String[] categoryFilter = entry.getValue();
for(String category : categoryFilter) {
ids.addAll(getCube(entry.getKey()).getKeysForValue(category));
}
private Set<Integer> addIdSetsForCategoryFilters(Query query, @Nullable Set<Integer> startingIds) {
VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
return query.getCategoryFilters().entrySet().parallelStream().map(entry->{
Set<Integer> ids = new TreeSet<>();
if(VariantUtils.pathIsVariantSpec(entry.getKey())) {
addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache);
} else {
String[] categoryFilter = entry.getValue();
for(String category : categoryFilter) {
ids.addAll(getCube(entry.getKey()).getKeysForValue(category));
}
return ids;
}).collect(Collectors.toSet());
filteredIdSets.addAll(idsThatMatchFilters);
}
}
return ids;
}).reduce(startingIds, this::nullSafeIntersect);
}

private void addIdSetsForVariantSpecCategoryFilters(String[] zygosities, String key, Set<Integer> ids, VariantBucketHolder<VariantMasks> bucketCache) {
Expand Down Expand Up @@ -661,4 +652,12 @@ public String[] getPatientIds() {
public VariantMasks getMasks(String path, VariantBucketHolder<VariantMasks> variantMasksVariantBucketHolder) {
return variantService.getMasks(path, variantMasksVariantBucketHolder);
}

/**
* BE CAREFUL WITH THIS METHOD! NOT A TRUE INTERSECTION
* This intersects the set, but if the second set is null, it just returns the first
*/
private <T> Set<T> nullSafeIntersect(@NotNull Set<T> first, @Nullable Set<T> second) {
return second == null ? first : Sets.intersection(first, second);
}
}

0 comments on commit d33bd3c

Please sign in to comment.