Merge master into branch

.github/workflows/github-actions-deploy-snapshots.yml

@@ -18,4 +18,4 @@ jobs:
       - name: Build with Maven
         run: mvn --update-snapshots deploy
         env:
-          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/label-checker.yml

@@ -0,0 +1,21 @@
+name: Label Checker
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+      - unlabeled
+
+jobs:
+
+  check_labels:
+    name: Check labels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: docker://agilepathway/pull-request-label-checker:latest
+        with:
+          one_of: breaking-change,enhancement,bug,documentation,ignore-for-release
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+

client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java

@@ -37,6 +37,7 @@ public Query(Query query) {
 	private List<String> fields = new ArrayList<>();
 	private List<String> requiredFields = new ArrayList<>();
 	private List<String> anyRecordOf = new ArrayList<>();
+	private List<List<String>> anyRecordOfMulti = new ArrayList<>();
 	private Map<String, DoubleFilter> numericFilters = new HashMap<>();
 	private Map<String, String[]> categoryFilters = new HashMap<>();
 	private List<VariantInfoFilter> variantInfoFilters = new ArrayList<>();
@@ -62,6 +63,14 @@ public List<String> getRequiredFields() {
 	public List<String> getAnyRecordOf() {
 		return anyRecordOf;
 	}
+	public List<List<String>> getAnyRecordOfMulti() {
+		return anyRecordOfMulti;
+	}
+	public List<List<String>> getAllAnyRecordOf() {
+		List<List<String>> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
+		anyRecordOfMultiCopy.add(anyRecordOf);
+		return anyRecordOfMultiCopy;
+	}
 
 	public Map<String, DoubleFilter> getNumericFilters() {
 		return numericFilters;
@@ -98,6 +107,9 @@ public void setRequiredFields(Collection<String> requiredFields) {
 	public void setAnyRecordOf(Collection<String> anyRecordOf) {
 		this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
 	}
+	public void setAnyRecordOfMulti(Collection<List<String>> anyRecordOfMulti) {
+		this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
+	}
 
 	public void setNumericFilters(Map<String, DoubleFilter> numericFilters) {
 		this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
@@ -191,7 +203,7 @@ public String toString() {
 		writePartFormat("Numeric filters", numericFilters, builder);
 		writePartFormat("Category filters", categoryFilters, builder);
 		writePartFormat("Variant Info filters", variantInfoFilters, builder, false);
-		writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true);
+		writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true);
 
 		return builder.toString();
 	}
@@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder
 
 				Integer count = countMap.get(firstLevel);
 				if(count == null) {
-					count = new Integer(1);
+					count = 1;
 				} else {
 					count = count + 1;
 				}

docker/pic-sure-hpds/Dockerfile

@@ -1,11 +1,2 @@
-FROM docker.io/alpine:3.16
-
-RUN apk add --no-cache --purge -uU bash &&     rm -rf /var/cache/apk/* /tmp/*
-
-RUN apk add --no-cache --purge -uU curl wget unzip
-
-RUN apk add --no-cache --purge openjdk11
-
-ADD hpds-war-2.0.0-SNAPSHOT-war-exec.jar /hpds.jar
-
-EXPOSE 8080
+FROM tomcat:9-jre11-openjdk-slim
+ADD hpds-war-2.0.0-SNAPSHOT.war /usr/local/tomcat/webapps/ROOT.war

processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java

@@ -6,6 +6,7 @@
 import java.util.Map.Entry;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 
 import com.google.common.util.concurrent.UncheckedExecutionException;
@@ -196,24 +197,27 @@ protected Set<Integer> applyBooleanLogic(List<Set<Integer>> filteredIdSets) {
 	 * @return
 	 */
 	protected List<Set<Integer>> idSetsForEachFilter(Query query) {
-		ArrayList<Set<Integer>> filteredIdSets = new ArrayList<Set<Integer>>();
+		final ArrayList<Set<Integer>> filteredIdSets = new ArrayList<>();
 
 		try {
-			addIdSetsForAnyRecordOf(query, filteredIdSets);
+			query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> {
+				addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets);
+			});
 			addIdSetsForRequiredFields(query, filteredIdSets);
 			addIdSetsForNumericFilters(query, filteredIdSets);
 			addIdSetsForCategoryFilters(query, filteredIdSets);
 		} catch (InvalidCacheLoadException e) {
 			log.warn("Invalid query supplied: " + e.getLocalizedMessage());
-			filteredIdSets.add(new HashSet<Integer>()); // if an invalid path is supplied, no patients should match.
+			filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match.
 		}
 
 		//AND logic to make sure all patients match each filter
 		if(filteredIdSets.size()>1) {
-			filteredIdSets = new ArrayList<Set<Integer>>(List.of(applyBooleanLogic(filteredIdSets)));
+			List<Set<Integer>> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets)));
+			return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets);
+		} else {
+			return addIdSetsForVariantInfoFilters(query, filteredIdSets);
 		}
-
-		return addIdSetsForVariantInfoFilters(query, filteredIdSets);
 	}
 
 	/**
@@ -261,22 +265,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList<Set<Integer>> fil
 		}
 	}
 
-	private void addIdSetsForAnyRecordOf(Query query, ArrayList<Set<Integer>> filteredIdSets) {
-		if(!query.getAnyRecordOf().isEmpty()) {
-			Set<Integer> patientsInScope = new ConcurrentSkipListSet<Integer>();
-			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
-			query.getAnyRecordOf().parallelStream().forEach(path->{
-				if(patientsInScope.size()<Math.max(
-						phenotypeMetaStore.getPatientIds().size(),
-						variantService.getPatientIds().length)) {
-					if(VariantUtils.pathIsVariantSpec(path)) {
-						addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1","1/1"}, path, patientsInScope, bucketCache);
-					} else {
-						patientsInScope.addAll(getCube(path).keyBasedIndex());
-					}
+	private void addIdSetsForAnyRecordOf(List<String> anyRecordOfFilters, ArrayList<Set<Integer>> filteredIdSets) {
+		if(!anyRecordOfFilters.isEmpty()) {
+			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
+			Set<Integer> anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> {
+				if (VariantUtils.pathIsVariantSpec(path)) {
+					TreeSet<Integer> patientsInScope = new TreeSet<>();
+					addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache);
+					return patientsInScope.stream();
+				} else {
+					return (Stream<Integer>) getCube(path).keyBasedIndex().stream();
 				}
-			});
-			filteredIdSets.add(patientsInScope);
+			}).collect(Collectors.toSet());
+			filteredIdSets.add(anyRecordOfPatientSet);
 		}
 	}
 
@@ -290,9 +291,9 @@ private void addIdSetsForNumericFilters(Query query, ArrayList<Set<Integer>> fil
 
 	private void addIdSetsForCategoryFilters(Query query, ArrayList<Set<Integer>> filteredIdSets) {
 		if(!query.getCategoryFilters().isEmpty()) {
-			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<VariantMasks>();
-			Set<Set<Integer>> idsThatMatchFilters = (Set<Set<Integer>>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{
-				Set<Integer> ids = new TreeSet<Integer>();
+			VariantBucketHolder<VariantMasks> bucketCache = new VariantBucketHolder<>();
+			Set<Set<Integer>> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{
+				Set<Integer> ids = new TreeSet<>();
 				if(VariantUtils.pathIsVariantSpec(entry.getKey())) {
 					addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache);
 				} else {

processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantListProcessor.java

@@ -178,7 +178,7 @@ public String runVcfExcerptQuery(Query query, boolean includePatientData) throws
 		}
 
 		//patient count columns
-		builder.append("\tPatients with this variant in subset\tPatients With this variant NOT in subset");
+		builder.append("\tPatients with this variant in subset\tPatients with this variant NOT in subset");
 
 		//then one column per patient.  We also need to identify the patient ID and
 		// map it to the right index in the bit mask fields.

service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java

@@ -225,8 +225,8 @@ private QueryStatus convertToQueryStatus(AsyncResult entity) {
 	@Path("/query/{resourceQueryId}/result")
 	@Produces(MediaType.TEXT_PLAIN_VALUE)
 	@Override
-	public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) {
-		AsyncResult result = queryService.getResultFor(queryId);
+	public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest resultRequest) {
+		AsyncResult result = queryService.getResultFor(queryId.toString());
 		if (result == null) {
 			// This happens sometimes when users immediately request the status for a query
 			// before it can be initialized. We wait a bit and try again before throwing an
@@ -237,7 +237,7 @@ public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryR
 				return Response.status(500).build();
 			}
 
-			result = queryService.getResultFor(queryId);
+			result = queryService.getResultFor(queryId.toString());
 			if (result == null) {
 				return Response.status(404).build();
 			}
@@ -253,8 +253,8 @@ public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryR
 	@POST
 	@Path("/query/{resourceQueryId}/status")
 	@Override
-	public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest request) {
-		return convertToQueryStatus(queryService.getStatusFor(queryId));
+	public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest request) {
+		return convertToQueryStatus(queryService.getStatusFor(queryId.toString()));
 	}
 
 	@POST
@@ -328,7 +328,7 @@ private Response _querySync(QueryRequest resultRequest) throws IOException {
 			QueryStatus status = query(resultRequest);
 			while (status.getResourceStatus().equalsIgnoreCase("RUNNING")
 					|| status.getResourceStatus().equalsIgnoreCase("PENDING")) {
-				status = queryStatus(status.getResourceResultId(), null);
+				status = queryStatus(UUID.fromString(status.getResourceResultId()), null);
 			}
 			log.info(status.toString());
 

war/pom.xml

@@ -20,30 +20,6 @@
 	<build>
 		<pluginManagement>
 			<plugins>
-				<plugin>
-					<groupId>org.apache.tomcat.maven</groupId>
-					<artifactId>tomcat7-maven-plugin</artifactId>
-					<version>2.0</version>
-					<executions>
-						<execution>
-							<id>default-cli</id>
-							<goals>
-								<goal>run</goal>
-							</goals>
-							<configuration>
-								<port>13000</port>
-								<path>/jaxrs-service</path>
-								<useSeparateTomcatClassLoader>true</useSeparateTomcatClassLoader>
-								<jpda>true</jpda>
-								<systemProperties>
-									<JAVA_OPTS>-Xms256m -Xmx512m</JAVA_OPTS>
-								</systemProperties>
-							</configuration>
-						</execution>
-					</executions>
-					<configuration>
-					</configuration>
-				</plugin>
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-compiler-plugin</artifactId>
@@ -90,22 +66,12 @@
 				</executions>
 			</plugin>
 			<plugin>
-				<groupId>org.apache.tomcat.maven</groupId>
-				<artifactId>tomcat7-maven-plugin</artifactId>
-				<executions>
-					<execution>
-						<id>tomcat-run</id>
-						<goals>
-							<goal>exec-war-only</goal>
-						</goals>
-						<phase>package</phase>
-						<configuration>
-							<attachArtifact>true</attachArtifact>
-							<buildDirectory>../docker/pic-sure-hpds</buildDirectory>
-							<path>/</path>
-						</configuration>
-					</execution>
-				</executions>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-war-plugin</artifactId>
+				<version>3.3.1</version>
+				<configuration>
+					<outputDirectory>docker/pic-sure-hpds</outputDirectory>
+				</configuration>
 			</plugin>
 		</plugins>
 	</build>