From 3dc96b53b121159b3c9c9142657a39ea5839f0a8 Mon Sep 17 00:00:00 2001 From: ramari16 Date: Wed, 30 Aug 2023 15:29:34 -0400 Subject: [PATCH 1/3] [ALS-4947] Add anyRecordOfMulti field to the query object (#77) --- .../dbmi/avillach/hpds/data/query/Query.java | 16 +++++- .../hpds/processing/AbstractProcessor.java | 49 ++++++++++--------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java index eb7ff4d7..8e28e74e 100644 --- a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java +++ b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java @@ -37,6 +37,7 @@ public Query(Query query) { private List fields = new ArrayList<>(); private List requiredFields = new ArrayList<>(); private List anyRecordOf = new ArrayList<>(); + private List> anyRecordOfMulti = new ArrayList<>(); private Map numericFilters = new HashMap<>(); private Map categoryFilters = new HashMap<>(); private List variantInfoFilters = new ArrayList<>(); @@ -62,6 +63,14 @@ public List getRequiredFields() { public List getAnyRecordOf() { return anyRecordOf; } + public List> getAnyRecordOfMulti() { + return anyRecordOfMulti; + } + public List> getAllAnyRecordOf() { + List> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti); + anyRecordOfMultiCopy.add(anyRecordOf); + return anyRecordOfMultiCopy; + } public Map getNumericFilters() { return numericFilters; @@ -98,6 +107,9 @@ public void setRequiredFields(Collection requiredFields) { public void setAnyRecordOf(Collection anyRecordOf) { this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>(); } + public void setAnyRecordOfMulti(Collection> anyRecordOfMulti) { + this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>(); + } public void setNumericFilters(Map numericFilters) { this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>(); @@ -191,7 +203,7 @@ public String toString() { writePartFormat("Numeric filters", numericFilters, builder); writePartFormat("Category filters", categoryFilters, builder); writePartFormat("Variant Info filters", variantInfoFilters, builder, false); - writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true); + writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true); return builder.toString(); } @@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder Integer count = countMap.get(firstLevel); if(count == null) { - count = new Integer(1); + count = 1; } else { count = count + 1; } diff --git a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java index e2f6c44a..e67d24cb 100644 --- a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java +++ b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java @@ -6,6 +6,7 @@ import java.util.Map.Entry; import java.util.concurrent.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import com.google.common.util.concurrent.UncheckedExecutionException; @@ -195,24 +196,27 @@ protected Set applyBooleanLogic(List> filteredIdSets) { * @return */ protected List> idSetsForEachFilter(Query query) { - ArrayList> filteredIdSets = new ArrayList>(); + final ArrayList> filteredIdSets = new ArrayList<>(); try { - addIdSetsForAnyRecordOf(query, filteredIdSets); + query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> { + addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets); + }); addIdSetsForRequiredFields(query, filteredIdSets); addIdSetsForNumericFilters(query, filteredIdSets); addIdSetsForCategoryFilters(query, filteredIdSets); } catch (InvalidCacheLoadException e) { log.warn("Invalid query supplied: " + e.getLocalizedMessage()); - filteredIdSets.add(new HashSet()); // if an invalid path is supplied, no patients should match. + filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match. } //AND logic to make sure all patients match each filter if(filteredIdSets.size()>1) { - filteredIdSets = new ArrayList>(List.of(applyBooleanLogic(filteredIdSets))); + List> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets))); + return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets); + } else { + return addIdSetsForVariantInfoFilters(query, filteredIdSets); } - - return addIdSetsForVariantInfoFilters(query, filteredIdSets); } /** @@ -260,22 +264,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList> fil } } - private void addIdSetsForAnyRecordOf(Query query, ArrayList> filteredIdSets) { - if(!query.getAnyRecordOf().isEmpty()) { - Set patientsInScope = new ConcurrentSkipListSet(); - VariantBucketHolder bucketCache = new VariantBucketHolder(); - query.getAnyRecordOf().parallelStream().forEach(path->{ - if(patientsInScope.size() anyRecordOfFilters, ArrayList> filteredIdSets) { + if(!anyRecordOfFilters.isEmpty()) { + VariantBucketHolder bucketCache = new VariantBucketHolder<>(); + Set anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> { + if (VariantUtils.pathIsVariantSpec(path)) { + TreeSet patientsInScope = new TreeSet<>(); + addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache); + return patientsInScope.stream(); + } else { + return (Stream) getCube(path).keyBasedIndex().stream(); } - }); - filteredIdSets.add(patientsInScope); + }).collect(Collectors.toSet()); + filteredIdSets.add(anyRecordOfPatientSet); } } @@ -289,9 +290,9 @@ private void addIdSetsForNumericFilters(Query query, ArrayList> fil private void addIdSetsForCategoryFilters(Query query, ArrayList> filteredIdSets) { if(!query.getCategoryFilters().isEmpty()) { - VariantBucketHolder bucketCache = new VariantBucketHolder(); - Set> idsThatMatchFilters = (Set>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{ - Set ids = new TreeSet(); + VariantBucketHolder bucketCache = new VariantBucketHolder<>(); + Set> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{ + Set ids = new TreeSet<>(); if(VariantUtils.pathIsVariantSpec(entry.getKey())) { addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache); } else { From a43a209d210571749be46ac59812f31042003e5e Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Mon, 18 Sep 2023 10:00:25 -0400 Subject: [PATCH 2/3] [ALS-4980] Update PicSureService to match contract with PIC-SURe API --- .../dbmi/avillach/hpds/service/PicSureService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java index 58b81399..12287521 100644 --- a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java +++ b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java @@ -225,8 +225,8 @@ private QueryStatus convertToQueryStatus(AsyncResult entity) { @Path("/query/{resourceQueryId}/result") @Produces(MediaType.TEXT_PLAIN_VALUE) @Override - public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) { - AsyncResult result = queryService.getResultFor(queryId); + public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest resultRequest) { + AsyncResult result = queryService.getResultFor(queryId.toString()); if (result == null) { // This happens sometimes when users immediately request the status for a query // before it can be initialized. We wait a bit and try again before throwing an @@ -237,7 +237,7 @@ public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryR return Response.status(500).build(); } - result = queryService.getResultFor(queryId); + result = queryService.getResultFor(queryId.toString()); if (result == null) { return Response.status(404).build(); } @@ -253,8 +253,8 @@ public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryR @POST @Path("/query/{resourceQueryId}/status") @Override - public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest request) { - return convertToQueryStatus(queryService.getStatusFor(queryId)); + public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest request) { + return convertToQueryStatus(queryService.getStatusFor(queryId.toString())); } @POST @@ -328,7 +328,7 @@ private Response _querySync(QueryRequest resultRequest) throws IOException { QueryStatus status = query(resultRequest); while (status.getResourceStatus().equalsIgnoreCase("RUNNING") || status.getResourceStatus().equalsIgnoreCase("PENDING")) { - status = queryStatus(status.getResourceResultId(), null); + status = queryStatus(UUID.fromString(status.getResourceResultId()), null); } log.info(status.toString()); From e8c62bcf3c0de5bfb5e229c33d9f548b3fd4d0b4 Mon Sep 17 00:00:00 2001 From: ramari16 Date: Fri, 22 Sep 2023 14:11:39 -0400 Subject: [PATCH 3/3] [ALS-5052] Update to use docker tomcat image (#82) --- docker/pic-sure-hpds/Dockerfile | 13 ++-------- war/pom.xml | 46 +++++---------------------------- 2 files changed, 8 insertions(+), 51 deletions(-) diff --git a/docker/pic-sure-hpds/Dockerfile b/docker/pic-sure-hpds/Dockerfile index 0b38a4de..1d1f395f 100644 --- a/docker/pic-sure-hpds/Dockerfile +++ b/docker/pic-sure-hpds/Dockerfile @@ -1,11 +1,2 @@ -FROM docker.io/alpine:3.16 - -RUN apk add --no-cache --purge -uU bash && rm -rf /var/cache/apk/* /tmp/* - -RUN apk add --no-cache --purge -uU curl wget unzip - -RUN apk add --no-cache --purge openjdk11 - -ADD hpds-war-1.0-SNAPSHOT-war-exec.jar /hpds.jar - -EXPOSE 8080 +FROM tomcat:9-jre11-openjdk-slim +ADD hpds-war-1.0-SNAPSHOT.war /usr/local/tomcat/webapps/ROOT.war \ No newline at end of file diff --git a/war/pom.xml b/war/pom.xml index e71e6efa..93c90038 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -20,30 +20,6 @@ - - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 - - - default-cli - - run - - - 13000 - /jaxrs-service - true - true - - -Xms256m -Xmx512m - - - - - - - org.apache.maven.plugins maven-compiler-plugin @@ -90,22 +66,12 @@ - org.apache.tomcat.maven - tomcat7-maven-plugin - - - tomcat-run - - exec-war-only - - package - - true - ../docker/pic-sure-hpds - / - - - + org.apache.maven.plugins + maven-war-plugin + 3.3.1 + + docker/pic-sure-hpds +