From 3dc96b53b121159b3c9c9142657a39ea5839f0a8 Mon Sep 17 00:00:00 2001 From: ramari16 Date: Wed, 30 Aug 2023 15:29:34 -0400 Subject: [PATCH 1/7] [ALS-4947] Add anyRecordOfMulti field to the query object (#77) --- .../dbmi/avillach/hpds/data/query/Query.java | 16 +++++- .../hpds/processing/AbstractProcessor.java | 49 ++++++++++--------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java index eb7ff4d7..8e28e74e 100644 --- a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java +++ b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java @@ -37,6 +37,7 @@ public Query(Query query) { private List fields = new ArrayList<>(); private List requiredFields = new ArrayList<>(); private List anyRecordOf = new ArrayList<>(); + private List> anyRecordOfMulti = new ArrayList<>(); private Map numericFilters = new HashMap<>(); private Map categoryFilters = new HashMap<>(); private List variantInfoFilters = new ArrayList<>(); @@ -62,6 +63,14 @@ public List getRequiredFields() { public List getAnyRecordOf() { return anyRecordOf; } + public List> getAnyRecordOfMulti() { + return anyRecordOfMulti; + } + public List> getAllAnyRecordOf() { + List> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti); + anyRecordOfMultiCopy.add(anyRecordOf); + return anyRecordOfMultiCopy; + } public Map getNumericFilters() { return numericFilters; @@ -98,6 +107,9 @@ public void setRequiredFields(Collection requiredFields) { public void setAnyRecordOf(Collection anyRecordOf) { this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>(); } + public void setAnyRecordOfMulti(Collection> anyRecordOfMulti) { + this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>(); + } public void setNumericFilters(Map numericFilters) { this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>(); @@ -191,7 +203,7 @@ public String toString() { writePartFormat("Numeric filters", numericFilters, builder); writePartFormat("Category filters", categoryFilters, builder); writePartFormat("Variant Info filters", variantInfoFilters, builder, false); - writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true); + writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true); return builder.toString(); } @@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder Integer count = countMap.get(firstLevel); if(count == null) { - count = new Integer(1); + count = 1; } else { count = count + 1; } diff --git a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java index e2f6c44a..e67d24cb 100644 --- a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java +++ b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java @@ -6,6 +6,7 @@ import java.util.Map.Entry; import java.util.concurrent.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import com.google.common.util.concurrent.UncheckedExecutionException; @@ -195,24 +196,27 @@ protected Set applyBooleanLogic(List> filteredIdSets) { * @return */ protected List> idSetsForEachFilter(Query query) { - ArrayList> filteredIdSets = new ArrayList>(); + final ArrayList> filteredIdSets = new ArrayList<>(); try { - addIdSetsForAnyRecordOf(query, filteredIdSets); + query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> { + addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets); + }); addIdSetsForRequiredFields(query, filteredIdSets); addIdSetsForNumericFilters(query, filteredIdSets); addIdSetsForCategoryFilters(query, filteredIdSets); } catch (InvalidCacheLoadException e) { log.warn("Invalid query supplied: " + e.getLocalizedMessage()); - filteredIdSets.add(new HashSet()); // if an invalid path is supplied, no patients should match. + filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match. } //AND logic to make sure all patients match each filter if(filteredIdSets.size()>1) { - filteredIdSets = new ArrayList>(List.of(applyBooleanLogic(filteredIdSets))); + List> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets))); + return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets); + } else { + return addIdSetsForVariantInfoFilters(query, filteredIdSets); } - - return addIdSetsForVariantInfoFilters(query, filteredIdSets); } /** @@ -260,22 +264,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList> fil } } - private void addIdSetsForAnyRecordOf(Query query, ArrayList> filteredIdSets) { - if(!query.getAnyRecordOf().isEmpty()) { - Set patientsInScope = new ConcurrentSkipListSet(); - VariantBucketHolder bucketCache = new VariantBucketHolder(); - query.getAnyRecordOf().parallelStream().forEach(path->{ - if(patientsInScope.size() anyRecordOfFilters, ArrayList> filteredIdSets) { + if(!anyRecordOfFilters.isEmpty()) { + VariantBucketHolder bucketCache = new VariantBucketHolder<>(); + Set anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> { + if (VariantUtils.pathIsVariantSpec(path)) { + TreeSet patientsInScope = new TreeSet<>(); + addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache); + return patientsInScope.stream(); + } else { + return (Stream) getCube(path).keyBasedIndex().stream(); } - }); - filteredIdSets.add(patientsInScope); + }).collect(Collectors.toSet()); + filteredIdSets.add(anyRecordOfPatientSet); } } @@ -289,9 +290,9 @@ private void addIdSetsForNumericFilters(Query query, ArrayList> fil private void addIdSetsForCategoryFilters(Query query, ArrayList> filteredIdSets) { if(!query.getCategoryFilters().isEmpty()) { - VariantBucketHolder bucketCache = new VariantBucketHolder(); - Set> idsThatMatchFilters = (Set>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{ - Set ids = new TreeSet(); + VariantBucketHolder bucketCache = new VariantBucketHolder<>(); + Set> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{ + Set ids = new TreeSet<>(); if(VariantUtils.pathIsVariantSpec(entry.getKey())) { addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache); } else { From a43a209d210571749be46ac59812f31042003e5e Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Mon, 18 Sep 2023 10:00:25 -0400 Subject: [PATCH 2/7] [ALS-4980] Update PicSureService to match contract with PIC-SURe API --- .../dbmi/avillach/hpds/service/PicSureService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java index 58b81399..12287521 100644 --- a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java +++ b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java @@ -225,8 +225,8 @@ private QueryStatus convertToQueryStatus(AsyncResult entity) { @Path("/query/{resourceQueryId}/result") @Produces(MediaType.TEXT_PLAIN_VALUE) @Override - public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) { - AsyncResult result = queryService.getResultFor(queryId); + public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest resultRequest) { + AsyncResult result = queryService.getResultFor(queryId.toString()); if (result == null) { // This happens sometimes when users immediately request the status for a query // before it can be initialized. We wait a bit and try again before throwing an @@ -237,7 +237,7 @@ public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryR return Response.status(500).build(); } - result = queryService.getResultFor(queryId); + result = queryService.getResultFor(queryId.toString()); if (result == null) { return Response.status(404).build(); } @@ -253,8 +253,8 @@ public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryR @POST @Path("/query/{resourceQueryId}/status") @Override - public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest request) { - return convertToQueryStatus(queryService.getStatusFor(queryId)); + public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest request) { + return convertToQueryStatus(queryService.getStatusFor(queryId.toString())); } @POST @@ -328,7 +328,7 @@ private Response _querySync(QueryRequest resultRequest) throws IOException { QueryStatus status = query(resultRequest); while (status.getResourceStatus().equalsIgnoreCase("RUNNING") || status.getResourceStatus().equalsIgnoreCase("PENDING")) { - status = queryStatus(status.getResourceResultId(), null); + status = queryStatus(UUID.fromString(status.getResourceResultId()), null); } log.info(status.toString()); From e8c62bcf3c0de5bfb5e229c33d9f548b3fd4d0b4 Mon Sep 17 00:00:00 2001 From: ramari16 Date: Fri, 22 Sep 2023 14:11:39 -0400 Subject: [PATCH 3/7] [ALS-5052] Update to use docker tomcat image (#82) --- docker/pic-sure-hpds/Dockerfile | 13 ++-------- war/pom.xml | 46 +++++---------------------------- 2 files changed, 8 insertions(+), 51 deletions(-) diff --git a/docker/pic-sure-hpds/Dockerfile b/docker/pic-sure-hpds/Dockerfile index 0b38a4de..1d1f395f 100644 --- a/docker/pic-sure-hpds/Dockerfile +++ b/docker/pic-sure-hpds/Dockerfile @@ -1,11 +1,2 @@ -FROM docker.io/alpine:3.16 - -RUN apk add --no-cache --purge -uU bash && rm -rf /var/cache/apk/* /tmp/* - -RUN apk add --no-cache --purge -uU curl wget unzip - -RUN apk add --no-cache --purge openjdk11 - -ADD hpds-war-1.0-SNAPSHOT-war-exec.jar /hpds.jar - -EXPOSE 8080 +FROM tomcat:9-jre11-openjdk-slim +ADD hpds-war-1.0-SNAPSHOT.war /usr/local/tomcat/webapps/ROOT.war \ No newline at end of file diff --git a/war/pom.xml b/war/pom.xml index e71e6efa..93c90038 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -20,30 +20,6 @@ - - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 - - - default-cli - - run - - - 13000 - /jaxrs-service - true - true - - -Xms256m -Xmx512m - - - - - - - org.apache.maven.plugins maven-compiler-plugin @@ -90,22 +66,12 @@ - org.apache.tomcat.maven - tomcat7-maven-plugin - - - tomcat-run - - exec-war-only - - package - - true - ../docker/pic-sure-hpds - / - - - + org.apache.maven.plugins + maven-war-plugin + 3.3.1 + + docker/pic-sure-hpds + From 50ed0ad4c3612b9b4be687ee2d64587609c8de1f Mon Sep 17 00:00:00 2001 From: gcolon021 Date: Tue, 26 Sep 2023 10:50:05 -0400 Subject: [PATCH 4/7] [ALS-5000] Update mvn compiler plugin --- war/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/war/pom.xml b/war/pom.xml index 93c90038..42999b00 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -26,7 +26,7 @@ 9 - 3.8.0 + 3.11.0 org.apache.maven.plugins From 9201e126e67ff6127ce9f612403bcb7e65d45f33 Mon Sep 17 00:00:00 2001 From: gcolon021 Date: Wed, 4 Oct 2023 18:05:09 -0400 Subject: [PATCH 5/7] [ALS-5000] Update maven compiler plugin Maven compiler plugin has a dependency on log4j. --- client-api/pom.xml | 10 +- pom.xml | 654 ++++++++++++++++++++++----------------------- 2 files changed, 328 insertions(+), 336 deletions(-) diff --git a/client-api/pom.xml b/client-api/pom.xml index ef02dcbb..06638ab2 100644 --- a/client-api/pom.xml +++ b/client-api/pom.xml @@ -6,21 +6,17 @@ edu.harvard.hms.dbmi.avillach.hpds 1.0-SNAPSHOT - edu.harvard.hms.dbmi.avillach.hpds client-api 1.0-SNAPSHOT - client-api http://www.example.com - UTF-8 1.7 1.7 - junit @@ -29,9 +25,9 @@ test - - + + @@ -45,7 +41,7 @@ maven-compiler-plugin - 3.8.0 + 3.11.0 maven-surefire-plugin diff --git a/pom.xml b/pom.xml index ab958be4..879eeaf9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,333 +1,329 @@ - - 4.0.0 - edu.harvard.hms.dbmi.avillach.hpds - pic-sure-hpds - 1.0-SNAPSHOT - pom - pic-sure-hpds - - common - service - etl - data - docker - processing - war - client-api - - - 1.8.6 - UTF-8 - 1.4.10 - - - - github - GitHub HMS-DBMI Apache Maven Packages - https://maven.pkg.github.com/hms-dbmi/pic-sure - - true - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.8 - - - copy-installed - install - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - pic-sure-hpds - - - - - - - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 - - - default-cli - - run - - - 13000 - /jaxrs-service - true - true - - -Xms256m -Xmx512m - - - - - - - - - org.apache.maven.plugins - maven-eclipse-plugin - - [artifactId]-[version] - true - true - 2.0 - - - - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - 11 + + 4.0.0 + edu.harvard.hms.dbmi.avillach.hpds + pic-sure-hpds + 1.0-SNAPSHOT + pom + pic-sure-hpds + + common + service + etl + data + docker + processing + war + client-api + + + 1.8.6 + UTF-8 + 1.4.10 + + + + github + GitHub HMS-DBMI Apache Maven Packages + https://maven.pkg.github.com/hms-dbmi/pic-sure + + true + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.8 + + + copy-installed + install + + copy + + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + pic-sure-hpds + + + + + + org.apache.tomcat.maven + tomcat7-maven-plugin + 2.0 + + + default-cli + + run + + + 13000 + /jaxrs-service + true + true + + -Xms256m -Xmx512m + + + + + - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - - - - - - - junit - junit - test - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - org.slf4j - slf4j-api - - - - - - edu.harvard.hms.dbmi.avillach.hpds - common - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - client-api - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - service - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - processing - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - etl - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - data - ${project.version} - - - edu.harvard.hms.dbmi.avillach - pic-sure-resource-api - 2.1.0-SNAPSHOT - - - ch.qos.logback - logback-core - 1.2.9 - - - ch.qos.logback - logback-classic - 1.2.9 - - - org.slf4j - slf4j-api - 1.7.25 - - - com.google.guava - guava - 30.0-jre - - - org.apache.commons - commons-math3 - 3.6.1 - - - de.siegmar - fastcsv - 1.0.2 - - - org.apache.commons - commons-csv - 1.5 - - - commons-io - commons-io - 2.7 - - - org.apache.cxf - cxf-rt-frontend-jaxrs - 3.2.5 - - - org.apache.cxf - cxf-rt-rs-client - 3.2.5 - - - org.codehaus.jackson - jackson-core-asl - ${jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${jackson.version} - - - javax.xml.bind - jaxb-api - 2.3.0 - - - com.sun.xml.bind - jaxb-core - 2.3.0 - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - - - javax.activation - activation - 1.1.1 - - - org.codehaus.jackson - jackson-jaxrs - ${jackson.version} - - - org.springframework - spring-web - 4.3.20.RELEASE - - - junit - junit - 4.13.1 - test - - - org.mockito - mockito-core - 3.8.0 - test - - - com.oracle.database.jdbc - ojdbc10 - 19.17.0.0 - - - org.springframework - spring-jdbc - 5.1.1.RELEASE - - - com.github.ben-manes.caffeine - caffeine - 3.1.1 - - - org.springframework - spring-test - 4.3.30.RELEASE - + + + org.apache.maven.plugins + maven-eclipse-plugin + + [artifactId]-[version] + true + true + 2.0 + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.11.0 + + 11 + + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + + + junit + junit + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.slf4j + slf4j-api + + + + + + edu.harvard.hms.dbmi.avillach.hpds + common + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + client-api + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + service + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + processing + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + etl + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + data + ${project.version} + + + edu.harvard.hms.dbmi.avillach + pic-sure-resource-api + 2.1.0-SNAPSHOT + + + ch.qos.logback + logback-core + 1.2.9 + + + ch.qos.logback + logback-classic + 1.2.9 + + + org.slf4j + slf4j-api + 1.7.25 + + + com.google.guava + guava + 30.0-jre + + + org.apache.commons + commons-math3 + 3.6.1 + + + de.siegmar + fastcsv + 1.0.2 + + + org.apache.commons + commons-csv + 1.5 + + + commons-io + commons-io + 2.7 + + + org.apache.cxf + cxf-rt-frontend-jaxrs + 3.2.5 + + + org.apache.cxf + cxf-rt-rs-client + 3.2.5 + + + org.codehaus.jackson + jackson-core-asl + ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${jackson.version} + + + javax.xml.bind + jaxb-api + 2.3.0 + + + com.sun.xml.bind + jaxb-core + 2.3.0 + + + com.sun.xml.bind + jaxb-impl + 2.3.0 + + + javax.activation + activation + 1.1.1 + + + org.codehaus.jackson + jackson-jaxrs + ${jackson.version} + + + org.springframework + spring-web + 4.3.20.RELEASE + + + junit + junit + 4.13.1 + test + + + org.mockito + mockito-core + 3.8.0 + test + + + com.oracle.database.jdbc + ojdbc10 + 19.17.0.0 + + + org.springframework + spring-jdbc + 5.1.1.RELEASE + + + com.github.ben-manes.caffeine + caffeine + 3.1.1 + + + org.springframework + spring-test + 4.3.30.RELEASE + - - - - - github - GitHub HMS-DBMI Apache Maven Packages - https://maven.pkg.github.com/hms-dbmi/pic-sure-hpds - - + + + + + github + GitHub HMS-DBMI Apache Maven Packages + https://maven.pkg.github.com/hms-dbmi/pic-sure-hpds + + \ No newline at end of file From 8119d456a5ad043452150ebea04d81b58a899724 Mon Sep 17 00:00:00 2001 From: gcolon021 Date: Fri, 6 Oct 2023 21:53:27 -0400 Subject: [PATCH 6/7] [ALS-5053] Add error pages to web.xml --- war/src/main/webapp/WEB-INF/web.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/war/src/main/webapp/WEB-INF/web.xml b/war/src/main/webapp/WEB-INF/web.xml index 04fbef21..dfa1d2ee 100644 --- a/war/src/main/webapp/WEB-INF/web.xml +++ b/war/src/main/webapp/WEB-INF/web.xml @@ -31,4 +31,12 @@ true + + 404 + /error-404.html + + + 500 + /error-500.html + From 093f6ff8584e87bef774938df6cc3cf38f3c7c6b Mon Sep 17 00:00:00 2001 From: gcolon021 Date: Fri, 6 Oct 2023 22:03:33 -0400 Subject: [PATCH 7/7] [ALS-5053] Add value to server.xml This value will intercept request processed by tomcat. I have disabled reports and show server information. This information should not be returned to the client. --- docker/pic-sure-hpds/server.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/pic-sure-hpds/server.xml b/docker/pic-sure-hpds/server.xml index ebf2760b..8b835346 100644 --- a/docker/pic-sure-hpds/server.xml +++ b/docker/pic-sure-hpds/server.xml @@ -27,6 +27,7 @@ +