From 0d9f3e428f8bce093c374caaaa9d4d07e37ee1f5 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:22:08 -0400 Subject: [PATCH 1/3] [ALS-5053] pen test medium stacktrace error (#85) * [ALS-5000] Update maven compiler plugin Maven compiler plugin has a dependency on log4j. * [ALS-5053] Add error pages to web.xml * [ALS-5053] Add value to server.xml This value will intercept request processed by tomcat. I have disabled reports and show server information. This information should not be returned to the client. --- client-api/pom.xml | 10 +- docker/pic-sure-hpds/server.xml | 1 + pom.xml | 654 ++++++++++++++-------------- war/pom.xml | 2 +- war/src/main/webapp/WEB-INF/web.xml | 8 + 5 files changed, 338 insertions(+), 337 deletions(-) diff --git a/client-api/pom.xml b/client-api/pom.xml index ef02dcbb..06638ab2 100644 --- a/client-api/pom.xml +++ b/client-api/pom.xml @@ -6,21 +6,17 @@ edu.harvard.hms.dbmi.avillach.hpds 1.0-SNAPSHOT - edu.harvard.hms.dbmi.avillach.hpds client-api 1.0-SNAPSHOT - client-api http://www.example.com - UTF-8 1.7 1.7 - junit @@ -29,9 +25,9 @@ test - - + + @@ -45,7 +41,7 @@ maven-compiler-plugin - 3.8.0 + 3.11.0 maven-surefire-plugin diff --git a/docker/pic-sure-hpds/server.xml b/docker/pic-sure-hpds/server.xml index ebf2760b..8b835346 100644 --- a/docker/pic-sure-hpds/server.xml +++ b/docker/pic-sure-hpds/server.xml @@ -27,6 +27,7 @@ + diff --git a/pom.xml b/pom.xml index ab958be4..879eeaf9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,333 +1,329 @@ - - 4.0.0 - edu.harvard.hms.dbmi.avillach.hpds - pic-sure-hpds - 1.0-SNAPSHOT - pom - pic-sure-hpds - - common - service - etl - data - docker - processing - war - client-api - - - 1.8.6 - UTF-8 - 1.4.10 - - - - github - GitHub HMS-DBMI Apache Maven Packages - https://maven.pkg.github.com/hms-dbmi/pic-sure - - true - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.8 - - - copy-installed - install - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - pic-sure-hpds - - - - - - - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 - - - default-cli - - run - - - 13000 - /jaxrs-service - true - true - - -Xms256m -Xmx512m - - - - - - - - - org.apache.maven.plugins - maven-eclipse-plugin - - [artifactId]-[version] - true - true - 2.0 - - - - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - 11 + + 4.0.0 + edu.harvard.hms.dbmi.avillach.hpds + pic-sure-hpds + 1.0-SNAPSHOT + pom + pic-sure-hpds + + common + service + etl + data + docker + processing + war + client-api + + + 1.8.6 + UTF-8 + 1.4.10 + + + + github + GitHub HMS-DBMI Apache Maven Packages + https://maven.pkg.github.com/hms-dbmi/pic-sure + + true + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.8 + + + copy-installed + install + + copy + + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + pic-sure-hpds + + + + + + org.apache.tomcat.maven + tomcat7-maven-plugin + 2.0 + + + default-cli + + run + + + 13000 + /jaxrs-service + true + true + + -Xms256m -Xmx512m + + + + + - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - - - - - - - junit - junit - test - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - org.slf4j - slf4j-api - - - - - - edu.harvard.hms.dbmi.avillach.hpds - common - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - client-api - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - service - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - processing - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - etl - ${project.version} - - - edu.harvard.hms.dbmi.avillach.hpds - data - ${project.version} - - - edu.harvard.hms.dbmi.avillach - pic-sure-resource-api - 2.1.0-SNAPSHOT - - - ch.qos.logback - logback-core - 1.2.9 - - - ch.qos.logback - logback-classic - 1.2.9 - - - org.slf4j - slf4j-api - 1.7.25 - - - com.google.guava - guava - 30.0-jre - - - org.apache.commons - commons-math3 - 3.6.1 - - - de.siegmar - fastcsv - 1.0.2 - - - org.apache.commons - commons-csv - 1.5 - - - commons-io - commons-io - 2.7 - - - org.apache.cxf - cxf-rt-frontend-jaxrs - 3.2.5 - - - org.apache.cxf - cxf-rt-rs-client - 3.2.5 - - - org.codehaus.jackson - jackson-core-asl - ${jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${jackson.version} - - - javax.xml.bind - jaxb-api - 2.3.0 - - - com.sun.xml.bind - jaxb-core - 2.3.0 - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - - - javax.activation - activation - 1.1.1 - - - org.codehaus.jackson - jackson-jaxrs - ${jackson.version} - - - org.springframework - spring-web - 4.3.20.RELEASE - - - junit - junit - 4.13.1 - test - - - org.mockito - mockito-core - 3.8.0 - test - - - com.oracle.database.jdbc - ojdbc10 - 19.17.0.0 - - - org.springframework - spring-jdbc - 5.1.1.RELEASE - - - com.github.ben-manes.caffeine - caffeine - 3.1.1 - - - org.springframework - spring-test - 4.3.30.RELEASE - + + + org.apache.maven.plugins + maven-eclipse-plugin + + [artifactId]-[version] + true + true + 2.0 + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.11.0 + + 11 + + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + + + junit + junit + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.slf4j + slf4j-api + + + + + + edu.harvard.hms.dbmi.avillach.hpds + common + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + client-api + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + service + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + processing + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + etl + ${project.version} + + + edu.harvard.hms.dbmi.avillach.hpds + data + ${project.version} + + + edu.harvard.hms.dbmi.avillach + pic-sure-resource-api + 2.1.0-SNAPSHOT + + + ch.qos.logback + logback-core + 1.2.9 + + + ch.qos.logback + logback-classic + 1.2.9 + + + org.slf4j + slf4j-api + 1.7.25 + + + com.google.guava + guava + 30.0-jre + + + org.apache.commons + commons-math3 + 3.6.1 + + + de.siegmar + fastcsv + 1.0.2 + + + org.apache.commons + commons-csv + 1.5 + + + commons-io + commons-io + 2.7 + + + org.apache.cxf + cxf-rt-frontend-jaxrs + 3.2.5 + + + org.apache.cxf + cxf-rt-rs-client + 3.2.5 + + + org.codehaus.jackson + jackson-core-asl + ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${jackson.version} + + + javax.xml.bind + jaxb-api + 2.3.0 + + + com.sun.xml.bind + jaxb-core + 2.3.0 + + + com.sun.xml.bind + jaxb-impl + 2.3.0 + + + javax.activation + activation + 1.1.1 + + + org.codehaus.jackson + jackson-jaxrs + ${jackson.version} + + + org.springframework + spring-web + 4.3.20.RELEASE + + + junit + junit + 4.13.1 + test + + + org.mockito + mockito-core + 3.8.0 + test + + + com.oracle.database.jdbc + ojdbc10 + 19.17.0.0 + + + org.springframework + spring-jdbc + 5.1.1.RELEASE + + + com.github.ben-manes.caffeine + caffeine + 3.1.1 + + + org.springframework + spring-test + 4.3.30.RELEASE + - - - - - github - GitHub HMS-DBMI Apache Maven Packages - https://maven.pkg.github.com/hms-dbmi/pic-sure-hpds - - + + + + + github + GitHub HMS-DBMI Apache Maven Packages + https://maven.pkg.github.com/hms-dbmi/pic-sure-hpds + + \ No newline at end of file diff --git a/war/pom.xml b/war/pom.xml index 93c90038..42999b00 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -26,7 +26,7 @@ 9 - 3.8.0 + 3.11.0 org.apache.maven.plugins diff --git a/war/src/main/webapp/WEB-INF/web.xml b/war/src/main/webapp/WEB-INF/web.xml index 04fbef21..dfa1d2ee 100644 --- a/war/src/main/webapp/WEB-INF/web.xml +++ b/war/src/main/webapp/WEB-INF/web.xml @@ -31,4 +31,12 @@ true + + 404 + /error-404.html + + + 500 + /error-500.html + From b78567061a50dca47ce99f92984bed86f3e7665c Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Sun, 15 Oct 2023 10:26:08 -0400 Subject: [PATCH 2/3] [ALS-5056] QueryRequest sumtype - Update jackson deps to support serialization of polymorphic types --- data/pom.xml | 16 ++++++++++++---- .../hpds/data/phenotype/ColumnMeta.java | 4 ++-- pom.xml | 19 ++++++++++++------- service/pom.xml | 16 ++++++++++------ service/src/main/resources/jaxrs-context.xml | 2 +- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/data/pom.xml b/data/pom.xml index 298160fc..f4afa848 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -8,6 +8,10 @@ 1.0-SNAPSHOT + + 2.15.2 + + data data @@ -26,12 +30,16 @@ commons-csv - org.codehaus.jackson - jackson-core-asl + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind - org.codehaus.jackson - jackson-mapper-asl + com.fasterxml.jackson.core + jackson-annotations edu.harvard.hms.dbmi.avillach diff --git a/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMeta.java b/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMeta.java index 3831c31c..98d2fa8d 100644 --- a/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMeta.java +++ b/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMeta.java @@ -3,8 +3,8 @@ import java.io.Serializable; import java.util.List; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) public class ColumnMeta implements Serializable{ diff --git a/pom.xml b/pom.xml index 879eeaf9..67701777 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ client-api - 1.8.6 + 2.15.2 UTF-8 1.4.10 @@ -245,13 +245,18 @@ 3.2.5 - org.codehaus.jackson - jackson-core-asl + com.fasterxml.jackson.core + jackson-core ${jackson.version} - org.codehaus.jackson - jackson-mapper-asl + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations ${jackson.version} @@ -275,8 +280,8 @@ 1.1.1 - org.codehaus.jackson - jackson-jaxrs + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider ${jackson.version} diff --git a/service/pom.xml b/service/pom.xml index 734e3a94..267d3092 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -45,12 +45,16 @@ guava - org.codehaus.jackson - jackson-core-asl + com.fasterxml.jackson.core + jackson-core - org.codehaus.jackson - jackson-mapper-asl + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations javax.xml.bind @@ -69,8 +73,8 @@ activation - org.codehaus.jackson - jackson-jaxrs + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider org.springframework diff --git a/service/src/main/resources/jaxrs-context.xml b/service/src/main/resources/jaxrs-context.xml index 08a237bf..006ed56d 100644 --- a/service/src/main/resources/jaxrs-context.xml +++ b/service/src/main/resources/jaxrs-context.xml @@ -14,7 +14,7 @@ - + From 47b827b05d27e47d63baa8a7a68a46a9be995861 Mon Sep 17 00:00:00 2001 From: ramari16 Date: Mon, 23 Oct 2023 15:43:55 -0400 Subject: [PATCH 3/3] [ALS-5050] Ignore invalid concept paths in anyRecordOf queries (#87) --- .../hpds/processing/AbstractProcessor.java | 7 +- .../processing/AbstractProcessorTest.java | 80 ++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java index e67d24cb..0a196f5b 100644 --- a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java +++ b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java @@ -273,7 +273,12 @@ private void addIdSetsForAnyRecordOf(List anyRecordOfFilters, ArrayList< addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache); return patientsInScope.stream(); } else { - return (Stream) getCube(path).keyBasedIndex().stream(); + try { + return (Stream) getCube(path).keyBasedIndex().stream(); + } catch (InvalidCacheLoadException e) { + // return an empty stream if this concept doesn't exist + return Stream.empty(); + } } }).collect(Collectors.toSet()); filteredIdSets.add(anyRecordOfPatientSet); diff --git a/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessorTest.java b/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessorTest.java index e22bea5e..47153ee3 100644 --- a/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessorTest.java +++ b/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessorTest.java @@ -1,7 +1,10 @@ package edu.harvard.hms.dbmi.avillach.hpds.processing; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import edu.harvard.hms.dbmi.avillach.hpds.data.genotype.FileBackedByteIndexedInfoStore; +import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.PhenoCube; import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query; import edu.harvard.hms.dbmi.avillach.hpds.storage.FileBackedByteIndexedStorage; import org.junit.Before; @@ -12,6 +15,7 @@ import org.mockito.junit.MockitoJUnitRunner; import java.util.*; +import java.util.concurrent.ExecutionException; import static org.mockito.ArgumentMatchers.any; import static org.junit.Assert.*; @@ -33,6 +37,9 @@ public class AbstractProcessorTest { @Mock private PatientVariantJoinHandler patientVariantJoinHandler; + @Mock + private LoadingCache> mockLoadingCache; + public static final String GENE_WITH_VARIANT_KEY = "Gene_with_variant"; private static final String VARIANT_SEVERITY_KEY = "Variant_severity"; public static final List EXAMPLE_GENES_WITH_VARIANT = List.of("CDH8", "CDH9", "CDH10"); @@ -61,7 +68,7 @@ public void setup() { new TreeMap<>(), new TreeSet<>() ), - null, + mockLoadingCache, infoStores, null, variantService, @@ -125,20 +132,87 @@ public void getPatientSubsetForQuery_twoVariantCategoryFilters_intersectFilters( when(patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(any(), argumentCaptor.capture())).thenReturn(List.of(Set.of(42))); Map categoryVariantInfoFilters = Map.of( - GENE_WITH_VARIANT_KEY, new String[] {EXAMPLE_GENES_WITH_VARIANT.get(0)}, - VARIANT_SEVERITY_KEY, new String[] {EXAMPLE_VARIANT_SEVERITIES.get(0)} + GENE_WITH_VARIANT_KEY, new String[] {EXAMPLE_GENES_WITH_VARIANT.get(0)}, + VARIANT_SEVERITY_KEY, new String[] {EXAMPLE_VARIANT_SEVERITIES.get(0)} + ); + Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); + variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; + + List variantInfoFilters = List.of(variantInfoFilter); + + Query query = new Query(); + query.setVariantInfoFilters(variantInfoFilters); + + TreeSet patientSubsetForQuery = abstractProcessor.getPatientSubsetForQuery(query); + assertFalse(patientSubsetForQuery.isEmpty()); + // Expected result is the intersection of the two filters + assertEquals(argumentCaptor.getValue(), new SparseVariantIndex(Set.of(4, 6))); + } + + @Test + public void getPatientSubsetForQuery_anyRecordOf_applyOrLogic() throws ExecutionException { + when(variantIndexCache.get(GENE_WITH_VARIANT_KEY, EXAMPLE_GENES_WITH_VARIANT.get(0))).thenReturn(new SparseVariantIndex(Set.of(2, 4, 6))); + when(variantIndexCache.get(VARIANT_SEVERITY_KEY, EXAMPLE_VARIANT_SEVERITIES.get(0))).thenReturn(new SparseVariantIndex(Set.of(4, 5, 6, 7))); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(VariantIndex.class); + ArgumentCaptor>> listArgumentCaptor = ArgumentCaptor.forClass(List.class); + when(patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(listArgumentCaptor.capture(), argumentCaptor.capture())).thenReturn(List.of(Set.of(42))); + + Map categoryVariantInfoFilters = Map.of( + GENE_WITH_VARIANT_KEY, new String[] {EXAMPLE_GENES_WITH_VARIANT.get(0)}, + VARIANT_SEVERITY_KEY, new String[] {EXAMPLE_VARIANT_SEVERITIES.get(0)} ); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; List variantInfoFilters = List.of(variantInfoFilter); + PhenoCube mockPhenoCube = mock(PhenoCube.class); + when(mockPhenoCube.keyBasedIndex()).thenReturn(List.of(42, 101)); + when(mockLoadingCache.get("good concept")).thenReturn(mockPhenoCube); + when(mockLoadingCache.get("bad concept")).thenThrow(CacheLoader.InvalidCacheLoadException.class); + + Query query = new Query(); + query.setVariantInfoFilters(variantInfoFilters); + query.setAnyRecordOf(List.of("good concept", "bad concept")); + + TreeSet patientSubsetForQuery = abstractProcessor.getPatientSubsetForQuery(query); + assertFalse(patientSubsetForQuery.isEmpty()); + // Expected result is the intersection of the two filters + assertEquals(argumentCaptor.getValue(), new SparseVariantIndex(Set.of(4, 6))); + assertEquals(listArgumentCaptor.getValue().get(0), Set.of(42, 101)); + } + + + + @Test + public void getPatientSubsetForQuery_anyRecordOfInvalidKey_returnEmpty() throws ExecutionException { + when(variantIndexCache.get(GENE_WITH_VARIANT_KEY, EXAMPLE_GENES_WITH_VARIANT.get(0))).thenReturn(new SparseVariantIndex(Set.of(2, 4, 6))); + when(variantIndexCache.get(VARIANT_SEVERITY_KEY, EXAMPLE_VARIANT_SEVERITIES.get(0))).thenReturn(new SparseVariantIndex(Set.of(4, 5, 6, 7))); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(VariantIndex.class); + ArgumentCaptor>> listArgumentCaptor = ArgumentCaptor.forClass(List.class); + when(patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(listArgumentCaptor.capture(), argumentCaptor.capture())).thenReturn(List.of(Set.of(42))); + + Map categoryVariantInfoFilters = Map.of( + GENE_WITH_VARIANT_KEY, new String[] {EXAMPLE_GENES_WITH_VARIANT.get(0)}, + VARIANT_SEVERITY_KEY, new String[] {EXAMPLE_VARIANT_SEVERITIES.get(0)} + ); + Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); + variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; + + List variantInfoFilters = List.of(variantInfoFilter); + + when(mockLoadingCache.get("bad concept")).thenThrow(CacheLoader.InvalidCacheLoadException.class); + Query query = new Query(); query.setVariantInfoFilters(variantInfoFilters); + query.setAnyRecordOf(List.of("bad concept")); TreeSet patientSubsetForQuery = abstractProcessor.getPatientSubsetForQuery(query); assertFalse(patientSubsetForQuery.isEmpty()); // Expected result is the intersection of the two filters assertEquals(argumentCaptor.getValue(), new SparseVariantIndex(Set.of(4, 6))); + assertEquals(listArgumentCaptor.getValue().get(0), Set.of()); } }