From 6594e4f76bb8ec59518175ad1f97e8bf7ac57d2d Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 14 Nov 2024 11:48:24 -0500 Subject: [PATCH 01/22] added test and fixed performance --- ...9-reduce-memory-overhead-for-searches.yaml | 8 +++ .../jpa/search/builder/SearchBuilder.java | 39 +++++++++-- .../builder/sql/SearchQueryExecutor.java | 3 + .../FhirResourceDaoR4SearchOptimizedTest.java | 69 ++++++++++++++++++- .../jpa/api/config/JpaStorageSettings.java | 2 +- 5 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6469-reduce-memory-overhead-for-searches.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6469-reduce-memory-overhead-for-searches.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6469-reduce-memory-overhead-for-searches.yaml new file mode 100644 index 000000000000..173bc12c1b54 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6469-reduce-memory-overhead-for-searches.yaml @@ -0,0 +1,8 @@ +--- +type: perf +issue: 6469 +title: "Searching for a large number of resources can use a lot of + memory, due to the nature of deduplication of results in memory. + We will instead push this responsibility to the db to save + reduce this overhead. +" diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 9a28921f1ae1..9b2560c05147 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -638,7 +638,8 @@ private void createChunkedQueryNormalSearch( boolean theCountOnlyFlag, RequestDetails theRequest, List thePidList, - List theSearchQueryExecutors) { + List theSearchQueryExecutors + ) { SearchQueryBuilder sqlBuilder = new SearchQueryBuilder( myContext, myStorageSettings, @@ -719,9 +720,13 @@ private void createChunkedQueryNormalSearch( } /* - * If offset is present, we want deduplicate the results by using GROUP BY + * If offset is present, we want to deduplicate the results by using GROUP BY; + * OR + * if the MaxResultsToFetch is null, we are requesting "everything", + * so we'll let the db do the deduplication (instead of in-memory) */ - if (theOffset != null) { + // todo - if the limit is not present, we should group + if (theOffset != null || myMaxResultsToFetch == null) { queryStack3.addGrouping(); queryStack3.setUseAggregate(true); } @@ -2403,7 +2408,14 @@ private void fetchNext() { if (nextLong != null) { JpaPid next = JpaPid.fromId(nextLong); - if (myPidSet.add(next) && doNotSkipNextPidForEverything()) { + // TODO - check if it's the unlimited case + if (doNotSkipNextPidForEverything() && !myPidSet.contains(next)) { + if (myMaxResultsToFetch != null) { + // we only add to the map if we aren't fetching "everything"; + // otherwise, we let the de-duplication happen in the database + // (see createChunkedQueryNormalSearch above) + myPidSet.add(next); + } myNext = next; myNonSkipCount++; break; @@ -2478,6 +2490,25 @@ private void fetchNext() { } } + /** + * Determine if the next value should be skipped or not. + * + * We skip if: + * * we are in everything mode + * AND + * * we've already seen next (and we add it to our list of seen ids) + * OR + * * we've already seen the result and we're fetching everything; we + * don't add it to the map in this case because we might be seeing + * millions of records and don't want to actually fill up our map; + * the database will do the deduplication for us. + */ + private boolean shouldSkip(JpaPid next) { + return !doNotSkipNextPidForEverything() && ( + (myMaxResultsToFetch == null && myPidSet.contains(next)) + || (!myPidSet.add(next))); + } + private Integer calculateMaxResultsToFetch() { if (myParams.getLoadSynchronousUpTo() != null) { return myParams.getLoadSynchronousUpTo(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java index f636ab7eb4cb..08499e47749d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java @@ -55,6 +55,8 @@ public class SearchQueryExecutor implements ISearchQueryExecutor { private ScrollableResultsIterator myResultSet; private Long myNext; + private Integer myMaxResultsToFetch; + /** * Constructor */ @@ -62,6 +64,7 @@ public SearchQueryExecutor(GeneratedSql theGeneratedSql, Integer theMaxResultsTo Validate.notNull(theGeneratedSql, "theGeneratedSql must not be null"); myGeneratedSql = theGeneratedSql; myQueryInitialized = false; + myMaxResultsToFetch = theMaxResultsToFetch; } /** diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java index e03d3725746c..b2ce52e759f8 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java @@ -15,6 +15,7 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.util.QueryParameterUtils; +import ca.uhn.fhir.jpa.util.SqlQuery; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.SummaryEnum; @@ -33,6 +34,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.BaseResource; import org.hl7.fhir.r4.model.BodyStructure; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; @@ -65,14 +67,18 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.leftPad; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.fail; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -556,6 +562,67 @@ public void testFetchOnlySmallBatches() { } + /** + * We want to use the db to deduplicate in the "fetch everything" + * case because it's more memory efficient. + */ + @Test + public void search_whenPastPreFetchLimit_usesDBToDeduplicate() { + // setup + IBundleProvider results; + List queries; + List ids; + + create200Patients(); + + myCaptureQueriesListener.clear(); + // set the prefetch thresholds low so we don't need to + // search for tons of resources + myStorageSettings.setSearchPreFetchThresholds(List.of(5, 10, -1)); + + // basic search map + SearchParameterMap map = new SearchParameterMap(); + map.setSort(new SortSpec(BaseResource.SP_RES_LAST_UPDATED)); + + // test + results = myPatientDao.search(map, null); + String uuid = results.getUuid(); + ourLog.debug("** Search returned UUID: {}", uuid); + assertNotNull(results); + ids = toUnqualifiedVersionlessIdValues(results, 0, 9, true); + assertEquals(9, ids.size()); + + // first search was < 10 (our max pre-fetch value); so we should + // expect no "group by" queries (we deduplicate in memory) + queries = findGroupByQueries(); + assertTrue(queries.isEmpty()); + myCaptureQueriesListener.clear(); + + ids = toUnqualifiedVersionlessIdValues(results, 10, 100, true); + assertEquals(90, ids.size()); + + // we are now requesting > 10 results, meaning we should be using the + // database to deduplicate any values not fetched yet; + // so we *do* expect to see a "group by" query + queries = findGroupByQueries(); + assertFalse(queries.isEmpty()); + assertEquals(1, queries.size()); + SqlQuery query = queries.get(0); + String sql = query.getSql(true, false); + // we expect a "GROUP BY t0.RES_ID" (but we'll be ambiguous about the table + // name, just in case) + Pattern p = Pattern.compile("GROUP BY .+\\.RES_ID"); + Matcher m = p.matcher(sql); + assertTrue(m.find()); + } + + private List findGroupByQueries() { + List queries = myCaptureQueriesListener.getSelectQueries(); + queries = queries.stream().filter(q -> q.getSql(true, false).toLowerCase().contains("group by")) + .collect(Collectors.toList()); + return queries; + } + @Test public void testFetchMoreThanFirstPageSizeInFirstPage() { create200Patients(); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java index 9b9e19034d04..cad57a018f1c 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java @@ -189,7 +189,7 @@ public class JpaStorageSettings extends StorageSettings { // start with a tiny number so our first page always loads quickly. // If they fetch the second page, fetch more. // Use prime sizes to avoid empty next links. - private List mySearchPreFetchThresholds = Arrays.asList(13, 503, 2003, -1); + private List mySearchPreFetchThresholds = Arrays.asList(13, 503, 2003, 1000003, -1); private List myWarmCacheEntries = new ArrayList<>(); private boolean myEnforceReferenceTargetTypes = true; private ClientIdStrategyEnum myResourceClientIdStrategy = ClientIdStrategyEnum.ALPHANUMERIC; From 0be9b67036a3d46ce19c8db73dadf2390f307355 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 14 Nov 2024 13:18:24 -0500 Subject: [PATCH 02/22] spotless --- .../ca/uhn/fhir/jpa/search/builder/SearchBuilder.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 9b2560c05147..06322a0a6603 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -638,8 +638,7 @@ private void createChunkedQueryNormalSearch( boolean theCountOnlyFlag, RequestDetails theRequest, List thePidList, - List theSearchQueryExecutors - ) { + List theSearchQueryExecutors) { SearchQueryBuilder sqlBuilder = new SearchQueryBuilder( myContext, myStorageSettings, @@ -2504,9 +2503,8 @@ private void fetchNext() { * the database will do the deduplication for us. */ private boolean shouldSkip(JpaPid next) { - return !doNotSkipNextPidForEverything() && ( - (myMaxResultsToFetch == null && myPidSet.contains(next)) - || (!myPidSet.add(next))); + return !doNotSkipNextPidForEverything() + && ((myMaxResultsToFetch == null && myPidSet.contains(next)) || (!myPidSet.add(next))); } private Integer calculateMaxResultsToFetch() { From b242620bb809d585f6fc8a6c5e04be090366944b Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 14 Nov 2024 13:33:45 -0500 Subject: [PATCH 03/22] cleanup --- .../jpa/search/builder/SearchBuilder.java | 23 ++----------------- .../builder/sql/SearchQueryExecutor.java | 3 --- .../jpa/api/config/JpaStorageSettings.java | 3 ++- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 06322a0a6603..56df4b840199 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -724,7 +724,6 @@ private void createChunkedQueryNormalSearch( * if the MaxResultsToFetch is null, we are requesting "everything", * so we'll let the db do the deduplication (instead of in-memory) */ - // todo - if the limit is not present, we should group if (theOffset != null || myMaxResultsToFetch == null) { queryStack3.addGrouping(); queryStack3.setUseAggregate(true); @@ -2407,12 +2406,12 @@ private void fetchNext() { if (nextLong != null) { JpaPid next = JpaPid.fromId(nextLong); - // TODO - check if it's the unlimited case if (doNotSkipNextPidForEverything() && !myPidSet.contains(next)) { if (myMaxResultsToFetch != null) { // we only add to the map if we aren't fetching "everything"; // otherwise, we let the de-duplication happen in the database - // (see createChunkedQueryNormalSearch above) + // (see createChunkedQueryNormalSearch above), becuase it saves + // memory that way myPidSet.add(next); } myNext = next; @@ -2489,24 +2488,6 @@ private void fetchNext() { } } - /** - * Determine if the next value should be skipped or not. - * - * We skip if: - * * we are in everything mode - * AND - * * we've already seen next (and we add it to our list of seen ids) - * OR - * * we've already seen the result and we're fetching everything; we - * don't add it to the map in this case because we might be seeing - * millions of records and don't want to actually fill up our map; - * the database will do the deduplication for us. - */ - private boolean shouldSkip(JpaPid next) { - return !doNotSkipNextPidForEverything() - && ((myMaxResultsToFetch == null && myPidSet.contains(next)) || (!myPidSet.add(next))); - } - private Integer calculateMaxResultsToFetch() { if (myParams.getLoadSynchronousUpTo() != null) { return myParams.getLoadSynchronousUpTo(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java index 08499e47749d..f636ab7eb4cb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java @@ -55,8 +55,6 @@ public class SearchQueryExecutor implements ISearchQueryExecutor { private ScrollableResultsIterator myResultSet; private Long myNext; - private Integer myMaxResultsToFetch; - /** * Constructor */ @@ -64,7 +62,6 @@ public SearchQueryExecutor(GeneratedSql theGeneratedSql, Integer theMaxResultsTo Validate.notNull(theGeneratedSql, "theGeneratedSql must not be null"); myGeneratedSql = theGeneratedSql; myQueryInitialized = false; - myMaxResultsToFetch = theMaxResultsToFetch; } /** diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java index cad57a018f1c..ab5cf7e20e1b 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/JpaStorageSettings.java @@ -188,7 +188,8 @@ public class JpaStorageSettings extends StorageSettings { // start with a tiny number so our first page always loads quickly. // If they fetch the second page, fetch more. - // Use prime sizes to avoid empty next links. + // we'll only fetch (by default) up to 1 million records, because after that, deduplication in local memory is + // prohibitive private List mySearchPreFetchThresholds = Arrays.asList(13, 503, 2003, 1000003, -1); private List myWarmCacheEntries = new ArrayList<>(); private boolean myEnforceReferenceTargetTypes = true; From 1d2bde268ecc5ff963a509c69d3b0219d29dec63 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Tue, 19 Nov 2024 13:53:09 -0500 Subject: [PATCH 04/22] code review fix --- .../jpa/search/builder/SearchBuilder.java | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 56df4b840199..4bbe54670c91 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -198,6 +198,26 @@ public class SearchBuilder implements ISearchBuilder { private String mySearchUuid; private int myFetchSize; private Integer myMaxResultsToFetch; + + /** + * Set of PIDs of results that have already been returned in a search. + * + * Searches use pre-fetch thresholds to avoid returning every result in the db + * (see {@link JpaStorageSettings mySearchPreFetchThresholds}). These threshold values + * dictate the usage of this set. + * + * Results from searches returning *less* than a prefetch threshold are put into this set + * for 2 purposes: + * 1) skipping already seen resources. ie, client requesting next "page" of + * results should skip previously returned results + * 2) deduplication of returned results. ie, searches can return duplicate resources (due to + * sort and filter criteria), so this set will be used to avoid returning duplicate results. + * + * NOTE: if a client requests *more* resources than *all* prefetch thresholds, + * we push the work of "deduplication" to the database. No newly seen resource + * will be stored in this set (to avoid this set exploding in size and the JVM running out memory). + * We will, however, still use it to skip previously seen results. + */ private Set myPidSet; private boolean myHasNextIteratorQuery = false; private RequestPartitionId myRequestPartitionId; @@ -2408,10 +2428,12 @@ private void fetchNext() { JpaPid next = JpaPid.fromId(nextLong); if (doNotSkipNextPidForEverything() && !myPidSet.contains(next)) { if (myMaxResultsToFetch != null) { - // we only add to the map if we aren't fetching "everything"; - // otherwise, we let the de-duplication happen in the database - // (see createChunkedQueryNormalSearch above), becuase it saves - // memory that way + /* + * We only add to the map if we aren't fetching "everything"; + * otherwise, we let the de-duplication happen in the database + * (see createChunkedQueryNormalSearch above), because it + * saves memory that way. + */ myPidSet.add(next); } myNext = next; From 2d9db1ce74c05a12b729c0192d1150b808a06644 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 21 Nov 2024 13:15:31 -0500 Subject: [PATCH 05/22] spotless --- .../main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 4bbe54670c91..a69137cc436b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -219,6 +219,7 @@ public class SearchBuilder implements ISearchBuilder { * We will, however, still use it to skip previously seen results. */ private Set myPidSet; + private boolean myHasNextIteratorQuery = false; private RequestPartitionId myRequestPartitionId; From 18635f727e05a30f8d79d908f75ac4cbfe5efa37 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Fri, 22 Nov 2024 16:54:31 -0500 Subject: [PATCH 06/22] fixing counts --- .../ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java | 1 + .../ca/uhn/fhir/jpa/search/builder/SearchBuilder.java | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java index 317b8fa21052..6b947683e9a7 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java @@ -130,6 +130,7 @@ public IBundleProvider executeQuery( List> contentAndTerms = theParams.get(Constants.PARAM_CONTENT); List> textAndTerms = theParams.get(Constants.PARAM_TEXT); + // TODO - this count query should not be grouped count = theSb.createCountQuery( theParams, theSearchUuid, theRequestDetails, theRequestPartitionId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index e457349252bd..3103b0452ebc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -762,7 +762,7 @@ private void createChunkedQueryNormalSearch( * if the MaxResultsToFetch is null, we are requesting "everything", * so we'll let the db do the deduplication (instead of in-memory) */ - if (theOffset != null || myMaxResultsToFetch == null) { + if (theOffset != null || (myMaxResultsToFetch == null && !theCountOnlyFlag)) { queryStack3.addGrouping(); queryStack3.setUseAggregate(true); } @@ -2507,7 +2507,11 @@ private void fetchNext() { } } - mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size()); + if (myMaxResultsToFetch == null) { + mySearchRuntimeDetails.setFoundIndexMatchesCount(myNonSkipCount); + } else { + mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size()); + } } finally { // search finished - fire hooks From 43c8341d9c6fb4346ad5e226449c77b8e6d1af31 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 25 Nov 2024 10:32:38 -0500 Subject: [PATCH 07/22] fixing tests --- .../jpa/search/SynchronousSearchSvcImpl.java | 1 - ...rResourceDaoR4ComboNonUniqueParamTest.java | 17 ++-- .../jpa/dao/r4/PartitioningSqlR4Test.java | 90 +++++++++---------- 3 files changed, 51 insertions(+), 57 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java index 6b947683e9a7..317b8fa21052 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java @@ -130,7 +130,6 @@ public IBundleProvider executeQuery( List> contentAndTerms = theParams.get(Constants.PARAM_CONTENT); List> textAndTerms = theParams.get(Constants.PARAM_TEXT); - // TODO - this count query should not be grouped count = theSb.createCountQuery( theParams, theSearchUuid, theRequestDetails, theRequestPartitionId); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java index 0b7dcd5878a1..cfc2fb36cc22 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ComboNonUniqueParamTest.java @@ -165,7 +165,7 @@ public void testStringAndToken_Create() { assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue()); assertThat(myCaptureQueriesListener.getSelectQueries().stream().map(t -> t.getSql(true, false)).toList()).contains( - "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '-2634469377090377342')" + "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '-2634469377090377342') GROUP BY t0.RES_ID" ); logCapturedMessages(); @@ -291,7 +291,7 @@ public void testStringAndToken_SearchWithExtraParameters() { assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue()); String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); - String expected = "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.HASH_COMPLETE = '-2634469377090377342') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND (((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_LOW_DATE_ORDINAL <= '20210202')) AND ((t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL >= '20210202')))))"; + String expected = "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_IDX_CMB_TOK_NU t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_DATE t2 ON (t1.RES_ID = t2.RES_ID) WHERE ((t0.HASH_COMPLETE = '-2634469377090377342') AND ((t2.HASH_IDENTITY = '5247847184787287691') AND (((t2.SP_VALUE_LOW_DATE_ORDINAL >= '20210202') AND (t2.SP_VALUE_LOW_DATE_ORDINAL <= '20210202')) AND ((t2.SP_VALUE_HIGH_DATE_ORDINAL <= '20210202') AND (t2.SP_VALUE_HIGH_DATE_ORDINAL >= '20210202'))))) GROUP BY t1.RES_ID"; assertEquals(expected, sql); logCapturedMessages(); @@ -323,7 +323,7 @@ public void testStringAndToken_MultipleAnd() { assertThat(actual).containsExactlyInAnyOrder(id1.toUnqualifiedVersionless().getValue()); String sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); - String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '7545664593829342272') AND ((t1.HASH_NORM_PREFIX = '6206712800146298788') AND (t1.SP_VALUE_NORMALIZED LIKE 'JAY%')))"; + String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '7545664593829342272') AND ((t1.HASH_NORM_PREFIX = '6206712800146298788') AND (t1.SP_VALUE_NORMALIZED LIKE 'JAY%'))) GROUP BY t0.RES_ID"; assertEquals(expected, sql); logCapturedMessages(); @@ -363,7 +363,7 @@ public void testStringAndDate_Create() { myCaptureQueriesListener.logSelectQueries(); assertThat(actual).contains(id1.toUnqualifiedVersionless().getValue()); - String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '7196518367857292879')"; + String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '7196518367857292879') GROUP BY t0.RES_ID"; assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); logCapturedMessages(); @@ -398,7 +398,7 @@ public void testStringAndReference_Create() { myCaptureQueriesListener.logSelectQueries(); assertThat(actual).contains(id1.toUnqualifiedVersionless().getValue()); - String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '2591238402961312979')"; + String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE = '2591238402961312979') GROUP BY t0.RES_ID"; assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); } @@ -461,9 +461,8 @@ public void testMultipleAndCombinations_EqualNumbers() { myCaptureQueriesListener.logSelectQueries(); assertThat(actual).contains("Patient/A"); - String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_IDX_CMB_TOK_NU t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND (t1.HASH_COMPLETE = '-8088946700286918311'))"; + String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_IDX_CMB_TOK_NU t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND (t1.HASH_COMPLETE = '-8088946700286918311')) GROUP BY t0.RES_ID"; assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); - } /** @@ -497,7 +496,7 @@ public void testMultipleAndCombinations_NonEqualNumbers() { myCaptureQueriesListener.logSelectQueries(); assertThat(actual).contains("Patient/A"); - String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND ((t1.HASH_NORM_PREFIX = '-3664262414674370905') AND (t1.SP_VALUE_NORMALIZED LIKE 'JONES%')))"; + String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 INNER JOIN HFJ_SPIDX_STRING t1 ON (t0.RES_ID = t1.RES_ID) WHERE ((t0.HASH_COMPLETE = '822090206952728926') AND ((t1.HASH_NORM_PREFIX = '-3664262414674370905') AND (t1.SP_VALUE_NORMALIZED LIKE 'JONES%'))) GROUP BY t0.RES_ID"; assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); } @@ -520,7 +519,7 @@ public void testOrQuery() { myCaptureQueriesListener.logSelectQueries(); assertThat(actual).contains("Observation/O1"); - String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE IN ('2445648980345828396','-6884698528022589694','-8034948665712960724') )"; + String expected = "SELECT t0.RES_ID FROM HFJ_IDX_CMB_TOK_NU t0 WHERE (t0.HASH_COMPLETE IN ('2445648980345828396','-6884698528022589694','-8034948665712960724') ) GROUP BY t0.RES_ID"; assertEquals(expected, myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false)); logCapturedMessages(); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java index 737c764724b4..851b0304d6f3 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java @@ -1289,7 +1289,7 @@ public void testSearch_IdParamOnly_PidId_SpecificPartition() { // Only the read columns should be used, no criteria use partition assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); } // Read in null Partition @@ -1342,7 +1342,7 @@ public void testSearch_IdParamSecond_PidId_SpecificPartition() { // Only the read columns should be used, no criteria use partition assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); // If this switches to 2 that would be fine + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(4); // If this switches to 2 that would be fine } // Read in null Partition @@ -1397,7 +1397,7 @@ public void testSearch_IdParamOnly_ForcedId_SpecificPartition() { // Only the read columns should be used, no criteria use partition assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); } // Read in null Partition @@ -1475,10 +1475,8 @@ public void testSearch_IdParamSecond_ForcedId_SpecificPartition() { IBundleProvider searchOutcome = myPatientDao.search(map, mySrd); assertEquals(0, searchOutcome.size()); } - } - @Test public void testSearch_MissingParamString_SearchAllPartitions() { myPartitionSettings.setIncludePartitionInSearchHashes(false); @@ -1500,7 +1498,7 @@ public void testSearch_MissingParamString_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'true'")); } @@ -1517,7 +1515,7 @@ public void testSearch_MissingParamString_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'false'")); } } @@ -1626,7 +1624,7 @@ public void testSearch_MissingParamReference_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")); assertEquals(1, StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")); } @@ -1653,7 +1651,7 @@ public void testSearch_MissingParamReference_SearchOnePartition_IncludePartition ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '-3438137196820602023'")).as(searchSql).isEqualTo(1); @@ -1681,7 +1679,7 @@ public void testSearch_MissingParamReference_SearchOnePartition_DontIncludeParti ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1); @@ -1707,7 +1705,7 @@ public void testSearch_MissingParamReference_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1); @@ -1732,7 +1730,7 @@ public void testSearch_NoParams_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); } @Test @@ -1752,7 +1750,7 @@ public void testSearch_NoParams_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); } @Test @@ -1822,7 +1820,7 @@ public void testSearch_DateParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date OR param @@ -1838,7 +1836,7 @@ public void testSearch_DateParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date AND param @@ -1854,7 +1852,7 @@ public void testSearch_DateParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // DateRangeParam @@ -1870,14 +1868,12 @@ public void testSearch_DateParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); - } - @Test public void testSearch_DateParam_SearchSpecificPartitions() { myPartitionSettings.setIncludePartitionInSearchHashes(false); @@ -1907,7 +1903,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); assertThat(StringUtils.countMatches(searchSql, "SP_VALUE_LOW")).as(searchSql).isEqualTo(2); // Date OR param @@ -1923,7 +1919,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date AND param @@ -1939,7 +1935,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // DateRangeParam @@ -1955,7 +1951,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); @@ -1987,7 +1983,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date OR param @@ -2003,7 +1999,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date AND param @@ -2019,7 +2015,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // DateRangeParam @@ -2035,7 +2031,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); @@ -2105,7 +2101,7 @@ public void testSearch_HasParam_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); ourLog.info("Search SQL:\n{}", searchSql); assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); } @@ -2129,7 +2125,7 @@ public void testSearch_StringParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2152,7 +2148,7 @@ public void testSearch_StringParam_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); searchSql = searchSql.toUpperCase(); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2176,7 +2172,7 @@ public void testSearch_StringParam_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2209,7 +2205,7 @@ public void testSearch_StringParam_SearchMultiplePartitions() { ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); assertThat(searchSql).contains("PARTITION_ID IN ('1','2')"); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); } // Match two partitions including null @@ -2226,7 +2222,7 @@ public void testSearch_StringParam_SearchMultiplePartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); assertThat(searchSql).contains("PARTITION_ID IS NULL"); assertThat(searchSql).contains("PARTITION_ID = '1'"); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(4, StringUtils.countMatches(searchSql, "PARTITION_ID")); } } @@ -2287,7 +2283,7 @@ public void testSearch_StringParam_SearchDefaultPartition_IncludePartitionInHash String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); searchSql = searchSql.toUpperCase(); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2315,7 +2311,7 @@ public void testSearch_StringParam_SearchOnePartition_IncludePartitionInHashes() String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2372,7 +2368,7 @@ public void testSearch_TagNotParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); // And with another param @@ -2389,7 +2385,7 @@ public void testSearch_TagNotParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(1); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, ".HASH_SYS_AND_VALUE =")).as(searchSql).isEqualTo(1); @@ -2413,7 +2409,7 @@ public void testSearch_TagNotParam_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); @@ -2441,7 +2437,7 @@ public void testSearch_TagNotParam_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2463,7 +2459,7 @@ public void testSearch_TagParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2489,7 +2485,7 @@ public void testSearch_TagParam_SearchOnePartition() { ourLog.info("Search SQL:\n{}", searchSql); assertEquals(2, StringUtils.countMatches(searchSql, "JOIN")); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2514,7 +2510,7 @@ public void testSearch_TagParamNot_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2539,7 +2535,7 @@ public void testSearch_TagParamNot_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2687,7 +2683,7 @@ public void testSearch_RefParam_TargetPid_SearchOnePartition() { assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")).as(searchSql).isEqualTo(1); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); // Same query, different partition addReadPartition(2); @@ -2724,7 +2720,7 @@ public void testSearch_RefParam_TargetPid_SearchDefaultPartition() { assertEquals(1, StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); // Same query, different partition addReadPartition(2); @@ -2759,7 +2755,7 @@ public void testSearch_RefParam_TargetForcedId_SearchOnePartition() { ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); // Same query, different partition addReadPartition(2); @@ -2829,7 +2825,7 @@ public void testSearch_RefParam_TargetForcedId_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); // Same query, different partition addReadPartition(2); From 5d396b1f6400e6e26cace1485dcc9529603b49f5 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 25 Nov 2024 13:52:27 -0500 Subject: [PATCH 08/22] fixing everything tests --- .../uhn/fhir/jpa/search/builder/SearchBuilder.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 3103b0452ebc..4814f9f87ceb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -124,6 +124,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.repository.query.Param; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.transaction.support.TransactionSynchronizationManager; @@ -2452,7 +2453,8 @@ private void fetchNext() { if (nextLong != null) { JpaPid next = JpaPid.fromId(nextLong); - if (doNotSkipNextPidForEverything() && !myPidSet.contains(next)) { + + if (!myPidSet.contains(next)) { if (myMaxResultsToFetch != null) { /* * We only add to the map if we aren't fetching "everything"; @@ -2462,9 +2464,11 @@ private void fetchNext() { */ myPidSet.add(next); } - myNext = next; - myNonSkipCount++; - break; + if (doNotSkipNextPidForEverything()) { + myNext = next; + myNonSkipCount++; + break; + } } else { mySkipCount++; } From 168ee41abc6c6a4261ceac4b0fd40d84da39d4df Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 25 Nov 2024 13:52:44 -0500 Subject: [PATCH 09/22] spotless --- .../main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 4814f9f87ceb..8d9dcf409d65 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -124,7 +124,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.repository.query.Param; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.transaction.support.TransactionSynchronizationManager; From c7a6cd98131916920002a21604e6cd4aa5c5c802 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 25 Nov 2024 16:27:09 -0500 Subject: [PATCH 10/22] intermittent fix --- .../dstu3/ResourceProviderDstu3Test.java | 155 +++++++++--------- 1 file changed, 77 insertions(+), 78 deletions(-) diff --git a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java index a4bc55eef464..47eb77ebaa8c 100644 --- a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java @@ -1,10 +1,5 @@ package ca.uhn.fhir.jpa.provider.dstu3; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertFalse; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; import ca.uhn.fhir.jpa.dao.data.ISearchDao; @@ -30,9 +25,7 @@ import ca.uhn.fhir.rest.param.DateRangeParam; import ca.uhn.fhir.rest.param.NumberParam; import ca.uhn.fhir.rest.param.ParamPrefixEnum; -import ca.uhn.fhir.rest.param.StringAndListParam; -import ca.uhn.fhir.rest.param.StringOrListParam; -import ca.uhn.fhir.rest.param.StringParam; +import ca.uhn.fhir.rest.server.IPagingProvider; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; @@ -133,6 +126,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -161,7 +155,11 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -184,10 +182,8 @@ public void after() throws Exception { mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(null); mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(QueryParameterUtils.DEFAULT_SYNC_SIZE); mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(false); - } - @Test public void testSearchBySourceTransactionId() { @@ -1485,53 +1481,51 @@ public void testEverythingEncounterType() { ourLog.info(ids.toString()); } - @Test - public void testEverythingInstanceWithContentFilter() { - Patient pt1 = new Patient(); - pt1.addName().setFamily("Everything").addGiven("Arthur"); - IIdType ptId1 = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless(); - Patient pt2 = new Patient(); - pt2.addName().setFamily("Everything").addGiven("Arthur"); - IIdType ptId2 = myPatientDao.create(pt2, mySrd).getId().toUnqualifiedVersionless(); + @Test + public void testEverythingInstanceWithContentFilter() { + Patient pt1 = new Patient(); + pt1.addName().setFamily("Everything").addGiven("Arthur"); + IIdType ptId1 = myPatientDao.create(pt1, mySrd).getId().toUnqualifiedVersionless(); - Device dev1 = new Device(); - dev1.setManufacturer("Some Manufacturer"); - IIdType devId1 = myDeviceDao.create(dev1, mySrd).getId().toUnqualifiedVersionless(); + Patient pt2 = new Patient(); + pt2.addName().setFamily("Everything").addGiven("Arthur"); + IIdType ptId2 = myPatientDao.create(pt2, mySrd).getId().toUnqualifiedVersionless(); - Device dev2 = new Device(); - dev2.setManufacturer("Some Manufacturer 2"); - myDeviceDao.create(dev2, mySrd).getId().toUnqualifiedVersionless(); + Device dev1 = new Device(); + dev1.setManufacturer("Some Manufacturer"); + IIdType devId1 = myDeviceDao.create(dev1, mySrd).getId().toUnqualifiedVersionless(); - Observation obs1 = new Observation(); - obs1.getText().setDivAsString("
OBSTEXT1
"); - obs1.getSubject().setReferenceElement(ptId1); - obs1.getCode().addCoding().setCode("CODE1"); - obs1.setValue(new StringType("obsvalue1")); - obs1.getDevice().setReferenceElement(devId1); - IIdType obsId1 = myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless(); + Device dev2 = new Device(); + dev2.setManufacturer("Some Manufacturer 2"); + myDeviceDao.create(dev2, mySrd).getId().toUnqualifiedVersionless(); - Observation obs2 = new Observation(); - obs2.getSubject().setReferenceElement(ptId1); - obs2.getCode().addCoding().setCode("CODE2"); - obs2.setValue(new StringType("obsvalue2")); - IIdType obsId2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless(); + // create an observation that links to Dev1 and Patient1 + Observation obs1 = new Observation(); + obs1.getText().setDivAsString("
OBSTEXT1
"); + obs1.getSubject().setReferenceElement(ptId1); + obs1.getCode().addCoding().setCode("CODE1"); + obs1.setValue(new StringType("obsvalue1")); + obs1.getDevice().setReferenceElement(devId1); + IIdType obsId1 = myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless(); - Observation obs3 = new Observation(); - obs3.getSubject().setReferenceElement(ptId2); - obs3.getCode().addCoding().setCode("CODE3"); - obs3.setValue(new StringType("obsvalue3")); - IIdType obsId3 = myObservationDao.create(obs3, mySrd).getId().toUnqualifiedVersionless(); + Observation obs2 = new Observation(); + obs2.getSubject().setReferenceElement(ptId1); + obs2.getCode().addCoding().setCode("CODE2"); + obs2.setValue(new StringType("obsvalue2")); + IIdType obsId2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless(); - List actual; - StringAndListParam param; + Observation obs3 = new Observation(); + obs3.getSubject().setReferenceElement(ptId2); + obs3.getCode().addCoding().setCode("CODE3"); + obs3.setValue(new StringType("obsvalue3")); + IIdType obsId3 = myObservationDao.create(obs3, mySrd).getId().toUnqualifiedVersionless(); - ourLog.info("Pt1:{} Pt2:{} Obs1:{} Obs2:{} Obs3:{}", ptId1.getIdPart(), ptId2.getIdPart(), obsId1.getIdPart(), obsId2.getIdPart(), obsId3.getIdPart()); + List actual; - param = new StringAndListParam(); - param.addAnd(new StringOrListParam().addOr(new StringParam("obsvalue1"))); + ourLog.info("Pt1:{} Pt2:{} Obs1:{} Obs2:{} Obs3:{}", ptId1.getIdPart(), ptId2.getIdPart(), obsId1.getIdPart(), obsId2.getIdPart(), obsId3.getIdPart()); - //@formatter:off + //@formatter:off Parameters response = myClient .operation() .onInstance(ptId1) @@ -1540,10 +1534,9 @@ public void testEverythingInstanceWithContentFilter() { .execute(); //@formatter:on - actual = toUnqualifiedVersionlessIds((Bundle) response.getParameter().get(0).getResource()); - assertThat(actual).containsExactlyInAnyOrder(ptId1, obsId1, devId1); - - } + actual = toUnqualifiedVersionlessIds((Bundle) response.getParameter().get(0).getResource()); + assertThat(actual).containsExactlyInAnyOrder(ptId1, obsId1, devId1); + } /** * See #147"Patient" @@ -2497,6 +2490,7 @@ public void testMetadataSuperParamsAreIncluded() { assertEquals(1, resp.getTotal()); } +// @Disabled @Test public void testMetaOperations() { String methodName = "testMetaOperations"; @@ -2627,35 +2621,40 @@ public void testPagingOverEverythingSet() throws InterruptedException { @Test public void testEverythingWithNoPagingProvider() { - myRestServer.setPagingProvider(null); - - Patient p = new Patient(); - p.setActive(true); - String pid = myPatientDao.create(p).getId().toUnqualifiedVersionless().getValue(); - - for (int i = 0; i < 20; i++) { - Observation o = new Observation(); - o.getSubject().setReference(pid); - o.addIdentifier().setSystem("foo").setValue(Integer.toString(i)); - myObservationDao.create(o); - } + IPagingProvider pagingProvider = myRestServer.getPagingProvider(); + try { + myRestServer.setPagingProvider(null); - mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(50); - mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(10); - mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(true); + Patient p = new Patient(); + p.setActive(true); + String pid = myPatientDao.create(p).getId().toUnqualifiedVersionless().getValue(); + + for (int i = 0; i < 20; i++) { + Observation o = new Observation(); + o.getSubject().setReference(pid); + o.addIdentifier().setSystem("foo").setValue(Integer.toString(i)); + myObservationDao.create(o); + } - Bundle response = myClient - .operation() - .onInstance(new IdType(pid)) - .named("everything") - .withSearchParameter(Parameters.class, "_count", new NumberParam(10)) - .returnResourceType(Bundle.class) - .useHttpGet() - .execute(); + mySearchCoordinatorSvcRaw.setLoadingThrottleForUnitTests(50); + mySearchCoordinatorSvcRaw.setSyncSizeForUnitTests(10); + mySearchCoordinatorSvcRaw.setNeverUseLocalSearchForUnitTests(true); + + Bundle response = myClient + .operation() + .onInstance(new IdType(pid)) + .named("everything") + .withSearchParameter(Parameters.class, "_count", new NumberParam(10)) + .returnResourceType(Bundle.class) + .useHttpGet() + .execute(); - assertThat(response.getEntry()).hasSize(10); - assertNull(response.getTotalElement().getValue()); - assertNull(response.getLink("next")); + assertThat(response.getEntry()).hasSize(10); + assertNull(response.getTotalElement().getValue()); + assertNull(response.getLink("next")); + } finally { + myRestServer.setPagingProvider(pagingProvider); + } } @Test From 99868e152873ca228c83397fc772552e0f3d6dba Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Wed, 27 Nov 2024 09:11:00 -0500 Subject: [PATCH 11/22] refactor --- .../jpa/search/SearchCoordinatorSvcImpl.java | 9 +- .../jpa/search/builder/SearchBuilder.java | 79 ++++++++-------- .../builder/models/SearchQueryProperties.java | 91 +++++++++++++++++++ .../r4/FhirResourceDaoR4SearchSqlTest.java | 48 +++++----- 4 files changed, 160 insertions(+), 67 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 477999b3d3eb..108bf89cbcf1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -66,6 +66,7 @@ import com.google.common.annotations.VisibleForTesting; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.time.DateUtils; import org.hl7.fhir.instance.model.api.IBaseResource; import org.springframework.beans.factory.BeanFactory; @@ -394,7 +395,6 @@ public IBundleProvider registerSearch( try { return direct.get(); - } catch (ResourceNotFoundInIndexException theE) { // some resources were not found in index, so we will inform this and resort to JPA search ourLog.warn( @@ -402,6 +402,13 @@ public IBundleProvider registerSearch( } } + // we set a max to fetch from the db for synchronous searches; + // otherwise, we would have to load everything into memory (or force the db to do so); + // So let's set a max value here +// Integer maxToLoad = ObjectUtils.defaultIfNull(loadSynchronousUpTo, myStorageSettings.getInternalSynchronousSearchSize()); +// ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); +// sb.setMaxResultsToFetch(maxToLoad); + ourLog.debug("Search {} is loading in synchronous mode", searchUuid); return mySynchronousSearchSvc.executeQuery( theParams, theRequestDetails, searchUuid, sb, loadSynchronousUpTo, theRequestPartitionId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 8d9dcf409d65..8afda1b9bdfb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -60,6 +60,7 @@ import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; import ca.uhn.fhir.jpa.search.SearchConstants; import ca.uhn.fhir.jpa.search.builder.models.ResolvedSearchQueryExecutor; +import ca.uhn.fhir.jpa.search.builder.models.SearchQueryProperties; import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor; @@ -360,7 +361,9 @@ public Long createCountQuery( return myFulltextSearchSvc.count(myResourceName, theParams.clone()); } - List queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null); + SearchQueryProperties properties = new SearchQueryProperties(); + + List queries = createQuery(theParams.clone(), properties, theRequest, null); if (queries.isEmpty()) { return 0L; } else { @@ -405,10 +408,7 @@ private void init(SearchParameterMap theParams, String theSearchUuid, RequestPar private List createQuery( SearchParameterMap theParams, - SortSpec sort, - Integer theOffset, - Integer theMaximumResults, - boolean theCountOnlyFlag, + SearchQueryProperties theSearchProperties, RequestDetails theRequest, SearchRuntimeDetails theSearchRuntimeDetails) { @@ -422,7 +422,7 @@ private List createQuery( List fulltextMatchIds = null; int resultCount = 0; if (myParams.isLastN()) { - fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults); + fulltextMatchIds = executeLastNAgainstIndex(theSearchProperties.getMaxResultsRequested()); resultCount = fulltextMatchIds.size(); } else if (myParams.getEverythingMode() != null) { fulltextMatchIds = queryHibernateSearchForEverythingPids(theRequest); @@ -479,8 +479,8 @@ private List createQuery( if (canSkipDatabase) { ourLog.trace("Query finished after HSearch. Skip db query phase"); - if (theMaximumResults != null) { - fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theMaximumResults); + if (theSearchProperties.getMaxResultsRequested() != null) { + fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theSearchProperties.getMaxResultsRequested()); } queries.add(fulltextExecutor); } else { @@ -494,12 +494,12 @@ private List createQuery( // for each list of (SearchBuilder.getMaximumPageSize()) // we create a chunked query and add it to 'queries' t -> doCreateChunkedQueries( - theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries)); + theParams, t, theSearchProperties, theRequest, queries)); } } else { // do everything in the database. createChunkedQuery( - theParams, sort, theOffset, theMaximumResults, theCountOnlyFlag, theRequest, null, queries); + theParams, theSearchProperties, theRequest, null, queries); } return queries; @@ -594,16 +594,16 @@ private List queryHibernateSearchForEverythingPids(RequestDetails theReq private void doCreateChunkedQueries( SearchParameterMap theParams, List thePids, - Integer theOffset, - SortSpec sort, - boolean theCount, + SearchQueryProperties theSearchQueryProperties, RequestDetails theRequest, ArrayList theQueries) { if (thePids.size() < getMaximumPageSize()) { thePids = normalizeIdListForInClause(thePids); } - createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids, theQueries); + // TODO - thesize was the 4th parameter... what is it supposed to be in createchunkedquery? + theSearchQueryProperties.setMaxResultsRequested(theParams.size()); + createChunkedQuery(theParams, theSearchQueryProperties, theRequest, thePids, theQueries); } /** @@ -653,27 +653,22 @@ private void extractTargetPidsFromIdParams(Set theTargetPids) { private void createChunkedQuery( SearchParameterMap theParams, - SortSpec sort, - Integer theOffset, - Integer theMaximumResults, - boolean theCountOnlyFlag, + SearchQueryProperties theSearchProperties, RequestDetails theRequest, List thePidList, List theSearchQueryExecutors) { if (myParams.getEverythingMode() != null) { createChunkedQueryForEverythingSearch( - theParams, theOffset, theMaximumResults, theCountOnlyFlag, thePidList, theSearchQueryExecutors); + theParams, theSearchProperties, thePidList, theSearchQueryExecutors); } else { createChunkedQueryNormalSearch( - theParams, sort, theOffset, theCountOnlyFlag, theRequest, thePidList, theSearchQueryExecutors); + theParams, theSearchProperties, theRequest, thePidList, theSearchQueryExecutors); } } private void createChunkedQueryNormalSearch( SearchParameterMap theParams, - SortSpec sort, - Integer theOffset, - boolean theCountOnlyFlag, + SearchQueryProperties theSearchProperites, RequestDetails theRequest, List thePidList, List theSearchQueryExecutors) { @@ -685,7 +680,8 @@ private void createChunkedQueryNormalSearch( myResourceName, mySqlBuilderFactory, myDialectProvider, - theCountOnlyFlag); + theSearchProperites.isDoCountOnlyFlag() + ); QueryStack queryStack3 = new QueryStack( theParams, myStorageSettings, myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings); @@ -762,7 +758,7 @@ private void createChunkedQueryNormalSearch( * if the MaxResultsToFetch is null, we are requesting "everything", * so we'll let the db do the deduplication (instead of in-memory) */ - if (theOffset != null || (myMaxResultsToFetch == null && !theCountOnlyFlag)) { + if (theSearchProperites.isDeduplicateInDBFlag()) { queryStack3.addGrouping(); queryStack3.setUseAggregate(true); } @@ -773,16 +769,16 @@ private void createChunkedQueryNormalSearch( * If we have a sort, we wrap the criteria search (the search that actually * finds the appropriate resources) in an outer search which is then sorted */ - if (sort != null) { - assert !theCountOnlyFlag; + if (theSearchProperites.hasSort()) { + assert !theSearchProperites.isDoCountOnlyFlag(); - createSort(queryStack3, sort, theParams); + createSort(queryStack3, theSearchProperites.getSortSpec(), theParams); } /* * Now perform the search */ - executeSearch(theOffset, theSearchQueryExecutors, sqlBuilder); + executeSearch(theSearchProperites.getOffset(), theSearchQueryExecutors, sqlBuilder); } private void executeSearch( @@ -797,9 +793,7 @@ private void executeSearch( private void createChunkedQueryForEverythingSearch( SearchParameterMap theParams, - Integer theOffset, - Integer theMaximumResults, - boolean theCountOnlyFlag, + SearchQueryProperties theSearchQueryProperties, List thePidList, List theSearchQueryExecutors) { @@ -811,12 +805,12 @@ private void createChunkedQueryForEverythingSearch( null, mySqlBuilderFactory, myDialectProvider, - theCountOnlyFlag); + theSearchQueryProperties.isDoCountOnlyFlag()); QueryStack queryStack3 = new QueryStack( theParams, myStorageSettings, myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings); - JdbcTemplate jdbcTemplate = initializeJdbcTemplate(theMaximumResults); + JdbcTemplate jdbcTemplate = initializeJdbcTemplate(theSearchQueryProperties.getMaxResultsRequested()); Set targetPids = new HashSet<>(); if (myParams.get(IAnyResource.SP_RES_ID) != null) { @@ -839,8 +833,8 @@ private void createChunkedQueryForEverythingSearch( myResourceName, mySqlBuilderFactory, myDialectProvider, - theCountOnlyFlag); - GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch); + theSearchQueryProperties.isDoCountOnlyFlag()); + GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theSearchQueryProperties.getOffset(), myMaxResultsToFetch); String sql = allTargetsSql.getSql(); Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]); @@ -874,7 +868,7 @@ public Long mapRow(ResultSet rs, int rowNum) throws SQLException { * If offset is present, we want deduplicate the results by using GROUP BY * ORDER BY is required to make sure we return unique results for each page */ - if (theOffset != null) { + if (theSearchQueryProperties.hasOffset()) { queryStack3.addGrouping(); queryStack3.addOrdering(); queryStack3.setUseAggregate(true); @@ -883,7 +877,7 @@ public Long mapRow(ResultSet rs, int rowNum) throws SQLException { /* * Now perform the search */ - executeSearch(theOffset, theSearchQueryExecutors, sqlBuilder); + executeSearch(theSearchQueryProperties.getOffset(), theSearchQueryExecutors, sqlBuilder); } private void addPidListPredicate(List thePidList, SearchQueryBuilder theSqlBuilder) { @@ -2586,8 +2580,13 @@ private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToF if (myParams.getEverythingMode() != null) { offset = 0; } - myQueryList = createQuery( - myParams, mySort, offset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails); + + SearchQueryProperties properties = new SearchQueryProperties(); + properties.setOffset(offset) + .setMaxResultsRequested(theMaxResultsToFetch) + .setDoCountOnlyFlag(false) + .setSortSpec(mySort); + myQueryList = createQuery(myParams, properties, myRequest, mySearchRuntimeDetails); } mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java new file mode 100644 index 000000000000..0c78f6814711 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java @@ -0,0 +1,91 @@ +package ca.uhn.fhir.jpa.search.builder.models; + +import ca.uhn.fhir.rest.api.SortSpec; + +public class SearchQueryProperties { + + /** + * True if this query is only to fetch the count (and not any results). + * + * True means this is a count only query + */ + private boolean myDoCountOnlyFlag; + /** + * Whether or not we do deduplication of results in memory + * (using a hashset, etc), or push this to the database + * (using GROUP BY, etc). + * + * True means use the database + */ + private boolean myDeduplicateInDBFlag; + + /** + * The maximum number of results to fetch (when we want it limited). + * Can be null if we are fetching everything or paging. + */ + private Integer myMaxResultsRequested; + /** + * The offset for the results to fetch. + * + * null if the first page, some number if it's a later page + */ + private Integer myOffset; + + /** + * The sort spec for this search + */ + private SortSpec mySortSpec; + + public boolean isDoCountOnlyFlag() { + return myDoCountOnlyFlag; + } + + public SearchQueryProperties setDoCountOnlyFlag(boolean theDoCountOnlyFlag) { + myDoCountOnlyFlag = theDoCountOnlyFlag; + return this; + } + + public boolean isDeduplicateInDBFlag() { + return myDeduplicateInDBFlag; + } + + public SearchQueryProperties setDeduplicateInDBFlag(boolean theDeduplicateInDBFlag) { + myDeduplicateInDBFlag = theDeduplicateInDBFlag; + return this; + } + + public Integer getMaxResultsRequested() { + return myMaxResultsRequested; + } + + public SearchQueryProperties setMaxResultsRequested(Integer theMaxResultsRequested) { + myMaxResultsRequested = theMaxResultsRequested; + return this; + } + + public Integer getOffset() { + return myOffset; + } + + public boolean hasOffset() { + return myOffset != null; + } + + public SearchQueryProperties setOffset(Integer theOffset) { + myOffset = theOffset; + return this; + } + + public SortSpec getSortSpec() { + return mySortSpec; + } + + public boolean hasSort() { + return mySortSpec != null; + } + + public SearchQueryProperties setSortSpec(SortSpec theSortSpec) { + mySortSpec = theSortSpec; + return this; + } +} diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java index cd9b1955cf82..3bed2cff933d 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchSqlTest.java @@ -64,44 +64,44 @@ static List sqlGenerationTestCases() { new SqlGenerationTestCase( "single string - no hfj_resource root", "Patient?name=FOO", - "SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.PARTITION_ID = ?) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)))" + "SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.PARTITION_ID = ?) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) fetch first ? rows only" ) , new SqlGenerationTestCase( "two regular params - should use hfj_resource as root", "Patient?name=smith&active=true", - "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?))", - "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) INNER JOIN HFJ_SPIDX_TOKEN t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID)) WHERE (((t0.PARTITION_ID = ?) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) AND ((t2.PARTITION_ID = ?) AND (t2.HASH_VALUE = ?)))" + "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?)) fetch first ? rows only", + "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) INNER JOIN HFJ_SPIDX_TOKEN t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID)) WHERE (((t0.PARTITION_ID = ?) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) AND ((t2.PARTITION_ID = ?) AND (t2.HASH_VALUE = ?))) fetch first ? rows only" ) , new SqlGenerationTestCase( "token not as a NOT IN subselect", "Encounter?class:not=not-there", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID = ?) AND ((t0.PARTITION_ID,t0.RES_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )))" + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID = ?) AND ((t0.PARTITION_ID,t0.RES_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))) fetch first ? rows only" ) , new SqlGenerationTestCase( "token not on chain join - NOT IN from hfj_res_link target columns", "Observation?encounter.class:not=not-there", - "SELECT t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", - "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.PARTITION_ID = ?) AND ((t0.TARGET_RES_PARTITION_ID,t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )))" + "SELECT t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.PARTITION_ID = ?) AND ((t0.TARGET_RES_PARTITION_ID,t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))) fetch first ? rows only" ) , new SqlGenerationTestCase( "bare sort", "Patient?_sort=name", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST" + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only" ) , new SqlGenerationTestCase( "sort with predicate", "Patient?active=true&_sort=name", - "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (t0.HASH_VALUE = ?) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST", - "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.PARTITION_ID = ?) AND (t0.HASH_VALUE = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST" + "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (t0.HASH_VALUE = ?) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", + "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.PARTITION_ID = ?) AND (t0.HASH_VALUE = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only" ) , new SqlGenerationTestCase( "chained sort", "Patient?_sort=Practitioner:general-practitioner.name", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RES_PARTITION_ID = t2.PARTITION_ID) AND (t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST" + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RES_PARTITION_ID = t2.PARTITION_ID) AND (t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only" ) ); } @@ -153,7 +153,7 @@ public void testSingleRegularSearchParam() { myPatientDao.search(map); assertEquals(1, myCaptureQueriesListener.countSelectQueries()); String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); - assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))", sql); + assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) fetch first ? rows only", sql); } @@ -162,7 +162,6 @@ public void testSingleRegularSearchParam() { */ @Test public void testTwoRegularSearchParams() { - myCaptureQueriesListener.clear(); SearchParameterMap map = SearchParameterMap.newSynchronous() .add(Patient.SP_NAME, new StringParam("FOO")) @@ -170,14 +169,11 @@ public void testTwoRegularSearchParams() { myPatientDao.search(map); assertEquals(1, myCaptureQueriesListener.countSelectQueries()); String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); - assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_SYS_AND_VALUE = ?))", sql); - - + assertEquals("SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_SYS_AND_VALUE = ?)) fetch first ? rows only", sql); } @Test public void testSearchByProfile_VersionedMode() { - // Put a tag in so we can search for it String code = "http://" + UUID.randomUUID(); Patient p = new Patient(); @@ -193,7 +189,7 @@ public void testSearchByProfile_VersionedMode() { assertEquals(3, myCaptureQueriesListener.countSelectQueries()); // Query 1 - Find resources: Make sure we search for tag type+system+code always String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); - assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_TAG t1 ON (t0.RES_ID = t1.RES_ID) INNER JOIN HFJ_TAG_DEF t2 ON (t1.TAG_ID = t2.TAG_ID) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t2.TAG_TYPE = ?) AND (t2.TAG_SYSTEM = ?) AND (t2.TAG_CODE = ?)))", sql); + assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_TAG t1 ON (t0.RES_ID = t1.RES_ID) INNER JOIN HFJ_TAG_DEF t2 ON (t1.TAG_ID = t2.TAG_ID) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t2.TAG_TYPE = ?) AND (t2.TAG_SYSTEM = ?) AND (t2.TAG_CODE = ?))) fetch first ? rows only", sql); // Query 2 - Load resourece contents sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(1).getSql(false, false); assertThat(sql).contains("where rsv1_0.RES_ID in (?)"); @@ -202,7 +198,6 @@ public void testSearchByProfile_VersionedMode() { assertThat(sql).contains("from HFJ_RES_TAG rt1_0 join HFJ_TAG_DEF"); assertThat(toUnqualifiedVersionlessIds(outcome)).containsExactly(id); - } @Test @@ -229,9 +224,11 @@ public void testSearchByProfile_InlineMode() { .add(Constants.PARAM_PROFILE, new UriParam(code)); IBundleProvider outcome = myPatientDao.search(map, mySrd); assertEquals(2, myCaptureQueriesListener.countSelectQueries()); + // Query 1 - Find resources: Just a standard token search in this mode String sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(false, false); - assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_URI t0 WHERE (t0.HASH_URI = ?)", sql); + assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_URI t0 WHERE (t0.HASH_URI = ?) fetch first ? rows only", sql); + // Query 2 - Load resourece contents sql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(1).getSql(false, false); assertThat(sql).contains("where rsv1_0.RES_ID in (?)"); @@ -255,11 +252,10 @@ public void testSearchByToken_IncludeHashIdentity(boolean theIncludeHashIdentity // Verify if (theIncludeHashIdentity) { - assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE ((t0.HASH_IDENTITY = '7001889285610424179') AND (t0.HASH_SYS_AND_VALUE = '-2780914544385068076'))", myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false)); + assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE ((t0.HASH_IDENTITY = '7001889285610424179') AND (t0.HASH_SYS_AND_VALUE = '-2780914544385068076')) fetch first '10000' rows only", myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false)); } else { - assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_SYS_AND_VALUE = '-2780914544385068076')", myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false)); + assertEquals("SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_SYS_AND_VALUE = '-2780914544385068076') fetch first '10000' rows only", myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false)); } - } public static class MyPartitionInterceptor { From f7ffe9d62be1615a3850d2f804b1399529f53e43 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Wed, 27 Nov 2024 14:40:33 -0500 Subject: [PATCH 12/22] limiting synchronous queries --- .../jpa/search/SearchCoordinatorSvcImpl.java | 7 +- .../jpa/search/SynchronousSearchSvcImpl.java | 2 +- .../jpa/search/builder/SearchBuilder.java | 93 ++++++++++++------- .../builder/models/SearchQueryProperties.java | 13 +++ .../builder/sql/SearchQueryBuilder.java | 1 - .../jpa/search/builder/tasks/SearchTask.java | 1 + .../dstu3/ResourceProviderDstu3Test.java | 1 - .../ca/uhn/fhir/jpa/dao/ISearchBuilder.java | 9 ++ 8 files changed, 89 insertions(+), 38 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 108bf89cbcf1..824ac6690522 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -378,6 +378,7 @@ public IBundleProvider registerSearch( final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(theCallingDao, theResourceType, resourceTypeClass); sb.setFetchSize(mySyncSize); + sb.setRequireTotal(theParams.getCount() != null); final Integer loadSynchronousUpTo = getLoadSynchronousUpToOrNull(theCacheControlDirective); boolean isOffsetQuery = theParams.isOffsetQuery(); @@ -405,9 +406,9 @@ public IBundleProvider registerSearch( // we set a max to fetch from the db for synchronous searches; // otherwise, we would have to load everything into memory (or force the db to do so); // So let's set a max value here -// Integer maxToLoad = ObjectUtils.defaultIfNull(loadSynchronousUpTo, myStorageSettings.getInternalSynchronousSearchSize()); -// ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); -// sb.setMaxResultsToFetch(maxToLoad); + Integer maxToLoad = ObjectUtils.defaultIfNull(loadSynchronousUpTo, myStorageSettings.getInternalSynchronousSearchSize()); + ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); + sb.setMaxResultsToFetch(maxToLoad); ourLog.debug("Search {} is loading in synchronous mode", searchUuid); return mySynchronousSearchSvc.executeQuery( diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java index 317b8fa21052..21c212f03000 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SynchronousSearchSvcImpl.java @@ -246,7 +246,7 @@ public IBundleProvider executeQuery( resources, theRequestDetails, myInterceptorBroadcaster); SimpleBundleProvider bundleProvider = new SimpleBundleProvider(resources); - if (hasACount) { + if (hasACount && theSb.requiresTotal()) { bundleProvider.setTotalResourcesRequestedReturned(receivedResourceCount); } if (theParams.isOffsetQuery()) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 8afda1b9bdfb..0fa1843cd50f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -202,7 +202,8 @@ public class SearchBuilder implements ISearchBuilder { private SearchParameterMap myParams; private String mySearchUuid; private int myFetchSize; - private Integer myMaxResultsToFetch; + + private boolean myRequiresTotal; /** * Set of PIDs of results that have already been returned in a search. @@ -228,6 +229,8 @@ public class SearchBuilder implements ISearchBuilder { private boolean myHasNextIteratorQuery = false; private RequestPartitionId myRequestPartitionId; + private SearchQueryProperties mySearchProperties; + @Autowired(required = false) private IFulltextSearchSvc myFulltextSearchSvc; @@ -273,6 +276,8 @@ public SearchBuilder( myResourceSearchViewDao = theResourceSearchViewDao; myContext = theContext; myIdHelperService = theIdHelperService; + + mySearchProperties = new SearchQueryProperties(); } @VisibleForTesting @@ -282,7 +287,21 @@ void setResourceName(String theName) { @Override public void setMaxResultsToFetch(Integer theMaxResultsToFetch) { - myMaxResultsToFetch = theMaxResultsToFetch; + mySearchProperties.setMaxResultsRequested(theMaxResultsToFetch); + } + + public void setShouldDeduplicateInDB(boolean theShouldDeduplicateInDB) { + mySearchProperties.setDeduplicateInDBFlag(theShouldDeduplicateInDB); + } + + @Override + public void setRequireTotal(boolean theRequireTotal) { + myRequiresTotal = theRequireTotal; + } + + @Override + public boolean requiresTotal() { + return myRequiresTotal; } private void searchForIdsWithAndOr( @@ -291,6 +310,7 @@ private void searchForIdsWithAndOr( @Nonnull SearchParameterMap theParams, RequestDetails theRequest) { myParams = theParams; + mySearchProperties.setSortSpec(myParams.getSort()); // Remove any empty parameters theParams.clean(); @@ -361,8 +381,11 @@ public Long createCountQuery( return myFulltextSearchSvc.count(myResourceName, theParams.clone()); } - SearchQueryProperties properties = new SearchQueryProperties(); - + SearchQueryProperties properties = mySearchProperties.clone(); + properties.setDoCountOnlyFlag(true); + properties.setSortSpec(null); // counts don't require sorts + properties.setMaxResultsRequested(null); + properties.setOffset(null); List queries = createQuery(theParams.clone(), properties, theRequest, null); if (queries.isEmpty()) { return 0L; @@ -402,16 +425,24 @@ private void init(SearchParameterMap theParams, String theSearchUuid, RequestPar myCriteriaBuilder = myEntityManager.getCriteriaBuilder(); // we mutate the params. Make a private copy. myParams = theParams.clone(); + mySearchProperties.setSortSpec(myParams.getSort()); mySearchUuid = theSearchUuid; myRequestPartitionId = theRequestPartitionId; } + /** + * The query created can be either a count query or the + * actual query. + * This is why it takes a SearchQueryProperties object + * (and doesn't use the local version of it). + * The properties may differ slightly for whichever + * query this is. + */ private List createQuery( SearchParameterMap theParams, SearchQueryProperties theSearchProperties, RequestDetails theRequest, SearchRuntimeDetails theSearchRuntimeDetails) { - ArrayList queries = new ArrayList<>(); if (checkUseHibernateSearch()) { @@ -479,7 +510,7 @@ private List createQuery( if (canSkipDatabase) { ourLog.trace("Query finished after HSearch. Skip db query phase"); - if (theSearchProperties.getMaxResultsRequested() != null) { + if (theSearchProperties.hasMaxResultsRequested()) { fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theSearchProperties.getMaxResultsRequested()); } queries.add(fulltextExecutor); @@ -668,7 +699,7 @@ private void createChunkedQuery( private void createChunkedQueryNormalSearch( SearchParameterMap theParams, - SearchQueryProperties theSearchProperites, + SearchQueryProperties theSearchProperties, RequestDetails theRequest, List thePidList, List theSearchQueryExecutors) { @@ -680,7 +711,7 @@ private void createChunkedQueryNormalSearch( myResourceName, mySqlBuilderFactory, myDialectProvider, - theSearchProperites.isDoCountOnlyFlag() + theSearchProperties.isDoCountOnlyFlag() ); QueryStack queryStack3 = new QueryStack( theParams, myStorageSettings, myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings); @@ -758,7 +789,7 @@ private void createChunkedQueryNormalSearch( * if the MaxResultsToFetch is null, we are requesting "everything", * so we'll let the db do the deduplication (instead of in-memory) */ - if (theSearchProperites.isDeduplicateInDBFlag()) { + if (theSearchProperties.isDeduplicateInDBFlag()) { queryStack3.addGrouping(); queryStack3.setUseAggregate(true); } @@ -769,24 +800,24 @@ private void createChunkedQueryNormalSearch( * If we have a sort, we wrap the criteria search (the search that actually * finds the appropriate resources) in an outer search which is then sorted */ - if (theSearchProperites.hasSort()) { - assert !theSearchProperites.isDoCountOnlyFlag(); + if (theSearchProperties.hasSort()) { + assert !theSearchProperties.isDoCountOnlyFlag(); - createSort(queryStack3, theSearchProperites.getSortSpec(), theParams); + createSort(queryStack3, theSearchProperties.getSortSpec(), theParams); } /* * Now perform the search */ - executeSearch(theSearchProperites.getOffset(), theSearchQueryExecutors, sqlBuilder); + executeSearch(theSearchProperties, theSearchQueryExecutors, sqlBuilder); } private void executeSearch( - Integer theOffset, List theSearchQueryExecutors, SearchQueryBuilder sqlBuilder) { - GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch); + SearchQueryProperties theProperties, List theSearchQueryExecutors, SearchQueryBuilder sqlBuilder) { + GeneratedSql generatedSql = sqlBuilder.generate(theProperties.getOffset(), theProperties.getMaxResultsRequested()); if (!generatedSql.isMatchNothing()) { SearchQueryExecutor executor = - mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch); + mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, theProperties.getMaxResultsRequested()); theSearchQueryExecutors.add(executor); } } @@ -834,7 +865,7 @@ private void createChunkedQueryForEverythingSearch( mySqlBuilderFactory, myDialectProvider, theSearchQueryProperties.isDoCountOnlyFlag()); - GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theSearchQueryProperties.getOffset(), myMaxResultsToFetch); + GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theSearchQueryProperties.getOffset(), mySearchProperties.getMaxResultsRequested()); String sql = allTargetsSql.getSql(); Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]); @@ -877,7 +908,7 @@ public Long mapRow(ResultSet rs, int rowNum) throws SQLException { /* * Now perform the search */ - executeSearch(theSearchQueryProperties.getOffset(), theSearchQueryExecutors, sqlBuilder); + executeSearch(theSearchQueryProperties, theSearchQueryExecutors, sqlBuilder); } private void addPidListPredicate(List thePidList, SearchQueryBuilder theSqlBuilder) { @@ -2406,15 +2437,15 @@ private void fetchNext() { // If we don't have a query yet, create one if (myResultsIterator == null) { - if (myMaxResultsToFetch == null) { - myMaxResultsToFetch = calculateMaxResultsToFetch(); + if (!mySearchProperties.hasMaxResultsRequested()) { + mySearchProperties.setMaxResultsRequested(calculateMaxResultsToFetch()); } /* * assigns the results iterator * and populates the myQueryList. */ - initializeIteratorQuery(myOffset, myMaxResultsToFetch); + initializeIteratorQuery(myOffset, mySearchProperties.getMaxResultsRequested()); } if (myNext == null) { @@ -2448,7 +2479,7 @@ private void fetchNext() { JpaPid next = JpaPid.fromId(nextLong); if (!myPidSet.contains(next)) { - if (myMaxResultsToFetch != null) { + if (mySearchProperties.hasMaxResultsRequested()) { /* * We only add to the map if we aren't fetching "everything"; * otherwise, we let the de-duplication happen in the database @@ -2468,13 +2499,12 @@ private void fetchNext() { } if (!myResultsIterator.hasNext()) { - if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) { + if (mySearchProperties.hasMaxResultsRequested() && (mySkipCount + myNonSkipCount == mySearchProperties.getMaxResultsRequested())) { if (mySkipCount > 0 && myNonSkipCount == 0) { - sendProcessingMsgAndFirePerformanceHook(); - - myMaxResultsToFetch += 1000; - initializeIteratorQuery(myOffset, myMaxResultsToFetch); + int maxResults = mySearchProperties.getMaxResultsRequested() + 1000; + mySearchProperties.setMaxResultsRequested(maxResults); + initializeIteratorQuery(myOffset, mySearchProperties.getMaxResultsRequested()); } } } @@ -2504,7 +2534,7 @@ private void fetchNext() { } } - if (myMaxResultsToFetch == null) { + if (!mySearchProperties.hasMaxResultsRequested()) { mySearchRuntimeDetails.setFoundIndexMatchesCount(myNonSkipCount); } else { mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size()); @@ -2565,7 +2595,7 @@ private void sendProcessingMsgAndFirePerformanceHook() { String msg = "Pass completed with no matching results seeking rows " + myPidSet.size() + "-" + mySkipCount + ". This indicates an inefficient query! Retrying with new max count of " - + myMaxResultsToFetch; + + mySearchProperties.getMaxResultsRequested(); firePerformanceWarning(myRequest, msg); } @@ -2581,11 +2611,10 @@ private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToF offset = 0; } - SearchQueryProperties properties = new SearchQueryProperties(); + SearchQueryProperties properties = mySearchProperties.clone(); properties.setOffset(offset) .setMaxResultsRequested(theMaxResultsToFetch) - .setDoCountOnlyFlag(false) - .setSortSpec(mySort); + .setDoCountOnlyFlag(false); myQueryList = createQuery(myParams, properties, myRequest, mySearchRuntimeDetails); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java index 0c78f6814711..0238bfcbb6d0 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java @@ -63,6 +63,10 @@ public SearchQueryProperties setMaxResultsRequested(Integer theMaxResultsRequest return this; } + public boolean hasMaxResultsRequested() { + return myMaxResultsRequested != null; + } + public Integer getOffset() { return myOffset; } @@ -88,4 +92,13 @@ public SearchQueryProperties setSortSpec(SortSpec theSortSpec) { mySortSpec = theSortSpec; return this; } + + public SearchQueryProperties clone() { + return new SearchQueryProperties() + .setMaxResultsRequested(myMaxResultsRequested) + .setSortSpec(mySortSpec) + .setOffset(myOffset) + .setDoCountOnlyFlag(myDoCountOnlyFlag) + .setDeduplicateInDBFlag(myDeduplicateInDBFlag); + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java index 188c6f970a7d..9c1cb0b0f802 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryBuilder.java @@ -510,7 +510,6 @@ public boolean isSelectPartitionId() { * Generate and return the SQL generated by this builder */ public GeneratedSql generate(@Nullable Integer theOffset, @Nullable Integer theMaxResultsToFetch) { - getOrCreateFirstPredicateBuilder(); mySelect.validate(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java index cbec057c33a6..fbb271086cc2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/tasks/SearchTask.java @@ -597,6 +597,7 @@ private void doSearch() { if (next == -1) { sb.setMaxResultsToFetch(null); + sb.setShouldDeduplicateInDB(true); } else { // we want at least 1 more than our requested amount // so we know that there are other results diff --git a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java index 47eb77ebaa8c..c75e78d07468 100644 --- a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java @@ -4072,7 +4072,6 @@ public void testSortFromResourceProvider() { ourLog.info(StringUtils.join(names, '\n')); assertThat(names).containsExactly("Daniel Adams", "Aaron Alexis", "Carol Allen", "Ruth Black", "Brian Brooks", "Amy Clark", "Susan Clark", "Anthony Coleman", "Lisa Coleman", "Steven Coleman", "Ruth Cook", "Betty Davis", "Joshua Diaz", "Brian Gracia", "Sarah Graham", "Stephan Graham"); - } /** diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/ISearchBuilder.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/ISearchBuilder.java index 7b79e60658df..7e9e66b68783 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/ISearchBuilder.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/ISearchBuilder.java @@ -77,6 +77,15 @@ Long createCountQuery( void setMaxResultsToFetch(Integer theMaxResultsToFetch); + void setShouldDeduplicateInDB(boolean theShouldDeduplicateInDB); + + void setRequireTotal(boolean theRequireTotal); + + /** + * True if the results should have a 'total' value + */ + boolean requiresTotal(); + void loadResourcesByPid( Collection thePids, Collection theIncludedPids, From d0419b731a7ac8711b66252565bcf017e75dec36 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Wed, 27 Nov 2024 14:41:34 -0500 Subject: [PATCH 13/22] spotless --- .../jpa/search/SearchCoordinatorSvcImpl.java | 3 +- .../jpa/search/builder/SearchBuilder.java | 35 ++++++++++--------- .../builder/models/SearchQueryProperties.java | 10 +++--- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 824ac6690522..bc6d3e93253c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -406,7 +406,8 @@ public IBundleProvider registerSearch( // we set a max to fetch from the db for synchronous searches; // otherwise, we would have to load everything into memory (or force the db to do so); // So let's set a max value here - Integer maxToLoad = ObjectUtils.defaultIfNull(loadSynchronousUpTo, myStorageSettings.getInternalSynchronousSearchSize()); + Integer maxToLoad = ObjectUtils.defaultIfNull( + loadSynchronousUpTo, myStorageSettings.getInternalSynchronousSearchSize()); ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); sb.setMaxResultsToFetch(maxToLoad); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 0fa1843cd50f..7150bdb81b18 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -511,7 +511,8 @@ private List createQuery( if (canSkipDatabase) { ourLog.trace("Query finished after HSearch. Skip db query phase"); if (theSearchProperties.hasMaxResultsRequested()) { - fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theSearchProperties.getMaxResultsRequested()); + fulltextExecutor = SearchQueryExecutors.limited( + fulltextExecutor, theSearchProperties.getMaxResultsRequested()); } queries.add(fulltextExecutor); } else { @@ -524,13 +525,11 @@ private List createQuery( SearchBuilder.getMaximumPageSize(), // for each list of (SearchBuilder.getMaximumPageSize()) // we create a chunked query and add it to 'queries' - t -> doCreateChunkedQueries( - theParams, t, theSearchProperties, theRequest, queries)); + t -> doCreateChunkedQueries(theParams, t, theSearchProperties, theRequest, queries)); } } else { // do everything in the database. - createChunkedQuery( - theParams, theSearchProperties, theRequest, null, queries); + createChunkedQuery(theParams, theSearchProperties, theRequest, null, queries); } return queries; @@ -689,8 +688,7 @@ private void createChunkedQuery( List thePidList, List theSearchQueryExecutors) { if (myParams.getEverythingMode() != null) { - createChunkedQueryForEverythingSearch( - theParams, theSearchProperties, thePidList, theSearchQueryExecutors); + createChunkedQueryForEverythingSearch(theParams, theSearchProperties, thePidList, theSearchQueryExecutors); } else { createChunkedQueryNormalSearch( theParams, theSearchProperties, theRequest, thePidList, theSearchQueryExecutors); @@ -711,8 +709,7 @@ private void createChunkedQueryNormalSearch( myResourceName, mySqlBuilderFactory, myDialectProvider, - theSearchProperties.isDoCountOnlyFlag() - ); + theSearchProperties.isDoCountOnlyFlag()); QueryStack queryStack3 = new QueryStack( theParams, myStorageSettings, myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings); @@ -813,8 +810,11 @@ private void createChunkedQueryNormalSearch( } private void executeSearch( - SearchQueryProperties theProperties, List theSearchQueryExecutors, SearchQueryBuilder sqlBuilder) { - GeneratedSql generatedSql = sqlBuilder.generate(theProperties.getOffset(), theProperties.getMaxResultsRequested()); + SearchQueryProperties theProperties, + List theSearchQueryExecutors, + SearchQueryBuilder sqlBuilder) { + GeneratedSql generatedSql = + sqlBuilder.generate(theProperties.getOffset(), theProperties.getMaxResultsRequested()); if (!generatedSql.isMatchNothing()) { SearchQueryExecutor executor = mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, theProperties.getMaxResultsRequested()); @@ -865,7 +865,8 @@ private void createChunkedQueryForEverythingSearch( mySqlBuilderFactory, myDialectProvider, theSearchQueryProperties.isDoCountOnlyFlag()); - GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theSearchQueryProperties.getOffset(), mySearchProperties.getMaxResultsRequested()); + GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate( + theSearchQueryProperties.getOffset(), mySearchProperties.getMaxResultsRequested()); String sql = allTargetsSql.getSql(); Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]); @@ -2499,7 +2500,8 @@ private void fetchNext() { } if (!myResultsIterator.hasNext()) { - if (mySearchProperties.hasMaxResultsRequested() && (mySkipCount + myNonSkipCount == mySearchProperties.getMaxResultsRequested())) { + if (mySearchProperties.hasMaxResultsRequested() + && (mySkipCount + myNonSkipCount == mySearchProperties.getMaxResultsRequested())) { if (mySkipCount > 0 && myNonSkipCount == 0) { sendProcessingMsgAndFirePerformanceHook(); int maxResults = mySearchProperties.getMaxResultsRequested() + 1000; @@ -2612,9 +2614,10 @@ private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToF } SearchQueryProperties properties = mySearchProperties.clone(); - properties.setOffset(offset) - .setMaxResultsRequested(theMaxResultsToFetch) - .setDoCountOnlyFlag(false); + properties + .setOffset(offset) + .setMaxResultsRequested(theMaxResultsToFetch) + .setDoCountOnlyFlag(false); myQueryList = createQuery(myParams, properties, myRequest, mySearchRuntimeDetails); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java index 0238bfcbb6d0..d2668624ca93 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/models/SearchQueryProperties.java @@ -95,10 +95,10 @@ public SearchQueryProperties setSortSpec(SortSpec theSortSpec) { public SearchQueryProperties clone() { return new SearchQueryProperties() - .setMaxResultsRequested(myMaxResultsRequested) - .setSortSpec(mySortSpec) - .setOffset(myOffset) - .setDoCountOnlyFlag(myDoCountOnlyFlag) - .setDeduplicateInDBFlag(myDeduplicateInDBFlag); + .setMaxResultsRequested(myMaxResultsRequested) + .setSortSpec(mySortSpec) + .setOffset(myOffset) + .setDoCountOnlyFlag(myDoCountOnlyFlag) + .setDeduplicateInDBFlag(myDeduplicateInDBFlag); } } From cdd5983d92680432ce24a24a06219371f0961bb5 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 28 Nov 2024 10:32:13 -0500 Subject: [PATCH 14/22] fixing intermittent --- .../fhir/jpa/mdm/svc/MdmResourceDaoSvcTest.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmResourceDaoSvcTest.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmResourceDaoSvcTest.java index 80f15e0089db..8ff23edb63cf 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmResourceDaoSvcTest.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmResourceDaoSvcTest.java @@ -9,6 +9,8 @@ import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; import ca.uhn.fhir.mdm.api.IMdmResourceDaoSvc; import ca.uhn.fhir.mdm.util.MdmResourceUtil; +import ca.uhn.fhir.rest.api.SortOrderEnum; +import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.param.StringOrListParam; @@ -23,8 +25,10 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -95,8 +99,9 @@ public void testSearchGoldenResourceOnSamePartition() { public void testSearchForMultiplePatientsByIdInPartitionedEnvironment() { // setup int resourceCount = 3; + // keep alphabetical String[] idPrefaces = new String[] { - "RED", "BLUE", "GREEN" + "BLUE", "GREEN", "RED" }; SearchParameterMap map; @@ -120,11 +125,17 @@ public void testSearchForMultiplePatientsByIdInPartitionedEnvironment() { patientIds.add(new StringParam("Patient/" + patientOnPartition.getIdElement().getIdPart() )); + await().atLeast(100, TimeUnit.MILLISECONDS); } // test map = SearchParameterMap.newSynchronous(); map.add("_id", patientIds); + // we'll use a sort to ensure consistent ordering of returned values + SortSpec sort = new SortSpec(); + sort.setOrder(SortOrderEnum.ASC); + sort.setParamName("_id"); + map.setSort(sort); result = myPatientDao.search(map, new SystemRequestDetails()); // verify @@ -132,6 +143,7 @@ public void testSearchForMultiplePatientsByIdInPartitionedEnvironment() { assertFalse(result.isEmpty()); List resources = result.getAllResources(); assertThat(resources).hasSize(resourceCount); + int count = 0; for (IBaseResource resource : resources) { String id = idPrefaces[count++]; From fcd93993d20958f9eb2abdaef0e029332c8d15a7 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 28 Nov 2024 15:34:02 -0500 Subject: [PATCH 15/22] fixing tests --- .../jpa/search/builder/SearchBuilder.java | 20 ++++++++++++++++--- .../FhirResourceDaoDstu2SearchFtTest.java | 1 - 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 4c30389464fb..319a2e3c5439 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -627,8 +627,7 @@ private void doCreateChunkedQueries( if (thePids.size() < getMaximumPageSize()) { thePids = normalizeIdListForInClause(thePids); } - // TODO - thesize was the 4th parameter... what is it supposed to be in createchunkedquery? - theSearchQueryProperties.setMaxResultsRequested(theParams.size()); + theSearchQueryProperties.setMaxResultsRequested(thePids.size()); createChunkedQuery(theParams, theSearchQueryProperties, theRequest, thePids, theQueries); } @@ -2481,7 +2480,7 @@ private void fetchNext() { JpaPid next = JpaPid.fromId(nextLong); if (!myPidSet.contains(next)) { - if (mySearchProperties.hasMaxResultsRequested()) { + if (!mySearchProperties.isDeduplicateInDBFlag()) { /* * We only add to the map if we aren't fetching "everything"; * otherwise, we let the de-duplication happen in the database @@ -2505,8 +2504,23 @@ private void fetchNext() { && (mySkipCount + myNonSkipCount == mySearchProperties.getMaxResultsRequested())) { if (mySkipCount > 0 && myNonSkipCount == 0) { sendProcessingMsgAndFirePerformanceHook(); + // need the next iterator; increase the maxsize + // (we should always do this) int maxResults = mySearchProperties.getMaxResultsRequested() + 1000; mySearchProperties.setMaxResultsRequested(maxResults); + + if (!mySearchProperties.isDeduplicateInDBFlag()) { + // if we're not using the database to deduplicate + // we should recheck our memory usage + // the prefetch size check is future proofing + int prefetchSize = myStorageSettings.getSearchPreFetchThresholds().size(); + if (prefetchSize > 0) { + if (myStorageSettings.getSearchPreFetchThresholds().get(prefetchSize - 1) < mySearchProperties.getMaxResultsRequested()) { + mySearchProperties.setDeduplicateInDBFlag(true); + } + } + } + initializeIteratorQuery(myOffset, mySearchProperties.getMaxResultsRequested()); } } diff --git a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchFtTest.java b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchFtTest.java index 5f1928d8ac73..573225d62cba 100644 --- a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchFtTest.java +++ b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/dao/dstu2/FhirResourceDaoDstu2SearchFtTest.java @@ -86,7 +86,6 @@ protected void doInTransactionWithoutResult(TransactionStatus theStatus) { map = new SearchParameterMap(); map.add(Constants.PARAM_TEXT, new StringParam("DIVBBB")); assertThat(toUnqualifiedVersionlessIds(myPatientDao.search(map))).containsExactly(pId1); - } @Test From 3f7050739d488fca242c97a16ba70e6ee504ca9d Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Thu, 28 Nov 2024 15:34:18 -0500 Subject: [PATCH 16/22] spotless --- .../ca/uhn/fhir/jpa/search/builder/SearchBuilder.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 319a2e3c5439..35b9db873561 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -2513,9 +2513,14 @@ private void fetchNext() { // if we're not using the database to deduplicate // we should recheck our memory usage // the prefetch size check is future proofing - int prefetchSize = myStorageSettings.getSearchPreFetchThresholds().size(); + int prefetchSize = myStorageSettings + .getSearchPreFetchThresholds() + .size(); if (prefetchSize > 0) { - if (myStorageSettings.getSearchPreFetchThresholds().get(prefetchSize - 1) < mySearchProperties.getMaxResultsRequested()) { + if (myStorageSettings + .getSearchPreFetchThresholds() + .get(prefetchSize - 1) + < mySearchProperties.getMaxResultsRequested()) { mySearchProperties.setDeduplicateInDBFlag(true); } } From 47d4dc6a84bf3d3838a23577e72288d3fe09c590 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Fri, 29 Nov 2024 16:58:32 -0500 Subject: [PATCH 17/22] fixing tests --- .../ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java | 7 +++++-- .../jpa/interceptor/PatientIdPartitionInterceptorTest.java | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index fbe580696fa1..426fe9628b8f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -405,8 +405,11 @@ public IBundleProvider registerSearch( // we set a max to fetch from the db for synchronous searches; // otherwise, we would have to load everything into memory (or force the db to do so); // So let's set a max value here - Integer maxToLoad = ObjectUtils.defaultIfNull( - loadSynchronousUpTo, myStorageSettings.getInternalSynchronousSearchSize()); + Integer maxToLoad = + ObjectUtils.firstNonNull( + loadSynchronousUpTo, + theParams.getCount() != null ? theParams.getCount() + 1 : null, + myStorageSettings.getInternalSynchronousSearchSize()); ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); sb.setMaxResultsToFetch(maxToLoad); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/PatientIdPartitionInterceptorTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/PatientIdPartitionInterceptorTest.java index aaaa937a3ddc..2fc45ddca16a 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/PatientIdPartitionInterceptorTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/interceptor/PatientIdPartitionInterceptorTest.java @@ -270,7 +270,7 @@ public void testSearchObservation_NoCompartmentMembership() { myCaptureQueriesListener.clear(); myObservationDao.search(SearchParameterMap.newSynchronous(), mySrd); myCaptureQueriesListener.logSelectQueries(); - assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE ((t0.RES_TYPE = 'Observation') AND (t0.RES_DELETED_AT IS NULL))", myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false)); + assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE ((t0.RES_TYPE = 'Observation') AND (t0.RES_DELETED_AT IS NULL)) fetch first '10000' rows only", myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false)); } @Test From 60c1a57cd56aa0c7f01ddb92c64503ca8c28b0b6 Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Fri, 29 Nov 2024 16:58:44 -0500 Subject: [PATCH 18/22] spotless --- .../java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 426fe9628b8f..b6e71b5452c4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -405,8 +405,7 @@ public IBundleProvider registerSearch( // we set a max to fetch from the db for synchronous searches; // otherwise, we would have to load everything into memory (or force the db to do so); // So let's set a max value here - Integer maxToLoad = - ObjectUtils.firstNonNull( + Integer maxToLoad = ObjectUtils.firstNonNull( loadSynchronousUpTo, theParams.getCount() != null ? theParams.getCount() + 1 : null, myStorageSettings.getInternalSynchronousSearchSize()); From abe951606e7f36a8d6aede105c85fc81c976c80e Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 2 Dec 2024 09:44:26 -0500 Subject: [PATCH 19/22] merging --- .../ca/uhn/fhir/jpa/search/builder/SearchBuilder.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index ae8e32b9fee9..4d7715f7da8a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -634,7 +634,7 @@ private List queryHibernateSearchForEverythingPids(RequestDetails theReq private void doCreateChunkedQueries( SearchParameterMap theParams, - List thePids, + List thePids, SearchQueryProperties theSearchQueryProperties, RequestDetails theRequest, ArrayList theQueries) { @@ -698,7 +698,8 @@ private void createChunkedQuery( List thePidList, List theSearchQueryExecutors) { if (myParams.getEverythingMode() != null) { - createChunkedQueryForEverythingSearch(theParams, theSearchProperties, thePidList, theSearchQueryExecutors); + createChunkedQueryForEverythingSearch( + theRequest, theParams, theSearchProperties, thePidList, theSearchQueryExecutors); } else { createChunkedQueryNormalSearch( theParams, theSearchProperties, theRequest, thePidList, theSearchQueryExecutors); @@ -842,7 +843,7 @@ private void createChunkedQueryForEverythingSearch( RequestDetails theRequest, SearchParameterMap theParams, SearchQueryProperties theSearchQueryProperties, - List thePidList, + List thePidList, List theSearchQueryExecutors) { SearchQueryBuilder sqlBuilder = new SearchQueryBuilder( @@ -2736,7 +2737,7 @@ private void fetchNext() { callPerformanceTracingHook(nextPid); } - if (nextLong != null) { + if (nextPid != null) { if (!myPidSet.contains(nextPid)) { if (!mySearchProperties.isDeduplicateInDBFlag()) { /* From f0867a0a2b19b8251cd53092a34894ee1b8da24e Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 2 Dec 2024 11:32:38 -0500 Subject: [PATCH 20/22] fixing tests --- .../test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java index 84a57c718c72..07be3d5e2d47 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java @@ -1919,7 +1919,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // DateRangeParam From 57f447937f4f49e77c4e672d0db6aa24a2cc8e3f Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 2 Dec 2024 11:44:05 -0500 Subject: [PATCH 21/22] Fixing tests --- .../jpa/dao/r4/PartitioningSqlR4Test.java | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java index 07be3d5e2d47..f2ede78fa6e1 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/PartitioningSqlR4Test.java @@ -1273,7 +1273,7 @@ public void testSearch_IdParamOnly_PidId_SpecificPartition() { // Only the read columns should be used, no criteria use partition assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); } // Read in null Partition @@ -1326,7 +1326,7 @@ public void testSearch_IdParamSecond_PidId_SpecificPartition() { // Only the read columns should be used, no criteria use partition assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(4); // If this switches to 2 that would be fine + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); // If this switches to 2 that would be fine } // Read in null Partition @@ -1381,7 +1381,7 @@ public void testSearch_IdParamOnly_ForcedId_SpecificPartition() { // Only the read columns should be used, no criteria use partition assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); } // Read in null Partition @@ -1482,7 +1482,7 @@ public void testSearch_MissingParamString_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'true'")); } @@ -1499,7 +1499,7 @@ public void testSearch_MissingParamString_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_MISSING = 'false'")); } } @@ -1608,7 +1608,7 @@ public void testSearch_MissingParamReference_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")); assertEquals(1, StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")); } @@ -1635,7 +1635,7 @@ public void testSearch_MissingParamReference_SearchOnePartition_IncludePartition ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '-3438137196820602023'")).as(searchSql).isEqualTo(1); @@ -1663,7 +1663,7 @@ public void testSearch_MissingParamReference_SearchOnePartition_DontIncludeParti ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1); @@ -1689,7 +1689,7 @@ public void testSearch_MissingParamReference_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HFJ_RES_PARAM_PRESENT")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "HASH_PRESENCE = '1919227773735728687'")).as(searchSql).isEqualTo(1); @@ -1714,7 +1714,7 @@ public void testSearch_NoParams_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); } @Test @@ -1734,7 +1734,7 @@ public void testSearch_NoParams_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); } @Test @@ -1804,7 +1804,7 @@ public void testSearch_DateParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date OR param @@ -1820,7 +1820,7 @@ public void testSearch_DateParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date AND param @@ -1836,7 +1836,7 @@ public void testSearch_DateParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // DateRangeParam @@ -1852,7 +1852,7 @@ public void testSearch_DateParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); @@ -1887,7 +1887,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); assertThat(StringUtils.countMatches(searchSql, "SP_VALUE_LOW")).as(searchSql).isEqualTo(2); // Date OR param @@ -1903,7 +1903,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date AND param @@ -1935,7 +1935,7 @@ public void testSearch_DateParam_SearchSpecificPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); @@ -1967,7 +1967,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(2, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date OR param @@ -1983,7 +1983,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // Date AND param @@ -1999,7 +1999,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(4, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); // DateRangeParam @@ -2015,7 +2015,7 @@ public void testSearch_DateParam_SearchDefaultPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); // NOTE: the query is changed, only one SP_VALUE_LOW and SP_VALUE_HIGH assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_LOW")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_HIGH")); @@ -2085,7 +2085,7 @@ public void testSearch_HasParam_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); ourLog.info("Search SQL:\n{}", searchSql); assertThat(searchSql).as(searchSql).contains("PARTITION_ID = '1'"); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); } @@ -2109,7 +2109,7 @@ public void testSearch_StringParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2132,7 +2132,7 @@ public void testSearch_StringParam_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); searchSql = searchSql.toUpperCase(); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2156,7 +2156,7 @@ public void testSearch_StringParam_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2189,7 +2189,7 @@ public void testSearch_StringParam_SearchMultiplePartitions() { ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); assertThat(searchSql).contains("PARTITION_ID IN ('1','2')"); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); } // Match two partitions including null @@ -2206,7 +2206,7 @@ public void testSearch_StringParam_SearchMultiplePartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); assertThat(searchSql).contains("PARTITION_ID IS NULL"); assertThat(searchSql).contains("PARTITION_ID = '1'"); - assertEquals(4, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); } } @@ -2267,7 +2267,7 @@ public void testSearch_StringParam_SearchDefaultPartition_IncludePartitionInHash String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); searchSql = searchSql.toUpperCase(); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2295,7 +2295,7 @@ public void testSearch_StringParam_SearchOnePartition_IncludePartitionInHashes() String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "SP_VALUE_NORMALIZED")); } @@ -2352,7 +2352,7 @@ public void testSearch_TagNotParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); // And with another param @@ -2369,7 +2369,7 @@ public void testSearch_TagNotParam_SearchAllPartitions() { searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, ".HASH_SYS_AND_VALUE =")).as(searchSql).isEqualTo(1); @@ -2393,7 +2393,7 @@ public void testSearch_TagNotParam_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); @@ -2421,7 +2421,7 @@ public void testSearch_TagNotParam_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2443,7 +2443,7 @@ public void testSearch_TagParam_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2469,7 +2469,7 @@ public void testSearch_TagParam_SearchOnePartition() { ourLog.info("Search SQL:\n{}", searchSql); assertEquals(2, StringUtils.countMatches(searchSql, "JOIN")); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2494,7 +2494,7 @@ public void testSearch_TagParamNot_SearchAllPartitions() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(1, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2519,7 +2519,7 @@ public void testSearch_TagParamNot_SearchOnePartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); assertEquals(1, StringUtils.countMatches(searchSql, "TAG_SYSTEM = 'http://system'")); } @@ -2667,7 +2667,7 @@ public void testSearch_RefParam_TargetPid_SearchOnePartition() { assertThat(StringUtils.countMatches(searchSql, "t0.PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")).as(searchSql).isEqualTo(1); assertThat(StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")).as(searchSql).isEqualTo(1); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); // Same query, different partition addReadPartition(2); @@ -2704,7 +2704,7 @@ public void testSearch_RefParam_TargetPid_SearchDefaultPartition() { assertEquals(1, StringUtils.countMatches(searchSql, "t0.PARTITION_ID IS NULL")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.SRC_PATH = 'Observation.subject'")); assertEquals(1, StringUtils.countMatches(searchSql, "t0.TARGET_RESOURCE_ID = '" + patientId.getIdPartAsLong() + "'")); - assertEquals(3, StringUtils.countMatches(searchSql, "PARTITION_ID")); + assertEquals(2, StringUtils.countMatches(searchSql, "PARTITION_ID")); // Same query, different partition addReadPartition(2); @@ -2739,7 +2739,7 @@ public void testSearch_RefParam_TargetForcedId_SearchOnePartition() { ourLog.info("Search SQL:\n{}", myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true)); String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, false); assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID = '1'")).as(searchSql).isEqualTo(1); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); // Same query, different partition addReadPartition(2); @@ -2809,7 +2809,7 @@ public void testSearch_RefParam_TargetForcedId_SearchDefaultPartition() { String searchSql = myCaptureQueriesListener.getSelectQueriesForCurrentThread().get(0).getSql(true, true); ourLog.info("Search SQL:\n{}", searchSql); assertThat(StringUtils.countMatches(searchSql.toUpperCase(Locale.US), "PARTITION_ID IS NULL")).as(searchSql).isEqualTo(1); - assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(3); + assertThat(StringUtils.countMatches(searchSql, "PARTITION_ID")).as(searchSql).isEqualTo(2); // Same query, different partition addReadPartition(2); From 32f77859c891ee6922b66f7e0745511482e1150a Mon Sep 17 00:00:00 2001 From: leif stawnyczy Date: Mon, 2 Dec 2024 16:38:48 -0500 Subject: [PATCH 22/22] changes to tests --- .../jpa/search/SearchCoordinatorSvcImpl.java | 1 + .../dstu3/ResourceProviderDstu3Test.java | 1 - .../r4/FhirResourceDaoR4SearchNoFtTest.java | 1 - .../r5/dbpartitionmode/TestDefinitions.java | 64 +++++++++---------- .../ca/uhn/fhir/jpa/test/OverlayTestApp.java | 33 +++++++++- .../java/ca/uhn/fhir/jpa/test/WebTest.java | 3 - 6 files changed, 65 insertions(+), 38 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index b6e71b5452c4..84149956509b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -408,6 +408,7 @@ public IBundleProvider registerSearch( Integer maxToLoad = ObjectUtils.firstNonNull( loadSynchronousUpTo, theParams.getCount() != null ? theParams.getCount() + 1 : null, + myStorageSettings.getFetchSizeDefaultMaximum(), myStorageSettings.getInternalSynchronousSearchSize()); ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); sb.setMaxResultsToFetch(maxToLoad); diff --git a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java index a0179f645f82..debff2e7be53 100644 --- a/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java @@ -2490,7 +2490,6 @@ public void testMetadataSuperParamsAreIncluded() { assertEquals(1, resp.getTotal()); } -// @Disabled @Test public void testMetaOperations() { String methodName = "testMetaOperations"; diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java index 9cec742dca9d..0c3f2feaed8f 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java @@ -28,7 +28,6 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap.EverythingModeEnum; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.config.TestHSearchAddInConfig; -import ca.uhn.fhir.jpa.util.SqlQuery; import ca.uhn.fhir.jpa.util.TestUtil; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.api.TemporalPrecisionEnum; diff --git a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/dbpartitionmode/TestDefinitions.java b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/dbpartitionmode/TestDefinitions.java index 5eb53eeb62b0..60715387ccc0 100644 --- a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/dbpartitionmode/TestDefinitions.java +++ b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/dbpartitionmode/TestDefinitions.java @@ -782,7 +782,7 @@ public void testSearch_ListParam() { } else { assertThat(getSelectSql(0)).contains(" INNER JOIN HFJ_RES_LINK t0 ON (t1.RES_ID = t0.TARGET_RESOURCE_ID) "); if (myIncludePartitionIdsInSql) { - assertThat(getSelectSql(0)).endsWith(" WHERE ((t0.PARTITION_ID = '1') AND (t0.SRC_PATH = 'List.entry.item') AND (t0.TARGET_RESOURCE_TYPE = 'Patient') AND (t0.SRC_RESOURCE_ID = '" + listIdLong + "'))"); + assertThat(getSelectSql(0)).endsWith(" WHERE ((t0.PARTITION_ID = '1') AND (t0.SRC_PATH = 'List.entry.item') AND (t0.TARGET_RESOURCE_TYPE = 'Patient') AND (t0.SRC_RESOURCE_ID = '" + listIdLong + "')) fetch first '10000' rows only"); } else { assertThat(getSelectSql(0)).endsWith(" WHERE ((t0.SRC_PATH = 'List.entry.item') AND (t0.TARGET_RESOURCE_TYPE = 'Patient') AND (t0.SRC_RESOURCE_ID = '" + listIdLong + "'))"); } @@ -816,7 +816,7 @@ public void testSearch_MultiPartition() { assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = '-9208284524139093953')) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID IN ('1','2') )) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST", getSelectSql(0)); assertThat(getSelectSql(1)).contains(" where (rht1_0.RES_ID,rht1_0.PARTITION_ID) in (('" + id0.getIdPartAsLong() + "','1'),('" + id1.getIdPartAsLong() + "','2'),('-1',NULL),('-1',NULL),('-1',NULL),('-1',NULL),('-1',NULL),('-1',NULL),('-1',NULL),('-1',NULL)) and mrt1_0.RES_VER=rht1_0.RES_VER"); } else if (myIncludePartitionIdsInSql) { - assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = '-9208284524139093953')) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID IN ('1','2') )) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST", getSelectSql(0)); + assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = '-9208284524139093953')) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID IN ('1','2') )) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first '10000' rows only", getSelectSql(0)); assertThat(getSelectSql(1)).contains(" where rht1_0.RES_ID in ('" + id0.getIdPartAsLong() + "','" + id1.getIdPartAsLong() + "','-1','-1','-1','-1','-1','-1','-1','-1') and mrt1_0.RES_VER=rht1_0.RES_VER"); } else { assertEquals("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = '-9208284524139093953')) WHERE ((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST", getSelectSql(0)); @@ -870,9 +870,9 @@ public void testSearch_Source(boolean theAccessMetaSourceInformationFromProvenan } } else if (myIncludePartitionIdsInSql) { if (theAccessMetaSourceInformationFromProvenanceTable) { - assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_VER_PROV t1 ON (t0.RES_ID = t1.RES_PID) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t1.SOURCE_URI = 'http://foo'))", getSelectSql(0)); + assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_VER_PROV t1 ON (t0.RES_ID = t1.RES_PID) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t1.SOURCE_URI = 'http://foo')) fetch first '10000' rows only", getSelectSql(0)); } else { - assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_VER t1 ON (t0.RES_ID = t1.RES_ID) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t1.SOURCE_URI = 'http://foo'))", getSelectSql(0)); + assertEquals("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 INNER JOIN HFJ_RES_VER t1 ON (t0.RES_ID = t1.RES_ID) WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t1.SOURCE_URI = 'http://foo')) fetch first '10000' rows only", getSelectSql(0)); } } else { if (theAccessMetaSourceInformationFromProvenanceTable) { @@ -1003,7 +1003,7 @@ public void testSearch_Token() { // Verify myCaptureQueriesListener.logSelectQueries(); if (myIncludePartitionIdsInSql) { - assertThat(getSelectSql(0)).endsWith(" WHERE ((t0.PARTITION_ID = '1') AND (t0.HASH_VALUE = '7943378963388545453'))"); + assertThat(getSelectSql(0)).endsWith(" WHERE ((t0.PARTITION_ID = '1') AND (t0.HASH_VALUE = '7943378963388545453')) fetch first '10000' rows only"); } else { assertThat(getSelectSql(0)).endsWith(" WHERE (t0.HASH_VALUE = '7943378963388545453')"); } @@ -1066,7 +1066,7 @@ public void testSearch_Includes_Forward_Star() { sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); if (myIncludePartitionIdsInSql) { - assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = '1'))"); + assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = '1')) fetch first '10000' rows only"); } else { assertThat(sql).isEqualTo("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE ((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL))"); } @@ -1130,7 +1130,7 @@ public void testSearch_Includes_Forward_Star_UsingCanonicalUrl() { sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); if (myIncludePartitionIdsInSql) { - assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'QuestionnaireResponse') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = '1'))"); + assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'QuestionnaireResponse') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = '1')) fetch first '10000' rows only"); } else { assertThat(sql).isEqualTo("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE ((t0.RES_TYPE = 'QuestionnaireResponse') AND (t0.RES_DELETED_AT IS NULL))"); } @@ -1177,7 +1177,7 @@ public void testSearch_Includes_Forward_Specific() { sql = myCaptureQueriesListener.getSelectQueries().get(0).getSql(true, false); if (myIncludePartitionIdsInSql) { - assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = '1'))"); + assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = '1')) fetch first '10000' rows only"); } else { assertThat(sql).isEqualTo("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE ((t0.RES_TYPE = 'Patient') AND (t0.RES_DELETED_AT IS NULL))"); } @@ -1335,7 +1335,7 @@ public void testSearch_Includes_Reverse_Star() { if (myIncludePartitionIdsInSql && myPartitionSettings.getDefaultPartitionId() == null) { assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID IS NULL) AND (t0.RES_ID = '" + ids.parentOrgPid() + "')))"); } else if (myIncludePartitionIdsInSql) { - assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID = '0') AND (t0.RES_ID = '" + ids.parentOrgPid() + "')))"); + assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID = '0') AND (t0.RES_ID = '" + ids.parentOrgPid() + "'))) fetch first '10000' rows only"); } else { assertThat(sql).isEqualTo("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.RES_ID = '" + ids.parentOrgPid() + "'))"); } @@ -1402,7 +1402,7 @@ public void testSearch_Includes_Reverse_Specific() { if (myIncludePartitionIdsInSql && myPartitionSettings.getDefaultPartitionId() == null) { assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID IS NULL) AND (t0.RES_ID = '" + ids.parentOrgPid() + "')))"); } else if (myIncludePartitionIdsInSql) { - assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID = '0') AND (t0.RES_ID = '" + ids.parentOrgPid() + "')))"); + assertThat(sql).isEqualTo("SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID = '0') AND (t0.RES_ID = '" + ids.parentOrgPid() + "'))) fetch first '10000' rows only"); } else { assertThat(sql).isEqualTo("SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = 'Organization') AND (t0.RES_DELETED_AT IS NULL)) AND (t0.RES_ID = '" + ids.parentOrgPid() + "'))"); } @@ -1762,8 +1762,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.allPartitions(), "single string - no hfj_resource root - all partitions", "Patient?name=FOO", - "SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))", + "SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))" ); SearchMultiPartitionTestCase.add( @@ -1771,8 +1771,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.fromPartitionIds(PARTITION_1, PARTITION_2), "single string - no hfj_resource root - multiple partitions", "Patient?name=FOO", - "SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.PARTITION_ID IN (?,?) ) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)))", + "SELECT t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.PARTITION_ID IN (?,?) ) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_STRING t0 WHERE ((t0.PARTITION_ID IN (?,?) ) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)))" ); @@ -1781,8 +1781,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.allPartitions(), "two regular params - should use hfj_resource as root - all partitions", "Patient?name=smith&active=true", - "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?))", - "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?))", + "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?)) fetch first ? rows only", + "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?)) fetch first ? rows only", "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) INNER JOIN HFJ_SPIDX_TOKEN t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID)) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?))" ); SearchMultiPartitionTestCase.add( @@ -1790,8 +1790,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.fromPartitionIds(PARTITION_1, PARTITION_2), "two regular params - should use hfj_resource as root - multiple partitions", "Patient?name=smith&active=true", - "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?))", - "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.PARTITION_ID IN (?,?) ) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) AND ((t2.PARTITION_ID IN (?,?) ) AND (t2.HASH_VALUE = ?)))", + "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?)) AND (t2.HASH_VALUE = ?)) fetch first ? rows only", + "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON (t1.RES_ID = t0.RES_ID) INNER JOIN HFJ_SPIDX_TOKEN t2 ON (t1.RES_ID = t2.RES_ID) WHERE (((t0.PARTITION_ID IN (?,?) ) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) AND ((t2.PARTITION_ID IN (?,?) ) AND (t2.HASH_VALUE = ?))) fetch first ? rows only", "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_STRING t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) INNER JOIN HFJ_SPIDX_TOKEN t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID)) WHERE (((t0.PARTITION_ID IN (?,?) ) AND ((t0.HASH_NORM_PREFIX = ?) AND (t0.SP_VALUE_NORMALIZED LIKE ?))) AND ((t2.PARTITION_ID IN (?,?) ) AND (t2.HASH_VALUE = ?)))" ); @@ -1800,8 +1800,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.allPartitions(), "token not as a NOT IN subselect - all partitions", "Encounter?class:not=not-there", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID,t0.RES_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))" ); SearchMultiPartitionTestCase.add( @@ -1809,8 +1809,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.fromPartitionIds(PARTITION_1, PARTITION_2), "token not as a NOT IN subselect - multiple partitions", "Encounter?class:not=not-there", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID IN (?,?) ) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )))", + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID IN (?,?) ) AND ((t0.RES_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))) fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND ((t0.PARTITION_ID IN (?,?) ) AND ((t0.PARTITION_ID,t0.RES_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )))" ); @@ -1819,8 +1819,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.allPartitions(), "token not on chain join - NOT IN from hfj_res_link target columns - all partitions", "Observation?encounter.class:not=not-there", - "SELECT t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", - "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", + "SELECT t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RES_PARTITION_ID,t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))" ); SearchMultiPartitionTestCase.add( @@ -1828,8 +1828,8 @@ static List searchMultiPartitionTestCases() { RequestPartitionId.fromPartitionIds(PARTITION_1, PARTITION_2), "token not on chain join - NOT IN from hfj_res_link target columns - multiple partitions", "Observation?encounter.class:not=not-there", - "SELECT t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))", - "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.PARTITION_ID IN (?,?) ) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )))", + "SELECT t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )) fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.PARTITION_ID IN (?,?) ) AND ((t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) ))) fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.SRC_RESOURCE_ID FROM HFJ_RES_LINK t0 WHERE ((t0.SRC_PATH = ?) AND ((t0.PARTITION_ID IN (?,?) ) AND ((t0.TARGET_RES_PARTITION_ID,t0.TARGET_RESOURCE_ID) NOT IN (SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_SPIDX_TOKEN t0 WHERE (t0.HASH_VALUE = ?)) )))" ); @@ -1848,22 +1848,22 @@ static List searchSortTestCases() { new SqlGenerationTestCase( "bare sort", "Patient?_sort=name", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST", + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_SPIDX_STRING t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.RES_ID) AND (t1.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t1.SP_VALUE_NORMALIZED ASC NULLS LAST" ) , new SqlGenerationTestCase( "sort with predicate", "Patient?active=true&_sort=name", - "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (t0.HASH_VALUE = ?) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST", - "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.PARTITION_ID = ?) AND (t0.HASH_VALUE = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST", + "SELECT t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (t0.HASH_VALUE = ?) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", + "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON (t1.RES_ID = t0.RES_ID) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.PARTITION_ID = ?) AND (t0.HASH_VALUE = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", "SELECT t1.PARTITION_ID,t1.RES_ID FROM HFJ_RESOURCE t1 INNER JOIN HFJ_SPIDX_TOKEN t0 ON ((t1.PARTITION_ID = t0.PARTITION_ID) AND (t1.RES_ID = t0.RES_ID)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.PARTITION_ID = t2.PARTITION_ID) AND (t1.RES_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.PARTITION_ID = ?) AND (t0.HASH_VALUE = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST" ) , new SqlGenerationTestCase( "chained sort", "Patient?_sort=Practitioner:general-practitioner.name", - "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST", - "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST", + "SELECT t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE ((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", + "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST fetch first ? rows only", "SELECT t0.PARTITION_ID,t0.RES_ID FROM HFJ_RESOURCE t0 LEFT OUTER JOIN HFJ_RES_LINK t1 ON ((t0.PARTITION_ID = t1.PARTITION_ID) AND (t0.RES_ID = t1.SRC_RESOURCE_ID) AND (t1.SRC_PATH = ?)) LEFT OUTER JOIN HFJ_SPIDX_STRING t2 ON ((t1.TARGET_RES_PARTITION_ID = t2.PARTITION_ID) AND (t1.TARGET_RESOURCE_ID = t2.RES_ID) AND (t2.HASH_IDENTITY = ?)) WHERE (((t0.RES_TYPE = ?) AND (t0.RES_DELETED_AT IS NULL)) AND (t0.PARTITION_ID = ?)) ORDER BY t2.SP_VALUE_NORMALIZED ASC NULLS LAST" ) ); diff --git a/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/OverlayTestApp.java b/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/OverlayTestApp.java index 53ec6b575c61..0a2e4d05b6af 100644 --- a/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/OverlayTestApp.java +++ b/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/OverlayTestApp.java @@ -11,7 +11,11 @@ import ca.uhn.fhir.rest.annotation.OptionalParam; import ca.uhn.fhir.rest.annotation.RequiredParam; import ca.uhn.fhir.rest.annotation.Search; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.param.DateRangeParam; +import ca.uhn.fhir.rest.param.StringAndListParam; +import ca.uhn.fhir.rest.param.TokenAndListParam; import ca.uhn.fhir.rest.param.TokenOrListParam; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.server.IResourceProvider; @@ -66,7 +70,7 @@ public static void main(String[] args) throws Exception { FhirContext ctx = FhirContext.forR4Cached(); RestfulServer restfulServer = new RestfulServer(ctx); restfulServer.registerProvider(new ProviderWithRequiredAndOptional()); - restfulServer.registerProvider(new HashMapResourceProvider<>(ctx, Patient.class)); + restfulServer.registerProvider(new PatientTestResourceProvider(ctx)); restfulServer.registerProvider(new HfqlRestProvider(hfqlExecutor)); ServletContextHandler proxyHandler = new ServletContextHandler(); @@ -222,4 +226,31 @@ public Class getResourceType() { } + public static class PatientTestResourceProvider extends HashMapResourceProvider { + + /** + * Constructor + * + * @param theFhirContext The FHIR context + */ + public PatientTestResourceProvider(FhirContext theFhirContext) { + super(theFhirContext, Patient.class); + } + + @Description(shortDefinition = "This is a provider endpoint with parameters for searching on patients to display") + @Search + public IBundleProvider findPatients(@RequiredParam(name = Patient.SP_ACTIVE) TokenAndListParam theType, + @Description(shortDefinition = "A portion of the given name of the patient") + @OptionalParam(name = "given") + StringAndListParam theGiven, + @Description(shortDefinition = "A portion of the family name of the patient") + @OptionalParam(name = "family") + StringAndListParam theFamily, + RequestDetails theRequestDetails + ) throws Exception { + return searchAll(theRequestDetails); + } + + } + } diff --git a/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/WebTest.java b/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/WebTest.java index 22129dd88b0e..69b48000f3c8 100644 --- a/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/WebTest.java +++ b/hapi-fhir-testpage-overlay/src/test/java/ca/uhn/fhir/jpa/test/WebTest.java @@ -288,7 +288,6 @@ public void testInvokeCustomOperation_Diff() throws IOException { assertThat(diffPage.asNormalizedText()).contains("\"resourceType\": \"Parameters\""); } - @Test public void testHfqlExecuteQuery() throws IOException { // Load home page @@ -317,7 +316,6 @@ public void testHfqlExecuteQuery() throws IOException { assertThat(table.asNormalizedText()).contains("Simpson"); } - private void registerAndUpdatePatient() { Patient p = new Patient(); Patient p2 = new Patient(); @@ -380,7 +378,6 @@ public Parameters diff(@IdParam IIdType theId) { Parameters parameters = new Parameters(); return parameters; } - } private static class MyServletContextHandler extends ServletContextHandler {