Fix #641 Multi prefix search for other codesystems

This fixes a bug when filtering FHIR ValueSets that are not using SNOMED CT. The bug fix enables searching with multiple words or prefixes. Before the fix only searching with one word or prefix was working.
IHTSDO · Nov 26, 2024 · 1560214 · 1560214
1 parent 2ea361e
commit 1560214
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 13 deletions.
diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java b/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java
@@ -943,7 +943,7 @@ private void addClause(Query queryClause, BoolQuery.Builder boolBuilder, boolean
 		}
 	}
 
-	private List<String> analyze(String text, StandardAnalyzer analyzer) {
+	public static List<String> analyze(String text, StandardAnalyzer analyzer) {
 		List<String> result = new ArrayList<>();
 		try {
 			TokenStream tokenStream = analyzer.tokenStream("contents", text);
@@ -953,12 +953,13 @@ private List<String> analyze(String text, StandardAnalyzer analyzer) {
 				result.add(attr.toString());
 			}
 		} catch (IOException e) {
-			logger.error("Failed to analyze text {}", text, e);
+			LoggerFactory.getLogger(DescriptionService.class)
+					.error("Failed to analyze text {}", text, e);
 		}
 		return result;
 	}
 
-	private String constructSimpleQueryString(String searchTerm) {
+	public static String constructSimpleQueryString(String searchTerm) {
 		return (searchTerm.trim().replace(" ", "* ") + "*").replace("**", "*");
 	}
 
@@ -1000,7 +1001,7 @@ private String constructRegexQuery(String term) {
 		return regexBuilder.toString();
 	}
 
-	private String constructSearchTerm(List<String> tokens) {
+	public static String constructSearchTerm(List<String> tokens) {
 		StringBuilder builder = new StringBuilder();
 		for (String token : tokens) {
 			builder.append(token);

diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java
@@ -1,10 +1,10 @@
 package org.snomed.snowstorm.fhir.services;
 
 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
-import co.elastic.clients.elasticsearch._types.query_dsl.PrefixQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.Operator;
 import com.google.common.base.Strings;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
-
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.hl7.fhir.r4.model.*;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -15,6 +15,7 @@
 import org.snomed.snowstorm.core.data.domain.QueryConcept;
 import org.snomed.snowstorm.core.data.domain.ReferenceSetMember;
 import org.snomed.snowstorm.core.data.services.ConceptService;
+import org.snomed.snowstorm.core.data.services.DescriptionService;
 import org.snomed.snowstorm.core.data.services.QueryService;
 import org.snomed.snowstorm.core.data.services.ReferenceSetMemberService;
 import org.snomed.snowstorm.core.data.services.pojo.MemberSearchRequest;
@@ -30,22 +31,24 @@
 import org.snomed.snowstorm.rest.pojo.SearchAfterPageRequest;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.*;
-import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
-import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.data.elasticsearch.client.elc.NativeQuery;
 import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
+import org.springframework.data.elasticsearch.client.elc.Queries;
+import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
+import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.stereotype.Service;
 
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.stream.Collectors;
 
+import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool;
 import static io.kaicode.elasticvc.api.ComponentService.LARGE_PAGE;
+import static io.kaicode.elasticvc.helper.QueryHelper.termQuery;
+import static io.kaicode.elasticvc.helper.QueryHelper.termsQuery;
 import static java.lang.Boolean.TRUE;
 import static java.lang.String.format;
-import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.*;
-import static io.kaicode.elasticvc.helper.QueryHelper.*;
 import static org.snomed.snowstorm.core.data.services.ReferenceSetMemberService.AGGREGATION_MEMBER_COUNTS_BY_REFERENCE_SET;
 import static org.snomed.snowstorm.core.util.CollectionUtils.orEmpty;
 import static org.snomed.snowstorm.fhir.services.FHIRHelper.*;
@@ -390,7 +393,10 @@ private BoolQuery.Builder getFhirConceptQuery(CodeSelectionCriteria codeSelectio
 		BoolQuery.Builder masterQuery = bool();
 		masterQuery.must(contentQuery.build()._toQuery());
 		if (termFilter != null) {
-			masterQuery.must(PrefixQuery.of(pq -> pq.field(FHIRConcept.Fields.DISPLAY).value(termFilter.toLowerCase()))._toQuery());
+			List<String> elasticAnalyzedWords = DescriptionService.analyze(termFilter, new StandardAnalyzer());
+			String searchTerm = DescriptionService.constructSearchTerm(elasticAnalyzedWords);
+			String query = DescriptionService.constructSimpleQueryString(searchTerm);
+			masterQuery.filter(Queries.queryStringQuery(FHIRConcept.Fields.DISPLAY, query, Operator.And, 2.0f)._toQuery());
 		}
 		return masterQuery;
 	}

diff --git a/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java b/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java
@@ -17,9 +17,9 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class FHIRValueSetProviderExpandGenericTest extends AbstractFHIRTest {
 
@@ -196,4 +196,54 @@ public void testExpandIncludesOtherValueSet() {
 		assertEquals(2, valueSet.getExpansion().getContains().size());
 	}
 
+	@Test
+	void testSearchMultipleWordsOrPrefixes() {
+		// "display": "additive, propagating",
+		System.out.println("----------");
+		ResponseEntity<String> response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs", HttpMethod.GET, null, String.class);
+		System.out.println("Search no filter");
+		assertExpand(response, 8, null);
+
+		System.out.println("----------");
+		response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=additive", HttpMethod.GET, null, String.class);
+		System.out.println("Search additive");
+		System.out.println(response.getBody());
+		assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']");
+
+		System.out.println("----------");
+		response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=additive propagating", HttpMethod.GET, null, String.class);
+		System.out.println("Search additive propagating");
+		System.out.println(response.getBody());
+		assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']");
+
+		System.out.println("----------");
+		response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=add", HttpMethod.GET, null, String.class);
+		System.out.println("Search add");
+		System.out.println(response.getBody());
+		assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']");
+
+		System.out.println("----------");
+		response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=add prop non", HttpMethod.GET, null, String.class);
+		System.out.println("Search add prop non");
+		System.out.println(response.getBody());
+		assertExpand(response, 1, "[AN|'additive, non-propagating']");
+
+		System.out.println("----------");
+	}
+
+	private void assertExpand(ResponseEntity<String> response, int expected, String expectedCodingString) {
+		assertEquals(HttpStatus.OK, response.getStatusCode());
+		assertNotNull(response.getBody());
+		ValueSet valueSet = fhirJsonParser.parseResource(ValueSet.class, response.getBody());
+		List<ValueSet.ValueSetExpansionContainsComponent> contains = valueSet.getExpansion().getContains();
+		assertEquals(expected, contains.size());
+		if (expectedCodingString != null) {
+			assertEquals(expectedCodingString, toCodingsString(contains));
+		}
+	}
+
+	private String toCodingsString(List<ValueSet.ValueSetExpansionContainsComponent> contains) {
+		return contains.stream().map(coding -> "%s|'%s'".formatted(coding.getCode(), coding.getDisplay())).toList().toString();
+	}
+
 }