From 30de66d58d52cc36c82d53c76695aa556fbb57e3 Mon Sep 17 00:00:00 2001 From: Kai Kewley Date: Mon, 25 Nov 2024 15:58:20 +0000 Subject: [PATCH] ISTO-127 Fix #641 Multi prefix search for other codesystems This fixes a bug when filtering FHIR ValueSets that are not using SNOMED CT. The bug fix enables searching with multiple words or prefixes. Before the fix only searching with one word or prefix was working. --- .../data/services/DescriptionService.java | 9 ++-- .../fhir/services/FHIRValueSetService.java | 20 ++++--- ...FHIRValueSetProviderExpandGenericTest.java | 54 ++++++++++++++++++- 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java b/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java index 289ddde6a..44474ec7f 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java +++ b/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java @@ -943,7 +943,7 @@ private void addClause(Query queryClause, BoolQuery.Builder boolBuilder, boolean } } - private List analyze(String text, StandardAnalyzer analyzer) { + public static List analyze(String text, StandardAnalyzer analyzer) { List result = new ArrayList<>(); try { TokenStream tokenStream = analyzer.tokenStream("contents", text); @@ -953,12 +953,13 @@ private List analyze(String text, StandardAnalyzer analyzer) { result.add(attr.toString()); } } catch (IOException e) { - logger.error("Failed to analyze text {}", text, e); + LoggerFactory.getLogger(DescriptionService.class) + .error("Failed to analyze text {}", text, e); } return result; } - private String constructSimpleQueryString(String searchTerm) { + public static String constructSimpleQueryString(String searchTerm) { return (searchTerm.trim().replace(" ", "* ") + "*").replace("**", "*"); } @@ -1000,7 +1001,7 @@ private String constructRegexQuery(String term) { return regexBuilder.toString(); } - private String constructSearchTerm(List tokens) { + public static String constructSearchTerm(List tokens) { StringBuilder builder = new StringBuilder(); for (String token : tokens) { builder.append(token); diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java index 7e17db113..76c4e875d 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java @@ -1,10 +1,10 @@ package org.snomed.snowstorm.fhir.services; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; -import co.elastic.clients.elasticsearch._types.query_dsl.PrefixQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Operator; import com.google.common.base.Strings; import it.unimi.dsi.fastutil.longs.LongArrayList; - +import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.hl7.fhir.r4.model.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -15,6 +15,7 @@ import org.snomed.snowstorm.core.data.domain.QueryConcept; import org.snomed.snowstorm.core.data.domain.ReferenceSetMember; import org.snomed.snowstorm.core.data.services.ConceptService; +import org.snomed.snowstorm.core.data.services.DescriptionService; import org.snomed.snowstorm.core.data.services.QueryService; import org.snomed.snowstorm.core.data.services.ReferenceSetMemberService; import org.snomed.snowstorm.core.data.services.pojo.MemberSearchRequest; @@ -30,10 +31,11 @@ import org.snomed.snowstorm.rest.pojo.SearchAfterPageRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.*; -import org.springframework.data.elasticsearch.core.ElasticsearchOperations; -import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.client.elc.NativeQuery; import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; +import org.springframework.data.elasticsearch.client.elc.Queries; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.stereotype.Service; import java.net.URLDecoder; @@ -41,11 +43,12 @@ import java.util.*; import java.util.stream.Collectors; +import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool; import static io.kaicode.elasticvc.api.ComponentService.LARGE_PAGE; +import static io.kaicode.elasticvc.helper.QueryHelper.termQuery; +import static io.kaicode.elasticvc.helper.QueryHelper.termsQuery; import static java.lang.Boolean.TRUE; import static java.lang.String.format; -import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.*; -import static io.kaicode.elasticvc.helper.QueryHelper.*; import static org.snomed.snowstorm.core.data.services.ReferenceSetMemberService.AGGREGATION_MEMBER_COUNTS_BY_REFERENCE_SET; import static org.snomed.snowstorm.core.util.CollectionUtils.orEmpty; import static org.snomed.snowstorm.fhir.services.FHIRHelper.*; @@ -392,7 +395,10 @@ private BoolQuery.Builder getFhirConceptQuery(CodeSelectionCriteria codeSelectio BoolQuery.Builder masterQuery = bool(); masterQuery.must(contentQuery.build()._toQuery()); if (termFilter != null) { - masterQuery.must(PrefixQuery.of(pq -> pq.field(FHIRConcept.Fields.DISPLAY).value(termFilter.toLowerCase()))._toQuery()); + List elasticAnalyzedWords = DescriptionService.analyze(termFilter, new StandardAnalyzer()); + String searchTerm = DescriptionService.constructSearchTerm(elasticAnalyzedWords); + String query = DescriptionService.constructSimpleQueryString(searchTerm); + masterQuery.filter(Queries.queryStringQuery(FHIRConcept.Fields.DISPLAY, query, Operator.And, 2.0f)._toQuery()); } return masterQuery; } diff --git a/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java b/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java index 9d7812e81..7550efa01 100644 --- a/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java +++ b/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java @@ -17,9 +17,9 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class FHIRValueSetProviderExpandGenericTest extends AbstractFHIRTest { @@ -196,4 +196,54 @@ public void testExpandIncludesOtherValueSet() { assertEquals(2, valueSet.getExpansion().getContains().size()); } + @Test + void testSearchMultipleWordsOrPrefixes() { + // "display": "additive, propagating", + System.out.println("----------"); + ResponseEntity response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs", HttpMethod.GET, null, String.class); + System.out.println("Search no filter"); + assertExpand(response, 8, null); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=additive", HttpMethod.GET, null, String.class); + System.out.println("Search additive"); + System.out.println(response.getBody()); + assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']"); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=additive propagating", HttpMethod.GET, null, String.class); + System.out.println("Search additive propagating"); + System.out.println(response.getBody()); + assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']"); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=add", HttpMethod.GET, null, String.class); + System.out.println("Search add"); + System.out.println(response.getBody()); + assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']"); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=add prop non", HttpMethod.GET, null, String.class); + System.out.println("Search add prop non"); + System.out.println(response.getBody()); + assertExpand(response, 1, "[AN|'additive, non-propagating']"); + + System.out.println("----------"); + } + + private void assertExpand(ResponseEntity response, int expected, String expectedCodingString) { + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertNotNull(response.getBody()); + ValueSet valueSet = fhirJsonParser.parseResource(ValueSet.class, response.getBody()); + List contains = valueSet.getExpansion().getContains(); + assertEquals(expected, contains.size()); + if (expectedCodingString != null) { + assertEquals(expectedCodingString, toCodingsString(contains)); + } + } + + private String toCodingsString(List contains) { + return contains.stream().map(coding -> "%s|'%s'".formatted(coding.getCode(), coding.getDisplay())).toList().toString(); + } + }