From db20fa452c68cbc313d563a7798f7047d9baa493 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Thu, 10 Oct 2024 09:08:11 -0400 Subject: [PATCH 01/10] [CHORE] GH actions fix --- .github/workflows/maven.yml | 2 +- .github/workflows/unit-tests.yaml | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 .github/workflows/unit-tests.yaml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 890be1e..3fc1e80 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: '21' - distribution: 'coretto' + distribution: 'temurin' cache: maven - name: Build with Maven run: mvn -B package --file pom.xml diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml deleted file mode 100644 index 5611efd..0000000 --- a/.github/workflows/unit-tests.yaml +++ /dev/null @@ -1,11 +0,0 @@ -steps: - - uses: actions/checkout@v4 - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'temurin' - cache: maven - - name: Build with Maven - run: mvn --batch-mode --update-snapshots verify - From fa91128d455f2d297547e24e428bb91039c67077 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 08:17:31 -0500 Subject: [PATCH 02/10] Add `@ActiveProfiles("test")` for testing classes This removes the spam from the DataSourceVerifier when running unit test. --- .../dictionary/datasource/DataSourceVerifier.java | 11 ++++++----- .../dictionary/concept/ConceptControllerTest.java | 2 ++ .../concept/ConceptDecoratorServiceTest.java | 3 ++- .../dictionary/concept/ConceptResultSetUtilTest.java | 5 ++--- .../concept/ConceptServiceIntegrationTest.java | 2 ++ .../dictionary/concept/ConceptServiceTest.java | 2 ++ .../dictionary/dashboard/DashboardConfigTest.java | 2 ++ .../dictionary/dashboard/DashboardControllerTest.java | 2 ++ .../dictionary/dashboard/DashboardServiceTest.java | 2 ++ .../dictionary/dataset/DatasetRepositoryTest.java | 2 -- .../dictionary/dataset/DatasetServiceTest.java | 2 ++ .../dictionary/facet/FacetControllerTest.java | 2 ++ .../avillach/dictionary/facet/FacetServiceTest.java | 2 ++ .../dictionary/facet/FilterPreProcessorTest.java | 2 ++ .../avillach/dictionary/info/InfoControllerTest.java | 2 ++ 15 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java index 9e36308..e3fc2a5 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java @@ -3,15 +3,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; -@Service +@Profile("!test") +@Configuration public class DataSourceVerifier { private static final Logger LOG = LoggerFactory.getLogger(DataSourceVerifier.class); @@ -28,11 +30,10 @@ public void verifyDataSourceConnection() { try (Connection connection = dataSource.getConnection()) { if (connection != null) { LOG.info("Datasource connection verified successfully."); - } else { - LOG.info("Failed to obtain a connection from the datasource."); } } catch (SQLException e) { - LOG.info("Error verifying datasource connection: {}", e.getMessage()); + LOG.info("Failed to obtain a connection from the datasource."); + LOG.debug("Error verifying datasource connection: {}", e.getMessage()); } } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java index a5d5fe6..363c180 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java @@ -16,12 +16,14 @@ import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; import java.util.Optional; @SpringBootTest(properties = {"concept.tree.max_depth=1"}) +@ActiveProfiles("test") class ConceptControllerTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java index 6800f57..c543f27 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java @@ -10,12 +10,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; -import javax.print.attribute.DocAttributeSet; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class ConceptDecoratorServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java index 3b1da9e..acd6668 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java @@ -1,17 +1,16 @@ package edu.harvard.dbmi.avillach.dictionary.concept; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; - class ConceptResultSetUtilTest { @Test void shouldParseValues() { - List actual = new ConceptResultSetUtil().parseValues("[\"Look, I'm valid json\"]"); + List actual = new JsonBlobParser().parseValues("[\"Look, I'm valid json\"]"); List expected = List.of("Look, I'm valid json"); Assertions.assertEquals(expected, actual); diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java index e51ac7b..5bef7ba 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java @@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.PostgreSQLContainer; @@ -21,6 +22,7 @@ @Testcontainers @SpringBootTest +@ActiveProfiles("test") class ConceptServiceIntegrationTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java index 0238e0c..610ba94 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java @@ -13,6 +13,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Pageable; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; @@ -20,6 +21,7 @@ @SpringBootTest +@ActiveProfiles("test") class ConceptServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java index 397a055..c5e280f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java @@ -4,10 +4,12 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class DashboardConfigTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java index ec11139..116e284 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java @@ -8,10 +8,12 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class DashboardControllerTest { @MockBean private DashboardService service; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java index 1825e21..03255bd 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java @@ -6,11 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; @SpringBootTest +@ActiveProfiles("test") class DashboardServiceTest { @MockBean DashboardRepository repository; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java index 036a36d..03f169d 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java @@ -1,6 +1,5 @@ package edu.harvard.dbmi.avillach.dictionary.dataset; -import edu.harvard.dbmi.avillach.dictionary.facet.FacetRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import java.util.Map; import java.util.Optional; -import static org.junit.jupiter.api.Assertions.*; @Testcontainers @SpringBootTest diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java index 22b8f12..8580060 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java @@ -6,11 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.Map; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class DatasetServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java index 331f176..96ea6eb 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java @@ -9,11 +9,13 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class FacetControllerTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java index e9c3aab..3b8775f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java @@ -8,12 +8,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class FacetServiceTest { @MockBean private FacetRepository repository; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java index 6d3d452..70e90a5 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java @@ -8,11 +8,13 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.MethodParameter; import org.springframework.http.HttpInputMessage; +import org.springframework.test.context.ActiveProfiles; import org.testcontainers.shaded.com.fasterxml.jackson.databind.type.SimpleType; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class FilterPreProcessorTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java index d401db4..53345a1 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java @@ -6,12 +6,14 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.UUID; @SpringBootTest +@ActiveProfiles("test") class InfoControllerTest { @Autowired From 80e7f06a1aaa3c39adf12881a2f64f411e8497c9 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 08:18:25 -0500 Subject: [PATCH 03/10] Add JsonBlobParser and refactor related utilities Created JsonBlobParser.java for JSON parsing and refactored ConceptResultSetUtil to use it for clearer separation of concerns. --- .../concept/ConceptResultSetUtil.java | 62 ++++-------------- .../dictionary/facet/FilterPreProcessor.java | 36 +++++----- .../dictionary/util/JsonBlobParser.java | 65 +++++++++++++++++++ 3 files changed, 96 insertions(+), 67 deletions(-) create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java index 9e8c220..0201691 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java @@ -2,29 +2,31 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; -import org.json.JSONArray; -import org.json.JSONException; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.math.BigDecimal; -import java.math.BigInteger; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; @Component public class ConceptResultSetUtil { private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class); + private final JsonBlobParser jsonBlobParser; + + @Autowired + public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) { + this.jsonBlobParser = jsonBlobParser; + } public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { return new CategoricalConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), - rs.getString("description"), rs.getString("values") == null ? List.of() : parseValues(rs.getString("values")), + rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")), rs.getBoolean("allowFiltering"), rs.getString("studyAcronym"), null, null ); } @@ -32,53 +34,11 @@ public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { return new ContinuousConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), - rs.getString("description"), rs.getBoolean("allowFiltering"), parseMin(rs.getString("values")), - parseMax(rs.getString("values")), rs.getString("studyAcronym"), null + rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")), + jsonBlobParser.parseMax(rs.getString("values")), rs.getString("studyAcronym"), null ); } - public List parseValues(String valuesArr) { - try { - ArrayList vals = new ArrayList<>(); - JSONArray arr = new JSONArray(valuesArr); - for (int i = 0; i < arr.length(); i++) { - vals.add(arr.getString(i)); - } - return vals; - } catch (JSONException ex) { - return List.of(); - } - } - public Float parseMin(String valuesArr) { - return parseFromIndex(valuesArr, 0); - } - private Float parseFromIndex(String valuesArr, int index) { - try { - JSONArray arr = new JSONArray(valuesArr); - if (arr.length() != 2) { - return 0F; - } - Object raw = arr.get(index); - return switch (raw) { - case Double d -> d.floatValue(); - case Integer i -> i.floatValue(); - case String s -> Double.valueOf(s).floatValue(); - case BigDecimal d -> d.floatValue(); - case BigInteger i -> i.floatValue(); - default -> 0f; - }; - } catch (JSONException ex) { - log.warn("Invalid json array for values: ", ex); - return 0F; - } catch (NumberFormatException ex) { - log.warn("Valid json array but invalid val within: ", ex); - return 0F; - } - } - - public Float parseMax(String valuesArr) { - return parseFromIndex(valuesArr, 1); - } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java index 98357dc..16510f1 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java @@ -35,26 +35,30 @@ public Object afterBodyRead( Class> converterType ) { if (body instanceof Filter filter) { - List newFacets = filter.facets(); - List newConsents = filter.consents(); - if (filter.facets() != null) { - newFacets = new ArrayList<>(filter.facets()); - newFacets.sort(Comparator.comparing(Facet::name)); - } - if (filter.consents() != null) { - newConsents = new ArrayList<>(newConsents); - newConsents.sort(Comparator.comparing(Function.identity())); - } - filter = new Filter(newFacets, filter.search(), newConsents); - - if (StringUtils.hasLength(filter.search())) { - filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); - } - return filter; + return processsFilter(filter); } return body; } + public static Filter processsFilter(Filter filter) { + List newFacets = filter.facets(); + List newConsents = filter.consents(); + if (filter.facets() != null) { + newFacets = new ArrayList<>(filter.facets()); + newFacets.sort(Comparator.comparing(Facet::name)); + } + if (filter.consents() != null) { + newConsents = new ArrayList<>(newConsents); + newConsents.sort(Comparator.comparing(Function.identity())); + } + filter = new Filter(newFacets, filter.search(), newConsents); + + if (StringUtils.hasLength(filter.search())) { + filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); + } + return filter; + } + @Override public Object handleEmptyBody( Object body, HttpInputMessage inputMessage, MethodParameter parameter, Type targetType, diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java new file mode 100644 index 0000000..f976e7c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java @@ -0,0 +1,65 @@ +package edu.harvard.dbmi.avillach.dictionary.util; + + +import org.json.JSONArray; +import org.json.JSONException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +@Component +public class JsonBlobParser { + + private final static Logger log = LoggerFactory.getLogger(JsonBlobParser.class); + + public List parseValues(String valuesArr) { + try { + ArrayList vals = new ArrayList<>(); + JSONArray arr = new JSONArray(valuesArr); + for (int i = 0; i < arr.length(); i++) { + vals.add(arr.getString(i)); + } + return vals; + } catch (JSONException ex) { + return List.of(); + } + } + + public Float parseMin(String valuesArr) { + return parseFromIndex(valuesArr, 0); + } + + private Float parseFromIndex(String valuesArr, int index) { + try { + JSONArray arr = new JSONArray(valuesArr); + if (arr.length() != 2) { + return 0F; + } + Object raw = arr.get(index); + return switch (raw) { + case Double d -> d.floatValue(); + case Integer i -> i.floatValue(); + case String s -> Double.valueOf(s).floatValue(); + case BigDecimal d -> d.floatValue(); + case BigInteger i -> i.floatValue(); + default -> 0f; + }; + } catch (JSONException ex) { + log.warn("Invalid json array for values: ", ex); + return 0F; + } catch (NumberFormatException ex) { + log.warn("Valid json array but invalid val within: ", ex); + return 0F; + } + } + + public Float parseMax(String valuesArr) { + return parseFromIndex(valuesArr, 1); + } + +} From f3cc9b50f6f5c10c6b1298831d0d85f6227cce1d Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 08:19:08 -0500 Subject: [PATCH 04/10] Remove unused imports from concept models This commit cleans up the code by removing unused imports in the ConceptShell and ContinuousConcept classes. This reduces clutter and improves code readability. --- .../dbmi/avillach/dictionary/concept/model/ConceptShell.java | 1 - .../avillach/dictionary/concept/model/ContinuousConcept.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java index 164b953..871c34f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java @@ -1,7 +1,6 @@ package edu.harvard.dbmi.avillach.dictionary.concept.model; import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset; -import jakarta.annotation.Nullable; import java.util.List; import java.util.Map; diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java index 8b465d3..021cb4f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java @@ -4,7 +4,6 @@ import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset; import jakarta.annotation.Nullable; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; From f0c430ca3b7a1c8fd395aa0f37056fff388b1c09 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 08:20:11 -0500 Subject: [PATCH 05/10] Add legacy search feature Implemented a new legacy search feature including service, controller, and related model classes. Updated the ConceptRepository to support legacy search queries and added test cases to ensure functionality. --- .../dictionary/concept/ConceptRepository.java | 52 ++++++++- .../dictionary/concept/ConceptService.java | 6 + .../legacysearch/LegacySearchController.java | 32 +++++ .../legacysearch/LegacySearchQueryMapper.java | 29 +++++ .../legacysearch/LegacySearchService.java | 24 ++++ .../legacysearch/MetadataResultSetUtil.java | 110 ++++++++++++++++++ .../legacysearch/SearchResultRowMapper.java | 38 ++++++ .../model/CategoricalMetadata.java | 23 ++++ .../model/ContinuousMetadata.java | 24 ++++ .../legacysearch/model/LegacyResponse.java | 6 + .../legacysearch/model/LegacySearchQuery.java | 7 ++ .../legacysearch/model/Metadata.java | 4 + .../dictionary/legacysearch/model/Result.java | 12 ++ .../legacysearch/model/Results.java | 8 ++ .../legacysearch/model/SearchResult.java | 6 + .../concept/ConceptRepositoryTest.java | 34 ++++++ 16 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index 410e379..eb08225 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -3,6 +3,8 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -35,6 +37,7 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys) private final NamedParameterJdbcTemplate template; private final ConceptRowMapper mapper; + private final SearchResultRowMapper searchResultRowMapper; private final ConceptFilterQueryGenerator filterGen; private final ConceptMetaExtractor conceptMetaExtractor; private final ConceptResultSetExtractor conceptResultSetExtractor; @@ -43,12 +46,13 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys) @Autowired public ConceptRepository( - NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen, - ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor, - @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields + NamedParameterJdbcTemplate template, ConceptRowMapper mapper, SearchResultRowMapper searchResultRowMapper, + ConceptFilterQueryGenerator filterGen, ConceptMetaExtractor conceptMetaExtractor, + ConceptResultSetExtractor conceptResultSetExtractor, @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields ) { this.template = template; this.mapper = mapper; + this.searchResultRowMapper = searchResultRowMapper; this.filterGen = filterGen; this.conceptMetaExtractor = conceptMetaExtractor; this.conceptResultSetExtractor = conceptResultSetExtractor; @@ -240,4 +244,46 @@ WITH RECURSIVE nodes AS ( return Optional.ofNullable(template.query(sql, params, conceptResultSetExtractor)); } + + public List getLegacySearchResults(Filter filter, Pageable pageable) { + QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable); + String sql = ALLOW_FILTERING_Q + ", " + filterQ.query() + """ + SELECT concept_node.concept_path AS conceptPath, + concept_node.display AS display, + concept_node.name AS name, + concept_node.concept_type AS conceptType, + ds.REF as dataset, + ds.abbreviation AS studyAcronym, + ds.full_name as dsFullName, + continuous_min.VALUE as min, + continuous_max.VALUE as max, + categorical_values.VALUE as values, + allow_filtering.allowFiltering AS allowFiltering, + meta_description.VALUE AS description, + stigmatized.value AS stigmatized, + parent.name AS parentName, + parent.display AS parentDisplay + FROM concept_node + INNER JOIN concepts_filtered_sorted ON concepts_filtered_sorted.concept_node_id = concept_node.concept_node_id + LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id + LEFT JOIN concept_node_meta AS meta_description + ON concept_node.concept_node_id = meta_description.concept_node_id AND + meta_description.KEY = 'description' + LEFT JOIN concept_node_meta AS continuous_min + ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max + ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values + ON concept_node.concept_node_id = categorical_values.concept_node_id AND + categorical_values.KEY = 'values' + LEFT JOIN concept_node_meta AS stigmatized ON concept_node.concept_node_id = stigmatized.concept_node_id AND + stigmatized.KEY = 'stigmatized' + LEFT JOIN concept_node AS parent ON parent.concept_node_id = concept_node.parent_id + LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id + ORDER BY concepts_filtered_sorted.rank DESC, concept_node.concept_node_id ASC + """; + MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields); + + return template.query(sql, params, searchResultRowMapper); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java index edd1e4d..a24500a 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java @@ -5,6 +5,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.data.domain.Pageable; @@ -70,4 +71,9 @@ public Optional conceptTree(String dataset, String conceptPath, int dep public Optional conceptDetailWithoutAncestors(String dataset, String conceptPath) { return getConcept(dataset, conceptPath, false); } + + public List getLegacySearchResults(Filter filter, Pageable pageable) { + return conceptRepository.getLegacySearchResults(filter, pageable); + } + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java new file mode 100644 index 0000000..c412200 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java @@ -0,0 +1,32 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacyResponse; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; + +import java.io.IOException; + +@Controller +public class LegacySearchController { + + private final LegacySearchService legacySearchService; + private final LegacySearchQueryMapper legacySearchQueryMapper; + + @Autowired + public LegacySearchController(LegacySearchService legacySearchService, LegacySearchQueryMapper legacySearchQueryMapper) { + this.legacySearchService = legacySearchService; + this.legacySearchQueryMapper = legacySearchQueryMapper; + } + + @RequestMapping(path = "/search") + public ResponseEntity legacySearch(@RequestBody String jsonString) throws IOException { + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + return ResponseEntity + .ok(new LegacyResponse(legacySearchService.getSearchResults(legacySearchQuery.filter(), legacySearchQuery.pageable()))); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java new file mode 100644 index 0000000..aed5eb6 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java @@ -0,0 +1,29 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.dbmi.avillach.dictionary.facet.FilterPreProcessor; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.List; + +@Component +public class LegacySearchQueryMapper { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public LegacySearchQuery mapFromJson(String jsonString) throws IOException { + JsonNode rootNode = objectMapper.readTree(jsonString); + JsonNode queryNode = rootNode.get("query"); + + String searchTerm = queryNode.get("searchTerm").asText(); + int limit = queryNode.get("limit").asInt(); + Filter filter = FilterPreProcessor.processsFilter(new Filter(List.of(), searchTerm, List.of())); + return new LegacySearchQuery(filter, PageRequest.of(0, limit)); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java new file mode 100644 index 0000000..03d4d01 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java @@ -0,0 +1,24 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptService; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; + +@Service +public class LegacySearchService { + + private final ConceptService conceptService; + + @Autowired + public LegacySearchService(ConceptService conceptService) { + this.conceptService = conceptService; + } + + public Results getSearchResults(Filter filter, Pageable pageable) { + return new Results(conceptService.getLegacySearchResults(filter, pageable)); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java new file mode 100644 index 0000000..8348f37 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java @@ -0,0 +1,110 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.CategoricalMetadata; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.ContinuousMetadata; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Result; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.ResultSet; +import java.sql.SQLException; + + +@Component +public class MetadataResultSetUtil { + + private final static Logger log = LoggerFactory.getLogger(MetadataResultSetUtil.class); + private final JsonBlobParser jsonBlobParser; + + @Autowired + public MetadataResultSetUtil(JsonBlobParser jsonBlobParser) { + this.jsonBlobParser = jsonBlobParser; + } + + public Result mapContinuousMetadata(ResultSet rs) throws SQLException { + String hashedVarId = hashVarId(rs.getString("conceptPath")); + String description = getDescription(rs); + String parentName = getParentName(rs); + String parentDisplay = getParentDisplay(rs); + + String max = String.valueOf(jsonBlobParser.parseMax(rs.getString("values"))); + String min = String.valueOf(jsonBlobParser.parseMin(rs.getString("values"))); + + ContinuousMetadata metadata = new ContinuousMetadata( + rs.getString("stigmatized"), rs.getString("display"), description, min, rs.getString("conceptPath"), parentName, + rs.getString("conceptPath"), rs.getString("name"), parentDisplay, description, // changed + "{}", "", parentName, max, description, rs.getString("dataset"), hashedVarId, rs.getString("conceptType"), rs.getString("name"), + rs.getString("dataset"), rs.getString("stigmatized"), rs.getString("display"), rs.getString("studyAcronym"), + rs.getString("dsFullName"), parentName, parentDisplay, rs.getString("conceptPath"), min, max + ); + return new Result( + metadata, jsonBlobParser.parseValues(rs.getString("values")), rs.getString("dataset"), parentName, rs.getString("name"), false, + true + ); + } + + + + public Result mapCategoricalMetadata(ResultSet rs) throws SQLException { + String hashedVarId = hashVarId(rs.getString("conceptPath")); + String description = getDescription(rs); + String parentName = getParentName(rs); + String parentDisplay = getParentDisplay(rs); + + CategoricalMetadata metadata = new CategoricalMetadata( + rs.getString("stigmatized"), rs.getString("display"), description, "", rs.getString("conceptPath"), parentName, + rs.getString("conceptPath"), rs.getString("name"), parentDisplay, description, // changed + "{}", "", parentName, "", description, rs.getString("dataset"), hashedVarId, rs.getString("conceptType"), rs.getString("name"), + rs.getString("dataset"), rs.getString("stigmatized"), rs.getString("display"), rs.getString("studyAcronym"), + rs.getString("dsFullName"), parentName, parentDisplay, rs.getString("conceptPath") + ); + + return new Result( + metadata, jsonBlobParser.parseValues(rs.getString("values")), rs.getString("dataset"), parentName, rs.getString("name"), true, + false + ); + } + + private static String hashVarId(String hpdsPath) { + String hashedVarId = ""; + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] encodedHash = digest.digest(hpdsPath.getBytes(StandardCharsets.UTF_8)); + hashedVarId = bytesToHex(encodedHash); + } catch (NoSuchAlgorithmException e) { + log.error(e.getMessage()); + } + + return hashedVarId; + } + + private static String bytesToHex(byte[] hash) { + StringBuilder hexString = new StringBuilder(2 * hash.length); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } + + private static String getParentDisplay(ResultSet rs) throws SQLException { + return rs.getString("parentDisplay") == null || rs.getString("parentDisplay").isBlank() ? "" : rs.getString("parentDisplay"); + } + + private static String getParentName(ResultSet rs) throws SQLException { + return rs.getString("parentName") == null || rs.getString("parentName").isBlank() ? "All Variables" : rs.getString("parentName"); + } + + private static String getDescription(ResultSet rs) throws SQLException { + return rs.getString("description") == null || rs.getString("description").isBlank() ? "" : rs.getString("description"); + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java new file mode 100644 index 0000000..cb51309 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java @@ -0,0 +1,38 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Result; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; + +@Component +public class SearchResultRowMapper implements RowMapper { + + private final MetadataResultSetUtil metadataResultSetUtil; + + @Autowired + public SearchResultRowMapper(MetadataResultSetUtil metadataResultSetUtil) { + this.metadataResultSetUtil = metadataResultSetUtil; + } + + @Override + public SearchResult mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapSearchResults(rs); + } + + private SearchResult mapSearchResults(ResultSet rs) throws SQLException { + Result result = switch (ConceptType.toConcept(rs.getString("conceptType"))) { + case Categorical -> this.metadataResultSetUtil.mapCategoricalMetadata(rs); + case Continuous -> this.metadataResultSetUtil.mapContinuousMetadata(rs); + }; + + return new SearchResult(result); + } + + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java new file mode 100644 index 0000000..79de460 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java @@ -0,0 +1,23 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record CategoricalMetadata( + @JsonProperty("columnmeta_is_stigmatized") String columnmetaIsStigmatized, @JsonProperty("columnmeta_name") String columnmetaName, + @JsonProperty("description") String description, @JsonProperty("columnmeta_min") String columnmetaMin, + @JsonProperty("HPDS_PATH") String hpdsPath, @JsonProperty("derived_group_id") String derivedGroupId, + @JsonProperty("columnmeta_hpds_path") String columnmetaHpdsPath, @JsonProperty("columnmeta_var_id") String columnmetaVarId, + @JsonProperty("columnmeta_var_group_description") String columnmetaVarGroupDescription, + @JsonProperty("derived_var_description") String derivedVarDescription, + @JsonProperty("derived_variable_level_data") String derivedVariableLevelData, @JsonProperty("data_hierarchy") String dataHierarchy, + @JsonProperty("derived_group_description") String derivedGroupDescription, @JsonProperty("columnmeta_max") String columnmetaMax, + @JsonProperty("columnmeta_description") String columnmetaDescription, @JsonProperty("derived_study_id") String derivedStudyId, + @JsonProperty("hashed_var_id") String hashedVarId, @JsonProperty("columnmeta_data_type") String columnmetaDataType, + @JsonProperty("derived_var_id") String derivedVarId, @JsonProperty("columnmeta_study_id") String columnmetaStudyId, + @JsonProperty("is_stigmatized") String isStigmatized, @JsonProperty("derived_var_name") String derivedVarName, + @JsonProperty("derived_study_abv_name") String derivedStudyAbvName, + @JsonProperty("derived_study_description") String derivedStudyDescription, + @JsonProperty("columnmeta_var_group_id") String columnmetaVarGroupId, @JsonProperty("derived_group_name") String derivedGroupName, + @JsonProperty("columnmeta_HPDS_PATH") String columnmetaHpdsPathAlternate +) implements Metadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java new file mode 100644 index 0000000..a067194 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java @@ -0,0 +1,24 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ContinuousMetadata( + @JsonProperty("columnmeta_is_stigmatized") String columnmetaIsStigmatized, @JsonProperty("columnmeta_name") String columnmetaName, + @JsonProperty("description") String description, @JsonProperty("columnmeta_min") String columnmetaMin, + @JsonProperty("HPDS_PATH") String hpdsPath, @JsonProperty("derived_group_id") String derivedGroupId, + @JsonProperty("columnmeta_hpds_path") String columnmetaHpdsPath, @JsonProperty("columnmeta_var_id") String columnmetaVarId, + @JsonProperty("columnmeta_var_group_description") String columnmetaVarGroupDescription, + @JsonProperty("derived_var_description") String derivedVarDescription, + @JsonProperty("derived_variable_level_data") String derivedVariableLevelData, @JsonProperty("data_hierarchy") String dataHierarchy, + @JsonProperty("derived_group_description") String derivedGroupDescription, @JsonProperty("columnmeta_max") String columnmetaMax, + @JsonProperty("columnmeta_description") String columnmetaDescription, @JsonProperty("derived_study_id") String derivedStudyId, + @JsonProperty("hashed_var_id") String hashedVarId, @JsonProperty("columnmeta_data_type") String columnmetaDataType, + @JsonProperty("derived_var_id") String derivedVarId, @JsonProperty("columnmeta_study_id") String columnmetaStudyId, + @JsonProperty("is_stigmatized") String isStigmatized, @JsonProperty("derived_var_name") String derivedVarName, + @JsonProperty("derived_study_abv_name") String derivedStudyAbvName, + @JsonProperty("derived_study_description") String derivedStudyDescription, + @JsonProperty("columnmeta_var_group_id") String columnmetaVarGroupId, @JsonProperty("derived_group_name") String derivedGroupName, + @JsonProperty("columnmeta_HPDS_PATH") String columnmetaHpdsPathAlternate, @JsonProperty("min") String min, + @JsonProperty("max") String max +) implements Metadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java new file mode 100644 index 0000000..7655c6c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java @@ -0,0 +1,6 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record LegacyResponse(@JsonProperty("results") Results results) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java new file mode 100644 index 0000000..1147a85 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java @@ -0,0 +1,7 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import org.springframework.data.domain.Pageable; + +public record LegacySearchQuery(Filter filter, Pageable pageable) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java new file mode 100644 index 0000000..d3d40d4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java @@ -0,0 +1,4 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +public sealed interface Metadata permits ContinuousMetadata, CategoricalMetadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java new file mode 100644 index 0000000..ba3a51d --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java @@ -0,0 +1,12 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record Result( + Metadata metadata, List values, @JsonProperty("studyId") String studyId, @JsonProperty("dtId") String dtId, + @JsonProperty("varId") String varId, @JsonProperty("is_categorical") boolean isCategorical, + @JsonProperty("is_continuous") boolean isContinuous +) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java new file mode 100644 index 0000000..fa37331 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java @@ -0,0 +1,8 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record Results(@JsonProperty("searchResults") List searchResults) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java new file mode 100644 index 0000000..d6a7f53 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java @@ -0,0 +1,6 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SearchResult(@JsonProperty("result") Result result) { +} diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java index f371012..ee55640 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java @@ -6,6 +6,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.facet.Facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -323,4 +324,37 @@ void shouldGetContConceptWithDecimalNotation() { Assertions.assertEquals(0.57f, concept.min()); Assertions.assertEquals(6.77f, concept.max()); } + + @Test + void shouldGetLegacySearchResults() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(30, searchResults.size()); + } + + @Test + void shouldGetLegacySearchResultsBySearch() { + List searchResults = + subject.getLegacySearchResults(new Filter(List.of(), "phs000007", List.of()), Pageable.unpaged()); + + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + + } + + @Test + void shouldGetLegacySearchResultsByPageSize() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.ofSize(5)); + + Assertions.assertEquals(5, searchResults.size()); + } + + @Test + void legacySearchResultShouldGetEqualCountToConceptSearch() { + // This test will ensure modifications made to the conceptSearch will be reflected in the legacy search result. + // They use near equivalent queries and updates made to one should be made to the other. + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + List concepts = subject.getConcepts(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(searchResults.size(), concepts.size()); + } } From 0545f11d0be4416447a036f426b9f22dfa1d1208 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 08:20:28 -0500 Subject: [PATCH 06/10] Add unit and integration tests for LegacySearch Introduces unit tests for `LegacySearchQueryMapper` to validate JSON parsing and string replacement functionality. Adds integration tests for `LegacySearchController` to verify search response correctness using a PostgreSQL container. --- ...LegacySearchControllerIntegrationTest.java | 57 +++++++++++++++++++ .../LegacySearchQueryMapperTest.java | 50 ++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java create mode 100644 src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java new file mode 100644 index 0000000..d68a4f4 --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java @@ -0,0 +1,57 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacyResponse; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.io.IOException; +import java.util.List; + +@SpringBootTest +@Testcontainers +class LegacySearchControllerIntegrationTest { + + @Autowired + LegacySearchController legacySearchController; + + @Container + static final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>("postgres:16").withReuse(true) + .withCopyFileToContainer(MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql"); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + @Test + void shouldGetLegacyResponseByStudyID() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"phs000007","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + ResponseEntity legacyResponseResponseEntity = legacySearchController.legacySearch(jsonString); + System.out.println(legacyResponseResponseEntity); + Assertions.assertEquals(HttpStatus.OK, legacyResponseResponseEntity.getStatusCode()); + LegacyResponse legacyResponseBody = legacyResponseResponseEntity.getBody(); + Assertions.assertNotNull(legacyResponseBody); + Results results = legacyResponseBody.results(); + List searchResults = results.searchResults(); + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + } + +} diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java new file mode 100644 index 0000000..e0661ab --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java @@ -0,0 +1,50 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.ActiveProfiles; + +import java.io.IOException; + +@SpringBootTest +@ActiveProfiles("test") +class LegacySearchQueryMapperTest { + + @Autowired + LegacySearchQueryMapper legacySearchQueryMapper; + + @Test + void shouldParseSearchRequest() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"age","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("age", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + + @Test + void shouldReplaceUnderscore() throws IOException { + String jsonString = + """ + {"query":{"searchTerm":"tutorial-biolincc_digitalis","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("tutorial-biolincc/digitalis", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + +} From 0d25f3e60630522e34fb0d64f86a3c73b0f74661 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 08:20:36 -0500 Subject: [PATCH 07/10] Add initial configuration and sample data files Introduce application properties for database configuration and dashboard settings. Add Docker commands for local development with weights configuration. Include weights.csv as sample data and dictonaryReqeust.http for testing API requests. --- dictionaryweights/README.md | 11 +++++++++++ .../main/resources/application-bdc-dev.properties | 9 +++++++++ dictionaryweights/weights.csv | 7 +++++++ dictonaryReqeust.http | 15 +++++++++++++++ src/main/resources/application-bdc-dev.properties | 14 ++++++++++++++ 5 files changed, 56 insertions(+) create mode 100644 dictionaryweights/README.md create mode 100644 dictionaryweights/src/main/resources/application-bdc-dev.properties create mode 100644 dictionaryweights/weights.csv create mode 100644 dictonaryReqeust.http create mode 100644 src/main/resources/application-bdc-dev.properties diff --git a/dictionaryweights/README.md b/dictionaryweights/README.md new file mode 100644 index 0000000..39e6352 --- /dev/null +++ b/dictionaryweights/README.md @@ -0,0 +1,11 @@ +## Docker commands for local development +### Docker build +```bash +docker build --no-cache --build-arg SPRING_PROFILE=bdc-dev -t weights:latest . +``` + +### Docker run +You will need a local weights.csv file. +```bash + docker run --rm -t --name dictionary-weights --network=host -v ./weights.csv:/weights.csv weights:latest +``` \ No newline at end of file diff --git a/dictionaryweights/src/main/resources/application-bdc-dev.properties b/dictionaryweights/src/main/resources/application-bdc-dev.properties new file mode 100644 index 0000000..5b79313 --- /dev/null +++ b/dictionaryweights/src/main/resources/application-bdc-dev.properties @@ -0,0 +1,9 @@ +spring.application.name=dictionaryweights +spring.main.web-application-type=none + +spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict +spring.datasource.username=username +spring.datasource.password=password +spring.datasource.driver-class-name=org.postgresql.Driver + +weights.filename=/weights.csv \ No newline at end of file diff --git a/dictionaryweights/weights.csv b/dictionaryweights/weights.csv new file mode 100644 index 0000000..d3cc913 --- /dev/null +++ b/dictionaryweights/weights.csv @@ -0,0 +1,7 @@ +concept_node.DISPLAY,2 +concept_node.CONCEPT_PATH,2 +dataset.FULL_NAME,1 +dataset.DESCRIPTION,1 +parent.DISPLAY,1 +grandparent.DISPLAY,1 +concept_node_meta_str,1 \ No newline at end of file diff --git a/dictonaryReqeust.http b/dictonaryReqeust.http new file mode 100644 index 0000000..e474278 --- /dev/null +++ b/dictonaryReqeust.http @@ -0,0 +1,15 @@ +# curl 'https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/picsure/proxy/dictionary-api/concepts?page_number=1&page_size=1' +# -H 'origin: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov' +# -H 'referer: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/' +# --data-raw '{"facets":[],"search":"","consents":[]}' +POST http://localhost:80/concepts?page_number=0&page_size=100 +Content-Type: application/json + +{"facets":[],"search":"lipid triglyceride"} + +### + +POST http://localhost:80/search +Content-Type: application/json + +{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"breast","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":10000000},"resourceUUID":null} \ No newline at end of file diff --git a/src/main/resources/application-bdc-dev.properties b/src/main/resources/application-bdc-dev.properties new file mode 100644 index 0000000..219dfbb --- /dev/null +++ b/src/main/resources/application-bdc-dev.properties @@ -0,0 +1,14 @@ +spring.application.name=dictionary +spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict +spring.datasource.username=username +spring.datasource.password=password +spring.datasource.driver-class-name=org.postgresql.Driver +server.port=80 + +dashboard.columns={abbreviation:'Abbreviation',name:'Name',clinvars:'Clinical Variables'} +dashboard.column-order=abbreviation,name,clinvars +dashboard.nonmeta-columns=abbreviation,name +dashboard.enable.extra_details=true +dashboard.enable.bdc_hack=true + +filtering.unfilterable_concepts=stigmatized \ No newline at end of file From edcd62968b2210afa1fa6d0e45a9ae42da0aa342 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 16:11:04 -0500 Subject: [PATCH 08/10] Add LegacySearchRepository and refactor search queries Introduced `LegacySearchRepository` to handle legacy search functionalities. Consolidated query logic by moving `ALLOW_FILTERING_Q` to `QueryUtility`. Removed legacy search code from `ConceptRepository` for better separation of concerns. --- .../dictionary/concept/ConceptRepository.java | 64 +-------------- .../legacysearch/LegacySearchRepository.java | 79 +++++++++++++++++++ .../dictionary/util/QueryUtility.java | 19 +++++ 3 files changed, 101 insertions(+), 61 deletions(-) create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index eb08225..1b2bd8b 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -4,7 +4,6 @@ import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper; -import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -17,33 +16,18 @@ import java.util.Map; import java.util.Optional; +import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + @Repository public class ConceptRepository { - private static final String ALLOW_FILTERING_Q = """ - WITH allow_filtering AS ( - SELECT - concept_node.concept_node_id AS concept_node_id, - (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering - FROM - concept_node - JOIN concept_node_meta ON - concept_node.concept_node_id = concept_node_meta.concept_node_id - AND concept_node_meta.KEY IN (:disallowed_meta_keys) - GROUP BY - concept_node.concept_node_id - ) - """; - private final NamedParameterJdbcTemplate template; private final ConceptRowMapper mapper; - private final SearchResultRowMapper searchResultRowMapper; private final ConceptFilterQueryGenerator filterGen; private final ConceptMetaExtractor conceptMetaExtractor; private final ConceptResultSetExtractor conceptResultSetExtractor; private final List disallowedMetaFields; - @Autowired public ConceptRepository( NamedParameterJdbcTemplate template, ConceptRowMapper mapper, SearchResultRowMapper searchResultRowMapper, @@ -52,7 +36,6 @@ public ConceptRepository( ) { this.template = template; this.mapper = mapper; - this.searchResultRowMapper = searchResultRowMapper; this.filterGen = filterGen; this.conceptMetaExtractor = conceptMetaExtractor; this.conceptResultSetExtractor = conceptResultSetExtractor; @@ -116,8 +99,7 @@ public Optional getConcept(String dataset, String conceptPath) { LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id WHERE - concept_node.concept_path = :conceptPath - AND ds.REF = :dataset + concept_node.concept_path IN :conceptPaths """; MapSqlParameterSource params = new MapSqlParameterSource().addValue("conceptPath", conceptPath).addValue("dataset", dataset) .addValue("disallowed_meta_keys", disallowedMetaFields); @@ -245,45 +227,5 @@ WITH RECURSIVE nodes AS ( } - public List getLegacySearchResults(Filter filter, Pageable pageable) { - QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable); - String sql = ALLOW_FILTERING_Q + ", " + filterQ.query() + """ - SELECT concept_node.concept_path AS conceptPath, - concept_node.display AS display, - concept_node.name AS name, - concept_node.concept_type AS conceptType, - ds.REF as dataset, - ds.abbreviation AS studyAcronym, - ds.full_name as dsFullName, - continuous_min.VALUE as min, - continuous_max.VALUE as max, - categorical_values.VALUE as values, - allow_filtering.allowFiltering AS allowFiltering, - meta_description.VALUE AS description, - stigmatized.value AS stigmatized, - parent.name AS parentName, - parent.display AS parentDisplay - FROM concept_node - INNER JOIN concepts_filtered_sorted ON concepts_filtered_sorted.concept_node_id = concept_node.concept_node_id - LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id - LEFT JOIN concept_node_meta AS meta_description - ON concept_node.concept_node_id = meta_description.concept_node_id AND - meta_description.KEY = 'description' - LEFT JOIN concept_node_meta AS continuous_min - ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' - LEFT JOIN concept_node_meta AS continuous_max - ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' - LEFT JOIN concept_node_meta AS categorical_values - ON concept_node.concept_node_id = categorical_values.concept_node_id AND - categorical_values.KEY = 'values' - LEFT JOIN concept_node_meta AS stigmatized ON concept_node.concept_node_id = stigmatized.concept_node_id AND - stigmatized.KEY = 'stigmatized' - LEFT JOIN concept_node AS parent ON parent.concept_node_id = concept_node.parent_id - LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id - ORDER BY concepts_filtered_sorted.rank DESC, concept_node.concept_node_id ASC - """; - MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields); - return template.query(sql, params, searchResultRowMapper); - } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java new file mode 100644 index 0000000..9ea09c4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java @@ -0,0 +1,79 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptFilterQueryGenerator; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.util.List; + +import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + +@Repository +public class LegacySearchRepository { + + private final ConceptFilterQueryGenerator filterGen; + private final NamedParameterJdbcTemplate template; + private final List disallowedMetaFields; + private final SearchResultRowMapper searchResultRowMapper; + + @Autowired + public LegacySearchRepository( + ConceptFilterQueryGenerator filterGen, NamedParameterJdbcTemplate template, + @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields, SearchResultRowMapper searchResultRowMapper + ) { + this.filterGen = filterGen; + this.template = template; + this.disallowedMetaFields = disallowedMetaFields; + this.searchResultRowMapper = searchResultRowMapper; + } + + public List getLegacySearchResults(Filter filter, Pageable pageable) { + QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable); + String sql = ALLOW_FILTERING_Q + ", " + filterQ.query() + """ + SELECT concept_node.concept_path AS conceptPath, + concept_node.display AS display, + concept_node.name AS name, + concept_node.concept_type AS conceptType, + ds.REF as dataset, + ds.abbreviation AS studyAcronym, + ds.full_name as dsFullName, + continuous_min.VALUE as min, + continuous_max.VALUE as max, + categorical_values.VALUE as values, + allow_filtering.allowFiltering AS allowFiltering, + meta_description.VALUE AS description, + stigmatized.value AS stigmatized, + parent.name AS parentName, + parent.display AS parentDisplay + FROM concept_node + INNER JOIN concepts_filtered_sorted ON concepts_filtered_sorted.concept_node_id = concept_node.concept_node_id + LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id + LEFT JOIN concept_node_meta AS meta_description + ON concept_node.concept_node_id = meta_description.concept_node_id AND + meta_description.KEY = 'description' + LEFT JOIN concept_node_meta AS continuous_min + ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max + ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values + ON concept_node.concept_node_id = categorical_values.concept_node_id AND + categorical_values.KEY = 'values' + LEFT JOIN concept_node_meta AS stigmatized ON concept_node.concept_node_id = stigmatized.concept_node_id AND + stigmatized.KEY = 'stigmatized' + LEFT JOIN concept_node AS parent ON parent.concept_node_id = concept_node.parent_id + LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id + ORDER BY concepts_filtered_sorted.rank DESC, concept_node.concept_node_id ASC + """; + MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields); + + return template.query(sql, params, searchResultRowMapper); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java new file mode 100644 index 0000000..69eedc4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java @@ -0,0 +1,19 @@ +package edu.harvard.dbmi.avillach.dictionary.util; + +public class QueryUtility { + + public static final String ALLOW_FILTERING_Q = """ + WITH allow_filtering AS ( + SELECT + concept_node.concept_node_id AS concept_node_id, + (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering + FROM + concept_node + JOIN concept_node_meta ON + concept_node.concept_node_id = concept_node_meta.concept_node_id + AND concept_node_meta.KEY IN (:disallowed_meta_keys) + GROUP BY + concept_node.concept_node_id + ) + """; +} From f7a47dd4e3ecf6b561ad03044cf9ae3a079d6af5 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Fri, 15 Nov 2024 16:12:46 -0500 Subject: [PATCH 09/10] Add FilterProcessor to handle filter processing logic Introduced a new FilterProcessor component to centralize filter processing logic. Enhanced getDescription, getParentName, and getParentDisplay methods in MetadataResultSetUtil for better validation using StringUtils. --- .../dictionary/facet/FilterPreProcessor.java | 35 +++++++------------ .../dictionary/filter/FilterProcessor.java | 34 ++++++++++++++++++ .../legacysearch/LegacySearchQueryMapper.java | 9 +++-- .../legacysearch/MetadataResultSetUtil.java | 15 ++++---- 4 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java index 16510f1..3c6eda1 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java @@ -1,22 +1,28 @@ package edu.harvard.dbmi.avillach.dictionary.facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.MethodParameter; import org.springframework.http.HttpInputMessage; import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.RequestBodyAdvice; import java.io.IOException; import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.function.Function; @ControllerAdvice public class FilterPreProcessor implements RequestBodyAdvice { + + private final FilterProcessor filterProcessor; + + @Autowired + public FilterPreProcessor(FilterProcessor filterProcessor) { + this.filterProcessor = filterProcessor; + } + + @Override public boolean supports(MethodParameter methodParameter, Type targetType, Class> converterType) { return true; @@ -35,29 +41,12 @@ public Object afterBodyRead( Class> converterType ) { if (body instanceof Filter filter) { - return processsFilter(filter); + return filterProcessor.processsFilter(filter); } return body; } - public static Filter processsFilter(Filter filter) { - List newFacets = filter.facets(); - List newConsents = filter.consents(); - if (filter.facets() != null) { - newFacets = new ArrayList<>(filter.facets()); - newFacets.sort(Comparator.comparing(Facet::name)); - } - if (filter.consents() != null) { - newConsents = new ArrayList<>(newConsents); - newConsents.sort(Comparator.comparing(Function.identity())); - } - filter = new Filter(newFacets, filter.search(), newConsents); - if (StringUtils.hasLength(filter.search())) { - filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); - } - return filter; - } @Override public Object handleEmptyBody( diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java new file mode 100644 index 0000000..f2478ae --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java @@ -0,0 +1,34 @@ +package edu.harvard.dbmi.avillach.dictionary.filter; + +import edu.harvard.dbmi.avillach.dictionary.facet.Facet; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; + +@Component +public class FilterProcessor { + + public Filter processsFilter(Filter filter) { + List newFacets = filter.facets(); + List newConsents = filter.consents(); + if (filter.facets() != null) { + newFacets = new ArrayList<>(filter.facets()); + newFacets.sort(Comparator.comparing(Facet::name)); + } + if (filter.consents() != null) { + newConsents = new ArrayList<>(newConsents); + newConsents.sort(Comparator.comparing(Function.identity())); + } + filter = new Filter(newFacets, filter.search(), newConsents); + + if (StringUtils.hasLength(filter.search())) { + filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); + } + return filter; + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java index aed5eb6..001436c 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java @@ -2,8 +2,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import edu.harvard.dbmi.avillach.dictionary.facet.FilterPreProcessor; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; @@ -15,6 +15,11 @@ public class LegacySearchQueryMapper { private static final ObjectMapper objectMapper = new ObjectMapper(); + private final FilterProcessor filterProcessor; + + public LegacySearchQueryMapper(FilterProcessor filterProcessor) { + this.filterProcessor = filterProcessor; + } public LegacySearchQuery mapFromJson(String jsonString) throws IOException { JsonNode rootNode = objectMapper.readTree(jsonString); @@ -22,7 +27,7 @@ public LegacySearchQuery mapFromJson(String jsonString) throws IOException { String searchTerm = queryNode.get("searchTerm").asText(); int limit = queryNode.get("limit").asInt(); - Filter filter = FilterPreProcessor.processsFilter(new Filter(List.of(), searchTerm, List.of())); + Filter filter = filterProcessor.processsFilter(new Filter(List.of(), searchTerm, List.of())); return new LegacySearchQuery(filter, PageRequest.of(0, limit)); } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java index 8348f37..bc8eea2 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java @@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -49,8 +50,6 @@ public Result mapContinuousMetadata(ResultSet rs) throws SQLException { ); } - - public Result mapCategoricalMetadata(ResultSet rs) throws SQLException { String hashedVarId = hashVarId(rs.getString("conceptPath")); String description = getDescription(rs); @@ -96,15 +95,15 @@ private static String bytesToHex(byte[] hash) { return hexString.toString(); } - private static String getParentDisplay(ResultSet rs) throws SQLException { - return rs.getString("parentDisplay") == null || rs.getString("parentDisplay").isBlank() ? "" : rs.getString("parentDisplay"); + private String getParentDisplay(ResultSet rs) throws SQLException { + return StringUtils.hasLength("parentDisplay") ? "" : rs.getString("parentDisplay"); } - private static String getParentName(ResultSet rs) throws SQLException { - return rs.getString("parentName") == null || rs.getString("parentName").isBlank() ? "All Variables" : rs.getString("parentName"); + private String getParentName(ResultSet rs) throws SQLException { + return StringUtils.hasLength(rs.getString("parentName")) ? "All Variables" : rs.getString("parentName"); } - private static String getDescription(ResultSet rs) throws SQLException { - return rs.getString("description") == null || rs.getString("description").isBlank() ? "" : rs.getString("description"); + private String getDescription(ResultSet rs) throws SQLException { + return StringUtils.hasLength(rs.getString("description")) ? "" : rs.getString("description"); } } From 2a7edaa5c34cd5ff092b38e1bb2670a240ca2f12 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Mon, 18 Nov 2024 09:29:33 -0500 Subject: [PATCH 10/10] Add LegacySearchRepositoryTest and refactor legacy search logic Introduced LegacySearchRepositoryTest to verify legacy search functionalities. Refactored legacy search logic out of ConceptService and into LegacySearchRepository. Updated related tests and cleaned up unused imports and methods. --- .../dictionary/concept/ConceptRepository.java | 10 ++- .../dictionary/concept/ConceptService.java | 5 -- .../legacysearch/LegacySearchService.java | 9 +-- .../concept/ConceptRepositoryTest.java | 32 -------- .../LegacySearchRepositoryTest.java | 76 +++++++++++++++++++ 5 files changed, 86 insertions(+), 46 deletions(-) create mode 100644 src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index 1b2bd8b..7f449bb 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -18,6 +18,7 @@ import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + @Repository public class ConceptRepository { @@ -30,9 +31,9 @@ public class ConceptRepository { @Autowired public ConceptRepository( - NamedParameterJdbcTemplate template, ConceptRowMapper mapper, SearchResultRowMapper searchResultRowMapper, - ConceptFilterQueryGenerator filterGen, ConceptMetaExtractor conceptMetaExtractor, - ConceptResultSetExtractor conceptResultSetExtractor, @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields + NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen, + ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor, + @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields ) { this.template = template; this.mapper = mapper; @@ -99,7 +100,8 @@ public Optional getConcept(String dataset, String conceptPath) { LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id WHERE - concept_node.concept_path IN :conceptPaths + concept_node.concept_path = :conceptPath + AND ds.REF = :dataset """; MapSqlParameterSource params = new MapSqlParameterSource().addValue("conceptPath", conceptPath).addValue("dataset", dataset) .addValue("disallowed_meta_keys", disallowedMetaFields); diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java index a24500a..dc35254 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java @@ -5,7 +5,6 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; -import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.data.domain.Pageable; @@ -72,8 +71,4 @@ public Optional conceptDetailWithoutAncestors(String dataset, String co return getConcept(dataset, conceptPath, false); } - public List getLegacySearchResults(Filter filter, Pageable pageable) { - return conceptRepository.getLegacySearchResults(filter, pageable); - } - } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java index 03d4d01..e7462c2 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java @@ -1,6 +1,5 @@ package edu.harvard.dbmi.avillach.dictionary.legacysearch; -import edu.harvard.dbmi.avillach.dictionary.concept.ConceptService; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; import org.springframework.beans.factory.annotation.Autowired; @@ -10,15 +9,15 @@ @Service public class LegacySearchService { - private final ConceptService conceptService; + private final LegacySearchRepository legacySearchRepository; @Autowired - public LegacySearchService(ConceptService conceptService) { - this.conceptService = conceptService; + public LegacySearchService(LegacySearchRepository legacySearchRepository) { + this.legacySearchRepository = legacySearchRepository; } public Results getSearchResults(Filter filter, Pageable pageable) { - return new Results(conceptService.getLegacySearchResults(filter, pageable)); + return new Results(legacySearchRepository.getLegacySearchResults(filter, pageable)); } } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java index ee55640..8d20516 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java @@ -6,7 +6,6 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.facet.Facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; -import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -325,36 +324,5 @@ void shouldGetContConceptWithDecimalNotation() { Assertions.assertEquals(6.77f, concept.max()); } - @Test - void shouldGetLegacySearchResults() { - List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); - - Assertions.assertEquals(30, searchResults.size()); - } - - @Test - void shouldGetLegacySearchResultsBySearch() { - List searchResults = - subject.getLegacySearchResults(new Filter(List.of(), "phs000007", List.of()), Pageable.unpaged()); - - searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); - - } - @Test - void shouldGetLegacySearchResultsByPageSize() { - List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.ofSize(5)); - - Assertions.assertEquals(5, searchResults.size()); - } - - @Test - void legacySearchResultShouldGetEqualCountToConceptSearch() { - // This test will ensure modifications made to the conceptSearch will be reflected in the legacy search result. - // They use near equivalent queries and updates made to one should be made to the other. - List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); - List concepts = subject.getConcepts(new Filter(List.of(), "", List.of()), Pageable.unpaged()); - - Assertions.assertEquals(searchResults.size(), concepts.size()); - } } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java new file mode 100644 index 0000000..9bb6bab --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java @@ -0,0 +1,76 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptRepository; +import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.util.List; + +@SpringBootTest +@Testcontainers +public class LegacySearchRepositoryTest { + + @Autowired + LegacySearchRepository subject; + + @Autowired + ConceptRepository conceptService; + + @Container + static final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>("postgres:16").withReuse(true) + .withCopyFileToContainer(MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql"); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + @Test + void shouldGetLegacySearchResults() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(30, searchResults.size()); + } + + @Test + void shouldGetLegacySearchResultsBySearch() { + List searchResults = + subject.getLegacySearchResults(new Filter(List.of(), "phs000007", List.of()), Pageable.unpaged()); + + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + + } + + @Test + void shouldGetLegacySearchResultsByPageSize() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.ofSize(5)); + + Assertions.assertEquals(5, searchResults.size()); + } + + @Test + void legacySearchResultShouldGetEqualCountToConceptSearch() { + // This test will ensure modifications made to the conceptSearch will be reflected in the legacy search result. + // They use near equivalent queries and updates made to one should be made to the other. + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + List concepts = conceptService.getConcepts(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(searchResults.size(), concepts.size()); + } + +}