[ALS-6276] Search improvements

hms-dbmi · Jul 13, 2024 · 2eb7a16 · 2eb7a16
1 parent f937b96
commit 2eb7a16
Show file tree

Hide file tree

Showing 19 changed files with 888 additions and 90 deletions.
diff --git a/JSONS.md b/JSONS.md
diff --git a/db/schema.sql b/db/schema.sql
@@ -37,6 +37,7 @@ CREATE TABLE dict.concept_node (
     CONCEPT_TYPE VARCHAR(32) NOT NULL DEFAULT 'Interior',
     CONCEPT_PATH VARCHAR(10000) NOT NULL DEFAULT 'INVALID',
     PARENT_ID INT,
+    SEARCHABLE_FIELDS TSVECTOR,
     PRIMARY KEY (CONCEPT_NODE_ID),
     CONSTRAINT fk_parent FOREIGN KEY (PARENT_ID) REFERENCES dict.CONCEPT_NODE(CONCEPT_NODE_ID),
     CONSTRAINT fk_study FOREIGN KEY (DATASET_ID) REFERENCES dict.dataset(DATASET_ID)

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -18,8 +18,6 @@ services:
       - dictionary-db
     restart: always
     env_file: .env
-    ports:
-      - "8080:8080"
     networks:
       - dictionary
       - hpdsNet

diff --git a/pom.xml b/pom.xml
@@ -41,6 +41,11 @@
 			<artifactId>spring-boot-starter-test</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.json</groupId>
+			<artifactId>json</artifactId>
+			<version>20240303</version>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-testcontainers</artifactId>

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java
@@ -43,20 +43,20 @@ public ResponseEntity<Page<Concept>> listConcepts(
         return ResponseEntity.ok(pageResp);
     }
 
-    @GetMapping(path = "/concepts/detail/{dataset}/{conceptPath}")
+    @PostMapping(path = "/concepts/detail/{dataset}")
     public ResponseEntity<Concept> conceptDetail(
         @PathVariable(name = "dataset") String dataset,
-        @PathVariable(name = "conceptPath") String conceptPath
+        @RequestBody() String conceptPath
     ) {
         return conceptService.conceptDetail(dataset, conceptPath)
             .map(ResponseEntity::ok)
             .orElse(ResponseEntity.notFound().build());
     }
 
-    @GetMapping(path = "/concepts/tree/{dataset}/{conceptPath}")
+    @PostMapping(path = "/concepts/tree/{dataset}")
     public ResponseEntity<Concept> conceptTree(
         @PathVariable(name = "dataset") String dataset,
-        @PathVariable(name = "conceptPath") String conceptPath,
+        @RequestBody() String conceptPath,
         @RequestParam(name = "depth", required = false, defaultValue = "2") Integer depth
     ) {
         if (depth < 0 || depth > MAX_DEPTH) {

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java
@@ -40,13 +40,15 @@ public List<Concept> getConcepts(Filter filter, Pageable pageable) {
                 concept_node.*,
                 ds.REF as dataset,
                 continuous_min.VALUE as min, continuous_max.VALUE as max,
-                categorical_values.VALUE as values
+                categorical_values.VALUE as values,
+                meta_description.VALUE AS description
             FROM
                 concept_node
                 LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id
-                LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'MIN'
-                LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'MAX'
-                LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'VALUES'
+                LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description'
+                LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
+                LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
+                LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
             WHERE concept_node.concept_node_id IN
             """;
         QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable);
@@ -69,13 +71,15 @@ public Optional<Concept> getConcept(String dataset, String conceptPath) {
                 concept_node.*,
                 ds.REF as dataset,
                 continuous_min.VALUE as min, continuous_max.VALUE as max,
-                categorical_values.VALUE as values
+                categorical_values.VALUE as values,
+                meta_description.VALUE AS description
             FROM
                 concept_node
                 LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id
-                LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'MIN'
-                LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'MAX'
-                LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'VALUES'
+                LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description'
+                LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
+                LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
+                LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
             WHERE
                 concept_node.concept_path = :conceptPath
                 AND ds.REF = :dataset

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowMapper.java
@@ -1,8 +1,11 @@
 package edu.harvard.dbmi.avillach.dictionary.concept;
 
 import edu.harvard.dbmi.avillach.dictionary.concept.model.*;
+import org.json.JSONArray;
+import org.json.JSONException;
 import org.springframework.jdbc.core.RowMapper;
 import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -14,7 +17,7 @@ public class ConceptRowMapper implements RowMapper<Concept> {
 
     @Override
     public Concept mapRow(ResultSet rs, int rowNum) throws SQLException {
-        return switch (ConceptType.valueOf(rs.getString("concept_type"))) {
+        return switch (ConceptType.toConcept(rs.getString("concept_type"))) {
             case Categorical -> mapCategorical(rs);
             case Continuous -> mapContinuous(rs);
         };
@@ -23,8 +26,8 @@ public Concept mapRow(ResultSet rs, int rowNum) throws SQLException {
     private CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
         return new CategoricalConcept(
             rs.getString("concept_path"), rs.getString("name"),
-            rs.getString("display"), rs.getString("dataset"),
-            List.of(rs.getString("values").split(",")),
+            rs.getString("display"), rs.getString("dataset"), rs.getString("description"),
+            rs.getString("values") == null ? List.of() : List.of(rs.getString("values").split(",")),
             null,
             null
         );
@@ -33,9 +36,27 @@ private CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
     private ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
         return new ContinuousConcept(
             rs.getString("concept_path"), rs.getString("name"),
-            rs.getString("display"), rs.getString("dataset"),
-            rs.getInt("min"), rs.getInt("max"),
+            rs.getString("display"), rs.getString("dataset"), rs.getString("description"),
+            parseMin(rs.getString("values")), parseMax(rs.getString("values")),
             null
         );
     }
+
+    private Integer parseMin(String valuesArr) {
+        try {
+            JSONArray arr = new JSONArray(valuesArr);
+            return arr.length() == 2 ? arr.getInt(0) : 0;
+        } catch (JSONException ex) {
+            return 0;
+        }
+    }
+
+    private Integer parseMax(String valuesArr) {
+        try {
+            JSONArray arr = new JSONArray(valuesArr);
+            return arr.length() == 2 ? arr.getInt(1) : 0;
+        } catch (JSONException ex) {
+            return 0;
+        }
+    }
 }
diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java
@@ -1,12 +1,13 @@
 package edu.harvard.dbmi.avillach.dictionary.concept.model;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import jakarta.annotation.Nullable;
 
 import java.util.List;
 import java.util.Map;
 
 public record CategoricalConcept(
-    String conceptPath, String name, String display, String dataset,
+    String conceptPath, String name, String display, String dataset, String description,
 
     List<String> values,
 
@@ -19,10 +20,11 @@ public record CategoricalConcept(
 ) implements Concept {
 
     public CategoricalConcept(CategoricalConcept core, Map<String, String> meta) {
-        this(core.conceptPath, core.name, core.display, core.dataset, core.values, core.children, core.meta);
+        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, core.children, meta);
     }
 
 
+    @JsonProperty("type")
     @Override
     public ConceptType type() {
         return ConceptType.Categorical;

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptType.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptType.java
@@ -1,5 +1,7 @@
 package edu.harvard.dbmi.avillach.dictionary.concept.model;
 
+import org.springframework.util.StringUtils;
+
 public enum ConceptType {
     /**
      * i.e. Eye color: brown, blue, hazel, etc.
@@ -10,6 +12,13 @@ public enum ConceptType {
      * i.e. Age: 0 - 150
      * Also known as numeric (to me)
      */
-    Continuous,
+    Continuous;
+
+    public static ConceptType toConcept(String in) {
+        return switch (StringUtils.capitalize(in)) {
+            case "Continuous" -> Continuous;
+            default -> Categorical;
+        };
+    }
 
 }
diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java
@@ -1,20 +1,22 @@
 package edu.harvard.dbmi.avillach.dictionary.concept.model;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import jakarta.annotation.Nullable;
 
 import java.util.Map;
 
 public record ContinuousConcept(
-    String conceptPath, String name, String display, String dataset,
+    String conceptPath, String name, String display, String dataset, String description,
 
     @Nullable Integer min, @Nullable Integer max,
     Map<String, String> meta
 ) implements Concept {
 
     public ContinuousConcept(ContinuousConcept core, Map<String, String> meta) {
-        this(core.conceptPath, core.name, core.display, core.dataset, core.min, core.max, meta);
+        this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, meta);
     }
 
+    @JsonProperty("type")
     @Override
     public ConceptType type() {
         return ConceptType.Continuous;

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java
@@ -46,7 +46,7 @@ public List<FacetCategory> getFacets(Filter filter) {
                 facet
                 LEFT JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id
                 LEFT JOIN facet as parent_facet ON facet.parent_id = parent_facet.facet_id
-                LEFT JOIN (
+                INNER JOIN (
                     SELECT
                         count(*) as facet_count, inner_facet_q.facet_id AS inner_facet_id
                     FROM

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterQueryGenerator.java
@@ -34,9 +34,23 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) {
         if (StringUtils.hasText(filter.search())) {
             clauses.add(createSearchFilter(filter.search(), params));
         }
-        if (clauses.isEmpty()) {
-            clauses = List.of("\tSELECT concept_node.concept_node_id FROM concept_node\n");
-        }
+        clauses.add("""
+            (
+                SELECT
+                    concept_node.concept_node_id, 0 as rank
+                FROM
+                    concept_node
+                    LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
+                    LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
+                    LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+                WHERE
+                    continuous_min.value <> '' OR
+                    continuous_max.value <> '' OR
+                    categorical_values.value <> ''
+            )
+            """
+        );
+
 
         String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n) ORDER BY concept_node_id\n";
         if (pageable.isPaged()) {
@@ -48,20 +62,25 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) {
                 .addValue("offset", pageable.getOffset());
         }
 
+        String superQuery = """
+            WITH q AS (%s) SELECT concept_node_id FROM q GROUP BY concept_node_id ORDER BY sum(rank) DESC"
+            """.formatted(query);
+
 
-        return new QueryParamPair(query, params);
+        return new QueryParamPair(superQuery, params);
     }
 
     private String createSearchFilter(String search, MapSqlParameterSource params) {
-        params.addValue("search", "%" + search + "%");
+        params.addValue("search", search);
         return """
             (
                 SELECT
-                    concept_node.concept_node_id AS concept_node_id
+                    concept_node.concept_node_id AS concept_node_id,
+                    ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery) as rank
                 FROM
                     concept_node
                 WHERE
-                    concept_node.concept_path LIKE :search
+                    concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
             )
             """;
     }
@@ -78,7 +97,7 @@ private List<String> createFacetFilter(List<Facet> facets, MapSqlParameterSource
                 return """
                 (
                     SELECT
-                        facet__concept_node.concept_node_id AS concept_node_id
+                        facet__concept_node.concept_node_id AS concept_node_id , 0 as rank
                     FROM facet
                         LEFT JOIN facet__concept_node ON facet__concept_node.facet_id = facet.facet_id
                         LEFT JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id

diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/MapExtractor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/MapExtractor.java
@@ -19,7 +19,7 @@ public MapExtractor(String keyName, String valueName) {
     @Override
     public Map<String, String> extractData(ResultSet rs) throws SQLException, DataAccessException {
         Map<String, String> map = new HashMap<>();
-        while (rs.next()) {
+        while (rs.next() && rs.getString(keyName) != null) {
             map.put(rs.getString(keyName), rs.getString(valueName));
         }
         return map;

diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
@@ -1,5 +1,7 @@
 spring.application.name=dictionary
-spring.datasource.url=jdbc:postgresql://${POSTGRES_HOST}:5432/${POSTGRES_DB}
+spring.datasource.url=jdbc:postgresql://${POSTGRES_HOST}:5432/${POSTGRES_DB}?currentSchema=dict
 spring.datasource.username=${POSTGRES_USER}
 spring.datasource.password=${POSTGRES_PASSWORD}
-spring.datasource.driver-class-name=org.postgresql.Driver
+spring.datasource.driver-class-name=org.postgresql.Driver
+server.port=80
+