Merge branch 'main' into ALS-7051

dictionaryweights/src/main/java/edu/harvard/dbmi/avillach/dictionaryweights/WeightUpdateCreator.java

@@ -22,7 +22,7 @@ public String createUpdate(List<Weight> weights) {
             .collect(Collectors.joining(", ' ',\n            "));
         return """
             UPDATE concept_node
-            SET SEARCHABLE_FIELDS = to_tsvector(data_table.search_str)
+            SET SEARCHABLE_FIELDS = to_tsvector(replace(data_table.search_str, '_', '/'))
             FROM
             (
                 SELECT
@@ -43,6 +43,8 @@ concept_node.concept_node_id AS id, string_agg(value, ' ') AS values
                             concept_node.concept_node_id
                     ) AS concept_node_meta_str ON concept_node_meta_str.id = concept_node.concept_node_id
                     LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+                    LEFT JOIN concept_node AS parent ON concept_node.parent_id = parent.concept_node_id
+                    LEFT JOIN concept_node AS grandparent ON parent.parent_id = grandparent.concept_node_id
             ) AS data_table
             WHERE concept_node.concept_node_id = data_table.search_key;
             """.formatted(searchableFields);

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java

@@ -11,6 +11,8 @@
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.*;
 
+import java.util.List;
+
 @Controller
 public class ConceptController {
 
@@ -43,6 +45,21 @@ public ResponseEntity<Page<Concept>> listConcepts(
         return ResponseEntity.ok(pageResp);
     }
 
+    @GetMapping(path = "/concepts/dump")
+    public ResponseEntity<Page<Concept>> dumpConcepts(
+        @RequestParam(name = "page_number", defaultValue = "0", required = false) int page,
+        @RequestParam(name = "page_size", defaultValue = "10", required = false) int size
+    ) {
+        PageRequest pagination = PageRequest.of(page, size);
+        PageImpl<Concept> pageResp = new PageImpl<>(
+            conceptService.listDetailedConcepts(new Filter(List.of(), "", List.of()), pagination),
+            pagination,
+            conceptService.countConcepts(new Filter(List.of(), "", List.of()))
+        );
+
+        return ResponseEntity.ok(pageResp);
+    }
+
     @PostMapping(path = "/concepts/detail/{dataset}")
     public ResponseEntity<Concept> conceptDetail(
         @PathVariable(name = "dataset") String dataset,

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java

@@ -15,6 +15,14 @@
 @Component
 public class ConceptFilterQueryGenerator {
 
+    private static final String CONSENT_QUERY = """
+                dataset.dataset_id IN (
+                    SELECT consent.dataset_id
+                    FROM consent
+                    WHERE consent.consent_code IN (:consents)
+                ) AND
+                """;
+
     /**
      * This generates a query that will return a list of concept_node IDs for the given filter.
      * <p>
@@ -31,12 +39,15 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) {
         MapSqlParameterSource params = new MapSqlParameterSource();
         List<String> clauses = new java.util.ArrayList<>(List.of());
         if (!CollectionUtils.isEmpty(filter.facets())) {
-            clauses.addAll(createFacetFilter(filter.facets(), params, filter.search()));
+            clauses.addAll(createFacetFilter(filter, params));
         }
         if (StringUtils.hasLength(filter.search())) {
             params.addValue("search", filter.search().trim());
         }
-        clauses.add(createValuelessNodeFilter(filter.search()));
+        if (!CollectionUtils.isEmpty(filter.consents())) {
+            params.addValue("consents", filter.consents());
+        }
+        clauses.add(createValuelessNodeFilter(filter.search(), filter.consents()));
 
 
         String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n)";
@@ -63,35 +74,39 @@ ORDER BY max(rank) DESC
         return new QueryParamPair(superQuery, params);
     }
 
-    private String createValuelessNodeFilter(String search) {
+    private String createValuelessNodeFilter(String search, List<String> consents) {
         String rankQuery = "0 as rank";
         String rankWhere = "";
         if (StringUtils.hasLength(search)) {
             rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery) as rank";
             rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND";
         }
+        String consentWhere = CollectionUtils.isEmpty(consents) ? "" : CONSENT_QUERY;
         // concept nodes that have no values and no min/max should not get returned by search
         return """
             SELECT
                 concept_node.concept_node_id,
                 %s
             FROM
                 concept_node
+                LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
                 LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
                 LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
                 LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
             WHERE
+                %s
                 %s
                 (
                     continuous_min.value <> '' OR
                     continuous_max.value <> '' OR
                     categorical_values.value <> ''
                 )
-            """.formatted(rankQuery, rankWhere);
+            """.formatted(rankQuery, rankWhere, consentWhere);
     }
 
-    private List<String> createFacetFilter(List<Facet> facets, MapSqlParameterSource params, String search) {
-        return facets.stream()
+    private List<String> createFacetFilter(Filter filter, MapSqlParameterSource params) {
+        String consentWhere = CollectionUtils.isEmpty(filter.consents()) ? "" : CONSENT_QUERY;
+        return filter.facets().stream()
             .collect(Collectors.groupingBy(Facet::category))
             .entrySet().stream()
             .map(facetsForCategory ->  {
@@ -101,7 +116,7 @@ private List<String> createFacetFilter(List<Facet> facets, MapSqlParameterSource
                     .addValue("category_%s".formatted(facetsForCategory.getKey()), facetsForCategory.getKey());
                 String rankQuery = "0";
                 String rankWhere = "";
-                if (StringUtils.hasLength(search)) {
+                if (StringUtils.hasLength(filter.search())) {
                     rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)";
                     rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND";
                 }
@@ -114,13 +129,15 @@ private List<String> createFacetFilter(List<Facet> facets, MapSqlParameterSource
                         LEFT JOIN facet__concept_node ON facet__concept_node.facet_id = facet.facet_id
                         JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id
                         JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node_id
+                        LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
                     WHERE
+                        %s
                         %s
                         facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s
                     GROUP BY
                         facet__concept_node.concept_node_id
                 )
-                """.formatted(rankQuery, rankWhere, facetsForCategory.getKey(), facetsForCategory.getKey());
+                """.formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey());
             })
             .toList();
     }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptMetaExtractor.java

@@ -0,0 +1,29 @@
+package edu.harvard.dbmi.avillach.dictionary.concept;
+
+import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.stereotype.Component;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+public class ConceptMetaExtractor implements ResultSetExtractor<Map<Concept, Map<String, String>>> {
+
+    @Override
+    public Map<Concept, Map<String, String>> extractData(ResultSet rs) throws SQLException, DataAccessException {
+        Map<Concept, Map<String, String>> sets = new HashMap<>();
+        while (rs.next()) {
+            Concept key = new ConceptShell(rs.getString("concept_path"), rs.getString("dataset_name"));
+            Map<String, String> meta = sets.getOrDefault(key, new HashMap<>());
+            meta.put(rs.getString("KEY"), rs.getString("VALUE"));
+            sets.put(key, meta);
+        }
+        return sets;
+    }
+}

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java

@@ -22,14 +22,20 @@ public class ConceptRepository {
     private final ConceptRowMapper mapper;
 
     private final ConceptFilterQueryGenerator filterGen;
+    private final ConceptMetaExtractor conceptMetaExtractor;
+    private final ConceptResultSetExtractor conceptResultSetExtractor;
+
 
     @Autowired
     public ConceptRepository(
-        NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen
+        NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
+        ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor
     ) {
         this.template = template;
         this.mapper = mapper;
         this.filterGen = filterGen;
+        this.conceptMetaExtractor = conceptMetaExtractor;
+        this.conceptResultSetExtractor = conceptResultSetExtractor;
     }
 
 
@@ -107,4 +113,109 @@ public Map<String, String> getConceptMeta(String dataset, String conceptPath) {
             .addValue("dataset", dataset);
         return template.query(sql, params, new MapExtractor("KEY", "VALUE"));
     }
+
+    public Map<Concept, Map<String, String>> getConceptMetaForConcepts(List<Concept> concepts) {
+        String sql = """
+            SELECT
+                concept_node_meta.KEY, concept_node_meta.VALUE,
+                concept_node.CONCEPT_PATH AS concept_path, dataset.REF AS dataset_name
+            FROM
+                concept_node
+                LEFT JOIN concept_node_meta ON concept_node.concept_node_id = concept_node_meta.concept_node_id
+                LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+            WHERE
+                (concept_node.CONCEPT_PATH, dataset.REF) IN (:pairs)
+            ORDER BY concept_node.CONCEPT_PATH, dataset.REF
+            """;
+        List<String[]> pairs = concepts.stream()
+            .map(c -> new String[]{c.conceptPath(), c.dataset()})
+            .toList();
+        MapSqlParameterSource params = new MapSqlParameterSource().addValue("pairs", pairs);
+
+        return template.query(sql, params, conceptMetaExtractor);
+
+    }
+
+    public Optional<Concept> getConceptTree(String dataset, String conceptPath, int depth) {
+        String sql = """
+                WITH core_query AS (
+                    WITH RECURSIVE nodes AS (
+                        SELECT
+                            concept_node_id, parent_id, 0 AS depth
+                        FROM
+                            concept_node
+                            LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+                        WHERE
+                            concept_node.CONCEPT_PATH = :path
+                            AND dataset.REF = :dataset
+                    UNION
+                        SELECT
+                            child_nodes.concept_node_id, child_nodes.parent_id, parent_node.depth+ 1
+                        FROM
+                            concept_node child_nodes
+                            INNER JOIN nodes parent_node ON child_nodes.parent_id = parent_node.concept_node_id
+                            LEFT JOIN dataset ON child_nodes.dataset_id = dataset.dataset_id
+                    )
+                    SELECT
+                        depth, child_nodes.concept_node_id
+                    FROM
+                        nodes parent_node
+                        INNER JOIN concept_node child_nodes ON child_nodes.parent_id = parent_node.concept_node_id
+                    WHERE
+                        depth < :depth
+                    UNION
+                    SELECT
+                        0 as depth, concept_node.concept_node_id
+                    FROM
+                        concept_node
+                        LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+                    WHERE
+                        concept_node.CONCEPT_PATH = :path
+                        AND dataset.REF = :dataset
+                    UNION
+                    SELECT
+                        -1 as depth, concept_node.concept_node_id
+                    FROM
+                        concept_node
+                    WHERE
+                        concept_node.concept_node_id = (
+                            SELECT
+                                parent_id
+                            FROM
+                                concept_node
+                                LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+                            WHERE
+                                concept_node.CONCEPT_PATH = :path
+                                AND dataset.REF = :dataset
+                        )
+                    ORDER BY depth ASC
+                )
+                SELECT
+                    concept_node.*,
+                    ds.REF AS dataset,
+                    continuous_min.VALUE AS min, continuous_max.VALUE AS max,
+                    categorical_values.VALUE AS values,
+                    meta_description.VALUE AS description,
+                    core_query.depth AS depth
+                FROM
+                    concept_node
+                    INNER JOIN core_query ON concept_node.concept_node_id = core_query.concept_node_id
+                    LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id
+                    LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description'
+                    LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
+                    LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
+                    LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+            """;
+        MapSqlParameterSource params = new MapSqlParameterSource()
+            .addValue("path", conceptPath)
+            .addValue("dataset", dataset)
+            .addValue("depth", depth);
+
+        if (depth < 0) {
+            return Optional.empty();
+        }
+
+        return Optional.ofNullable(template.query(sql, params, conceptResultSetExtractor));
+
+    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetExtractor.java

@@ -0,0 +1,52 @@
+package edu.harvard.dbmi.avillach.dictionary.concept;
+
+import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.stereotype.Component;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.*;
+
+@Component
+public class ConceptResultSetExtractor implements ResultSetExtractor<Concept> {
+    @Autowired
+    private ConceptResultSetUtil conceptResultSetUtil;
+
+    private record ConceptWithId(Concept c, int id) {};
+
+    @Override
+    public Concept extractData(ResultSet rs) throws SQLException, DataAccessException {
+        Map<Integer, List<ConceptWithId>> conceptsByParentId = new HashMap<>();
+        ConceptWithId root = null;
+        while (rs.next()) {
+            Concept c = switch (ConceptType.toConcept(rs.getString("concept_type"))) {
+                case Categorical -> conceptResultSetUtil.mapCategorical(rs);
+                case Continuous -> conceptResultSetUtil.mapContinuous(rs);
+            };
+            ConceptWithId conceptWithId = new ConceptWithId(c, rs.getInt("concept_node_id"));
+            if (root == null) { root = conceptWithId; }
+
+            int parentId = rs.getInt("parent_id");
+            // weirdness: null value for int is 0, so to check for missing parent value, you need the wasNull check
+            if (!rs.wasNull()) {
+                List<ConceptWithId> concepts = conceptsByParentId.getOrDefault(parentId, new ArrayList<>());
+                concepts.add(conceptWithId);
+                conceptsByParentId.put(parentId, concepts);
+            }
+        }
+
+
+        return root == null ? null : seedChildren(root, conceptsByParentId);
+    }
+
+    private Concept seedChildren(ConceptWithId root, Map<Integer, List<ConceptWithId>> conceptsByParentId) {
+        List<Concept> children = conceptsByParentId.getOrDefault(root.id, List.of()).stream()
+            .map(conceptWithId -> seedChildren(conceptWithId, conceptsByParentId))
+            .toList();
+        return root.c.withChildren(children);
+    }
+}