diff --git a/dictionaryweights/src/main/java/edu/harvard/dbmi/avillach/dictionaryweights/WeightUpdateCreator.java b/dictionaryweights/src/main/java/edu/harvard/dbmi/avillach/dictionaryweights/WeightUpdateCreator.java index 1de1d56..f87ae14 100644 --- a/dictionaryweights/src/main/java/edu/harvard/dbmi/avillach/dictionaryweights/WeightUpdateCreator.java +++ b/dictionaryweights/src/main/java/edu/harvard/dbmi/avillach/dictionaryweights/WeightUpdateCreator.java @@ -22,7 +22,7 @@ public String createUpdate(List weights) { .collect(Collectors.joining(", ' ',\n ")); return """ UPDATE concept_node - SET SEARCHABLE_FIELDS = to_tsvector(data_table.search_str) + SET SEARCHABLE_FIELDS = to_tsvector(replace(data_table.search_str, '_', '/')) FROM ( SELECT @@ -43,6 +43,8 @@ concept_node.concept_node_id AS id, string_agg(value, ' ') AS values concept_node.concept_node_id ) AS concept_node_meta_str ON concept_node_meta_str.id = concept_node.concept_node_id LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id + LEFT JOIN concept_node AS parent ON concept_node.parent_id = parent.concept_node_id + LEFT JOIN concept_node AS grandparent ON parent.parent_id = grandparent.concept_node_id ) AS data_table WHERE concept_node.concept_node_id = data_table.search_key; """.formatted(searchableFields); diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java index 59aa992..8da6fd8 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java @@ -11,6 +11,8 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; +import java.util.List; + @Controller public class ConceptController { @@ -43,6 +45,21 @@ public ResponseEntity> listConcepts( return ResponseEntity.ok(pageResp); } + @GetMapping(path = "/concepts/dump") + public ResponseEntity> dumpConcepts( + @RequestParam(name = "page_number", defaultValue = "0", required = false) int page, + @RequestParam(name = "page_size", defaultValue = "10", required = false) int size + ) { + PageRequest pagination = PageRequest.of(page, size); + PageImpl pageResp = new PageImpl<>( + conceptService.listDetailedConcepts(new Filter(List.of(), "", List.of()), pagination), + pagination, + conceptService.countConcepts(new Filter(List.of(), "", List.of())) + ); + + return ResponseEntity.ok(pageResp); + } + @PostMapping(path = "/concepts/detail/{dataset}") public ResponseEntity conceptDetail( @PathVariable(name = "dataset") String dataset, diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java index 205d144..1a9938d 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java @@ -15,6 +15,14 @@ @Component public class ConceptFilterQueryGenerator { + private static final String CONSENT_QUERY = """ + dataset.dataset_id IN ( + SELECT consent.dataset_id + FROM consent + WHERE consent.consent_code IN (:consents) + ) AND + """; + /** * This generates a query that will return a list of concept_node IDs for the given filter. *

@@ -31,12 +39,15 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) { MapSqlParameterSource params = new MapSqlParameterSource(); List clauses = new java.util.ArrayList<>(List.of()); if (!CollectionUtils.isEmpty(filter.facets())) { - clauses.addAll(createFacetFilter(filter.facets(), params, filter.search())); + clauses.addAll(createFacetFilter(filter, params)); } if (StringUtils.hasLength(filter.search())) { params.addValue("search", filter.search().trim()); } - clauses.add(createValuelessNodeFilter(filter.search())); + if (!CollectionUtils.isEmpty(filter.consents())) { + params.addValue("consents", filter.consents()); + } + clauses.add(createValuelessNodeFilter(filter.search(), filter.consents())); String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n)"; @@ -63,13 +74,14 @@ ORDER BY max(rank) DESC return new QueryParamPair(superQuery, params); } - private String createValuelessNodeFilter(String search) { + private String createValuelessNodeFilter(String search, List consents) { String rankQuery = "0 as rank"; String rankWhere = ""; if (StringUtils.hasLength(search)) { rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery) as rank"; rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND"; } + String consentWhere = CollectionUtils.isEmpty(consents) ? "" : CONSENT_QUERY; // concept nodes that have no values and no min/max should not get returned by search return """ SELECT @@ -77,21 +89,24 @@ private String createValuelessNodeFilter(String search) { %s FROM concept_node + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE + %s %s ( continuous_min.value <> '' OR continuous_max.value <> '' OR categorical_values.value <> '' ) - """.formatted(rankQuery, rankWhere); + """.formatted(rankQuery, rankWhere, consentWhere); } - private List createFacetFilter(List facets, MapSqlParameterSource params, String search) { - return facets.stream() + private List createFacetFilter(Filter filter, MapSqlParameterSource params) { + String consentWhere = CollectionUtils.isEmpty(filter.consents()) ? "" : CONSENT_QUERY; + return filter.facets().stream() .collect(Collectors.groupingBy(Facet::category)) .entrySet().stream() .map(facetsForCategory -> { @@ -101,7 +116,7 @@ private List createFacetFilter(List facets, MapSqlParameterSource .addValue("category_%s".formatted(facetsForCategory.getKey()), facetsForCategory.getKey()); String rankQuery = "0"; String rankWhere = ""; - if (StringUtils.hasLength(search)) { + if (StringUtils.hasLength(filter.search())) { rankQuery = "ts_rank(searchable_fields, (phraseto_tsquery(:search)::text || ':*')::tsquery)"; rankWhere = "concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND"; } @@ -114,13 +129,15 @@ private List createFacetFilter(List facets, MapSqlParameterSource LEFT JOIN facet__concept_node ON facet__concept_node.facet_id = facet.facet_id JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id WHERE + %s %s facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s GROUP BY facet__concept_node.concept_node_id ) - """.formatted(rankQuery, rankWhere, facetsForCategory.getKey(), facetsForCategory.getKey()); + """.formatted(rankQuery, rankWhere, consentWhere, facetsForCategory.getKey(), facetsForCategory.getKey()); }) .toList(); } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptMetaExtractor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptMetaExtractor.java new file mode 100644 index 0000000..cf5c8bf --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptMetaExtractor.java @@ -0,0 +1,29 @@ +package edu.harvard.dbmi.avillach.dictionary.concept; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +@Component +public class ConceptMetaExtractor implements ResultSetExtractor>> { + + @Override + public Map> extractData(ResultSet rs) throws SQLException, DataAccessException { + Map> sets = new HashMap<>(); + while (rs.next()) { + Concept key = new ConceptShell(rs.getString("concept_path"), rs.getString("dataset_name")); + Map meta = sets.getOrDefault(key, new HashMap<>()); + meta.put(rs.getString("KEY"), rs.getString("VALUE")); + sets.put(key, meta); + } + return sets; + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index 71172c2..11ab373 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -22,14 +22,20 @@ public class ConceptRepository { private final ConceptRowMapper mapper; private final ConceptFilterQueryGenerator filterGen; + private final ConceptMetaExtractor conceptMetaExtractor; + private final ConceptResultSetExtractor conceptResultSetExtractor; + @Autowired public ConceptRepository( - NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen + NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen, + ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor ) { this.template = template; this.mapper = mapper; this.filterGen = filterGen; + this.conceptMetaExtractor = conceptMetaExtractor; + this.conceptResultSetExtractor = conceptResultSetExtractor; } @@ -107,4 +113,109 @@ public Map getConceptMeta(String dataset, String conceptPath) { .addValue("dataset", dataset); return template.query(sql, params, new MapExtractor("KEY", "VALUE")); } + + public Map> getConceptMetaForConcepts(List concepts) { + String sql = """ + SELECT + concept_node_meta.KEY, concept_node_meta.VALUE, + concept_node.CONCEPT_PATH AS concept_path, dataset.REF AS dataset_name + FROM + concept_node + LEFT JOIN concept_node_meta ON concept_node.concept_node_id = concept_node_meta.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id + WHERE + (concept_node.CONCEPT_PATH, dataset.REF) IN (:pairs) + ORDER BY concept_node.CONCEPT_PATH, dataset.REF + """; + List pairs = concepts.stream() + .map(c -> new String[]{c.conceptPath(), c.dataset()}) + .toList(); + MapSqlParameterSource params = new MapSqlParameterSource().addValue("pairs", pairs); + + return template.query(sql, params, conceptMetaExtractor); + + } + + public Optional getConceptTree(String dataset, String conceptPath, int depth) { + String sql = """ + WITH core_query AS ( + WITH RECURSIVE nodes AS ( + SELECT + concept_node_id, parent_id, 0 AS depth + FROM + concept_node + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id + WHERE + concept_node.CONCEPT_PATH = :path + AND dataset.REF = :dataset + UNION + SELECT + child_nodes.concept_node_id, child_nodes.parent_id, parent_node.depth+ 1 + FROM + concept_node child_nodes + INNER JOIN nodes parent_node ON child_nodes.parent_id = parent_node.concept_node_id + LEFT JOIN dataset ON child_nodes.dataset_id = dataset.dataset_id + ) + SELECT + depth, child_nodes.concept_node_id + FROM + nodes parent_node + INNER JOIN concept_node child_nodes ON child_nodes.parent_id = parent_node.concept_node_id + WHERE + depth < :depth + UNION + SELECT + 0 as depth, concept_node.concept_node_id + FROM + concept_node + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id + WHERE + concept_node.CONCEPT_PATH = :path + AND dataset.REF = :dataset + UNION + SELECT + -1 as depth, concept_node.concept_node_id + FROM + concept_node + WHERE + concept_node.concept_node_id = ( + SELECT + parent_id + FROM + concept_node + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id + WHERE + concept_node.CONCEPT_PATH = :path + AND dataset.REF = :dataset + ) + ORDER BY depth ASC + ) + SELECT + concept_node.*, + ds.REF AS dataset, + continuous_min.VALUE AS min, continuous_max.VALUE AS max, + categorical_values.VALUE AS values, + meta_description.VALUE AS description, + core_query.depth AS depth + FROM + concept_node + INNER JOIN core_query ON concept_node.concept_node_id = core_query.concept_node_id + LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id + LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description' + LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' + """; + MapSqlParameterSource params = new MapSqlParameterSource() + .addValue("path", conceptPath) + .addValue("dataset", dataset) + .addValue("depth", depth); + + if (depth < 0) { + return Optional.empty(); + } + + return Optional.ofNullable(template.query(sql, params, conceptResultSetExtractor)); + + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetExtractor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetExtractor.java new file mode 100644 index 0000000..ce85966 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetExtractor.java @@ -0,0 +1,52 @@ +package edu.harvard.dbmi.avillach.dictionary.concept; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + +@Component +public class ConceptResultSetExtractor implements ResultSetExtractor { + @Autowired + private ConceptResultSetUtil conceptResultSetUtil; + + private record ConceptWithId(Concept c, int id) {}; + + @Override + public Concept extractData(ResultSet rs) throws SQLException, DataAccessException { + Map> conceptsByParentId = new HashMap<>(); + ConceptWithId root = null; + while (rs.next()) { + Concept c = switch (ConceptType.toConcept(rs.getString("concept_type"))) { + case Categorical -> conceptResultSetUtil.mapCategorical(rs); + case Continuous -> conceptResultSetUtil.mapContinuous(rs); + }; + ConceptWithId conceptWithId = new ConceptWithId(c, rs.getInt("concept_node_id")); + if (root == null) { root = conceptWithId; } + + int parentId = rs.getInt("parent_id"); + // weirdness: null value for int is 0, so to check for missing parent value, you need the wasNull check + if (!rs.wasNull()) { + List concepts = conceptsByParentId.getOrDefault(parentId, new ArrayList<>()); + concepts.add(conceptWithId); + conceptsByParentId.put(parentId, concepts); + } + } + + + return root == null ? null : seedChildren(root, conceptsByParentId); + } + + private Concept seedChildren(ConceptWithId root, Map> conceptsByParentId) { + List children = conceptsByParentId.getOrDefault(root.id, List.of()).stream() + .map(conceptWithId -> seedChildren(conceptWithId, conceptsByParentId)) + .toList(); + return root.c.withChildren(children); + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java new file mode 100644 index 0000000..bd543b4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java @@ -0,0 +1,52 @@ +package edu.harvard.dbmi.avillach.dictionary.concept; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; +import org.json.JSONArray; +import org.json.JSONException; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +@Component +public class ConceptResultSetUtil { + + public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { + return new CategoricalConcept( + rs.getString("concept_path"), rs.getString("name"), + rs.getString("display"), rs.getString("dataset"), rs.getString("description"), + rs.getString("values") == null ? List.of() : List.of(rs.getString("values").split(",")), + null, + null + ); + } + + public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { + return new ContinuousConcept( + rs.getString("concept_path"), rs.getString("name"), + rs.getString("display"), rs.getString("dataset"), rs.getString("description"), + parseMin(rs.getString("values")), parseMax(rs.getString("values")), + null + ); + } + + public Integer parseMin(String valuesArr) { + try { + JSONArray arr = new JSONArray(valuesArr); + return arr.length() == 2 ? arr.getInt(0) : 0; + } catch (JSONException ex) { + return 0; + } + } + + public Integer parseMax(String valuesArr) { + try { + JSONArray arr = new JSONArray(valuesArr); + return arr.length() == 2 ? arr.getInt(1) : 0; + } catch (JSONException ex) { + return 0; + } + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowMapper.java index fc632d0..17e984c 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowMapper.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowMapper.java @@ -3,6 +3,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.*; import org.json.JSONArray; import org.json.JSONException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @@ -15,48 +16,14 @@ @Component public class ConceptRowMapper implements RowMapper { + @Autowired + ConceptResultSetUtil conceptResultSetUtil; + @Override public Concept mapRow(ResultSet rs, int rowNum) throws SQLException { return switch (ConceptType.toConcept(rs.getString("concept_type"))) { - case Categorical -> mapCategorical(rs); - case Continuous -> mapContinuous(rs); + case Categorical -> conceptResultSetUtil.mapCategorical(rs); + case Continuous -> conceptResultSetUtil.mapContinuous(rs); }; } - - private CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { - return new CategoricalConcept( - rs.getString("concept_path"), rs.getString("name"), - rs.getString("display"), rs.getString("dataset"), rs.getString("description"), - rs.getString("values") == null ? List.of() : List.of(rs.getString("values").split(",")), - null, - null - ); - } - - private ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { - return new ContinuousConcept( - rs.getString("concept_path"), rs.getString("name"), - rs.getString("display"), rs.getString("dataset"), rs.getString("description"), - parseMin(rs.getString("values")), parseMax(rs.getString("values")), - null - ); - } - - private Integer parseMin(String valuesArr) { - try { - JSONArray arr = new JSONArray(valuesArr); - return arr.length() == 2 ? arr.getInt(0) : 0; - } catch (JSONException ex) { - return 0; - } - } - - private Integer parseMax(String valuesArr) { - try { - JSONArray arr = new JSONArray(valuesArr); - return arr.length() == 2 ? arr.getInt(1) : 0; - } catch (JSONException ex) { - return 0; - } - } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java index 3a7e094..9e7dca0 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java @@ -2,6 +2,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import org.springframework.beans.factory.annotation.Autowired; @@ -9,7 +10,10 @@ import org.springframework.stereotype.Service; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; @Service public class ConceptService { @@ -25,6 +29,16 @@ public List listConcepts(Filter filter, Pageable page) { return conceptRepository.getConcepts(filter, page); } + public List listDetailedConcepts(Filter filter, Pageable page) { + List concepts = conceptRepository.getConcepts(filter, page); + Map> metas = conceptRepository.getConceptMetaForConcepts(concepts); + return concepts.stream().map(concept -> (Concept) switch (concept) { + case ContinuousConcept cont -> new ContinuousConcept(cont, metas.getOrDefault(cont, Map.of())); + case CategoricalConcept cat -> new CategoricalConcept(cat, metas.getOrDefault(cat, Map.of())); + case ConceptShell ignored -> throw new RuntimeException("Concept shell escaped to API"); + }).toList(); + } + public long countConcepts(Filter filter) { return conceptRepository.countConcepts(filter); } @@ -36,12 +50,13 @@ public Optional conceptDetail(String dataset, String conceptPath) { return switch (core) { case ContinuousConcept cont -> new ContinuousConcept(cont, meta); case CategoricalConcept cat -> new CategoricalConcept(cat, meta); + case ConceptShell ignored -> throw new RuntimeException("Concept shell escaped to API"); }; } ); } public Optional conceptTree(String dataset, String conceptPath, int depth) { - return Optional.empty(); + return conceptRepository.getConceptTree(dataset, conceptPath, depth); } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java index 26b8485..ef3b60c 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Map; +import java.util.Objects; public record CategoricalConcept( String conceptPath, String name, String display, String dataset, String description, @@ -23,10 +24,34 @@ public CategoricalConcept(CategoricalConcept core, Map meta) { this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, core.children, meta); } + public CategoricalConcept(CategoricalConcept core, List children) { + this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, children, core.meta); + } + + public CategoricalConcept(String conceptPath, String dataset) { + this(conceptPath, "", "", dataset, "", List.of(), List.of(), null); + } + @JsonProperty("type") @Override public ConceptType type() { return ConceptType.Categorical; } + + @Override + public CategoricalConcept withChildren(List children) { + return new CategoricalConcept(this, children); + } + + @Override + public boolean equals(Object object) { + return conceptEquals(object); + } + + @Override + public int hashCode() { + return Objects.hash(conceptPath, dataset); + } + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java index d81aea7..17d70dd 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java @@ -3,8 +3,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.annotation.Nullable; +import java.util.List; import java.util.Map; +import java.util.Objects; @JsonIgnoreProperties(ignoreUnknown = true) @@ -18,7 +21,7 @@ @JsonSubTypes.Type(value = CategoricalConcept.class, name = "Categorical"), }) public sealed interface Concept - permits CategoricalConcept, ContinuousConcept { + permits CategoricalConcept, ConceptShell, ContinuousConcept { /** * @return The complete concept path for this concept (// delimited) @@ -48,5 +51,15 @@ public sealed interface Concept Map meta(); + @Nullable + List children(); + Concept withChildren(List children); + + default boolean conceptEquals(Object object) { + if (this == object) return true; + if (!(object instanceof Concept)) return false; + Concept that = (Concept) object; + return Objects.equals(dataset(), that.dataset()) && Objects.equals(conceptPath(), that.conceptPath()); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java new file mode 100644 index 0000000..d904632 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java @@ -0,0 +1,49 @@ +package edu.harvard.dbmi.avillach.dictionary.concept.model; + +import jakarta.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public record ConceptShell(String conceptPath, String dataset) implements Concept { + @Override + public String name() { + return "Shell. Not for external use."; + } + + @Override + public String display() { + return "Shell. Not for external use."; + } + + @Override + public ConceptType type() { + return ConceptType.Continuous; + } + + @Override + public Map meta() { + return Map.of(); + } + + @Override + public List children() { + return List.of(); + } + + @Override + public ConceptShell withChildren(List children) { + return this; + } + + @Override + public boolean equals(Object object) { + return conceptEquals(object); + } + + @Override + public int hashCode() { + return Objects.hash(conceptPath, dataset); + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java index 3b6c93c..48a9f2f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java @@ -3,17 +3,37 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.Objects; public record ContinuousConcept( String conceptPath, String name, String display, String dataset, String description, @Nullable Integer min, @Nullable Integer max, - Map meta + Map meta, + @Nullable + List children ) implements Concept { public ContinuousConcept(ContinuousConcept core, Map meta) { - this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, meta); + this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, meta, core.children); + } + + public ContinuousConcept(ContinuousConcept core, List children) { + this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, core.meta, children); + } + + public ContinuousConcept(String conceptPath, String dataset) { + this(conceptPath, "", "", dataset, "", null, null, null, List.of()); + } + + public ContinuousConcept( + String conceptPath, String name, String display, String dataset, String description, + @Nullable Integer min, @Nullable Integer max, Map meta + ) { + this(conceptPath, name, display, dataset, description, min, max, meta, null); } @JsonProperty("type") @@ -21,4 +41,19 @@ public ContinuousConcept(ContinuousConcept core, Map meta) { public ConceptType type() { return ConceptType.Continuous; } + + @Override + public ContinuousConcept withChildren(List children) { + return new ContinuousConcept(this, children); + } + + @Override + public boolean equals(Object object) { + return conceptEquals(object); + } + + @Override + public int hashCode() { + return Objects.hash(conceptPath, dataset); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/Facet.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/Facet.java index 1dd7ac4..58dc1a9 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/Facet.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/Facet.java @@ -13,4 +13,12 @@ public record Facet( public Facet(Facet core, Map meta) { this(core.name(), core.display(), core.description(), core.fullName(), core.count(), core.children(), core.category(), meta); } + + public Facet(String name, String category) { + this(name, "", "", "", null, null, category, null); + } + + public Facet withChildren(List children) { + return new Facet(this.name, this.display, this.description, this.fullName, this.count, children, this.category, this.meta); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetCategoryExtractor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetCategoryExtractor.java index 7fc3b49..8f6d3c1 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetCategoryExtractor.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetCategoryExtractor.java @@ -2,6 +2,7 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.util.StringUtils; import java.sql.ResultSet; import java.sql.SQLException; @@ -10,10 +11,17 @@ public class FacetCategoryExtractor implements ResultSetExtractor> { + private record Pair(String parent, String category) { + Pair(Facet facet) { + this(facet.name(), facet.category()); + } + }; + @Override public List extractData(ResultSet rs) throws SQLException, DataAccessException { List facets = new ArrayList<>(); Map categories = new HashMap<>(); + Map> childrenForParent = new HashMap<>(); while (rs.next()) { // build out all the facets and make shells of the facet categories @@ -21,15 +29,26 @@ public List extractData(ResultSet rs) throws SQLException, DataAc Facet facet = new Facet( rs.getString("name"), rs.getString("display"), rs.getString("description"), rs.getString("full_name"), rs.getInt("facet_count"), - null, category, null + List.of(), category, null ); FacetCategory facetCategory = new FacetCategory( category, rs.getString("category_display"), rs.getString("category_description"), List.of() ); - facets.add(facet); + String parentName = rs.getString("parent_name"); + if (StringUtils.hasLength(parentName)) { + Pair key = new Pair(parentName, category); + List facetsForParent = childrenForParent.getOrDefault(key, new ArrayList<>()); + facetsForParent.add(facet); + childrenForParent.put(key, facetsForParent); + } else { + facets.add(facet); + } categories.put(category, facetCategory); } + facets = facets.stream() + .map(f -> f.withChildren(childrenForParent.getOrDefault(new Pair(f), List.of()))) + .toList(); // group facets by category, then add them to their respective category Map> grouped = facets.stream().collect(Collectors.groupingBy(Facet::category)); return categories.entrySet().stream() diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetMapper.java index 7b52aa0..c74f988 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetMapper.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetMapper.java @@ -17,7 +17,7 @@ public Facet mapRow(ResultSet rs, int rowNum) throws SQLException { rs.getString("description"), rs.getString("full_name"), null, - null, + List.of(), rs.getString("category"), null ); diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java index 7abd75c..b408a67 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java @@ -10,32 +10,44 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.stream.Stream; @Component public class FacetQueryGenerator { + private static final String CONSENT_QUERY = """ + dataset.dataset_id IN ( + SELECT consent.dataset_id + FROM consent + WHERE consent.consent_code IN (:consents) + ) AND + """; + public String createFacetSQLAndPopulateParams(Filter filter, MapSqlParameterSource params) { Map> groupedFacets = (filter.facets() == null ? Stream.of() : filter.facets().stream()) .collect(Collectors.groupingBy(Facet::category)); + String consentWhere = ""; + if (!CollectionUtils.isEmpty(filter.consents())) { + params.addValue("consents", filter.consents()); + consentWhere = CONSENT_QUERY; + } if (CollectionUtils.isEmpty(filter.facets())) { if (StringUtils.hasLength(filter.search())) { - return createNoFacetSQLWithSearch(filter.search(), params); + return createNoFacetSQLWithSearch(filter.search(), consentWhere, params); } else { - return createNoFacetSQLNoSearch(params); + return createNoFacetSQLNoSearch(params, consentWhere); } } else if (groupedFacets.size() == 1) { if (StringUtils.hasLength(filter.search())) { - return createSingleCategorySQLWithSearch(filter.facets(), filter.search(), params); + return createSingleCategorySQLWithSearch(filter.facets(), filter.search(), consentWhere, params); } else { - return createSingleCategorySQLNoSearch(filter.facets(), params); + return createSingleCategorySQLNoSearch(filter.facets(), consentWhere, params); } } else { if (StringUtils.hasLength(filter.search())) { - return createMultiCategorySQLWithSearch(groupedFacets, filter.search(), params); + return createMultiCategorySQLWithSearch(groupedFacets, filter.search(), consentWhere, params); } else { - return createMultiCategorySQLNoSearch(groupedFacets, params); + return createMultiCategorySQLNoSearch(groupedFacets, consentWhere, params); } } } @@ -48,7 +60,7 @@ private Map createSQLSafeCategoryKeys(List categories) { return keys; } - private String createMultiCategorySQLWithSearch(Map> facets, String search, MapSqlParameterSource params) { + private String createMultiCategorySQLWithSearch(Map> facets, String search, String consentWhere, MapSqlParameterSource params) { Map categoryKeys = createSQLSafeCategoryKeys(facets.keySet().stream().toList()); params.addValue("search", search); @@ -104,15 +116,20 @@ private String createMultiCategorySQLWithSearch(Map> facets, facet.facet_id, count(*) as facet_count FROM facet + LEFT JOIN facet_category fc ON fc.facet_category_id = facet.facet_category_id JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id WHERE - fcn.concept_node_id IN (%s) + %s + fcn.concept_node_id IN (%s) AND + fc.name = :facet_category_%s GROUP BY facet.facet_id ORDER BY facet_count DESC ) - """.formatted(allConceptsForCategory); + """.formatted(consentWhere, allConceptsForCategory, categoryKeys.get(category)); }) .collect(Collectors.joining("\n\tUNION\n")); @@ -132,7 +149,10 @@ facet.facet_id, count(*) as facet_count facet JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id WHERE + %s fc.name NOT IN (:all_selected_facet_categories) AND fcn.concept_node_id IN (%s) GROUP BY @@ -140,12 +160,12 @@ AND fcn.concept_node_id IN (%s) ORDER BY facet_count DESC ) - """.formatted(allConceptsForUnselectedCategories); + """.formatted(consentWhere, allConceptsForUnselectedCategories); return conceptsQuery + selectedFacetsQuery + unselectedFacetsQuery; } - private String createMultiCategorySQLNoSearch(Map> facets, MapSqlParameterSource params) { + private String createMultiCategorySQLNoSearch(Map> facets, String consentWhere, MapSqlParameterSource params) { Map categoryKeys = createSQLSafeCategoryKeys(facets.keySet().stream().toList()); /* @@ -187,13 +207,13 @@ private String createMultiCategorySQLNoSearch(Map> facets, M and INTERSECT them. This creates the concepts for this category */ String selectedFacetsQuery = facets.keySet().stream().map(category -> { - params.addValue("facet_category_" + categoryKeys.get(category), category); - String allConceptsForCategory = categoryKeys.values().stream() - .filter(key -> !categoryKeys.get(category).equals(key)) - .map(key -> "SELECT * FROM facet_category_" + key + "_concepts") - .collect(Collectors.joining(" INTERSECT ")); - params.addValue("", ""); - return """ + params.addValue("facet_category_" + categoryKeys.get(category), category); + String allConceptsForCategory = categoryKeys.values().stream() + .filter(key -> !categoryKeys.get(category).equals(key)) + .map(key -> "SELECT * FROM facet_category_" + key + "_concepts") + .collect(Collectors.joining(" INTERSECT ")); + params.addValue("", ""); + return """ ( SELECT facet.facet_id, count(*) as facet_count @@ -201,7 +221,10 @@ facet.facet_id, count(*) as facet_count facet JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id WHERE + %s fcn.concept_node_id IN (%s) AND fc.name = :facet_category_%s GROUP BY @@ -209,7 +232,7 @@ fcn.concept_node_id IN (%s) ORDER BY facet_count DESC ) - """.formatted(allConceptsForCategory, categoryKeys.get(category)); + """.formatted(consentWhere, allConceptsForCategory, categoryKeys.get(category)); }) .collect(Collectors.joining("\n\tUNION\n")); @@ -229,7 +252,10 @@ facet.facet_id, count(*) as facet_count facet JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id WHERE + %s fc.name NOT IN (:all_selected_facet_categories) AND fcn.concept_node_id IN (%s) GROUP BY @@ -237,12 +263,12 @@ AND fcn.concept_node_id IN (%s) ORDER BY facet_count DESC ) - """.formatted(allConceptsForUnselectedCategories); + """.formatted(consentWhere, allConceptsForUnselectedCategories); return conceptsQuery + selectedFacetsQuery + unselectedFacetsQuery; } - private String createSingleCategorySQLWithSearch(List facets, String search, MapSqlParameterSource params) { + private String createSingleCategorySQLWithSearch(List facets, String search, String consentWhere, MapSqlParameterSource params) { params.addValue("facet_category_name", facets.getFirst().category()); params.addValue("facets", facets.stream().map(Facet::name).toList()); params.addValue("search", search); @@ -262,11 +288,13 @@ facet.facet_id, count(*) as facet_count facet JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id - JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE + %s fc.name = :facet_category_name AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND ( @@ -287,7 +315,8 @@ WITH matching_concepts AS ( FROM facet JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id - JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node + JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id + JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' @@ -306,19 +335,22 @@ facet.facet_id, count(*) as facet_count FROM facet JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id JOIN matching_concepts ON fcn.concept_node_id = matching_concepts.concept_node_id WHERE + %s fc.name <> :facet_category_name GROUP BY facet.facet_id ORDER BY facet_count DESC ) - """; + """.formatted(consentWhere, consentWhere); } - private String createSingleCategorySQLNoSearch(List facets, MapSqlParameterSource params) { + private String createSingleCategorySQLNoSearch(List facets, String consentWhere, MapSqlParameterSource params) { params.addValue("facet_category_name", facets.getFirst().category()); params.addValue("facets", facets.stream().map(Facet::name).toList()); // return all the facets in the matched category that are displayable @@ -332,11 +364,13 @@ facet.facet_id, count(*) as facet_count facet JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id - JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE + %s fc.name = :facet_category_name AND ( continuous_min.value <> '' OR @@ -375,19 +409,22 @@ facet.facet_id, count(*) as facet_count FROM facet JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id + LEFT JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id JOIN matching_concepts ON fcn.concept_node_id = matching_concepts.concept_node_id WHERE + %s fc.name <> :facet_category_name GROUP BY facet.facet_id ORDER BY facet_count DESC ) - """; + """.formatted(consentWhere, consentWhere); } - private String createNoFacetSQLWithSearch(String search, MapSqlParameterSource params) { + private String createNoFacetSQLWithSearch(String search, String consentWhere, MapSqlParameterSource params) { // return all the facets that match concepts that // match search // are displayable @@ -400,10 +437,12 @@ facet.facet_id, count(*) as facet_count JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE + %s concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND ( continuous_min.value <> '' OR @@ -414,11 +453,11 @@ facet.facet_id, count(*) as facet_count facet.facet_id ORDER BY facet_count DESC - """; + """.formatted(consentWhere); } - private String createNoFacetSQLNoSearch(MapSqlParameterSource params) { + private String createNoFacetSQLNoSearch(MapSqlParameterSource params, String consents) { // return all the facets that match displayable concepts // this is the easy one! return """ @@ -429,17 +468,21 @@ facet.facet_id, count(*) as facet_count JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' WHERE - continuous_min.value <> '' OR - continuous_max.value <> '' OR - categorical_values.value <> '' + %s + ( + continuous_min.value <> '' OR + continuous_max.value <> '' OR + categorical_values.value <> '' + ) GROUP BY facet.facet_id ORDER BY facet_count DESC - """; + """.formatted(consents); } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java index c0335a5..1cfd49d 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java @@ -74,7 +74,29 @@ public Optional getFacet(String facetCategory, String facet) { MapSqlParameterSource params = new MapSqlParameterSource() .addValue("facetCategory", facetCategory) .addValue("facetName", facet); - return template.query(sql, params, mapper).stream().findFirst(); + return template.query(sql, params, mapper).stream().findFirst() + .map(f -> f.withChildren(getFacetChildren(f.category(), f.name()))); + } + + private List getFacetChildren(String facetCategory, String parentFacetName) { + String sql = """ + SELECT + facet_category.name AS category, + facet.name, facet.display, facet.description, + facet_meta_full_name.value AS full_name + FROM + facet + LEFT JOIN facet as parent_facet ON facet.parent_id = parent_facet.facet_id + LEFT JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id + LEFT JOIN facet_meta AS facet_meta_full_name ON facet.facet_id = facet_meta_full_name.facet_id AND facet_meta_full_name.KEY = 'full_name' + WHERE + parent_facet.name = :facetName + AND facet_category.name = :facetCategory + """; + MapSqlParameterSource params = new MapSqlParameterSource() + .addValue("facetCategory", facetCategory) + .addValue("facetName", parentFacetName); + return template.query(sql, params, mapper); } public Map getFacetMeta(String facetCategory, String facet) { diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java new file mode 100644 index 0000000..594407a --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java @@ -0,0 +1,49 @@ +package edu.harvard.dbmi.avillach.dictionary.facet; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import org.springframework.core.MethodParameter; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.RequestBodyAdvice; + +import java.io.IOException; +import java.lang.reflect.Type; + +@ControllerAdvice +public class FilterPreProcessor implements RequestBodyAdvice { + @Override + public boolean supports( + MethodParameter methodParameter, Type targetType, Class> converterType + ) { + return true; + } + + @Override + public HttpInputMessage beforeBodyRead( + HttpInputMessage inputMessage, MethodParameter parameter, Type targetType, + Class> converterType + ) throws IOException { + return inputMessage; + } + + @Override + public Object afterBodyRead( + Object body, HttpInputMessage inputMessage, MethodParameter parameter, + Type targetType, Class> converterType + ) { + if (body instanceof Filter filter && StringUtils.hasLength(filter.search())) { + return new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); + } + return body; + } + + @Override + public Object handleEmptyBody( + Object body, HttpInputMessage inputMessage, MethodParameter parameter, Type targetType, + Class> converterType + ) { + return body; + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/Filter.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/Filter.java index 5f69069..c6f7fa3 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/Filter.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/Filter.java @@ -1,9 +1,11 @@ package edu.harvard.dbmi.avillach.dictionary.filter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import edu.harvard.dbmi.avillach.dictionary.facet.Facet; import jakarta.annotation.Nullable; import java.util.List; -public record Filter(@Nullable List facets, @Nullable String search) { +@JsonIgnoreProperties(ignoreUnknown = true) +public record Filter(@Nullable List facets, @Nullable String search, @Nullable List consents) { } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java index 28b3aba..5a738f4 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java @@ -12,6 +12,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -38,7 +39,7 @@ void shouldListConcepts() { ); Filter filter = new Filter( List.of(new Facet("questionare", "Questionare", "?", "Questionare", 1, null, "category", null)), - "foo" + "foo", List.of() ); Mockito.when(conceptService.listConcepts(filter, Pageable.ofSize(10).withPage(1))) .thenReturn(expected); @@ -143,4 +144,25 @@ void shouldNotGetConceptTreeWhenConceptDNE() { Assertions.assertEquals(HttpStatus.NOT_FOUND, actual.getStatusCode()); } + + @Test + void shouldDumpConcepts() { + Concept fooBar = new CategoricalConcept( + "/foo//bar", "bar", "Bar", "my_dataset", "foo!", List.of("a", "b"), List.of(), + Map.of("key", "value") + ); + Concept fooBaz = new ContinuousConcept( + "/foo//baz", "baz", "Baz", "my_dataset", "foo!", 0, 100, + Map.of("key", "value") + ); + List concepts = List.of(fooBar, fooBaz); + PageRequest page = PageRequest.of(0, 10); + Mockito.when(conceptService.listDetailedConcepts(new Filter(List.of(), "", List.of()), page)) + .thenReturn(concepts); + + ResponseEntity> actual = subject.dumpConcepts(0, 10); + + Assertions.assertEquals(concepts, actual.getBody().getContent()); + Assertions.assertEquals(HttpStatus.OK, actual.getStatusCode()); + } } \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java index d087486..98d4995 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java @@ -2,6 +2,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.facet.Facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; @@ -46,14 +47,14 @@ static void mySQLProperties(DynamicPropertyRegistry registry) { @Test void shouldListAllConcepts() { - List actual = subject.getConcepts(new Filter(List.of(), ""), Pageable.unpaged()); + List actual = subject.getConcepts(new Filter(List.of(), "", List.of()), Pageable.unpaged()); Assertions.assertEquals(29, actual.size()); } @Test void shouldListFirstTwoConcepts() { - List actual = subject.getConcepts(new Filter(List.of(), ""), Pageable.ofSize(2).first()); + List actual = subject.getConcepts(new Filter(List.of(), "", List.of()), Pageable.ofSize(2).first()); List expected = List.of( new ContinuousConcept("\\phs000007\\pht000021\\phv00003844\\FL200\\", "phv00003844", "FL200", "phs000007", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY", 0, 3, null), new CategoricalConcept("\\Variant Data Type\\Low coverage WGS\\", "Low coverage WGS", "Low coverage WGS", "1", "Low coverage WGS", List.of("TRUE"), null, null) @@ -64,7 +65,7 @@ void shouldListFirstTwoConcepts() { @Test void shouldListNextTwoConcepts() { - List actual = subject.getConcepts(new Filter(List.of(), ""), Pageable.ofSize(2).first().next()); + List actual = subject.getConcepts(new Filter(List.of(), "", List.of()), Pageable.ofSize(2).first().next()); List expected = List.of( new CategoricalConcept("\\phs002385\\RACEG\\", "RACEG", "RACEG", "phs002385", "Race (regrouped)", List.of("Not Reported"), null, null), new CategoricalConcept("\\Variant Data Type\\Low coverage WGS\\", "Low coverage WGS", "Low coverage WGS", "1", "Low coverage WGS", List.of("TRUE"), null, null) @@ -76,7 +77,7 @@ void shouldListNextTwoConcepts() { @Test void shouldFilterConceptsByFacet() { List actual = - subject.getConcepts(new Filter(List.of(new Facet("phs000007", "", "", "", 1, null, "study_ids_dataset_ids", null)), ""), Pageable.unpaged()); + subject.getConcepts(new Filter(List.of(new Facet("phs000007", "", "", "", 1, null, "study_ids_dataset_ids", null)), "", List.of()), Pageable.unpaged()); List expected = List.of( new ContinuousConcept("\\phs000007\\pht000022\\phv00004260\\FM219\\", "phv00004260", "FM219", "phs000007", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY", 0, 1, null), new ContinuousConcept("\\phs000007\\pht000021\\phv00003844\\FL200\\", "phv00003844", "FL200", "phs000007", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY", 0, 3, null), @@ -88,7 +89,7 @@ void shouldFilterConceptsByFacet() { @Test void shouldFilterBySearch() { - List actual = subject.getConcepts(new Filter(List.of(), "COLA"), Pageable.unpaged()); + List actual = subject.getConcepts(new Filter(List.of(), "COLA", List.of()), Pageable.unpaged()); List expected = List.of( new ContinuousConcept("\\phs000007\\pht000022\\phv00004260\\FM219\\", "phv00004260", "FM219", "phs000007", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY", 0, 1, null), new ContinuousConcept("\\phs000007\\pht000021\\phv00003844\\FL200\\", "phv00003844", "FL200", "phs000007", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY", 0, 3, null), @@ -101,7 +102,7 @@ void shouldFilterBySearch() { @Test void shouldFilterByBothSearchAndFacet() { List actual = - subject.getConcepts(new Filter(List.of(new Facet("phs002715", "", "", "", 1, null, "study_ids_dataset_ids", null)), "phs002715"), Pageable.unpaged()); + subject.getConcepts(new Filter(List.of(new Facet("phs002715", "", "", "", 1, null, "study_ids_dataset_ids", null)), "phs002715", List.of()), Pageable.unpaged()); List expected = List.of( new CategoricalConcept("\\phs002715\\age\\", "AGE_CATEGORY", "age", "phs002715", "Participant's age (category)", List.of("21"), null, null), new CategoricalConcept("\\phs002715\\nsrr_ever_smoker\\", "nsrr_ever_smoker", "nsrr_ever_smoker", "phs002715", "Smoker status", List.of("yes"), null, null) @@ -112,14 +113,14 @@ void shouldFilterByBothSearchAndFacet() { @Test void shouldGetCount() { - long actual = subject.countConcepts(new Filter(List.of(), "")); + long actual = subject.countConcepts(new Filter(List.of(), "", List.of())); Assertions.assertEquals(29L, actual); } @Test void shouldGetCountWithFilter() { - Long actual = subject.countConcepts(new Filter(List.of(new Facet("phs002715", "", "", "", 1, null, "study_ids_dataset_ids", null)), "")); + Long actual = subject.countConcepts(new Filter(List.of(new Facet("phs002715", "", "", "", 1, null, "study_ids_dataset_ids", null)), "", List.of())); Assertions.assertEquals(2L, actual); } @@ -145,7 +146,90 @@ void shouldNotGetConceptThatDNE() { Optional actual = subject.getConcept("invalid.invalid", "fake"); Assertions.assertEquals(Optional.empty(), actual); - actual = subject.getConcept("fake", "\\\\\\\\B\\\\\\\\2\\\\\\\\Z\\\\\\\\"); + actual = subject.getConcept("fake", "\\\\B\\\\2\\\\Z\\\\"); Assertions.assertEquals(Optional.empty(), actual); } + + @Test + void shouldGetMetaForMultipleConcepts() { + List concepts = List.of( + new ContinuousConcept("\\phs000007\\pht000022\\phv00004260\\FM219\\", "", "", "phs000007", "", null, null, Map.of()), + new ContinuousConcept("\\phs000007\\pht000033\\phv00008849\\D080\\", "", "", "phs000007", "", null, null, Map.of()) + ); + + Map> actual = subject.getConceptMetaForConcepts(concepts); + Map> expected = Map.of( + new ConceptShell("\\phs000007\\pht000022\\phv00004260\\FM219\\", "phs000007"), Map.of( + "unique_identifier", "no", + "stigmatizing", "no", + "bdc_open_access", "yes", + "values", "[0, 1]", + "description", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY", + "free_text", "no" + ), + new ConceptShell("\\phs000007\\pht000033\\phv00008849\\D080\\", "phs000007"), Map.of( + "unique_identifier", "no", + "stigmatizing", "no", + "bdc_open_access", "yes", + "values", "[0, 5]", + "description", "# 12 OZ CUPS OF CAFFEINATED COLA/DAY", + "free_text", "no" + ) + ); + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldGetTree() { + Concept d0 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\", "1"); + Concept d1 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\", "1"); + Concept d2 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\", "1"); + Concept d3 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\J45 Asthma\\", "1"); + Concept d4A = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\J45 Asthma\\J45.5 Severe persistent asthma\\", "1"); + Concept d4B = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\J45 Asthma\\J45.9 Other and unspecified );asthma\\", "1"); + d3 = d3.withChildren(List.of(d4A, d4B)); + d2.withChildren(List.of(d3)); + d1.withChildren(List.of(d2)); + d0.withChildren(List.of(d1)); + + Optional actual = subject.getConceptTree("1", "\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\", 3); + Optional expected = Optional.of(d0); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldGetTreeForDepthThatExceedsOntology() { + Concept d0 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\", "1"); + Concept d1 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\", "1"); + Concept d2 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\", "1"); + Concept d3 = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\J45 Asthma\\", "1"); + Concept d4A = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\J45 Asthma\\J45.5 Severe persistent asthma\\", "1"); + Concept d4B = new CategoricalConcept("\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\J40-J47 Chronic lower respiratory diseases (J40-J47)\\J45 Asthma\\J45.9 Other and unspecified );asthma\\", "1"); + d3 = d3.withChildren(List.of(d4A, d4B)); + d2.withChildren(List.of(d3)); + d1.withChildren(List.of(d2)); + d0.withChildren(List.of(d1)); + + Optional actual = subject.getConceptTree("1", "\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\", 30); + Optional expected = Optional.of(d0); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldReturnEmptyTreeForDNE() { + Optional actual = subject.getConceptTree("1", "\\ACT Top Secret ICD-69\\", 30); + Optional expected = Optional.empty(); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldReturnEmptyForNegativeDepth() { + Optional actual = subject.getConceptTree("1", "\\ACT Diagnosis ICD-10\\J00-J99 Diseases of the respiratory system (J00-J99)\\", -1); + Optional expected = Optional.empty(); + + Assertions.assertEquals(expected, actual); + } } \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java index f47920f..1757774 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java @@ -2,6 +2,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import org.junit.jupiter.api.Assertions; @@ -31,7 +32,7 @@ void shouldListConcepts() { List expected = List.of( new CategoricalConcept("A", "a", "A", "invalid.invalid", null, List.of(), null, null) ); - Filter filter = new Filter(List.of(), ""); + Filter filter = new Filter(List.of(), "", List.of()); Pageable page = Pageable.ofSize(10).first(); Mockito.when(repository.getConcepts(filter, page)) .thenReturn(expected); @@ -43,7 +44,7 @@ void shouldListConcepts() { @Test void shouldCountConcepts() { - Filter filter = new Filter(List.of(), ""); + Filter filter = new Filter(List.of(), "", List.of()); Mockito.when(repository.countConcepts(filter)) .thenReturn(1L); @@ -81,4 +82,33 @@ void shouldShowDetailForCategorical() { Assertions.assertEquals(expected, actual); } + + @Test + void shouldShowDetailForMultiple() { + ConceptShell shellA = new ConceptShell("pathA", "dataset"); + CategoricalConcept conceptA = new CategoricalConcept("pathA", "", "", "dataset", null, List.of("a"), List.of(), null); + Map metaA = Map.of("VALUES", "a", "stigmatizing", "true"); + + ConceptShell shellB = new ConceptShell("pathB", "dataset"); + ContinuousConcept conceptB = new ContinuousConcept("pathB", "", "", "dataset", null, 0, 1, null); + Map metaB = Map.of("MIN", "0", "MAX", "1", "stigmatizing", "true"); + + Map> metas = Map.of(shellA, metaA, shellB, metaB); + List concepts = List.of(conceptA, conceptB); + Filter emptyFilter = new Filter(List.of(), "", List.of()); + + + Mockito.when(repository.getConceptMetaForConcepts(concepts)) + .thenReturn(metas); + Mockito.when(repository.getConcepts(emptyFilter, Pageable.unpaged())) + .thenReturn(concepts); + + List actual = subject.listDetailedConcepts(emptyFilter, Pageable.unpaged()); + List expected = List.of( + new CategoricalConcept(conceptA, metaA), + new ContinuousConcept(conceptB, metaB) + ); + + Assertions.assertEquals(expected, actual); + } } \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java index ba813ad..9181493 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java @@ -75,7 +75,7 @@ void shouldIncludeTypeInList() throws JsonProcessingException { ); String actual = new ObjectMapper().writeValueAsString(concepts); - String expected = "[{\"conceptPath\":\"/foo//baz\",\"name\":\"baz\",\"display\":\"Baz\",\"dataset\":\"study_a\",\"description\":null,\"min\":0,\"max\":1,\"meta\":{},\"type\":\"Continuous\"},{\"conceptPath\":\"/foo//bar\",\"name\":\"bar\",\"display\":\"Bar\",\"dataset\":\"study_a\",\"description\":null,\"values\":[\"a\",\"b\"],\"children\":null,\"meta\":{},\"type\":\"Categorical\"}]"; + String expected = "[{\"conceptPath\":\"/foo//baz\",\"name\":\"baz\",\"display\":\"Baz\",\"dataset\":\"study_a\",\"description\":null,\"min\":0,\"max\":1,\"meta\":{},\"children\":null,\"type\":\"Continuous\"},{\"conceptPath\":\"/foo//bar\",\"name\":\"bar\",\"display\":\"Bar\",\"dataset\":\"study_a\",\"description\":null,\"values\":[\"a\",\"b\"],\"children\":null,\"meta\":{},\"type\":\"Categorical\"}]"; Assertions.assertEquals(expected, actual); } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java index 1be266f..99a3995 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java @@ -31,7 +31,7 @@ void shouldListFacets() { Filter filter = new Filter( List.of(new Facet("questionare", "Questionare", "?", "Examination", 1, null, "category", null)), - "foo" + "foo", List.of() ); Mockito.when(facetService.getFacets(filter)) .thenReturn(List.of(expected)); diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java new file mode 100644 index 0000000..8df45aa --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java @@ -0,0 +1,299 @@ +package edu.harvard.dbmi.avillach.dictionary.facet; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +@Testcontainers +@SpringBootTest +class FacetQueryGeneratorTest { + + @Autowired + NamedParameterJdbcTemplate template; + + @Autowired + FacetQueryGenerator subject; + + @Container + static final PostgreSQLContainer databaseContainer = + new PostgreSQLContainer<>("postgres:16") + .withReuse(true) + .withCopyFileToContainer( + MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql" + ); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + record IdCountPair(int facetId, int facetCount) {} + + static class IdCountPairMapper implements RowMapper { + + @Override + public IdCountPair mapRow(ResultSet rs, int rowNum) throws SQLException { + return new IdCountPair(rs.getInt("facet_id"), rs.getInt("facet_count")); + } + } + + @Test + void shouldCountFacetsWithNoSearchAndNoSelectedFacetsAndNoConsents() { + Filter filter = new Filter(List.of(), "", List.of()); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(22, 13), new IdCountPair(31, 3), new IdCountPair(27, 3), + new IdCountPair(26, 3), new IdCountPair(28, 3), new IdCountPair(23, 2), + new IdCountPair(25, 2), new IdCountPair(21, 1), new IdCountPair(20, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithNoSearchAndNoSelectedFacetsAndConsents() { + Filter filter = new Filter(List.of(), "", List.of("c2")); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(27, 3), new IdCountPair(20, 1), new IdCountPair(21, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithSearchAndNoSelectedFacetsAndNoConsents() { + Filter filter = new Filter(List.of(), "age", List.of()); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(23, 1), + new IdCountPair(25, 1), + new IdCountPair(26, 1), + new IdCountPair(28, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithSearchAndNoSelectedFacetsAndConsents() { + Filter filter = new Filter(List.of(), "age", List.of("c1")); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(25, 1), + new IdCountPair(26, 1), + new IdCountPair(28, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithSearchAndOneSelectedFacetsAndNoConsents() { + Filter filter = new Filter( + List.of(new Facet("phs002715", "study_ids_dataset_ids")), + "age", List.of() + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(23, 1), + new IdCountPair(25, 1), + new IdCountPair(26, 1), + new IdCountPair(28, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithSearchAndOneSelectedFacetsAndConsents() { + Filter filter = new Filter( + List.of(new Facet("phs002715", "study_ids_dataset_ids")), + "age", List.of("c1") + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(25, 1), + new IdCountPair(26, 1), + new IdCountPair(28, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsNoSearchAndOneSelectedFacetsAndNoConsents() { + Filter filter = new Filter( + List.of(new Facet("phs002715", "study_ids_dataset_ids")), + "", List.of() + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(28, 3), + new IdCountPair(26, 3), + new IdCountPair(31, 3), + new IdCountPair(22, 13), + new IdCountPair(23, 2), + new IdCountPair(25, 2), + new IdCountPair(27, 3) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsNoSearchAndOneSelectedFacetsAndConsents() { + Filter filter = new Filter( + List.of(new Facet("phs002715", "study_ids_dataset_ids")), + "", List.of("c2") + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(27, 3) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithSearchAndTwoSelectedFacetsInDifferentCatsAndNoConsents() { + Filter filter = new Filter( + List.of( + new Facet("phs000007", "study_ids_dataset_ids"), + new Facet("LOINC", "nsrr_harmonized") + ), + "cola", List.of() + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(21, 1), + new IdCountPair(27, 1), + new IdCountPair(20, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsWithSearchAndTwoSelectedFacetsInDifferentCatsAndConsents() { + Filter filter = new Filter( + List.of( + new Facet("phs000007", "study_ids_dataset_ids"), + new Facet("LOINC", "nsrr_harmonized") + ), + "cola", List.of("c1") + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(21, 1), + new IdCountPair(27, 1), + new IdCountPair(20, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsNoSearchAndTwoSelectedFacetsInDifferentCatsAndNoConsents() { + Filter filter = new Filter( + List.of( + new Facet("phs000007", "study_ids_dataset_ids"), + new Facet("LOINC", "nsrr_harmonized") + ), + "", List.of() + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(21, 1), + new IdCountPair(27, 1), + new IdCountPair(20, 1) + ); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldCountFacetsNoSearchAndTwoSelectedFacetsInDifferentCatsAndConsents() { + Filter filter = new Filter( + List.of( + new Facet("phs000007", "study_ids_dataset_ids"), + new Facet("LOINC", "nsrr_harmonized") + ), + "", List.of("c1") + ); + + MapSqlParameterSource params = new MapSqlParameterSource(); + String query = subject.createFacetSQLAndPopulateParams(filter, params); + + List actual = template.query(query, params, new IdCountPairMapper()); + List expected = List.of( + new IdCountPair(21, 1), + new IdCountPair(27, 1), + new IdCountPair(20, 1) + ); + + Assertions.assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java index 5b77d8e..4978476 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepositoryTest.java @@ -43,64 +43,57 @@ static void mySQLProperties(DynamicPropertyRegistry registry) { @Test void shouldGetAllFacets() { - Filter filter = new Filter(List.of(), ""); - - List actual = subject.getFacets(filter); - - Assertions.assertEquals(2, actual.size()); - } - - @Test - void shouldFilterFacetsBySearch() { - Filter filter = new Filter(List.of(), "X"); + Filter filter = new Filter(List.of(), "", List.of()); List actual = subject.getFacets(filter); - List expected = List.of( - new FacetCategory( - "site", "Site", "Filter variables by site", + new FacetCategory("study_ids_dataset_ids", "Study IDs/Dataset IDs", "", List.of( - new Facet("bch", "BCH", "Boston Childrens Hospital", "Boston Childrens Hospital", 1, null, "category", null) + new Facet("1", "GIC", null, null, 13, List.of(), "study_ids_dataset_ids", null), + new Facet("phs000284", "CFS", null, "Chronic Fatigue Syndrome", 3, List.of(), "study_ids_dataset_ids", null), + new Facet("phs000007", "FHS", null, "Framingham Heart Study", 3, List.of(), "study_ids_dataset_ids", null), + new Facet("phs002385", "HCT_for_SCD", null, null, 3, List.of(), "study_ids_dataset_ids", null), + new Facet("phs002808", "nuMoM2b", null, null, 3, List.of(), "study_ids_dataset_ids", null), + new Facet("2", "National Health and Nutrition Examination Survey", null, null, 2, List.of(), "study_ids_dataset_ids", null), + new Facet("phs002715", "NSRR CFS", null, "National Sleep Research Resource", 2, List.of(), "study_ids_dataset_ids", null), + new Facet("3", "1000 Genomes Project", null, null, 0, List.of(), "study_ids_dataset_ids", null), + new Facet("phs003463", "RECOVER_Adult", null, null, 0, List.of(), "study_ids_dataset_ids", null), + new Facet("phs003543", "NSRR_HSHC", null, null, 0, List.of(), "study_ids_dataset_ids", null), + new Facet("phs003566", "SPRINT", null, null, 0, List.of(), "study_ids_dataset_ids", null), + new Facet("phs001963", "DEMENTIA-SEQ", null, null, 0, List.of( + new Facet("NEST_1", "My Nested Facet 1", null, null, 0, List.of(), "study_ids_dataset_ids", null), + new Facet("NEST_2", "My Nested Facet 2", null, null, 0, List.of(), "study_ids_dataset_ids", null) + ), "study_ids_dataset_ids", null) ) ), - new FacetCategory( - "data_source", "Data Source", "What does this data relate to (image, questionnaire...)", + new FacetCategory("nsrr_harmonized", "Common Data Element Collection", "", List.of( - new Facet("imaging", "Imaging", "Data derived from an image", "Data derived from an image", 1, null, "data_source", null), - new Facet("questionnaire", "questionnaire", "Data derived from a questionnaire", "Data derived from a questionnaire", 1, null, "data_source", null), - new Facet("lab_test", "Lab Test", "Data derived from a lab test", "Data derived from a lab test", 1, null, "data_source", null) + new Facet("LOINC", "LOINC", null, null, 1, List.of(), "nsrr_harmonized", null), + new Facet("PhenX", "PhenX", null, null, 1, List.of(), "nsrr_harmonized", null), + new Facet("gad_7", "Generalized Anxiety Disorder Assessment (GAD-7)", null, null, 0, List.of(), "nsrr_harmonized", null), + new Facet("taps_tool", "NIDA CTN Common Data Elements = TAPS Tool", null, null, 0, List.of(), "nsrr_harmonized", null) ) ) ); + + Assertions.assertEquals(expected, actual); } @Test - void shouldFilterFacetsByFacet() { - Filter filter = new Filter(List.of(new Facet("bch", "BCH", "Boston Childrens Hospital", "Boston Childrens Hospital", 1, null, "category", null)), ""); - List actual = subject.getFacets(filter); - - List expected = List.of( - new FacetCategory( - "site", "Site", "Filter variables by site", - List.of( - new Facet("bch", "BCH", "Boston Childrens Hospital", "Boston Childrens Hospital", 1, null, "category", null), - new Facet("narnia", "Narnia", "Narnia", "Narnia", 1, null, "category", null) - ) - ), - new FacetCategory( - "data_source", "Data Source", "What does this data relate to (image, questionnaire...)", - List.of( - new Facet("imaging", "Imaging", "Data derived from an image", "Data derived from an image", 1, null, "data_source", null), - new Facet("questionnaire", "questionnaire", "Data derived from a questionnaire", "Data derived from a questionnaire", 1, null, "data_source", null), - new Facet("lab_test", "Lab Test", "Data derived from a lab test", "Data derived from a lab test", 1, null, "data_source", null) - ) - ) - ); + void shouldGetFacetWithChildren() { + Optional actual = subject.getFacet("study_ids_dataset_ids", "phs001963"); + Facet expected = new Facet("phs001963", "DEMENTIA-SEQ", null, null, null, List.of( + new Facet("NEST_1", "My Nested Facet 1", null, null, null, List.of(), "study_ids_dataset_ids", null), + new Facet("NEST_2", "My Nested Facet 2", null, null, null, List.of(), "study_ids_dataset_ids", null) + ), "study_ids_dataset_ids", null); + + Assertions.assertTrue(actual.isPresent()); + Assertions.assertEquals(expected, actual.get()); } @Test void shouldGetFacet() { Optional actual = subject.getFacet("study_ids_dataset_ids", "phs000007"); - Optional expected = Optional.of(new Facet("phs000007", "FHS", null, "Framingham Heart Study", null, null, "study_ids_dataset_ids", null)); + Optional expected = Optional.of(new Facet("phs000007", "FHS", null, "Framingham Heart Study", null, List.of(), "study_ids_dataset_ids", null)); Assertions.assertEquals(expected, actual); } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java index cb5269f..55bad26 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java @@ -22,7 +22,7 @@ class FacetServiceTest { @Test void shouldGetFacets() { - Filter filter = new Filter(List.of(), ""); + Filter filter = new Filter(List.of(), "", List.of()); List expected = List.of(new FacetCategory("n", "d", "", List.of(new Facet("f_n", "f_d", "", "", 1, null, "n", null)))); Mockito.when(repository.getFacets(filter)) diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java new file mode 100644 index 0000000..628c72e --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java @@ -0,0 +1,42 @@ +package edu.harvard.dbmi.avillach.dictionary.facet; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.MethodParameter; +import org.springframework.http.HttpInputMessage; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.type.SimpleType; + +import java.util.List; + +@SpringBootTest +class FilterPreProcessorTest { + + @Autowired + private FilterPreProcessor subject; + + @Test + void shouldProcessFilter() { + Object processedFilter = subject.afterBodyRead( + new Filter(List.of(), "I_love_underscores", List.of()), + Mockito.mock(HttpInputMessage.class), Mockito.mock(MethodParameter.class), + SimpleType.constructUnsafe(Filter.class), null + ); + + Assertions.assertEquals(new Filter(List.of(), "I/love/underscores", List.of()), processedFilter); + } + + @Test + void shouldNotProcessOtherBodies() { + Object actual = subject.afterBodyRead( + "I'm an object!", + Mockito.mock(HttpInputMessage.class), Mockito.mock(MethodParameter.class), + SimpleType.constructUnsafe(Filter.class), null + ); + + Assertions.assertEquals("I'm an object!", actual); + } +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/filter/ConceptFilterQueryGeneratorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/filter/ConceptFilterQueryGeneratorTest.java index cba90e7..3a6ba45 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/filter/ConceptFilterQueryGeneratorTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/filter/ConceptFilterQueryGeneratorTest.java @@ -46,7 +46,7 @@ static void mySQLProperties(DynamicPropertyRegistry registry) { @Test void shouldGenerateForFacetAndSearchNoMatch() { - Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), "smoke"); + Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), "smoke", List.of()); QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); List actual = template.queryForList(pair.query(), pair.params(), Integer.class); @@ -57,7 +57,7 @@ void shouldGenerateForFacetAndSearchNoMatch() { @Test void shouldGenerateForFHSFacet() { - Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), ""); + Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), "", List.of()); QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); List actual = template.queryForList(pair.query(), pair.params(), Integer.class); @@ -66,9 +66,42 @@ void shouldGenerateForFHSFacet() { Assertions.assertEquals(expected, actual); } + @Test + void shouldGenerateForFHSFacetWithConsent1() { + Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), "", List.of("c1")); + QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); + + List actual = template.queryForList(pair.query(), pair.params(), Integer.class); + List expected = List.of(229, 232, 235); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldGenerateForFHSFacetWithConsent1And2() { + Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), "", List.of("c1", "c2")); + QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); + + List actual = template.queryForList(pair.query(), pair.params(), Integer.class); + List expected = List.of(229, 232, 235); + + Assertions.assertEquals(expected, actual); + } + + @Test + void shouldGenerateForFHSFacetWithConsent3() { + Filter f = new Filter(List.of(new Facet("phs000007", "FHS", "", "", null, null, "study_ids_dataset_ids", null)), "", List.of("c3")); + QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); + + List actual = template.queryForList(pair.query(), pair.params(), Integer.class); + List expected = List.of(); + + Assertions.assertEquals(expected, actual); + } + @Test void shouldGenerateForFacetAndSearchMatch() { - Filter f = new Filter(List.of(new Facet("phs002715", "NSRR", "", "", null, null, "study_ids_dataset_ids", null)), "smoke"); + Filter f = new Filter(List.of(new Facet("phs002715", "NSRR", "", "", null, null, "study_ids_dataset_ids", null)), "smoke", List.of()); QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); List actual = template.queryForList(pair.query(), pair.params(), Integer.class); @@ -79,7 +112,7 @@ void shouldGenerateForFacetAndSearchMatch() { @Test void shouldGenerateForNSRRFacet() { - Filter f = new Filter(List.of(new Facet("phs002715", "NSRR", "", "", null, null, "study_ids_dataset_ids", null)), ""); + Filter f = new Filter(List.of(new Facet("phs002715", "NSRR", "", "", null, null, "study_ids_dataset_ids", null)), "", List.of()); QueryParamPair pair = subject.generateFilterQuery(f, Pageable.unpaged()); List actual = template.queryForList(pair.query(), pair.params(), Integer.class); diff --git a/src/test/resources/seed.sql b/src/test/resources/seed.sql index db1b91f..f811557 100644 --- a/src/test/resources/seed.sql +++ b/src/test/resources/seed.sql @@ -608,6 +608,8 @@ COPY public.facet (facet_id, facet_category_id, name, display, description, pare 31 1 phs002808 nuMoM2b \N \N 32 1 phs003566 SPRINT \N \N 33 1 phs001963 DEMENTIA-SEQ \N \N +55 1 NEST_1 My Nested Facet 1 \N 33 +56 1 NEST_2 My Nested Facet 2 \N 33 19 2 gad_7 Generalized Anxiety Disorder Assessment (GAD-7) \N \N 18 2 taps_tool NIDA CTN Common Data Elements = TAPS Tool \N \N \.