Skip to content

Commit

Permalink
[ALS-IDK] Stigmatizing variables
Browse files Browse the repository at this point in the history
  • Loading branch information
@Luke-Sikina
Luke Sikina authored and Luke-Sikina committed Sep 24, 2024
1 parent a2ad148 commit a0545f4
Show file tree
Hide file tree
Showing 13 changed files with 124 additions and 86 deletions.
45 changes: 35 additions & 10 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Pageable;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
Expand All @@ -17,35 +18,52 @@
@Repository
public class ConceptRepository {

private final NamedParameterJdbcTemplate template;
private static final String ALLOW_FILTERING_Q = """
WITH allow_filtering AS (
SELECT
concept_node.concept_node_id AS concept_node_id,
(string_agg(concept_node_meta.value, ' ') NOT LIKE '%yes%') AS allowFiltering
FROM
concept_node
JOIN concept_node_meta ON
concept_node.concept_node_id = concept_node_meta.concept_node_id
AND concept_node_meta.KEY IN (:disallowed_meta_keys)
GROUP BY
concept_node.concept_node_id
)
""";

private final NamedParameterJdbcTemplate template;
private final ConceptRowMapper mapper;

private final ConceptFilterQueryGenerator filterGen;
private final ConceptMetaExtractor conceptMetaExtractor;
private final ConceptResultSetExtractor conceptResultSetExtractor;
private final List<String> disallowedMetaFields;


@Autowired
public ConceptRepository(
NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor
ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor,
@Value("${filtering.unfilterable_concepts}") List<String> disallowedMetaFields
) {
this.template = template;
this.mapper = mapper;
this.filterGen = filterGen;
this.conceptMetaExtractor = conceptMetaExtractor;
this.conceptResultSetExtractor = conceptResultSetExtractor;
this.disallowedMetaFields = disallowedMetaFields;
}


public List<Concept> getConcepts(Filter filter, Pageable pageable) {
String sql = """
String sql = ALLOW_FILTERING_Q + """
SELECT
concept_node.*,
ds.REF as dataset,
continuous_min.VALUE as min, continuous_max.VALUE as max,
categorical_values.VALUE as values,
allow_filtering.allowFiltering AS allowFiltering,
meta_description.VALUE AS description
FROM
concept_node
Expand All @@ -54,12 +72,13 @@ public List<Concept> getConcepts(Filter filter, Pageable pageable) {
LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
WHERE concept_node.concept_node_id IN (
""";
QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable);
sql = sql + filterQ.query() + "\n)";
MapSqlParameterSource params = filterQ.params();
MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields);

return template.query(sql, params, mapper);
}
Expand All @@ -72,12 +91,13 @@ public long countConcepts(Filter filter) {
}

public Optional<Concept> getConcept(String dataset, String conceptPath) {
String sql = """
String sql = ALLOW_FILTERING_Q + """
SELECT
concept_node.*,
ds.REF as dataset,
continuous_min.VALUE as min, continuous_max.VALUE as max,
categorical_values.VALUE as values,
allow_filtering.allowFiltering AS allowFiltering,
meta_description.VALUE AS description
FROM
concept_node
Expand All @@ -86,13 +106,15 @@ public Optional<Concept> getConcept(String dataset, String conceptPath) {
LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
WHERE
concept_node.concept_path = :conceptPath
AND ds.REF = :dataset
""";
MapSqlParameterSource params = new MapSqlParameterSource()
.addValue("conceptPath", conceptPath)
.addValue("dataset", dataset);
.addValue("dataset", dataset)
.addValue("disallowed_meta_keys", disallowedMetaFields);
return template.query(sql, params, mapper).stream().findFirst();
}

Expand Down Expand Up @@ -137,8 +159,8 @@ public Map<Concept, Map<String, String>> getConceptMetaForConcepts(List<Concept>
}

public Optional<Concept> getConceptTree(String dataset, String conceptPath, int depth) {
String sql = """
WITH core_query AS (
String sql = ALLOW_FILTERING_Q + """
, core_query AS (
WITH RECURSIVE nodes AS (
SELECT
concept_node_id, parent_id, 0 AS depth
Expand Down Expand Up @@ -196,6 +218,7 @@ WITH RECURSIVE nodes AS (
continuous_min.VALUE AS min, continuous_max.VALUE AS max,
categorical_values.VALUE AS values,
meta_description.VALUE AS description,
allow_filtering.allowFiltering AS allowFiltering,
core_query.depth AS depth
FROM
concept_node
Expand All @@ -205,11 +228,13 @@ WITH RECURSIVE nodes AS (
LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
""";
MapSqlParameterSource params = new MapSqlParameterSource()
.addValue("path", conceptPath)
.addValue("dataset", dataset)
.addValue("depth", depth);
.addValue("depth", depth)
.addValue("disallowed_meta_keys", disallowedMetaFields);

if (depth < 0) {
return Optional.empty();
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
rs.getString("concept_path"), rs.getString("name"),
rs.getString("display"), rs.getString("dataset"), rs.getString("description"),
rs.getString("values") == null ? List.of() : parseValues(rs.getString("values")),
rs.getBoolean("allowFiltering"),
null,
null
);
Expand All @@ -29,6 +30,7 @@ public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
return new ContinuousConcept(
rs.getString("concept_path"), rs.getString("name"),
rs.getString("display"), rs.getString("dataset"), rs.getString("description"),
rs.getBoolean("allowFiltering"),
parseMin(rs.getString("values")), parseMax(rs.getString("values")),
null
);
Expand Down
20 changes: 8 additions & 12 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
public record CategoricalConcept(
String conceptPath, String name, String display, String dataset, String description,

List<String> values,
List<String> values, boolean allowFiltering,

@Nullable
List<Concept> children,
Expand All @@ -28,21 +28,17 @@ public record CategoricalConcept(

public CategoricalConcept(
String conceptPath, String name, String display, String dataset, String description, List<String> values,
@Nullable List<Concept> children, @Nullable Map<String, String> meta
boolean allowFiltering, @Nullable List<Concept> children, @Nullable Map<String, String> meta
) {
this(conceptPath, name, display, dataset, description, values, children, meta, null, null);
this(conceptPath, name, display, dataset, description, values, allowFiltering, children, meta, null, null);
}

public CategoricalConcept(CategoricalConcept core, Map<String, String> meta) {
this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, core.children, meta);
}

public CategoricalConcept(CategoricalConcept core, List<Concept> children) {
this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, children, core.meta);
this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.values, core.allowFiltering, core.children, meta);
}

public CategoricalConcept(String conceptPath, String dataset) {
this(conceptPath, "", "", dataset, "", List.of(), List.of(), null);
this(conceptPath, "", "", dataset, "", List.of(), false, List.of(), null);
}


Expand All @@ -54,20 +50,20 @@ public ConceptType type() {

@Override
public CategoricalConcept withChildren(List<Concept> children) {
return new CategoricalConcept(this, children);
return new CategoricalConcept(conceptPath, name, display, dataset, description, values, allowFiltering, children, meta);
}

@Override
public Concept withTable(Concept table) {
return new CategoricalConcept(
conceptPath, name, display, dataset, description, values, children, meta, table, study
conceptPath, name, display, dataset, description, values, allowFiltering, children, meta, table, study
);
}

@Override
public Concept withStudy(Concept study) {
return new CategoricalConcept(
conceptPath, name, display, dataset, description, values, children, meta, table, study
conceptPath, name, display, dataset, description, values, allowFiltering, children, meta, table, study
);
}

Expand Down
4 changes: 4 additions & 0 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ public sealed interface Concept
@Nullable
List<Concept> children();

default boolean allowFiltering() {
return false;
}

Concept withChildren(List<Concept> children);

Concept withTable(Concept table);
Expand Down
24 changes: 10 additions & 14 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import java.util.Objects;

public record ContinuousConcept(
String conceptPath, String name, String display, String dataset, String description,
String conceptPath, String name, String display, String dataset, String description, boolean allowFiltering,

@Nullable Integer min, @Nullable Integer max,
Map<String, String> meta,
Expand All @@ -24,29 +24,25 @@ public record ContinuousConcept(
) implements Concept {

public ContinuousConcept(
String conceptPath, String name, String display, String dataset, String description,
String conceptPath, String name, String display, String dataset, String description, boolean allowFiltering,
@Nullable Integer min, @Nullable Integer max, Map<String, String> meta, @Nullable List<Concept> children
) {
this(conceptPath, name, display, dataset, description, min, max, meta, children, null, null);
this(conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children, null, null);
}

public ContinuousConcept(ContinuousConcept core, Map<String, String> meta) {
this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, meta, core.children);
}

public ContinuousConcept(ContinuousConcept core, List<Concept> children) {
this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.min, core.max, core.meta, children);
this(core.conceptPath, core.name, core.display, core.dataset, core.description, core.allowFiltering, core.min, core.max, meta, core.children);
}

public ContinuousConcept(String conceptPath, String dataset) {
this(conceptPath, "", "", dataset, "", null, null, null, List.of());
this(conceptPath, "", "", dataset, "", true, null, null, null, List.of());
}

public ContinuousConcept(
String conceptPath, String name, String display, String dataset, String description,
String conceptPath, String name, String display, String dataset, String description, boolean allowFiltering,
@Nullable Integer min, @Nullable Integer max, Map<String, String> meta
) {
this(conceptPath, name, display, dataset, description, min, max, meta, null);
this(conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, null);
}

@JsonProperty("type")
Expand All @@ -57,20 +53,20 @@ public ConceptType type() {

@Override
public ContinuousConcept withChildren(List<Concept> children) {
return new ContinuousConcept(this, children);
return new ContinuousConcept(conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children);
}

@Override
public Concept withTable(Concept table) {
return new ContinuousConcept(
conceptPath, name, display, dataset, description, min, max, meta, children, table, study
conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children, table, study
);
}

@Override
public Concept withStudy(Concept study) {
return new ContinuousConcept(
conceptPath, name, display, dataset, description, min, max, meta, children, table, study
conceptPath, name, display, dataset, description, allowFiltering, min, max, meta, children, table, study
);
}

Expand Down
4 changes: 3 additions & 1 deletion src/main/resources/application-bdc.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,6 @@ spring.datasource.url=jdbc-secretsmanager:postgresql://${DATASOURCE_URL}/picsure
spring.datasource.username=${DATASOURCE_USERNAME}
server.port=80

dashboard.enable.extra_details=true
dashboard.enable.extra_details=true

filtering.unfilterable_concepts=stigmatizing
4 changes: 3 additions & 1 deletion src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,6 @@ dashboard.columns={abbreviation:'Abbreviation',name:'Name',clinvars:'Clinical Va
dashboard.column-order=abbreviation,name,clinvars
dashboard.nonmeta-columns=abbreviation,name
dashboard.enable.extra_details=true
dashboard.enable.bdc_hack=true
dashboard.enable.bdc_hack=true

filtering.unfilterable_concepts=stigmatizing
Loading

0 comments on commit a0545f4

Please sign in to comment.