From 477bf4b350bf05b2a780e8192da085c8002eb523 Mon Sep 17 00:00:00 2001 From: schuemie Date: Mon, 13 Feb 2017 15:14:56 +0100 Subject: [PATCH] Added option to map to non-standard (classification) concepts) --- src/org/ohdsi/usagi/TargetConcept.java | 12 ++++++++++++ src/org/ohdsi/usagi/UsagiSearchEngine.java | 16 ++++++++++------ .../ohdsi/usagi/indexBuilding/BuildIndex.java | 3 ++- src/org/ohdsi/usagi/ui/FilterPanel.java | 15 ++++++++------- src/org/ohdsi/usagi/ui/ImportDialog.java | 4 ++-- src/org/ohdsi/usagi/ui/MappingDetailPanel.java | 14 +++++++++++--- src/org/ohdsi/usagi/ui/MappingTablePanel.java | 4 +++- 7 files changed, 48 insertions(+), 20 deletions(-) diff --git a/src/org/ohdsi/usagi/TargetConcept.java b/src/org/ohdsi/usagi/TargetConcept.java index 2ca09ba..dd91af9 100644 --- a/src/org/ohdsi/usagi/TargetConcept.java +++ b/src/org/ohdsi/usagi/TargetConcept.java @@ -38,6 +38,7 @@ public class TargetConcept { public String validEndDate; public String invalidReason; public String additionalInformation; + public String standardConcept; public static TargetConcept EMPTY_CONCEPT = createEmptyConcept(); public TargetConcept(Row row) { @@ -52,6 +53,10 @@ public TargetConcept(Row row) { validStartDate = row.get("VALID_START_DATE"); validEndDate = row.get("VALID_END_DATE"); invalidReason = row.get("INVALID_REASON"); + if (row.getFieldNames().contains("STANDARDCONCEPT")) + standardConcept = row.get("STANDARDCONCEPT"); + else + standardConcept = ""; for (String domain : row.get("DOMAINS").split("/")) domains.add(domain); additionalInformation = row.get("ADDITIONAL_INFORMATION"); @@ -65,6 +70,10 @@ public TargetConcept(Row row) { validStartDate = row.get("VALIDSTARTDATE"); validEndDate = row.get("VALIDENDDATE"); invalidReason = row.get("INVALIDREASON"); + if (row.getFieldNames().contains("STANDARDCONCEPT")) + standardConcept = row.get("STANDARDCONCEPT"); + else + standardConcept = ""; for (String domain : row.get("DOMAINS").split("/")) domains.add(domain); additionalInformation = row.get("ADDITIONALINFORMATION"); @@ -82,6 +91,7 @@ private static TargetConcept createEmptyConcept() { concept.validStartDate = ""; concept.validEndDate = ""; concept.invalidReason = ""; + concept.standardConcept = ""; concept.additionalInformation = "This is an empty concept"; return concept; } @@ -100,6 +110,7 @@ public TargetConcept(TargetConcept concept) { validEndDate = concept.validEndDate; invalidReason = concept.invalidReason; domains.addAll(concept.domains); + standardConcept = concept.standardConcept; additionalInformation = concept.additionalInformation; } @@ -115,6 +126,7 @@ public Row toRow() { row.add("validEndDate", validEndDate); row.add("invalidReason", invalidReason); row.add("domains", StringUtilities.join(domains, "/")); + row.add("standardConcept", standardConcept); if (additionalInformation == null) row.add("additionalInformation", ""); else diff --git a/src/org/ohdsi/usagi/UsagiSearchEngine.java b/src/org/ohdsi/usagi/UsagiSearchEngine.java index dc04c46..62f9440 100644 --- a/src/org/ohdsi/usagi/UsagiSearchEngine.java +++ b/src/org/ohdsi/usagi/UsagiSearchEngine.java @@ -91,7 +91,8 @@ public class UsagiSearchEngine { private QueryParser conceptClassQueryParser; private QueryParser vocabularyQueryParser; private QueryParser keywordsQueryParser; - private QueryParser invalidQueryParser; +// private QueryParser invalidQueryParser; + private QueryParser standardConceptParser; private QueryParser domainQueryParser; private int numDocs; private FieldType textVectorField = getTextVectorFieldType(); @@ -150,6 +151,7 @@ public void addConceptToIndex(TargetConcept concept) { document.add(new StringField("VALID_START_DATE", concept.validStartDate, Store.YES)); document.add(new StringField("VALID_END_DATE", concept.validEndDate, Store.YES)); document.add(new StringField("INVALID_REASON", concept.invalidReason, Store.YES)); + document.add(new StringField("STANDARD_CONCEPT", concept.standardConcept, Store.YES)); document.add(new TextField("DOMAINS", StringUtilities.join(concept.domains, "\n"), Store.YES)); document.add(new StringField("ADDITIONAL_INFORMATION", concept.additionalInformation, Store.YES)); writer.addDocument(document); @@ -260,7 +262,8 @@ public void openIndexForSearching() { vocabularyQueryParser = new QueryParser(Version.LUCENE_4_9, "VOCABULARY", new KeywordAnalyzer()); keywordsQueryParser = new QueryParser(Version.LUCENE_4_9, "TERM", analyzer); domainQueryParser = new QueryParser(Version.LUCENE_4_9, "DOMAINS", analyzer); - invalidQueryParser = new QueryParser(Version.LUCENE_4_9, "INVALID_REASON", new KeywordAnalyzer()); +// invalidQueryParser = new QueryParser(Version.LUCENE_4_9, "INVALID_REASON", new KeywordAnalyzer()); + standardConceptParser = new QueryParser(Version.LUCENE_4_9, "STANDARD_CONCEPT", new KeywordAnalyzer()); numDocs = reader.numDocs(); } catch (Exception e) { throw new RuntimeException(e); @@ -317,7 +320,7 @@ public void close() { } public List search(String searchTerm, boolean useMlt, Collection filterConceptIds, String filterDomain, String filterConceptClass, - String filterVocabulary, boolean filterInvalid) { + String filterVocabulary, boolean filterStandard) { List results = new ArrayList(); try { Query query; @@ -365,9 +368,9 @@ public List search(String searchTerm, boolean useMlt, Collection< Query vocabularyQuery = vocabularyQueryParser.parse("\"" + filterVocabulary.toString() + "\""); booleanQuery.add(vocabularyQuery, Occur.MUST); } - if (filterInvalid) { - Query invalidQuery = invalidQueryParser.parse("\"\""); - booleanQuery.add(invalidQuery, Occur.MUST); + if (filterStandard) { + Query standardQuery = standardConceptParser.parse("S"); + booleanQuery.add(standardQuery, Occur.MUST); } TopDocs topDocs = searcher.search(booleanQuery, 100); @@ -402,6 +405,7 @@ private TargetConcept docIdToTargetConcept(int docId) { targetConcept.validStartDate = document.get("VALID_START_DATE"); targetConcept.validEndDate = document.get("VALID_END_DATE"); targetConcept.invalidReason = document.get("INVALID_REASON"); + targetConcept.standardConcept = document.get("STANDARD_CONCEPT"); for (String domain : document.get("DOMAINS").split("\n")) targetConcept.domains.add(domain); targetConcept.additionalInformation = document.get("ADDITIONAL_INFORMATION"); diff --git a/src/org/ohdsi/usagi/indexBuilding/BuildIndex.java b/src/org/ohdsi/usagi/indexBuilding/BuildIndex.java index 567d74f..3ec880b 100644 --- a/src/org/ohdsi/usagi/indexBuilding/BuildIndex.java +++ b/src/org/ohdsi/usagi/indexBuilding/BuildIndex.java @@ -151,7 +151,7 @@ public void run() { Row conceptRow = multiRowSet.get("concept").get(0); if (conceptRow.getCells().size() > 2) // Extra check to catch badly formatted rows (which are in a vocab we don't care about) - if (conceptRow.get("STANDARD_CONCEPT").equals("S")) { + if (conceptRow.get("STANDARD_CONCEPT").equals("S") || conceptRow.get("STANDARD_CONCEPT").equals("C")) { vocabularies.add(conceptRow.get("VOCABULARY_ID")); conceptClassIds.add(conceptRow.get("CONCEPT_CLASS_ID")); domainIds.add(conceptRow.get("DOMAIN_ID")); @@ -179,6 +179,7 @@ public void run() { concept.invalidReason = conceptRow.get("INVALID_REASON"); concept.validEndDate = conceptRow.get("VALID_END_DATE"); concept.validStartDate = conceptRow.get("VALID_START_DATE"); + concept.standardConcept = conceptRow.get("STANDARD_CONCEPT"); concept.vocabulary = conceptRow.get("VOCABULARY_ID"); if (loincToInfo != null && concept.vocabulary.equals("LOINC")) { String info = loincToInfo.get(concept.conceptCode); diff --git a/src/org/ohdsi/usagi/ui/FilterPanel.java b/src/org/ohdsi/usagi/ui/FilterPanel.java index cb96598..25dacda 100644 --- a/src/org/ohdsi/usagi/ui/FilterPanel.java +++ b/src/org/ohdsi/usagi/ui/FilterPanel.java @@ -35,7 +35,7 @@ public class FilterPanel extends JPanel { private static final long serialVersionUID = 1378433878412231259L; private JCheckBox filterByAutoCheckBox; - private JCheckBox filterInvalidCheckBox; + private JCheckBox filterStandardCheckBox; private JCheckBox filterByConceptClassCheckBox; private JCheckBox filterByVocabularyCheckBox; private JCheckBox filterByDomainCheckBox; @@ -70,16 +70,17 @@ public void actionPerformed(ActionEvent arg0) { c.gridy = 0; c.weightx = 0.1; c.gridwidth = 2; - filterInvalidCheckBox = new JCheckBox("Filter invalid concepts", false); - filterInvalidCheckBox.setToolTipText("Limit the search to only valid concepts"); - filterInvalidCheckBox.addActionListener(new ActionListener() { + filterStandardCheckBox = new JCheckBox("Filter standard concepts", false); + filterStandardCheckBox.setToolTipText("Limit the search to only standard concepts"); + filterStandardCheckBox.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { notifyListeners(); } }); - add(filterInvalidCheckBox, c); + filterStandardCheckBox.setSelected(true); + add(filterStandardCheckBox, c); c.gridx = 0; c.gridy = 1; @@ -197,8 +198,8 @@ public boolean getFilterByAuto() { return filterByAutoCheckBox.isSelected(); } - public boolean getFilterInvalid() { - return filterInvalidCheckBox.isSelected(); + public boolean getFilterStandard() { + return filterStandardCheckBox.isSelected(); } public boolean getFilterByConceptClass() { diff --git a/src/org/ohdsi/usagi/ui/ImportDialog.java b/src/org/ohdsi/usagi/ui/ImportDialog.java index 79e1d08..2a14c3d 100644 --- a/src/org/ohdsi/usagi/ui/ImportDialog.java +++ b/src/org/ohdsi/usagi/ui/ImportDialog.java @@ -375,7 +375,7 @@ public void run() { try { Global.usagiSearchEngine.createDerivedIndex(sourceCodes, null); - boolean filterInvalid = filterPanel.getFilterInvalid(); + boolean filterStandard = filterPanel.getFilterStandard(); String filterConceptClass = null; if (filterPanel.getFilterByConceptClass()) filterConceptClass = filterPanel.getConceptClass(); @@ -394,7 +394,7 @@ public void run() { CodeMapping codeMapping = new CodeMapping(sourceCode); List concepts = Global.usagiSearchEngine.search(sourceCode.sourceName, true, filterConceptIds, filterDomain, - filterConceptClass, filterVocabulary, filterInvalid); + filterConceptClass, filterVocabulary, filterStandard); if (concepts.size() > 0) { codeMapping.targetConcepts.add(concepts.get(0).concept); codeMapping.matchScore = concepts.get(0).matchScore; diff --git a/src/org/ohdsi/usagi/ui/MappingDetailPanel.java b/src/org/ohdsi/usagi/ui/MappingDetailPanel.java index 36bee32..3bd8ebc 100644 --- a/src/org/ohdsi/usagi/ui/MappingDetailPanel.java +++ b/src/org/ohdsi/usagi/ui/MappingDetailPanel.java @@ -206,6 +206,10 @@ public void valueChanged(ListSelectionEvent event) { } }); +// searchTable.hideColumn("Synonym"); + searchTable.hideColumn("Valid start date"); + searchTable.hideColumn("Valid end date"); + searchTable.hideColumn("Invalid reason"); panel.add(new JScrollPane(searchTable)); JPanel buttonPanel = new JPanel(); @@ -384,7 +388,7 @@ public void run() { if (filterPanel.getFilterByAuto()) filterConceptIds = codeMapping.sourceCode.sourceAutoAssignedConceptIds; - boolean filterInvalid = filterPanel.getFilterInvalid(); + boolean filterStandard = filterPanel.getFilterStandard(); String filterConceptClass = null; if (filterPanel.getFilterByConceptClass()) filterConceptClass = filterPanel.getConceptClass(); @@ -400,7 +404,7 @@ public void run() { if (Global.usagiSearchEngine.isOpenForSearching()) { List searchResults = Global.usagiSearchEngine.search(query, true, filterConceptIds, filterDomain, filterConceptClass, - filterVocabulary, filterInvalid); + filterVocabulary, filterStandard); searchTableModel.setScoredConcepts(searchResults); searchTable.scrollRectToVisible(new Rectangle(searchTable.getCellRect(0, 0, true))); @@ -505,7 +509,7 @@ class TargetConceptTableModel extends AbstractTableModel { private String scoreColumnName = "Score"; private String[] columnNames = { "Synonym", "Concept ID", "Concept name", "Domain", "Concept class", "Vocabulary", "Concept code", - "Valid start date", "Valid end date", "Invalid reason" }; + "Valid start date", "Valid end date", "Invalid reason", "Standard concept" }; private List targetConcepts = new ArrayList(); private boolean hasScoreColumn; private Double[] scoreColumn; @@ -585,6 +589,8 @@ public Object getValueAt(int row, int col) { return targetConcept.validEndDate; case 9: return targetConcept.invalidReason; + case 10: + return targetConcept.standardConcept; default: return ""; } @@ -617,6 +623,8 @@ public Class getColumnClass(int col) { return String.class; case 9: return String.class; + case 10: + return String.class; default: return String.class; } diff --git a/src/org/ohdsi/usagi/ui/MappingTablePanel.java b/src/org/ohdsi/usagi/ui/MappingTablePanel.java index 12b0fc1..c39dca1 100644 --- a/src/org/ohdsi/usagi/ui/MappingTablePanel.java +++ b/src/org/ohdsi/usagi/ui/MappingTablePanel.java @@ -94,7 +94,7 @@ class CodeMapTableModel extends AbstractTableModel { private String[] defaultColumnNames = { "Status", "Source code", "Source term", "Frequency", "Match score", "Synonym", "Concept ID", "Concept name", "Domain", "Concept class", "Vocabulary", "Concept code", "Valid start date", - "Valid end date", "Invalid reason" }; + "Valid end date", "Invalid reason", "Standard concept" }; private String[] columnNames = defaultColumnNames; private int addInfoColCount = 0; private int ADD_INFO_START_COL = 4; @@ -180,6 +180,8 @@ public Object getValueAt(int row, int col) { return targetConcept.validEndDate; case 14: return targetConcept.invalidReason; + case 15: + return targetConcept.standardConcept; default: return ""; }