Added option to map to non-standard (classification) concepts)

src/org/ohdsi/usagi/TargetConcept.java

@@ -38,6 +38,7 @@ public class TargetConcept {
 	public String				validEndDate;
 	public String				invalidReason;
 	public String				additionalInformation;
+	public String				standardConcept;
 	public static TargetConcept	EMPTY_CONCEPT	= createEmptyConcept();
 
 	public TargetConcept(Row row) {
@@ -52,6 +53,10 @@ public TargetConcept(Row row) {
 			validStartDate = row.get("VALID_START_DATE");
 			validEndDate = row.get("VALID_END_DATE");
 			invalidReason = row.get("INVALID_REASON");
+			if (row.getFieldNames().contains("STANDARDCONCEPT"))
+				standardConcept = row.get("STANDARDCONCEPT");
+			else
+				standardConcept = "";
 			for (String domain : row.get("DOMAINS").split("/"))
 				domains.add(domain);
 			additionalInformation = row.get("ADDITIONAL_INFORMATION");
@@ -65,6 +70,10 @@ public TargetConcept(Row row) {
 			validStartDate = row.get("VALIDSTARTDATE");
 			validEndDate = row.get("VALIDENDDATE");
 			invalidReason = row.get("INVALIDREASON");
+			if (row.getFieldNames().contains("STANDARDCONCEPT"))
+				standardConcept = row.get("STANDARDCONCEPT");
+			else
+				standardConcept = "";
 			for (String domain : row.get("DOMAINS").split("/"))
 				domains.add(domain);
 			additionalInformation = row.get("ADDITIONALINFORMATION");
@@ -82,6 +91,7 @@ private static TargetConcept createEmptyConcept() {
 		concept.validStartDate = "";
 		concept.validEndDate = "";
 		concept.invalidReason = "";
+		concept.standardConcept = "";
 		concept.additionalInformation = "This is an empty concept";
 		return concept;
 	}
@@ -100,6 +110,7 @@ public TargetConcept(TargetConcept concept) {
 		validEndDate = concept.validEndDate;
 		invalidReason = concept.invalidReason;
 		domains.addAll(concept.domains);
+		standardConcept = concept.standardConcept;
 		additionalInformation = concept.additionalInformation;
 	}
 
@@ -115,6 +126,7 @@ public Row toRow() {
 		row.add("validEndDate", validEndDate);
 		row.add("invalidReason", invalidReason);
 		row.add("domains", StringUtilities.join(domains, "/"));
+		row.add("standardConcept", standardConcept);
 		if (additionalInformation == null)
 			row.add("additionalInformation", "");
 		else

src/org/ohdsi/usagi/UsagiSearchEngine.java

@@ -91,7 +91,8 @@ public class UsagiSearchEngine {
 	private QueryParser		conceptClassQueryParser;
 	private QueryParser		vocabularyQueryParser;
 	private QueryParser		keywordsQueryParser;
-	private QueryParser		invalidQueryParser;
+//	private QueryParser		invalidQueryParser;
+	private QueryParser		standardConceptParser;
 	private QueryParser		domainQueryParser;
 	private int				numDocs;
 	private FieldType		textVectorField			= getTextVectorFieldType();
@@ -150,6 +151,7 @@ public void addConceptToIndex(TargetConcept concept) {
 			document.add(new StringField("VALID_START_DATE", concept.validStartDate, Store.YES));
 			document.add(new StringField("VALID_END_DATE", concept.validEndDate, Store.YES));
 			document.add(new StringField("INVALID_REASON", concept.invalidReason, Store.YES));
+			document.add(new StringField("STANDARD_CONCEPT", concept.standardConcept, Store.YES));
 			document.add(new TextField("DOMAINS", StringUtilities.join(concept.domains, "\n"), Store.YES));
 			document.add(new StringField("ADDITIONAL_INFORMATION", concept.additionalInformation, Store.YES));
 			writer.addDocument(document);
@@ -260,7 +262,8 @@ public void openIndexForSearching() {
 			vocabularyQueryParser = new QueryParser(Version.LUCENE_4_9, "VOCABULARY", new KeywordAnalyzer());
 			keywordsQueryParser = new QueryParser(Version.LUCENE_4_9, "TERM", analyzer);
 			domainQueryParser = new QueryParser(Version.LUCENE_4_9, "DOMAINS", analyzer);
-			invalidQueryParser = new QueryParser(Version.LUCENE_4_9, "INVALID_REASON", new KeywordAnalyzer());
+//			invalidQueryParser = new QueryParser(Version.LUCENE_4_9, "INVALID_REASON", new KeywordAnalyzer());
+			standardConceptParser = new QueryParser(Version.LUCENE_4_9, "STANDARD_CONCEPT", new KeywordAnalyzer());
 			numDocs = reader.numDocs();
 		} catch (Exception e) {
 			throw new RuntimeException(e);
@@ -317,7 +320,7 @@ public void close() {
 	}
 
 	public List<ScoredConcept> search(String searchTerm, boolean useMlt, Collection<Integer> filterConceptIds, String filterDomain, String filterConceptClass,
-			String filterVocabulary, boolean filterInvalid) {
+			String filterVocabulary, boolean filterStandard) {
 		List<ScoredConcept> results = new ArrayList<ScoredConcept>();
 		try {
 			Query query;
@@ -365,9 +368,9 @@ public List<ScoredConcept> search(String searchTerm, boolean useMlt, Collection<
 				Query vocabularyQuery = vocabularyQueryParser.parse("\"" + filterVocabulary.toString() + "\"");
 				booleanQuery.add(vocabularyQuery, Occur.MUST);
 			}
-			if (filterInvalid) {
-				Query invalidQuery = invalidQueryParser.parse("\"\"");
-				booleanQuery.add(invalidQuery, Occur.MUST);
+			if (filterStandard) {
+				Query standardQuery = standardConceptParser.parse("S");
+				booleanQuery.add(standardQuery, Occur.MUST);
 			}
 			TopDocs topDocs = searcher.search(booleanQuery, 100);
 
@@ -402,6 +405,7 @@ private TargetConcept docIdToTargetConcept(int docId) {
 			targetConcept.validStartDate = document.get("VALID_START_DATE");
 			targetConcept.validEndDate = document.get("VALID_END_DATE");
 			targetConcept.invalidReason = document.get("INVALID_REASON");
+			targetConcept.standardConcept = document.get("STANDARD_CONCEPT");
 			for (String domain : document.get("DOMAINS").split("\n"))
 				targetConcept.domains.add(domain);
 			targetConcept.additionalInformation = document.get("ADDITIONAL_INFORMATION");

src/org/ohdsi/usagi/indexBuilding/BuildIndex.java

@@ -151,7 +151,7 @@ public void run() {
 
 						Row conceptRow = multiRowSet.get("concept").get(0);
 						if (conceptRow.getCells().size() > 2) // Extra check to catch badly formatted rows (which are in a vocab we don't care about)
-							if (conceptRow.get("STANDARD_CONCEPT").equals("S")) {
+							if (conceptRow.get("STANDARD_CONCEPT").equals("S") || conceptRow.get("STANDARD_CONCEPT").equals("C")) {
 								vocabularies.add(conceptRow.get("VOCABULARY_ID"));
 								conceptClassIds.add(conceptRow.get("CONCEPT_CLASS_ID"));
 								domainIds.add(conceptRow.get("DOMAIN_ID"));
@@ -179,6 +179,7 @@ public void run() {
 									concept.invalidReason = conceptRow.get("INVALID_REASON");
 									concept.validEndDate = conceptRow.get("VALID_END_DATE");
 									concept.validStartDate = conceptRow.get("VALID_START_DATE");
+									concept.standardConcept = conceptRow.get("STANDARD_CONCEPT");
 									concept.vocabulary = conceptRow.get("VOCABULARY_ID");
 									if (loincToInfo != null && concept.vocabulary.equals("LOINC")) {
 										String info = loincToInfo.get(concept.conceptCode);

src/org/ohdsi/usagi/ui/FilterPanel.java

@@ -35,7 +35,7 @@ public class FilterPanel extends JPanel {
 
 	private static final long			serialVersionUID	= 1378433878412231259L;
 	private JCheckBox					filterByAutoCheckBox;
-	private JCheckBox					filterInvalidCheckBox;
+	private JCheckBox					filterStandardCheckBox;
 	private JCheckBox					filterByConceptClassCheckBox;
 	private JCheckBox					filterByVocabularyCheckBox;
 	private JCheckBox					filterByDomainCheckBox;
@@ -70,16 +70,17 @@ public void actionPerformed(ActionEvent arg0) {
 		c.gridy = 0;
 		c.weightx = 0.1;
 		c.gridwidth = 2;
-		filterInvalidCheckBox = new JCheckBox("Filter invalid concepts", false);
-		filterInvalidCheckBox.setToolTipText("Limit the search to only valid concepts");
-		filterInvalidCheckBox.addActionListener(new ActionListener() {
+		filterStandardCheckBox = new JCheckBox("Filter standard concepts", false);
+		filterStandardCheckBox.setToolTipText("Limit the search to only standard concepts");
+		filterStandardCheckBox.addActionListener(new ActionListener() {
 
 			@Override
 			public void actionPerformed(ActionEvent arg0) {
 				notifyListeners();
 			}
 		});
-		add(filterInvalidCheckBox, c);
+		filterStandardCheckBox.setSelected(true);
+		add(filterStandardCheckBox, c);
 
 		c.gridx = 0;
 		c.gridy = 1;
@@ -197,8 +198,8 @@ public boolean getFilterByAuto() {
 		return filterByAutoCheckBox.isSelected();
 	}
 
-	public boolean getFilterInvalid() {
-		return filterInvalidCheckBox.isSelected();
+	public boolean getFilterStandard() {
+		return filterStandardCheckBox.isSelected();
 	}
 
 	public boolean getFilterByConceptClass() {

src/org/ohdsi/usagi/ui/ImportDialog.java

@@ -375,7 +375,7 @@ public void run() {
 			try {
 				Global.usagiSearchEngine.createDerivedIndex(sourceCodes, null);
 
-				boolean filterInvalid = filterPanel.getFilterInvalid();
+				boolean filterStandard = filterPanel.getFilterStandard();
 				String filterConceptClass = null;
 				if (filterPanel.getFilterByConceptClass())
 					filterConceptClass = filterPanel.getConceptClass();
@@ -394,7 +394,7 @@ public void run() {
 
 					CodeMapping codeMapping = new CodeMapping(sourceCode);
 					List<ScoredConcept> concepts = Global.usagiSearchEngine.search(sourceCode.sourceName, true, filterConceptIds, filterDomain,
-							filterConceptClass, filterVocabulary, filterInvalid);
+							filterConceptClass, filterVocabulary, filterStandard);
 					if (concepts.size() > 0) {
 						codeMapping.targetConcepts.add(concepts.get(0).concept);
 						codeMapping.matchScore = concepts.get(0).matchScore;

src/org/ohdsi/usagi/ui/MappingDetailPanel.java

@@ -206,6 +206,10 @@ public void valueChanged(ListSelectionEvent event) {
 			}
 
 		});
+//		searchTable.hideColumn("Synonym");
+		searchTable.hideColumn("Valid start date");
+		searchTable.hideColumn("Valid end date");
+		searchTable.hideColumn("Invalid reason");
 		panel.add(new JScrollPane(searchTable));
 
 		JPanel buttonPanel = new JPanel();
@@ -384,7 +388,7 @@ public void run() {
 			if (filterPanel.getFilterByAuto())
 				filterConceptIds = codeMapping.sourceCode.sourceAutoAssignedConceptIds;
 
-			boolean filterInvalid = filterPanel.getFilterInvalid();
+			boolean filterStandard = filterPanel.getFilterStandard();
 			String filterConceptClass = null;
 			if (filterPanel.getFilterByConceptClass())
 				filterConceptClass = filterPanel.getConceptClass();
@@ -400,7 +404,7 @@ public void run() {
 
 			if (Global.usagiSearchEngine.isOpenForSearching()) {
 				List<ScoredConcept> searchResults = Global.usagiSearchEngine.search(query, true, filterConceptIds, filterDomain, filterConceptClass,
-						filterVocabulary, filterInvalid);
+						filterVocabulary, filterStandard);
 
 				searchTableModel.setScoredConcepts(searchResults);
 				searchTable.scrollRectToVisible(new Rectangle(searchTable.getCellRect(0, 0, true)));
@@ -505,7 +509,7 @@ class TargetConceptTableModel extends AbstractTableModel {
 
 		private String				scoreColumnName		= "Score";
 		private String[]			columnNames			= { "Synonym", "Concept ID", "Concept name", "Domain", "Concept class", "Vocabulary", "Concept code",
-																"Valid start date", "Valid end date", "Invalid reason" };
+																"Valid start date", "Valid end date", "Invalid reason", "Standard concept" };
 		private List<TargetConcept>	targetConcepts		= new ArrayList<TargetConcept>();
 		private boolean				hasScoreColumn;
 		private Double[]			scoreColumn;
@@ -585,6 +589,8 @@ public Object getValueAt(int row, int col) {
 					return targetConcept.validEndDate;
 				case 9:
 					return targetConcept.invalidReason;
+				case 10:
+					return targetConcept.standardConcept;
 				default:
 					return "";
 			}
@@ -617,6 +623,8 @@ public Class<?> getColumnClass(int col) {
 					return String.class;
 				case 9:
 					return String.class;
+				case 10:
+					return String.class;
 				default:
 					return String.class;
 			}

src/org/ohdsi/usagi/ui/MappingTablePanel.java

@@ -94,7 +94,7 @@ class CodeMapTableModel extends AbstractTableModel {
 
 		private String[]			defaultColumnNames	= { "Status", "Source code", "Source term", "Frequency", "Match score", "Synonym", "Concept ID",
 																"Concept name", "Domain", "Concept class", "Vocabulary", "Concept code", "Valid start date",
-																"Valid end date", "Invalid reason" };
+																"Valid end date", "Invalid reason", "Standard concept" };
 		private String[]			columnNames			= defaultColumnNames;
 		private int					addInfoColCount		= 0;
 		private int					ADD_INFO_START_COL	= 4;
@@ -180,6 +180,8 @@ public Object getValueAt(int row, int col) {
 						return targetConcept.validEndDate;
 					case 14:
 						return targetConcept.invalidReason;
+					case 15:
+						return targetConcept.standardConcept;
 					default:
 						return "";
 				}