Added option to filter by ATC code. Fixes #22

src/org/ohdsi/usagi/AtcToRxNorm.java

@@ -0,0 +1,21 @@
+package org.ohdsi.usagi;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.sleepycat.persist.model.Entity;
+import com.sleepycat.persist.model.PrimaryKey;
+
+@Entity
+public class AtcToRxNorm {
+	@PrimaryKey
+	public String		atc;
+	public Set<Integer>	conceptIds	= new HashSet<Integer>();
+
+	public AtcToRxNorm() {
+	}
+
+	public AtcToRxNorm(String atc) {
+		this.atc = atc;
+	}
+}

src/org/ohdsi/usagi/BerkeleyDbEngine.java

@@ -2,7 +2,9 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import org.ohdsi.utilities.DirectoryUtilities;
 
@@ -22,6 +24,7 @@ public class BerkeleyDbEngine {
 	private EntityStore							store;
 	private ConceptDataAccessor					conceptDataAccessor;
 	private MapsToRelationshipDataAccessor		mapsToRelationshipDataAccessor;
+	private AtcToRxNormDataAccessor				atcToRxNormDataAccessor;
 	private ParentChildRelationshipDataAccessor	parentChildRelationshipDataAccessor;
 	private String								databaseFolder;
 	private boolean								isOpenForReading	= false;
@@ -60,6 +63,7 @@ private void open(boolean create) {
 			store = new EntityStore(dbEnvironment, "EntityStore", storeConfig);
 			conceptDataAccessor = new ConceptDataAccessor();
 			mapsToRelationshipDataAccessor = new MapsToRelationshipDataAccessor();
+			atcToRxNormDataAccessor = new AtcToRxNormDataAccessor();
 			parentChildRelationshipDataAccessor = new ParentChildRelationshipDataAccessor();
 		} catch (DatabaseException dbe) {
 			throw new RuntimeException(dbe);
@@ -81,15 +85,24 @@ public BerkeleyDbStats getStats() {
 	}
 
 	public void put(Concept concept) {
-		conceptDataAccessor.primaryIndex.put(concept);
+		conceptDataAccessor.primaryIndex.putNoReturn(concept);
 	}
 
 	public void put(MapsToRelationship mapsToRelationship) {
-		mapsToRelationshipDataAccessor.primaryIndex.put(mapsToRelationship);
+		mapsToRelationshipDataAccessor.primaryIndex.putNoReturn(mapsToRelationship);
+	}
+
+	public void putAtcToRxNorm(String atc, int conceptId) {
+		AtcToRxNorm atcToRxNorm = atcToRxNormDataAccessor.primaryIndex.get(atc);
+		if (atcToRxNorm == null) {
+			atcToRxNorm = new AtcToRxNorm(atc);
+		}
+		atcToRxNorm.conceptIds.add(conceptId);
+		atcToRxNormDataAccessor.primaryIndex.putNoReturn(atcToRxNorm);
 	}
 
 	public void put(ParentChildRelationShip parentChildRelationship) {
-		parentChildRelationshipDataAccessor.primaryIndex.put(parentChildRelationship);
+		parentChildRelationshipDataAccessor.primaryIndex.putNoReturn(parentChildRelationship);
 	}
 
 	public EntityCursor<Concept> getConceptCursor() {
@@ -113,6 +126,14 @@ public List<MapsToRelationship> getMapsToRelationshipsByConceptId2(int conceptId
 		return relationships;
 	}
 
+	public Set<Integer> getRxNormConceptIds(String atc) {
+		AtcToRxNorm atcToRxNorm = atcToRxNormDataAccessor.primaryIndex.get(atc);
+		if (atcToRxNorm == null)
+			return Collections.emptySet();
+		else
+			return atcToRxNorm.conceptIds;
+	}
+
 	public List<ParentChildRelationShip> getParentChildRelationshipsByParentConceptId(int conceptId) {
 		EntityIndex<Integer, ParentChildRelationShip> subIndex = parentChildRelationshipDataAccessor.secondaryIndexParent.subIndex(conceptId);
 		EntityCursor<ParentChildRelationShip> cursor = subIndex.entities();
@@ -160,7 +181,6 @@ private class ConceptDataAccessor {
 		public ConceptDataAccessor() throws DatabaseException {
 			primaryIndex = store.getPrimaryIndex(Integer.class, Concept.class);
 		}
-
 	}
 
 	private class MapsToRelationshipDataAccessor {
@@ -173,6 +193,14 @@ public MapsToRelationshipDataAccessor() throws DatabaseException {
 		}
 	}
 
+	private class AtcToRxNormDataAccessor {
+		public PrimaryIndex<String, AtcToRxNorm> primaryIndex;
+
+		public AtcToRxNormDataAccessor() throws DatabaseException {
+			primaryIndex = store.getPrimaryIndex(String.class, AtcToRxNorm.class);
+		}
+	}
+
 	private class ParentChildRelationshipDataAccessor {
 		public PrimaryIndex<Integer, ParentChildRelationShip>				primaryIndex;
 		public SecondaryIndex<Integer, Integer, ParentChildRelationShip>	secondaryIndexParent;

src/org/ohdsi/usagi/indexBuilding/BerkeleyDbBuilder.java

@@ -30,29 +30,36 @@
 import org.ohdsi.utilities.files.Row;
 
 public class BerkeleyDbBuilder {
-	private BerkeleyDbEngine	dbEngine;
-	private BuildThread			buildThread;
+	private BerkeleyDbEngine		dbEngine;
+	private BuildThread				buildThread;
+	private IntHashSet				validConceptIds;
+	private Map<Integer, String>	conceptIdToAtcCode;
 
 	public void buildIndex(String vocabFolder, String loincFileName, BuildThread buildThread) {
 		this.buildThread = buildThread;
 		dbEngine = new BerkeleyDbEngine(Global.folder);
 		dbEngine.createDatabase();
-		IntHashSet validConceptIds = loadValidConceptIds(vocabFolder + "/CONCEPT.csv");
-		loadAncestors(vocabFolder + "/CONCEPT_ANCESTOR.csv", validConceptIds);
-		loadRelationships(vocabFolder + "/CONCEPT_RELATIONSHIP.csv", validConceptIds);
+		loadValidConceptIdsAndAtcCodes(vocabFolder + "/CONCEPT.csv");
+		loadAncestors(vocabFolder + "/CONCEPT_ANCESTOR.csv");
+		loadRelationships(vocabFolder + "/CONCEPT_RELATIONSHIP.csv");
 		loadConcepts(vocabFolder + "/CONCEPT.csv", loincFileName);
 		dbEngine.shutdown();
 	}
 
-	private IntHashSet loadValidConceptIds(String conceptFileName) {
-		IntHashSet validConceptIds = new IntHashSet();
-		for (Row row : new ReadAthenaFile(conceptFileName)) 
-			if (row.get("invalid_reason") == null)
+	private IntHashSet loadValidConceptIdsAndAtcCodes(String conceptFileName) {
+		validConceptIds = new IntHashSet();
+		conceptIdToAtcCode = new HashMap<Integer, String>();
+
+		for (Row row : new ReadAthenaFile(conceptFileName))
+			if (row.get("invalid_reason") == null) {
 				validConceptIds.add(row.getInt("concept_id"));
+				if (row.get("vocabulary_id").equals("ATC"))
+					conceptIdToAtcCode.put(row.getInt("concept_id"), row.get("concept_code"));
+			}
 		return validConceptIds;
 	}
 
-	private void loadRelationships(String conceptRelationshipFileName, IntHashSet validConceptIds) {
+	private void loadRelationships(String conceptRelationshipFileName) {
 		buildThread.report("Loading relationship information");
 		int count = 0;
 		for (Row row : new ReadAthenaFile(conceptRelationshipFileName)) {
@@ -61,13 +68,20 @@ private void loadRelationships(String conceptRelationshipFileName, IntHashSet va
 				MapsToRelationship mapsToRelationship = new MapsToRelationship(row);
 				dbEngine.put(mapsToRelationship);
 			}
+			if (row.get("relationship_id").equals("ATC - RxNorm") && row.get("invalid_reason") == null && validConceptIds.contains(row.getInt("concept_id_1"))
+					&& validConceptIds.contains(row.getInt("concept_id_2"))) {
+				String atc = conceptIdToAtcCode.get(row.getInt("concept_id_1"));
+				if (atc != null) {
+					dbEngine.putAtcToRxNorm(atc, row.getInt("concept_id_2"));
+				}
+			}
 			count++;
 			if (count % 100000 == 0)
 				System.out.println("Processed " + count + " relationships");
 		}
 	}
 
-	private void loadAncestors(String conceptAncestorFileName, IntHashSet validConceptIds) {
+	private void loadAncestors(String conceptAncestorFileName) {
 		File file = new File(conceptAncestorFileName);
 		if (file.exists()) {
 			buildThread.report("Loading parent-child information");

src/org/ohdsi/usagi/indexBuilding/IndexBuildCoordinator.java

@@ -17,10 +17,10 @@
 public class IndexBuildCoordinator {
 
 	public static void main(String[] args) {
-		Global.folder = "c:/temp/";
-		String vocabFolder = "S:/Data/OMOP Standard Vocabulary V5/Vocabulary-20161230";
-		String loincFolder = "S:/Data/LOINC/loinc.csv";
-		///String loincFolder = null;
+		Global.folder = "c:/data/usagi/";
+		String vocabFolder = "C:\\Data\\OMOP Standard Vocabulary V5\\Vocabulary-20180823";
+//		String loincFolder = "c:/Data/LOINC/loinc.csv";
+		String loincFolder = null;
 		IndexBuildCoordinator buildIndex = new IndexBuildCoordinator();
 		buildIndex.buildIndexes(vocabFolder, loincFolder);
 	}

src/org/ohdsi/usagi/ui/FilterPanel.java

@@ -54,7 +54,7 @@ public FilterPanel() {
 		c.gridy = 0;
 		c.weightx = 0.1;
 		c.gridwidth = 2;
-		filterByAutoCheckBox = new JCheckBox("Filter by user selected concepts", false);
+		filterByAutoCheckBox = new JCheckBox("Filter by user selected concepts / ATC code", false);
 		filterByAutoCheckBox.setToolTipText("Limit the search to those concept IDs specified in the input file");
 		filterByAutoCheckBox.addActionListener(new ActionListener() {
 

src/org/ohdsi/usagi/ui/ImportDialog.java

@@ -56,16 +56,20 @@
 import org.ohdsi.utilities.StringUtilities;
 import org.ohdsi.utilities.collections.Pair;
 import org.ohdsi.utilities.files.ReadCSVFile;
+import org.ohdsi.utilities.files.ReadTextFile;
 
 public class ImportDialog extends JDialog {
 
 	private static final long		serialVersionUID		= 8119661833870381094L;
+	private static String			CONCEPT_IDS				= "Auto concept ID column";
+	private static String			ATC						= "ATC column";
 	private List<String>			columnNames				= new ArrayList<String>();
 	private String[]				comboBoxOptions;
 	private List<List<String>>		data					= new ArrayList<List<String>>();
 	private FilterPanel				filterPanel;
 	private JPanel					columnMappingPanel;
 	private JScrollPane				columnMappingScrollPane;
+	private JComboBox<String>		conceptIdsOrAtc;
 	private JComboBox<String>		sourceCodeColumn;
 	private JComboBox<String>		sourceNameColumn;
 	private JComboBox<String>		sourceFrequencyColumn;
@@ -232,7 +236,8 @@ private Component createColumnMappingPanel() {
 		c.gridy = 3;
 		c.anchor = GridBagConstraints.WEST;
 		c.weightx = 1;
-		columnMappingPanel.add(new JLabel("Auto concept ID column"), c);
+		conceptIdsOrAtc = new JComboBox<String>(new String[] { CONCEPT_IDS, ATC });
+		columnMappingPanel.add(conceptIdsOrAtc, c);
 		c.gridx = 1;
 		c.gridy = 3;
 		c.anchor = GridBagConstraints.EAST;
@@ -290,10 +295,27 @@ private void importData() {
 				return;
 			}
 			if (filterPanel.getFilterByAuto() && autoConceptIdColumn.getSelectedItem().toString().equals("")) {
-				JOptionPane.showMessageDialog(this, "Must select an auto concept ID column when filtering by automatically selected concept IDs",
+				JOptionPane.showMessageDialog(this,
+						"Must select an auto concept ID column / ATC column when filtering by automatically selected concept IDs / ATC code",
 						"Cannot complete import", JOptionPane.ERROR_MESSAGE);
 				return;
 			}
+			if (filterPanel.getFilterByAuto() && conceptIdsOrAtc.getSelectedItem().toString().equals(ATC)) {
+				boolean atcLoaded = false;
+				for (String line : new ReadTextFile(Global.folder + "/VocabularyIds.txt"))
+					if (line.equals("ATC")) {
+						atcLoaded = true;
+						break;
+					}
+				if (!atcLoaded) {
+					JOptionPane.showMessageDialog(this,
+							"Filtering by ATC codes is selected, but the vocabulary does not contain ATC concept. Please reload the vocabulary from Athena.",
+							"ATC vocabulary missing",
+							JOptionPane.ERROR_MESSAGE);
+					return;
+				}
+			}
+
 			List<SourceCode> sourceCodes = createSourceCodes();
 
 			JDialog dialog = new JDialog(this, "Progress Dialog", false);
@@ -351,10 +373,17 @@ private List<SourceCode> createSourceCodes() {
 				sourceCode.sourceCode = row.get(sourceCodeIndex);
 			if (sourceFrequencyIndex != -1)
 				sourceCode.sourceFrequency = Integer.parseInt(row.get(sourceFrequencyIndex));
+			else
+				sourceCode.sourceFrequency = -1;
 			if (sourceAutoIndex != -1)
-				for (String conceptId : row.get(sourceAutoIndex).split(";"))
-					if (!conceptId.equals(""))
-						sourceCode.sourceAutoAssignedConceptIds.add(Integer.parseInt(conceptId));
+				if (conceptIdsOrAtc.getSelectedItem().toString().equals(CONCEPT_IDS)) {
+					for (String conceptId : row.get(sourceAutoIndex).split(";"))
+						if (!conceptId.equals(""))
+							sourceCode.sourceAutoAssignedConceptIds.add(Integer.parseInt(conceptId));
+				} else {
+					Set<Integer> conceptIds = Global.dbEngine.getRxNormConceptIds(row.get(sourceAutoIndex));
+					sourceCode.sourceAutoAssignedConceptIds.addAll(conceptIds);
+				}
 			for (int additionalInfoIndex : additionalInfoIndexes)
 				sourceCode.sourceAdditionalInfo.add(new Pair<String, String>(columnNames.get(additionalInfoIndex), row.get(additionalInfoIndex)));
 			sourceCodes.add(sourceCode);
@@ -399,7 +428,7 @@ public void run() {
 				if (threadCount <= 0) {
 					threadCount = 1;
 				}
-				
+
 				// Note: Lucene's and BerkeleyDB's search objects are thread safe, so do not need to be recreated for each thread.
 				ForkJoinPool forkJoinPool = new ForkJoinPool(threadCount);
 				forkJoinPool.submit(() -> sourceCodes.parallelStream().forEach(sourceCode -> {