WIP - multi-threaded indexing - dataset prefix fix

#1321 gbif/pipelines#217
CatalogueOfLife · Jun 27, 2024 · 7cd5f7c · 7cd5f7c
1 parent e8f9b00
commit 7cd5f7c
Showing 1 changed file with 11 additions and 3 deletions.
diff --git a/matching-ws/src/main/java/life/catalogue/matching/DatasetIndex.java b/matching-ws/src/main/java/life/catalogue/matching/DatasetIndex.java
@@ -82,10 +82,10 @@ void init() {
       }
 
       // load identifier indexes
-      this.identifierSearchers = initialiseAdditionalIndexes("identifiers");
+      this.identifierSearchers = initialiseAdditionalIndexes("identifiers", prefixMapping);
 
       // load ancillary indexes
-      this.ancillarySearchers = initialiseAdditionalIndexes("ancillary");
+      this.ancillarySearchers = initialiseAdditionalIndexes("ancillary", prefixMapping);
 
     } else {
       log.warn("Lucene index not found at {}", mainIndexPath);
@@ -107,7 +107,7 @@ protected void reinit() {
     }
   }
 
-  private HashMap<Dataset, IndexSearcher> initialiseAdditionalIndexes(String directoryName) {
+  private HashMap<Dataset, IndexSearcher> initialiseAdditionalIndexes(String directoryName, Map<Integer, Dataset> prefixMapping) {
     HashMap<Dataset, IndexSearcher> searchers = new HashMap<>();
     if (Path.of(indexPath + "/" + directoryName).toFile().exists()) {
       try (DirectoryStream<Path> stream = Files.newDirectoryStream(Path.of(indexPath + "/" + directoryName))) {
@@ -119,6 +119,14 @@ private HashMap<Dataset, IndexSearcher> initialiseAdditionalIndexes(String direc
               ObjectMapper mapper = new ObjectMapper();
               Dataset dataset = mapper.readValue(new FileReader(entry.resolve("metadata.json").toFile()),
                 Dataset.class);
+
+              // apply prefix mapping
+              Dataset prefixDatasetConfig = prefixMapping.get(dataset.getKey());
+              if (prefixDatasetConfig != null) {
+                dataset.setPrefix(prefixDatasetConfig.getPrefix());
+                dataset.setPrefixMapping(prefixDatasetConfig.getPrefixMapping());
+              }
+
               searchers.put(dataset, new IndexSearcher(reader));
             } catch (IOException e) {
               log.warn("Cannot open {} lucene index {}", directoryName, entry, e);