From e327e1ce08bfd8e9a0edf493bc0df1bfb8c5a264 Mon Sep 17 00:00:00 2001 From: damien-git Date: Fri, 11 Aug 2023 09:19:23 -0400 Subject: [PATCH 1/3] [VUFIND-1630] New normalizer for titles based on SolrMarc titleSortLower (#43) --- build.xml | 3 +++ .../java/org/vufind/util/TitleNormalizer.java | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 common/java/org/vufind/util/TitleNormalizer.java diff --git a/build.xml b/build.xml index 8facbef..6110a0b 100644 --- a/build.xml +++ b/build.xml @@ -45,6 +45,9 @@ + + + diff --git a/common/java/org/vufind/util/TitleNormalizer.java b/common/java/org/vufind/util/TitleNormalizer.java new file mode 100644 index 0000000..7324516 --- /dev/null +++ b/common/java/org/vufind/util/TitleNormalizer.java @@ -0,0 +1,18 @@ +package org.vufind.util; + +import java.util.EnumSet; + +import org.solrmarc.index.extractor.formatter.FieldFormatter; +import org.solrmarc.tools.DataUtil; + +public class TitleNormalizer implements Normalizer +{ + @Override + public byte[] normalize(String s) + { + EnumSet cleanValue = DataUtil.getCleanValForParam("titleSortLower"); + String normalizedTitle = DataUtil.cleanByVal(s, cleanValue); + byte[] bytes = normalizedTitle == null ? null : normalizedTitle.getBytes(); + return bytes; + } +} From 1d991f32a9fcf6808c033623af1cefef691c694d Mon Sep 17 00:00:00 2001 From: Demian Katz Date: Thu, 19 Oct 2023 06:23:12 -0400 Subject: [PATCH 2/3] Fix deprecations reported when building with Solr 9.3. (#44) --- browse-handler/java/org/vufind/solr/handler/AuthDB.java | 4 ++-- browse-handler/java/org/vufind/solr/handler/BibDB.java | 4 ++-- browse-indexing/PrintBrowseHeadings.java | 2 +- browse-indexing/StoredFieldLeech.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/browse-handler/java/org/vufind/solr/handler/AuthDB.java b/browse-handler/java/org/vufind/solr/handler/AuthDB.java index b8129b3..2929576 100644 --- a/browse-handler/java/org/vufind/solr/handler/AuthDB.java +++ b/browse-handler/java/org/vufind/solr/handler/AuthDB.java @@ -61,7 +61,7 @@ public Document getAuthorityRecord(String heading) 1)); if (results.totalHits.value > 0) { - return searcher.getIndexReader().document(results.scoreDocs[0].doc); + return searcher.getIndexReader().storedFields().document(results.scoreDocs[0].doc); } else { return null; } @@ -78,7 +78,7 @@ public List getPreferredHeadings(String heading) List result = new ArrayList<> (); for (int i = 0; i < results.totalHits.value; i++) { - result.add(searcher.getIndexReader().document(results.scoreDocs[i].doc)); + result.add(searcher.getIndexReader().storedFields().document(results.scoreDocs[i].doc)); } return result; diff --git a/browse-handler/java/org/vufind/solr/handler/BibDB.java b/browse-handler/java/org/vufind/solr/handler/BibDB.java index c81e55f..c8ae45c 100644 --- a/browse-handler/java/org/vufind/solr/handler/BibDB.java +++ b/browse-handler/java/org/vufind/solr/handler/BibDB.java @@ -224,7 +224,7 @@ public void collect(int docnum) { int docid = docnum + context.docBase; try { - Document doc = db.getIndexReader().document(docid); + Document doc = db.getIndexReader().storedFields().document(docid); for (String bibField : bibExtras) { String[] vals = doc.getValues(bibField); if (vals.length > 0) { @@ -321,7 +321,7 @@ public void collect(int docnum) { int docid = docnum + context.docBase; try { - Document doc = db.getIndexReader().document(docid); + Document doc = db.getIndexReader().storedFields().document(docid); for (String bibField : bibExtras) { for (String v : doc.getValues(bibField)) { bibinfo.get(bibField).add(v); diff --git a/browse-indexing/PrintBrowseHeadings.java b/browse-indexing/PrintBrowseHeadings.java index e5e4156..a9853a5 100644 --- a/browse-indexing/PrintBrowseHeadings.java +++ b/browse-indexing/PrintBrowseHeadings.java @@ -107,7 +107,7 @@ private boolean isLinkedFromBibData(String heading) } for (int i = 0; i < hits.scoreDocs.length; i++) { - Document doc = authSearcher.getIndexReader().document(hits.scoreDocs[i].doc); + Document doc = authSearcher.getIndexReader().storedFields().document(hits.scoreDocs[i].doc); String[] preferred = doc.getValues(System.getProperty("field.preferred", "preferred")); if (preferred.length > 0) { diff --git a/browse-indexing/StoredFieldLeech.java b/browse-indexing/StoredFieldLeech.java index 8c47085..5982c5e 100644 --- a/browse-indexing/StoredFieldLeech.java +++ b/browse-indexing/StoredFieldLeech.java @@ -47,7 +47,7 @@ public StoredFieldLeech(String indexPath, String field) throws Exception private void loadDocument(IndexReader reader, int docid) throws Exception { - Document doc = reader.document(currentDoc, fieldSelection); + Document doc = reader.storedFields().document(currentDoc, fieldSelection); String[] sort_key = doc.getValues(sortField); String[] value = doc.getValues(valueField); From a9fc2477004777c42e675d643279e6647bfa3647 Mon Sep 17 00:00:00 2001 From: Demian Katz Date: Thu, 19 Oct 2023 13:15:21 -0400 Subject: [PATCH 3/3] Access stored fields more efficiently. (#45) --- browse-handler/java/org/vufind/solr/handler/AuthDB.java | 4 +++- browse-indexing/PrintBrowseHeadings.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/browse-handler/java/org/vufind/solr/handler/AuthDB.java b/browse-handler/java/org/vufind/solr/handler/AuthDB.java index 2929576..3f97ebb 100644 --- a/browse-handler/java/org/vufind/solr/handler/AuthDB.java +++ b/browse-handler/java/org/vufind/solr/handler/AuthDB.java @@ -7,6 +7,7 @@ import java.util.Map; import org.apache.lucene.document.Document; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -77,8 +78,9 @@ public List getPreferredHeadings(String heading) List result = new ArrayList<> (); + StoredFields storedFields = searcher.getIndexReader().storedFields(); for (int i = 0; i < results.totalHits.value; i++) { - result.add(searcher.getIndexReader().storedFields().document(results.scoreDocs[i].doc)); + result.add(storedFields.document(results.scoreDocs[i].doc)); } return result; diff --git a/browse-indexing/PrintBrowseHeadings.java b/browse-indexing/PrintBrowseHeadings.java index a9853a5..23f8eee 100644 --- a/browse-indexing/PrintBrowseHeadings.java +++ b/browse-indexing/PrintBrowseHeadings.java @@ -106,8 +106,9 @@ private boolean isLinkedFromBibData(String heading) } } + StoredFields storedFields = authSearcher.getIndexReader().storedFields(); for (int i = 0; i < hits.scoreDocs.length; i++) { - Document doc = authSearcher.getIndexReader().storedFields().document(hits.scoreDocs[i].doc); + Document doc = storedFields.document(hits.scoreDocs[i].doc); String[] preferred = doc.getValues(System.getProperty("field.preferred", "preferred")); if (preferred.length > 0) {