BM25 tokenizer lowercase (#9745)

run-llama · Dec 29, 2023 · 9c30dbe · 9c30dbe
1 parent 7f32ca4
commit 9c30dbe
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/llama_index/retrievers/bm25_retriever.py b/llama_index/retrievers/bm25_retriever.py
@@ -15,6 +15,8 @@
 
 
 def tokenize_remove_stopwords(text: str) -> List[str]:
+    # lowercase and stem words
+    text = text.lower()
     stemmer = PorterStemmer()
     words = list(simple_extract_keywords(text))
     return [stemmer.stem(word) for word in words]