fix: preprocess reference in BM25 algorithm

Author: brunoarine

findlike/wrappers.py

@@ -59,6 +59,7 @@ def fit(self, documents: list[str]):
         self._model = BM25Okapi(self.tokenized_documents_)
 
     def get_scores(self, source: str):
-        tokenized_source = self.processor.tokenizer(source)
+        clean_source = self.processor.preprocessor(source)
+        tokenized_source = self.processor.tokenizer(clean_source)
         scores = self._model.get_scores(tokenized_source)
         return scores

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "findlike"
-version = "1.4.1"
+version = "1.4.2"
 authors = [{ name = "Bruno Arine", email = "[email protected]" }]
 description = "findlike is a package to retrieve similar documents"
 readme = "README.md"