perf: update regex pattern for Vietnamese

datvodinh · May 25, 2024 · 1f6bc2c · 1f6bc2c
1 parent 4e9755d
commit 1f6bc2c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/rag_chatbot/core/ingestion/ingestion.py b/rag_chatbot/core/ingestion/ingestion.py
@@ -19,7 +19,7 @@ def __init__(self, setting: RAGSettings | None = None) -> None:
 
     def _filter_text(self, text):
         # Define the regex pattern.
-        pattern = r'[a-zA-Z0-9 `~!@#$%^&*()_\-+=\[\]{}|\\;:\'",.<>/?]+'
+        pattern = r'[a-zA-Z0-9 \u00C0-\u01B0\u1EA0-\u1EF9`~!@#$%^&*()_\-+=\[\]{}|\\;:\'",.<>/?]+'
         matches = re.findall(pattern, text)
         # Join all matched substrings into a single string
         filtered_text = ' '.join(matches)