remove short workds

PyCampES · Apr 1, 2024 · fd2e364 · fd2e364
1 parent e4cf437
commit fd2e364
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/src/ficamp/classifier/preprocessing.py b/src/ficamp/classifier/preprocessing.py
@@ -48,6 +48,11 @@ def remove_isolated_digits(s: str) -> str:
     return " ".join(clean)
 
 
+def remove_short_words(s: str) -> str:
+    """Remove words made only of digits"""
+    return " ".join((word for word in s.split() if len(word) >= 2))
+
+
 def preprocess(s: str) -> str:
     "Clean up transaction description"
     steps = (
@@ -58,6 +63,7 @@ def preprocess(s: str) -> str:
         remove_digits,
         remove_punctuation,
         remove_isolated_digits,
+        remove_short_words,
     )
     out = s
     for func in steps:

diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py
@@ -8,6 +8,7 @@
     remove_pipes,
     remove_punctuation,
     remove_isolated_digits,
+    remove_short_words,
 )
 
 
@@ -82,6 +83,17 @@ def test_remove_isolated_digits(inp, exp):
     assert remove_isolated_digits(inp) == exp
 
 
+@pytest.mark.parametrize(
+    ("inp,exp"),
+    (
+        ("hello a world", "hello world"),
+        ("hello aa world", "hello aa world"),
+    ),
+)
+def test_remove_short_words(inp, exp):
+    assert remove_short_words(inp) == exp
+
+
 @pytest.mark.parametrize(
     ("inp,exp"),
     (
@@ -98,6 +110,7 @@ def test_remove_isolated_digits(inp, exp):
         ("CSID:NL0213324324324 HELLO,world1332", "csid hello"),
         ("CSID:NL021332432 N26 HELLO,world1332", "csid n26 hello"),
         ("CSID:NL021332432 4324 HELLO,world1332", "csid hello"),
+        ("CSID:NL021332432 n. HELLO,world1332", "csid hello"),
     ),
 )
 def test_preprocess(inp, exp):