remove some verbs from tweets

irLinja · Mar 1, 2020 · cfe28c3 · cfe28c3
1 parent 9e4129a
commit cfe28c3
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/twc.py b/twc.py
@@ -75,6 +75,7 @@ def clean_tweet(tweet):
     tweet = remove_emoji(tweet)
     normalizer = Normalizer()
     tweet = normalizer.normalize(tweet)
+    tweet = re.sub(r'ن?می[‌]\S+','',tweet) # removes verbs such as می‌شود or نمی‌گویند
     tokens = word_tokenize(tweet)
     tokens = [token for token in tokens if token not in stopwords.persian]
     tokens = [token for token in tokens if token not in stopwords.english]