Merge pull request behrouzbakhtiari#11 from ahangarha/improve-clean-t…

…weet Improve clean tweet
irLinja · Mar 1, 2020 · f8f8c82 · f8f8c82
2 parents a439fd6 + cfe28c3
commit f8f8c82
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/twc.py b/twc.py
@@ -70,10 +70,12 @@ def remove_emoji(tweet):
 def clean_tweet(tweet):
     tweet = str(tweet)
     tweet = tweet.lower()
+    tweet = tweet.replace("#", "") # remove # so we preserve hashtags for the cloud
     tweet = tp.clean(tweet)
     tweet = remove_emoji(tweet)
     normalizer = Normalizer()
     tweet = normalizer.normalize(tweet)
+    tweet = re.sub(r'ن?می[‌]\S+','',tweet) # removes verbs such as می‌شود or نمی‌گویند
     tokens = word_tokenize(tweet)
     tokens = [token for token in tokens if token not in stopwords.persian]
     tokens = [token for token in tokens if token not in stopwords.english]