Update check_tokens.py

matbahasa · Sep 13, 2019 · 5bfa032 · 5bfa032
1 parent 419742f
commit 5bfa032
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/check_tokens.py b/check_tokens.py
@@ -39,7 +39,7 @@
             tokens.append(x)
 
 #remove all punctuations
-tokens = [re.sub("[\>\<\/\#\”\“\'\`\(\)\:\;\!\?\"\,\s\.\[,\]]+", "", w) for w in tokens]
+tokens = [re.sub("[\>\<\/\#\”\“\'\`\(\)\:\;\!\?\"\,\s\.\[\]]+", "", w) for w in tokens]
 
 #open MALINDO Morph and make a list of all surface forms
 m = codecs.open('/home/david/MALINDO_Morph/malindo_dic_20180817.tsv', encoding='utf-8', mode='r')