For short strings, it is more accurate to use all ngrams rather than …

…a sample
Mimino666 · May 7, 2021 · 96dd84a · 96dd84a
1 parent a1598f1
commit 96dd84a
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/langdetect/detector.py b/langdetect/detector.py
@@ -149,6 +149,17 @@ def _detect_block(self):
         if not ngrams:
             raise LangDetectException(ErrorCode.CantDetectError, 'No features in text.')
 
+        if len(ngrams) < self.ITERATION_LIMIT / 3:
+            # More accurate to take them all
+            prob = self._init_probability()
+            for i, ngram in enumerate(ngrams):
+                self._update_lang_prob(prob, ngram, self.alpha)
+                if i % 5 == 0:
+                    self._normalize_prob(prob)
+            self._normalize_prob(prob)
+            self.langprob = prob
+            return
+
         self.langprob = [0.0] * len(self.langlist)
 
         self.random.seed(self.seed)