Merge pull request #18 from opendatalab/dev

fix: Fix missing short text in complex code blocks

magic_html/extractors/base_extractor.py

@@ -760,6 +760,9 @@ def delete_by_link_density(
             elemtext = trim(elem.text_content())
             result, templist = link_density_test(elem, elemtext, favor_precision)
             if result is True and img_div_check(elem):
+                # 保留table中的链接
+                if tagname in ['ul', 'li', 'div', 'p'] and ancestor_node_check(elem, ['td']):
+                    continue
                 deletions.append(elem)
             elif backtracking is True and len(templist) > 0:  # if?
                 myelems[elemtext].append(elem)

magic_html/readability_plus.py

@@ -448,6 +448,14 @@ def sanitize(self, node, candidates):
                     reason = "less than 3x <p>s than <input>s"
                     to_remove = True
                 elif content_length < MIN_LEN and counts["img"] == 0:
+                    # 代码块内容过短，导致删除
+                    if el.tag in ['code', 'pre']:
+                        continue
+                    if ancestor_node_check(el, ['code', 'pre']):
+                        continue
+                    # 保留table中的链接
+                    if el.tag in ['ul', 'li', 'div', 'p'] and ancestor_node_check(el, ['td']):
+                        continue
                     reason = (
                             "too short content length %s without a single image"
                             % content_length

magic_html/utils.py

@@ -173,6 +173,13 @@ def fromstring_bytes(htmlobject):
     return tree
 
 
+def ancestor_node_check(node: HtmlElement, tags: list):
+    for tag in tags:
+        if node.xpath(f'ancestor::{tag}[1]'):
+            return True
+    return False
+
+
 def load_html(htmlobject):
     if isinstance(htmlobject, HtmlElement):
         return htmlobject

requirements.txt

@@ -1,5 +1,5 @@
 Brotli
-cchardet
+cchardet==2.2.0a2
 charset_normalizer
 lxml<5.2.0
 numpy