Try some refactor to handle more cases.

papyri/common_ast.py

@@ -81,6 +81,16 @@ def __hash__(self):
         )
 
 
+class UnserializableNode(Node):
+    _dont_serialise = True
+
+    def cbor(self, encoder):
+        assert False
+
+    def to_json(self) -> bytes:
+        assert False
+
+
 TAG_MAP: Dict[Any, int] = {}
 REV_TAG_MAP: Dict[int, Any] = {}
 

papyri/examples.py

@@ -123,9 +123,9 @@
 Substitutions
 ~~~~~~~~~~~~~
 
-In this paragraph: |SubstitutionRef| Should be replaced...
+In this paragraph: |ALIAS| Should be replaced...
 
-.. |SubstitutionDef| replace:: ASUBSTITUTIONDEF
+.. |ALIAS| replace:: '-- SUBSTITUTION works--'
 
 
 

papyri/gen.py

@@ -2260,11 +2260,27 @@ def collect_api_docs(self, root: str, limit_to: List[str]) -> None:
 
             # substitution_defs: Dict[str, Union(MImage, ReplaceNode)] = {}
             substitution_defs = {}
+            sections = []
             for section in doc_blob.sections:
-                for child in doc_blob.content.get(section, []):
+                sections.append(doc_blob.content.get(section, []))
+            # arbitrary is usually raw RST that typically is a module docstring that can't be
+            # parsed by numpydoc
+            sections.extend(arbitrary)
+            for section in sections:
+                for child in section:
                     if isinstance(child, SubstitutionDef):
+                        if child.value in substitution_defs:
+                            self.log.warn(
+                                "The same substitution definition was found twice: %s",
+                                child.value,
+                            )
                         substitution_defs[child.value] = child.children
 
+            if substitution_defs:
+                self.log.debug(
+                    "Found substitution def for %s : %s", qa, substitution_defs
+                )
+
             # def flat(l) -> List[str]:
             #    return [y for x in l for y in x]
             for lr1 in _local_refs:

papyri/myst_ast.py

@@ -3,7 +3,7 @@
 """
 from typing import List, Union, Optional, Dict
 
-from .common_ast import Node, register
+from .common_ast import Node, register, UnserializableNode
 
 from . import take2
 from . import signature
@@ -151,16 +151,6 @@ def from_unprocessed(cls, up):
         return cls(up.name, up.args, up.options, up.value, up.children)
 
 
-class UnserializableNode(Node):
-    _dont_serialise = True
-
-    def cbor(self, encoder):
-        assert False
-
-    def to_json(self) -> bytes:
-        assert False
-
-
 class UnprocessedDirective(UnserializableNode):
     """
     Placeholder for yet unprocessed directives,  after they are parsed by tree-sitter,

papyri/take2.py

@@ -64,7 +64,7 @@
 import cbor2
 from there import print
 
-from .common_ast import Node, REV_TAG_MAP, register
+from .common_ast import Node, REV_TAG_MAP, register, UnserializableNode
 from .miniserde import get_type_hints
 
 from .utils import dedent_but_first
@@ -157,8 +157,7 @@ class Leaf(Node):
     value: str
 
 
-@register(4027)
-class SubstitutionDef(Node):
+class SubstitutionDef(UnserializableNode):
     """
     A Substitution Definition should be in the form of
 
@@ -181,21 +180,25 @@ def __init__(self, value, children):
         assert isinstance(children[0], UnprocessedDirective)
 
         if children[0].name == "image":
+            assert len(children) == 1
             self.children = [MImage(url=children[0].args, alt="")]
         elif children[0].name == "replace":
+            assert len(children) == 1
             self.children = [ReplaceNode(value=self.value, text=children[0].args)]
         else:
-            raise NotImplementedError
+            raise NotImplementedError("Substitution def not implemented for ", children)
 
 
-@register(4041)
-class SubstitutionRef(Leaf):
+class SubstitutionRef(UnserializableNode):
     """
     This will be in the for |XXX|, and need to be replaced.
     """
 
     value: str
 
+    def __init__(self, value):
+        self.value = value
+
 
 @register(4018)
 class Unimplemented(Node):

papyri/tree.py

@@ -691,6 +691,7 @@ def replace_SubstitutionDef(self, node):
         This node should be removed since the actual reference has already been
         resolved.
         """
+        log.warning("discard substitution node %s", node)
         return []
 
     def replace_SubstitutionRef(self, directive):
@@ -699,15 +700,17 @@ def replace_SubstitutionRef(self, directive):
         # an image (from the image directive.)
         if directive.value in self.substitution_defs:
             if isinstance(self.substitution_defs[directive.value][0], ReplaceNode):
-                return [MText(self.substitution_defs[directive.value][0].text)]
+                return self.substitution_defs[directive.value][0].children
             elif isinstance(self.substitution_defs[directive.value][0], MImage):
                 return [self.substitution_defs[directive.value][0]]
             else:
                 raise NotImplementedError(
-                    f"SubstitutionDef for custom inline directive {self.substitution_defs[directive.value][0]} not implemented."
+                    f"SubstitutionDef for custom inline directive "
+                    f"{self.substitution_defs[directive.value][0]} "
+                    "not implemented."
                 )
         else:
-            return [MText(directive.value)]
+            return [directive]
 
     def _resolve(self, loc, text):
         """

papyri/ts.py

@@ -512,7 +512,7 @@ def visit_paragraph(self, node):
     def visit_line_block(self, node):
         # TODO
         # e.g: numpy/doc/source/user/c-info.how-to-extend.rst
-        print("Skipping node", self.as_text(node))
+        log.warning("Skipping unimplemented line_block node: %s", self.as_text(node))
         return []
 
     def visit_substitution_reference(self, node):