WIP: Add substitution definition code and tests

TODO:
- [ ] Add tests for substitution definitions in ascii_expected (currently failing)
- [ ] Implement substitution definitions for all section types in gen.py (including narrative and module docstrings)
- [ ] Implement support for custom inline directives

papyri/crosslink.py

@@ -49,7 +49,7 @@ def find_all_refs(
 
     # TODO
     # here we can't compute just the dictionary and use frozenset(....values())
-    # as we may have multiple version of lisbraries; this is something that will
+    # as we may have multiple version of libraries; this is something that will
     # need to be fixed in the long run
     known_refs = []
     ref_map = {}
@@ -160,7 +160,7 @@ def process(
             "Receives",
             # "Notes",
             # "Signature",
-            #'Extended Summary',
+            # "Extended Summary",
             #'References'
             #'See Also'
             #'Examples'
@@ -181,7 +181,7 @@ def flat(l):
         local_refs = frozenset(flat(_local_refs))
 
         visitor = PostDVR(
-            self.qa, known_refs, local_refs, aliases, version=version, config={}
+            self.qa, known_refs, local_refs, {}, aliases, version=version, config={}
         )
         for section in ["Extended Summary", "Summary", "Notes"] + sections_:
             if section not in self.content:
@@ -313,6 +313,7 @@ def _ingest_examples(
                 f"TBD (examples, {path}), supposed to be QA",
                 known_refs,
                 set(),
+                {},
                 aliases,
                 version=version,
                 config={},
@@ -506,6 +507,7 @@ def relink(self) -> None:
                 f"TBD, supposed to be QA relink {key}",
                 known_refs,
                 set(),
+                {},
                 aliases,
                 version="?",
             )

papyri/gen.py

@@ -79,6 +79,7 @@
     RefInfo,
     Section,
     SeeAlsoItem,
+    SubstitutionDef,
     parse_rst_section,
 )
 from .toc import make_tree
@@ -1473,6 +1474,7 @@ def collect_narrative_docs(self):
                         key,
                         set(),
                         local_refs=set(),
+                        substitution_defs={},
                         aliases={},
                         version=self._meta["version"],
                         config=self.config.directives,
@@ -1931,6 +1933,7 @@ def collect_examples(self, folder: Path, config):
                     example.name,
                     frozenset(),
                     local_refs=frozenset(),
+                    substitution_defs={},
                     aliases={},
                     version=self.version,
                     config=self.config.directives,
@@ -2255,20 +2258,30 @@ def collect_api_docs(self, root: str, limit_to: List[str]) -> None:
                             if new_ref:
                                 _local_refs = _local_refs + new_ref
 
+            # substitution_defs: Dict[str, Union(MImage, ReplaceNode)] = {}
+            substitution_defs = {}
+            for section in doc_blob.sections:
+                for child in doc_blob.content.get(section, []):
+                    if isinstance(child, SubstitutionDef):
+                        substitution_defs[child.value] = child.children
+
             # def flat(l) -> List[str]:
             #    return [y for x in l for y in x]
             for lr1 in _local_refs:
                 assert isinstance(lr1, str)
             # lr: FrozenSet[str] = frozenset(flat(_local_refs))
             lr: FrozenSet[str] = frozenset(_local_refs)
+
             dv = DVR(
                 qa,
                 known_refs,
                 local_refs=lr,
+                substitution_defs=substitution_defs,
                 aliases={},
                 version=self.version,
                 config=self.config.directives,
             )
+
             doc_blob.arbitrary = [dv.visit(s) for s in arbitrary]
             doc_blob.example_section_data = dv.visit(doc_blob.example_section_data)
             doc_blob._content = {k: dv.visit(v) for (k, v) in doc_blob._content.items()}

papyri/myst_ast.py

@@ -103,7 +103,7 @@ def __init__(self, value):
 @register(4045)
 class MParagraph(Node):
     type = "paragraph"
-    children: List[Union["PhrasingContent", "take2.MUnimpl"]]
+    children: List[Union["PhrasingContent", "take2.MUnimpl", "MImage"]]
     # position: Any
     # data: Any
 
@@ -267,6 +267,18 @@ class MRoot(Node):
     ]
 
 
+class ReplaceNode(Node):
+    # We may want to return links too.
+    type = "replace"
+    value: str
+    text: str
+    # children: Union[
+    #     MText,
+    #     MInlineCode,
+    #     MInlineMath,
+    # ]
+
+
 StaticPhrasingContent = Union[
     MText,
     MInlineCode,

papyri/take2.py

@@ -159,14 +159,33 @@ class Leaf(Node):
 
 @register(4027)
 class SubstitutionDef(Node):
+    """
+    A Substitution Definition should be in the form of
+
+    .. raw:: rst
+
+        .. |value| inline_directive:: text to be inserted
+
+    Currently, the inline_directive can only be ``replace`` or ``image``. In the
+    future, we want to support any inline directives, including custom
+    user-defined directives.
+    """
+
     value: str
-    children: List[Union[MMystDirective, UnprocessedDirective]]
+    children: List[Union[ReplaceNode, MImage]]
 
     def __init__(self, value, children):
         self.value = value
         assert isinstance(children, list)
-        self.children = children
-        pass
+        assert len(children) == 1
+        assert isinstance(children[0], UnprocessedDirective)
+
+        if children[0].name == "image":
+            self.children = [MImage(url=children[0].args, alt="")]
+        elif children[0].name == "replace":
+            self.children = [ReplaceNode(value=self.value, text=children[0].args)]
+        else:
+            raise NotImplementedError
 
 
 @register(4041)
@@ -201,6 +220,8 @@ def __repr__(self):
     MBlockquote,
     MTarget,
     MThematicBreak,
+    MImage,
+    ReplaceNode,
 )
 
 

papyri/tests/expected/papyri.examples.expected

@@ -142,27 +142,9 @@ now.
 
 ### Substitutions
 
-In this paragraph:
-{
-  "type": "SubstitutionRef",
-  "value": "|SubstitutionRef|"
-} Should be
-replaced...
+In this paragraph: ASUBSTITUTIONDEF Should be replaced...
 
-{
-  "type": "SubstitutionDef",
-  "value": "|SubstitutionDef|",
-  "children": [
-    {
-      "type": "mystDirective",
-      "name": "replace",
-      "args": "ASUBSTITUTIONDEF",
-      "options": {},
-      "value": "",
-      "children": []
-    }
-  ]
-}## Quotes
+## Quotes
 
 Quotes are not implemented yet in the parser, this section below will appear
 empty

papyri/tests/test_parse.py

@@ -156,3 +156,40 @@ def test_parse_reference():
     [text, reference] = paragraph.children
     assert reference.value == "reference <to this>"
     assert text.value == "This is a "
+
+
+def test_parse_substitution_definition():
+    data1 = dedent(
+        """
+    A substitution definition block contains an embedded inline-compatible
+    directive (without the leading ".. "), such as "image" or "replace". For
+    example, the |biohazard| symbol must be used on containers used to dispose
+    of medical waste.
+
+    .. |biohazard| image :: https://upload.wikimedia.org/wikipedia/commons/c/c0/Biohazard_symbol.svg
+    """
+    )
+    data2 = dedent(
+        """
+    A substitution definition block contains an embedded inline-compatible
+    directive (without the leading ".. "), such as "image" or "replace". For
+    example, the |biohazard| symbol must be used on containers used to dispose
+    of medical waste.
+
+    .. |biohazard| replace :: **biohazard.png**
+    """
+    )
+    [section1] = parse(data1.encode(), "test_parse_substitution_definition")
+    [_, subsdef1] = section1.children
+    [node1] = subsdef1.children
+    [section2] = parse(data2.encode(), "test_parse_substitution_definition")
+    [_, subsdef2] = section2.children
+    [node2] = subsdef2.children
+    print(f"{node2=}")
+    assert subsdef1.value == "|biohazard|"
+    assert node1.type == "image"
+    assert node1.url == "https://upload.wikimedia.org/wikipedia/commons/c/c0/Biohazard_symbol.svg"
+    assert node1.alt == ""
+    assert subsdef2.value == "|biohazard|"
+    assert node2.type == "replace"
+    assert node2.text == "**biohazard.png**"

papyri/tests/utils.py

@@ -25,6 +25,7 @@ def _process(sample: Path):
         str(sample),
         frozenset(),
         local_refs=set(),
+        substitution_defs={},
         aliases={},
         version="TestSuite",
         config={},

papyri/tree.py

@@ -22,11 +22,13 @@
     MLink,
     MText,
     MList,
+    MImage,
     MListItem,
     MInlineMath,
     MInlineCode,
     MCode,
     MParagraph,
+    ReplaceNode,
     UnprocessedDirective,
 )
 from .utils import full_qual, FullQual, Cannonical, obj_from_qualname
@@ -259,6 +261,10 @@ def resolve_(
 
 
 class TreeVisitor:
+    """
+    Checks or modifies nodes in place.
+    """
+
     def __init__(self, find):
         self.skipped = set()
         self.find = find
@@ -296,7 +302,6 @@ def generic_visit(self, node):
                     for k, v in self.generic_visit(c).items():
                         acc.setdefault(k, []).extend(v)
             return acc
-
         elif hasattr(node, "value"):
             if type(node) not in self.skipped:
                 self.skipped.add(type(node))
@@ -332,6 +337,10 @@ def _call_method(self, method, node):
         return method(node)
 
     def generic_visit(self, node) -> List[Node]:
+        """
+        Detects node type and dispatches to "visit_<type>" or "replace_<type>"
+        methods.
+        """
         assert node is not None
         assert not isinstance(node, str)
         assert isinstance(node, Node), node
@@ -345,6 +354,7 @@ def generic_visit(self, node) -> List[Node]:
             if method := getattr(self, "replace_" + name, None):
                 self._replacements.update([name])
                 new_nodes = self._call_method(method, node)
+
             elif name in [
                 "Code",
                 "Comment",
@@ -354,6 +364,7 @@ def generic_visit(self, node) -> List[Node]:
                 "Link",
                 "MCode",
                 "MComment",
+                "MImage",
                 "MInlineCode",
                 "MInlineMath",
                 "MMath",
@@ -364,6 +375,7 @@ def generic_visit(self, node) -> List[Node]:
                 "SeeAlsoItem",
                 "SubstitutionRef",
                 "Unimplemented",
+                "ReplaceNode",
             ]:
                 return [node]
             else:
@@ -521,6 +533,7 @@ def __init__(
         qa: str,
         known_refs: FrozenSet[RefInfo],
         local_refs,
+        substitution_defs,
         aliases,
         version,
         config=None,
@@ -556,6 +569,7 @@ def __init__(
 
         self.known_refs = frozenset(known_refs)
         self.local_refs = frozenset(local_refs)
+        self.substitution_defs = substitution_defs
         self.qa = qa
         self.local: List[str] = []
         self.total: List[Tuple[Any, str]] = []
@@ -659,15 +673,42 @@ def replace_UnprocessedDirective(self, directive: UnprocessedDirective):
         return [MMystDirective.from_unprocessed(directive)]
 
     def replace_MMystDirective(self, myst_directive: MMystDirective):
+        """
+        Detects MyST directives that are not handled, and adds them to the
+        list of missing directives.
+        """
         meth = getattr(self, "_" + myst_directive.name + "_handler", None)
         if meth:
-            assert False, "shod have gone vian unprocessed"
+            assert False, "should have gone via unprocessed"
         if myst_directive.name not in _MISSING_DIRECTIVES:
             _MISSING_DIRECTIVES.append(myst_directive.name)
             log.debug("TODO: %s", myst_directive.name)
 
         return [myst_directive]
 
+    def replace_SubstitutionDef(self, node):
+        """
+        This node should be removed since the actual reference has already been
+        resolved.
+        """
+        return []
+
+    def replace_SubstitutionRef(self, directive):
+        # Return the corresponding substitution definition.
+        # TODO: This should be either Text (from the replace directive) or
+        # an image (from the image directive.)
+        if directive.value in self.substitution_defs:
+            if isinstance(self.substitution_defs[directive.value][0], ReplaceNode):
+                return [MText(self.substitution_defs[directive.value][0].text)]
+            elif isinstance(self.substitution_defs[directive.value][0], MImage):
+                return [self.substitution_defs[directive.value][0]]
+            else:
+                raise NotImplementedError(
+                    f"SubstitutionDef for custom inline directive {self.substitution_defs[directive.value][0]} not implemented."
+                )
+        else:
+            return [MText(directive.value)]
+
     def _resolve(self, loc, text):
         """
         Resolve `text` within local references `loc`
@@ -691,6 +732,9 @@ def _import_solver(cls, maybe_qa: str):
                 return target_qa
 
     def replace_Directive(self, directive: Directive):
+        """
+        Returns a new node to replace the directive.
+        """
         domain, role = directive.domain, directive.role
         if domain is None:
             domain = "py"