diff --git a/papyri/common_ast.py b/papyri/common_ast.py index 6e2b5a20..6af4fd8f 100644 --- a/papyri/common_ast.py +++ b/papyri/common_ast.py @@ -81,6 +81,16 @@ def __hash__(self): ) +class UnserializableNode(Node): + _dont_serialise = True + + def cbor(self, encoder): + assert False + + def to_json(self) -> bytes: + assert False + + TAG_MAP: Dict[Any, int] = {} REV_TAG_MAP: Dict[int, Any] = {} diff --git a/papyri/examples.py b/papyri/examples.py index 2e901a79..dfee6d30 100644 --- a/papyri/examples.py +++ b/papyri/examples.py @@ -123,9 +123,9 @@ Substitutions ~~~~~~~~~~~~~ -In this paragraph: |SubstitutionRef| Should be replaced... +In this paragraph: |ALIAS| Should be replaced... -.. |SubstitutionDef| replace:: ASUBSTITUTIONDEF +.. |ALIAS| replace:: '-- SUBSTITUTION works--' diff --git a/papyri/gen.py b/papyri/gen.py index cd3b2d16..4dd89746 100644 --- a/papyri/gen.py +++ b/papyri/gen.py @@ -2260,11 +2260,27 @@ def collect_api_docs(self, root: str, limit_to: List[str]) -> None: # substitution_defs: Dict[str, Union(MImage, ReplaceNode)] = {} substitution_defs = {} + sections = [] for section in doc_blob.sections: - for child in doc_blob.content.get(section, []): + sections.append(doc_blob.content.get(section, [])) + # arbitrary is usually raw RST that typically is a module docstring that can't be + # parsed by numpydoc + sections.extend(arbitrary) + for section in sections: + for child in section: if isinstance(child, SubstitutionDef): + if child.value in substitution_defs: + self.log.warn( + "The same substitution definition was found twice: %s", + child.value, + ) substitution_defs[child.value] = child.children + if substitution_defs: + self.log.debug( + "Found substitution def for %s : %s", qa, substitution_defs + ) + # def flat(l) -> List[str]: # return [y for x in l for y in x] for lr1 in _local_refs: diff --git a/papyri/myst_ast.py b/papyri/myst_ast.py index 9bc8ba2d..79e73a27 100644 --- a/papyri/myst_ast.py +++ b/papyri/myst_ast.py @@ -3,7 +3,7 @@ """ from typing import List, Union, Optional, Dict -from .common_ast import Node, register +from .common_ast import Node, register, UnserializableNode from . import take2 from . import signature @@ -151,16 +151,6 @@ def from_unprocessed(cls, up): return cls(up.name, up.args, up.options, up.value, up.children) -class UnserializableNode(Node): - _dont_serialise = True - - def cbor(self, encoder): - assert False - - def to_json(self) -> bytes: - assert False - - class UnprocessedDirective(UnserializableNode): """ Placeholder for yet unprocessed directives, after they are parsed by tree-sitter, diff --git a/papyri/take2.py b/papyri/take2.py index 6df155d9..6a28fc6b 100644 --- a/papyri/take2.py +++ b/papyri/take2.py @@ -64,7 +64,7 @@ import cbor2 from there import print -from .common_ast import Node, REV_TAG_MAP, register +from .common_ast import Node, REV_TAG_MAP, register, UnserializableNode from .miniserde import get_type_hints from .utils import dedent_but_first @@ -157,8 +157,7 @@ class Leaf(Node): value: str -@register(4027) -class SubstitutionDef(Node): +class SubstitutionDef(UnserializableNode): """ A Substitution Definition should be in the form of @@ -181,21 +180,25 @@ def __init__(self, value, children): assert isinstance(children[0], UnprocessedDirective) if children[0].name == "image": + assert len(children) == 1 self.children = [MImage(url=children[0].args, alt="")] elif children[0].name == "replace": + assert len(children) == 1 self.children = [ReplaceNode(value=self.value, text=children[0].args)] else: - raise NotImplementedError + raise NotImplementedError("Substitution def not implemented for ", children) -@register(4041) -class SubstitutionRef(Leaf): +class SubstitutionRef(UnserializableNode): """ This will be in the for |XXX|, and need to be replaced. """ value: str + def __init__(self, value): + self.value = value + @register(4018) class Unimplemented(Node): diff --git a/papyri/tree.py b/papyri/tree.py index a137d3d3..aed8dd7a 100644 --- a/papyri/tree.py +++ b/papyri/tree.py @@ -691,6 +691,7 @@ def replace_SubstitutionDef(self, node): This node should be removed since the actual reference has already been resolved. """ + log.warning("discard substitution node %s", node) return [] def replace_SubstitutionRef(self, directive): @@ -699,15 +700,17 @@ def replace_SubstitutionRef(self, directive): # an image (from the image directive.) if directive.value in self.substitution_defs: if isinstance(self.substitution_defs[directive.value][0], ReplaceNode): - return [MText(self.substitution_defs[directive.value][0].text)] + return self.substitution_defs[directive.value][0].children elif isinstance(self.substitution_defs[directive.value][0], MImage): return [self.substitution_defs[directive.value][0]] else: raise NotImplementedError( - f"SubstitutionDef for custom inline directive {self.substitution_defs[directive.value][0]} not implemented." + f"SubstitutionDef for custom inline directive " + f"{self.substitution_defs[directive.value][0]} " + "not implemented." ) else: - return [MText(directive.value)] + return [directive] def _resolve(self, loc, text): """ diff --git a/papyri/ts.py b/papyri/ts.py index 74283f2c..ece44571 100644 --- a/papyri/ts.py +++ b/papyri/ts.py @@ -512,7 +512,7 @@ def visit_paragraph(self, node): def visit_line_block(self, node): # TODO # e.g: numpy/doc/source/user/c-info.how-to-extend.rst - print("Skipping node", self.as_text(node)) + log.warning("Skipping unimplemented line_block node: %s", self.as_text(node)) return [] def visit_substitution_reference(self, node):