Skip to content

Commit

Permalink
refactor: split out files, write xrefs for more nodes
Browse files Browse the repository at this point in the history
agoose77 committed Aug 1, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 8a8769d commit bee4d3b
Showing 4 changed files with 158 additions and 97 deletions.
97 changes: 1 addition & 96 deletions src/sphinx_ext_mystmd/__init__.py
Original file line number Diff line number Diff line change
@@ -1,99 +1,4 @@
from sphinx.builders import Builder
import json
import os.path
import pathlib
import hashlib

from .transform import MySTNodeVisitor


def to_text(node):
if "value" in node:
return node['value']
elif "children" in node:
return "".join([to_text(n) for n in node['children']])
else:
return ""


def find_by_type(type_, node):
for child in node["children"]:
if child["type"] == type_:
yield child
elif "children" in child:
yield from find_by_type(type_, child)


class MySTBuilder(Builder):
name = "myst"

def slugify(self, path):
return path.replace("/", "-")

def _get_xref_path(self, doc_name):
target_stem = self.slugify(doc_name)
return os.path.join(self.outdir, "content", f"{target_stem}.myst.json")

def prepare_writing(self, docnames):
print(f"About to write {docnames}")

def get_outdated_docs(self):
for docname in self.env.found_docs:
if docname not in self.env.all_docs:
yield docname
continue
target_path = self._get_xref_path(docname)
try:
targetmtime = os.path.getmtime(target_path)
except Exception:
targetmtime = 0
try:
srcmtime = os.path.getmtime(self.env.doc2path(docname))
if srcmtime > targetmtime:
yield docname
except OSError:
# source doesn't exist anymore
pass

def write_doc(self, docname, doctree):
visitor = MySTNodeVisitor(doctree)
doctree.walkabout(visitor)
slug = self.slugify(docname)
xref_path = self._get_xref_path(docname)

json_xref_dst = pathlib.Path(xref_path)
json_xref_dst.parent.mkdir(exist_ok=True)

with open(self.env.doc2path(docname), "rb") as f:
sha256 = hashlib.sha256(f.read()).hexdigest()
title = to_text(next(find_by_type("heading", visitor.result)))
with open(json_xref_dst, "w") as f:
json.dump(
{
"kind": "Article",
"sha256": sha256,
"slug": slug,
"location": f"/{docname}",
"dependencies": [],
"frontmatter": {"title": title, "content_includes_title": True},
"mdast": visitor.result,
"references": {"cite": {"order": [], "data": {}}},
},
f,
indent=2,
)

def finish(self):
references = [
{"kind": "page", "url": f"/{slug}", "data": f"/content/{slug}.myst.json"}
for slug in (self.slugify(n) for n in self.env.found_docs)
]
xref = {"version": "1", "myst": "1.2.9", "references": references}
with open(os.path.join(self.outdir, "myst.xref.json"), "w") as f:
json.dump(xref, f, indent=2)

def get_target_uri(self, docname, typ=None):
return self.slugify(docname)
from .builder import MySTBuilder


def setup(app):
122 changes: 122 additions & 0 deletions src/sphinx_ext_mystmd/builder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
from sphinx.builders import Builder
from sphinx.util import logging

import json
import os.path
import pathlib
import hashlib

from .transform import MySTNodeVisitor
from .utils import to_text, find_by_type, breadth_first_walk


logger = logging.getLogger(__name__)


class MySTBuilder(Builder):
name = "myst"

def _slugify(self, path):
return path.replace("/", "-")

def _get_xref_path(self, doc_name):
target_stem = self._slugify(doc_name)
return os.path.join(self.outdir, "content", f"{target_stem}.myst.json")

def prepare_writing(self, docnames):
logger.info(f"About to write {docnames}")

def get_outdated_docs(self):
for docname in self.env.found_docs:
if docname not in self.env.all_docs:
yield docname
continue
target_path = self._get_xref_path(docname)
try:
targetmtime = os.path.getmtime(target_path)
except Exception:
targetmtime = 0
try:
srcmtime = os.path.getmtime(self.env.doc2path(docname))
if srcmtime > targetmtime:
yield docname
except OSError:
# source doesn't exist anymore
pass

def write_doc(self, docname, doctree):
visitor = MySTNodeVisitor(doctree)
doctree.walkabout(visitor)
slug = self._slugify(docname)
xref_path = self._get_xref_path(docname)

json_xref_dst = pathlib.Path(xref_path)
json_xref_dst.parent.mkdir(exist_ok=True)

with open(self.env.doc2path(docname), "rb") as f:
sha256 = hashlib.sha256(f.read()).hexdigest()
title = to_text(next(find_by_type("heading", visitor.result)))
with open(json_xref_dst, "w") as f:
json.dump(
{
"kind": "Article",
"sha256": sha256,
"slug": slug,
"location": f"/{docname}",
"dependencies": [],
"frontmatter": {"title": title, "content_includes_title": True},
"mdast": visitor.result,
"references": {"cite": {"order": [], "data": {}}},
},
f,
indent=2,
)

def _xref_kind_for_node(self, node):
if node['type'] == 'container':
return node.get('kind', 'figure')
if "kind" in node:
return f"{node['type']}:{node['kind']}"
return node["type"]

def _get_written_target_references(self, doc):
path = self._get_xref_path(doc)
slug = self._slugify(doc)

with open(path, "r") as f:
data = json.load(f)

mdast = data["mdast"]
for node in breadth_first_walk(mdast):
if "identifier" in node:
yield {
"identifier": node["identifier"],
"kind": self._xref_kind_for_node(node),
"data": f"/{slug}.json",
"url": f"/{slug}",
}

def finish(self):
page_references = [
{
"kind": "page",
"url": f"/{self._slugify(n)}",
"data": self._get_xref_path(n),
}
for n in self.env.found_docs
]
target_references = [
ref
for refs in (
self._get_written_target_references(n) for n in self.env.found_docs
)
for ref in refs
]
references = [*page_references, *target_references]

xref = {"version": "1", "myst": "1.2.9", "references": references}
with open(os.path.join(self.outdir, "myst.xref.json"), "w") as f:
json.dump(xref, f, indent=2)

def get_target_uri(self, docname, typ=None):
return self._slugify(docname)
1 change: 0 additions & 1 deletion src/sphinx_ext_mystmd/transform.py
Original file line number Diff line number Diff line change
@@ -231,7 +231,6 @@ def visit_superscript(self, node):
self.push_myst_node({"type": "html", "value": "</sup>"})

def visit_literal(self, node):
print("LITERAL", repr(node))
return self.enter_myst_node({"type": "inlineCode", "children": []}, node)

def visit_literal_emphasis(self, node):
35 changes: 35 additions & 0 deletions src/sphinx_ext_mystmd/utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import re
import collections


def normalize_label(
@@ -31,3 +32,37 @@ def create_html_id(identifier):
),
),
)


def to_text(node):
if "value" in node:
return node["value"]
elif "children" in node:
return "".join([to_text(n) for n in node["children"]])
else:
return ""


def depth_first_walk(node):
queue = [node]
while queue:
this = queue.pop()
if "children" in this:
queue.extend(this['children'])
yield this


def breadth_first_walk(node):
queue = collections.deque([node])
while queue:
this = queue.popleft()
if "children" in this:
queue.extend(this['children'])
yield this



def find_by_type(type_, node):
for node in breadth_first_walk(node):
if node["type"] == type_:
yield node

0 comments on commit bee4d3b

Please sign in to comment.