Misc Ascii and browser fixes. (#333)

This move some of the coloration outside of the templating engine, this should let us add an option to make colors optionals, and should make testing easier, as now we can have expected colorless outputs and test the codebase against it. This also add some fixes to the urwid frontend – which still crashes in a number of places, so I'm wondering if we shoudl rewrite it using textual. One of the other current limitation with urwind is the lack of ability to reflow paragraph is they contain multiple entries.
jupyter · Nov 27, 2023 · 914c5f9 · 914c5f9
1 parent 620b585
commit 914c5f9
Show file tree

Hide file tree

Showing 10 changed files with 580 additions and 84 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -107,7 +107,7 @@ jobs:
     - name: Test with pytest
       run: |
         pip install scipy dask
-        pytest --cov=papyri --cov-append
+        pytest --cov=papyri --cov-append -m "not postingest"
         ls -al
     - uses: actions/download-artifact@v3
       with:
@@ -121,6 +121,9 @@ jobs:
     - name: Ingest
       run: |
         coverage run -a -m papyri install ~/.papyri/data/*.zip
+    - name: Test with pytest postintest
+      run: |
+        pytest -m "postingest"
     - name: Relink
       run: |
         coverage run -a -m papyri relink

diff --git a/papyri/__init__.py b/papyri/__init__.py
@@ -518,13 +518,16 @@ def drop():
 
 
 @app.command()
-def ascii(name: str):
-    _intro()
+def ascii(name: str, color: bool = True):
+    # _intro()
     import trio
 
     from .render import ascii_render
 
-    trio.run(ascii_render, name)
+    async def _():
+        await ascii_render(name, color=color)
+
+    trio.run(_)
 
 
 @app.command()

diff --git a/papyri/browser.py b/papyri/browser.py
@@ -47,6 +47,9 @@ def __init__(self, attr, text, cb):
         self.text = text
         self.cb = cb
 
+    def selectable(self):
+        return True
+
 
 class TextWithLink(urwid.Text):
     # _selectable = True
@@ -292,7 +295,7 @@ def render_Directive(self, d):
             c = converter()
 
             return ("math", c.convert(cont))
-        return ("directive", f"{d.domain}:{d.role}:`{cont}`")
+        return Text(("directive", f"{d.domain}:{d.role}:`{cont}`"))
 
     def render_Example(self, ex):
         acc = []
@@ -307,15 +310,15 @@ def render_Example(self, ex):
 
     def render_Link(self, link):
         if link.reference.kind == "local":
-            return ("local", link.value)
+            return Text(("local", link.value))
         return Link("link", link.value, lambda: self.cb(link.reference))
 
     def render_BlockQuote(self, quote):
         return urwid.Padding(
             urwid.Pile([self.render(c) for c in quote.children]), left=4, right=4
         )
 
-    def render_Admonition(self, adm):
+    def render_MAdmonition(self, adm):
         kind = adm.kind
         title = (f"{kind} : {adm.title}",)
         if kind == "versionchanged":
@@ -332,6 +335,13 @@ def render_Admonition(self, adm):
             ),
         )
 
+    def render_MText(self, text):
+        return urwid.Text(text.value)
+
+    def render_MParagraph(self, paragraph):
+        stuff = [self.render(c) for c in paragraph.children]
+        return urwid.Pile(stuff)
+
     def render_BlockMath(self, math):
         from flatlatex import converter
 

diff --git a/papyri/gen.py b/papyri/gen.py
@@ -33,10 +33,7 @@
 
 import jedi
 
-try:
-    import tomllib
-except ModuleNotFoundError:
-    import tomli as tomllib
+import tomllib
 import tomli_w
 from IPython.core.oinspect import find_file
 from IPython.utils.path import compress_user
@@ -1993,7 +1990,9 @@ def collect_api_docs(self, root: str, limit_to: List[str]):
                         "error with %s %s", qa, list(ecollector._errors.keys())
                     )
                 else:
-                    self.log.info("only expected error with %s", qa)
+                    self.log.info(
+                        "only expected error with %s, %s", qa, ecollector._errors.keys()
+                    )
                 continue
 
             try:

diff --git a/papyri/render.py b/papyri/render.py
@@ -9,7 +9,7 @@
 import uuid
 from collections import OrderedDict, defaultdict
 from dataclasses import dataclass
-from functools import lru_cache
+from functools import lru_cache, partial
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple
 
@@ -68,7 +68,7 @@ def minify(s: str) -> str:
 
 
 def unreachable(*obj):
-    return str(obj[1])
+    return str(obj)
     assert False, f"Unreachable: {obj=}"
 
 
@@ -1190,7 +1190,7 @@ def old_render_one(
 
 
 @lru_cache
-def _ascii_env():
+def _ascii_env(color=True):
     env = Environment(
         loader=CleanLoader(Path(os.path.dirname(__file__)) / "templates"),
         lstrip_blocks=True,
@@ -1200,6 +1200,28 @@ def _ascii_env():
     env.globals["len"] = len
     env.globals["unreachable"] = unreachable
     env.globals["sidebar"] = False
+    env.globals["str"] = str
+    env.filters["pad"] = lambda ss: "│" + "\n│".join(
+        [x.strip() for x in ss.split("\n")]
+    )
+
+    def esc(control, value):
+        return f"\x1b[{control};m{value}\x1b[0;m"
+
+    for control, value in [
+        ("bold", 1),
+        ("underline", 4),
+        ("black", 30),
+        ("red", 31),
+        ("green", 32),
+        ("yellow", 33),
+        ("blue", 34),
+        ("magenta", 35),
+        ("cyan", 36),
+        ("white", 37),
+    ]:
+        env.globals[control] = partial(esc, value) if color else lambda x: x
+
     try:
         c = converter()
 
@@ -1240,11 +1262,12 @@ async def _ascii_render(key: Key, store: GraphStore, *, env, template):
     )
 
 
-async def ascii_render(name, store=None):
+async def ascii_render(name, store=None, color=True):
     gstore = GraphStore(ingest_dir, {})
     key = next(iter(gstore.glob((None, None, "module", name))))
 
-    env, template = _ascii_env()
+    env, template = _ascii_env(color=color)
+
     builtins.print(await _ascii_render(key, gstore, env=env, template=template))
 
 

diff --git a/papyri/templates/ascii.tpl.j2 b/papyri/templates/ascii.tpl.j2
@@ -24,30 +24,9 @@
 {%- endif -%}
 {%- endmacro %}
 {#--#}
-{% macro render_inner(type, obj) -%}
-   |{%- if type == 'Directive' -%}
-   |   |:{{blue(obj.domain)}}:{{blue(obj.role)}}:`{{blue(obj.value)}}`
-   |{%- elif type == 'Link' %}
-   |   |{% if obj.kind=='local' %}
-   |   |    {{-green(obj.value)-}}
-   |   |{% elif obj.kind=='exists' %}
-   |   |    {{-blue(obj.value)-}}
-   |   |{% elif obj.kind=='missing' %}
-   |   |    {{-unreachable(obj)}}
-   |   |{% else %}
-   |   |{% endif %}
-   |{% elif type == 'Math' %}
-   |    ${{obj.value}}$
-   |{% elif type == 'Verbatim' %}
-   |    {{-yellow(obj.text)-}}
-   |{% else %}
-   |   |Some A {{type}}
-   |{%- endif -%}
-{%- endmacro %}
-
 {% macro block_paragraph(pgr) -%}
    |{%- for c in pgr.children -%}
-   |    |{{-render_inner(c.__class__.__name__, c)-}}
+   |    |{{-render_II(c) | trim-}}
    |{%- endfor -%}
 {% endmacro %}
 
@@ -61,41 +40,81 @@
     {% endif %}
 {% endmacro %}
 
+{% macro render_list(objs) -%}
+   |{% for item in objs -%}
+   |{{- render_II(item) -}}
+   |{% endfor -%}
+{%- endmacro %}
+
 {% macro render_II(obj) -%}
-   |{% if obj.__class__.__name__ == 'Paragraph' %}
-   |   |{{block_paragraph(obj)}}
-   |{% else %}
+   |{%- if obj.__class__.__name__ == 'Paragraph' -%}
+   |   |{{- block_paragraph(obj) -}}
+   |{%- else -%}
    |{%- set type = obj.__class__.__name__ -%}
-   |   |{%- if type in ('Directive', 'Link', 'Math', 'Verbatim') -%}
-   |   |   |{{unreachable()}}
-   |   |{% elif type == 'MMystDirective' %}
+   |   |{%- if type in ('Math', 'Verbatim') -%}
+   |   |   |{{unreachable(obj)}}
+   |   |{%- elif type == 'MAdmonitionTitle' -%}
+   |   |   |    ..
+   |   |   |    {%- for item in obj.children %}
+   |   |   |    |  {{render_II(item)|trim}}
+   |   |   |    {%- endfor %}{{'::\n\n'}}
+   |   |{%- elif type == 'MAdmonition' -%}{{'\n'}}
+   |   |   |    {% for item in obj.children %}
+   |   |   |    |{{render_II(item)|wordwrap|pad}}
+   |   |   |    {%endfor %}
+   |   |   |    |{{'\n'}}
+   |   |{%- elif type == 'Directive' -%}
+   |   |   |{{blue(':'+str(obj.domain)+':'+str(obj.role)+':`'+obj.value+'`')}}
+   |   |{%- elif type == 'MMystDirective' -%}
    |   |   |{{'\n'}}{{myst_directive(obj)}}
-   |   |{% elif type == 'Verbatim' %}
+   |   |{%- elif type == 'Verbatim' -%}
    |   |   |{{obj}}
-   |   |{% elif type == 'Paragraph' %}
+   |   |{%- elif type == 'Paragraph' -%}
    |   |   |{{block_paragraph(obj)}}
-   |   |{% elif type == 'BlockVerbatim' %}
+   |   |{%- elif type == 'BlockVerbatim' -%}
    |   |   |{{obj.value}}
-   |   |{% elif type == 'DefList' %}
+   |   |{%- elif type == 'DefList' -%}
    |   |   |    {% for item in obj.children %}
    |   |   |    {{render_II(item.dt)}}
    |   |   |        {{render_II(item.dd)}}
-   |   |   |    {%endfor %}  
-   |   |{% elif type == 'DefListItem' %}
+   |   |   |    {%endfor %}
+   |   |{%- elif type == 'DefListItem' %}
    |   |    {{-block(obj)-}}
-   |   |{% elif type == 'Example' %}
+   |   |{%- elif type == 'Example' %}
    |   |   |   |{{block(obj)}}
+   |   |{%- elif type == 'MLink' %}
+   |   |   |{{render_list(obj.children)}}
+   |   |{%- elif type == 'MInlineCode' -%}
+   |   |   |{{bold(obj.value)}}
+   |   |{%- elif type == 'MText' -%}
+   |   |   |{{obj.value}}
+   |   |{%- elif type == 'MParagraph'  -%}
+   |   |   |{{- render_list(obj.children) -}}
    |   |{% elif type == 'Parameters' %}
-   |   |   {% for item in obj.children %}
+   |   |   {%- for item in obj.children %}
    |   |       {{ render_II(item) }}
-   |   |   {% endfor %}
-   |   |{% elif type == 'Section' %}
-   |   |   {% for item in obj %}
-   |   |       {{ render_II(item) }}
-   |   |   {% endfor %}
-   |   |{% else %}
-   |   |   |Some B {{type}} Not implemented yet
-   |   |    {#{{unreachable(obj)}}#}
+   |   |   {%- endfor %}
+   |   |{%- elif type == 'Section' %}
+   |   |   {%- for item in obj %}
+   |   |       {{- render_II(item) -}}
+   |   |   {%- endfor %}
+   |   |{%- elif type == 'Directive' -%}
+   |   |   |:{{blue(obj.domain)}}:{{blue(obj.role)}}:`{{blue(obj.value)}}`
+   |   |{%- elif type == 'Link' %}
+   |   |   |{% if obj.kind=='local' %}
+   |   |   |    {{-green(obj.value)-}}
+   |   |   |{% elif obj.exists %}
+   |   |   |    {{-blue(obj.value)-}}
+   |   |   |{% else %}
+   |   |   |    {{-unreachable(obj)}}
+   |   |   |{% endif %}
+   |   |{% elif type == 'Math' %}
+   |   |    ${{obj.value}}$
+   |   |{% elif type == 'Verbatim' %}
+   |   |    {{-yellow(obj.text)-}}
+   |   |{%- else %}
+   |   |   |Some B {{type}} Not implemented yet:
+   |   |   |{{unreachable(obj)}}
    |   |{%- endif -%}
    |{%- endif -%}
 {%- endmacro %}
@@ -106,28 +125,16 @@
    |{% endif %}
    |{% for prg in prgs %}
    |   |   |{%- for obj in prg.children -%}
-   |   |   |    {{- render_inner(obj.__class__.__name__, obj) -}} 
+   |   |   |    {{- render_II(obj) -}}
    |   |   |{%- endfor -%}
    |{%- endfor -%}
 {%- endmacro %}
 
-        
+
 {%- macro esc(val, stuff) -%}
 [{{val}};m{{stuff}}[0;m
 {%- endmacro %}
 
-{% macro bold(stuff) -%}{{esc(1,stuff)}}{%- endmacro %}
-{% macro underline(stuff) -%}{{esc(4,stuff)}}{%- endmacro %}
-
-{% macro black(stuff) -%}{{esc(30,stuff)}}{%- endmacro %}
-{% macro red(stuff) -%}{{esc(31,stuff)}}{%- endmacro %}
-{% macro green(stuff) -%}{{esc(32,stuff)}}{%- endmacro %}
-{% macro yellow(stuff) -%}{{esc(33,stuff)}}{%- endmacro %}
-{% macro blue(stuff) -%}{{esc(34,stuff)}}{%- endmacro %}
-{% macro magenta(stuff) -%}{{esc(35,stuff)}}{%- endmacro %}
-{% macro cyan(stuff) -%}{{esc(36,stuff)}}{%- endmacro %}
-{% macro white(stuff) -%}{{esc(37,stuff)}}{%- endmacro %}
-
 {%- macro ref(r) -%}
    {%- if r.exists -%}
    {{-blue(r.name)-}}
@@ -136,8 +143,6 @@
    {%- endif -%}
 {% endmacro -%}
 
-{{-green("green")}} refer to items within the same document, {{blue("blue")}} external {{underline("known")}} entities, {{red("red")}} entities we were not able to find and {{yellow("yellow")}} for code, and other "raw" items, typically between double backticks.
-
 {% if doc.signature %}
     |{{bold(underline(name))}}{{bold(underline(doc.signature.to_signature()))}}
 {% endif %}
@@ -148,10 +153,8 @@
    |{{bold(section)}}
    | 
    |{% endif %}
-   |{% if section in ['Summary', 'Extended Summary', 'Notes'] %}
-   |
-   |     |{{render_II(doc.content[section]) | wordwrap}}
-   |
+   |{%- if section in ['Summary', 'Extended Summary', 'Notes'] -%}
+   |     |{{render_II(doc.content[section]) | wordwrap| trim}}
    |{% elif section in ['Signature'] %}
    |{{unreachable()}}
    |{% elif section in ['Examples'] %}
@@ -183,7 +186,7 @@
    |   |   |   |    {{green(p[1])}}
    |   |   |   |{% endif %}
    |   |   |   |   {% for item in p[2] %}
-   |   |   |   |       {{render_II(item)| wordwrap | indent(width=8)}}
+   |   |   |   |        {{render_II(item)| wordwrap |trim| indent(width=8)}}
    |   |   |   |   {% endfor -%}
    |   |   |{%- endfor -%}
    |   |{%- endfor -%}
@@ -197,8 +200,8 @@
 {% if doc.see_also %}
    |{{bold("See Also")}}
    |{% for sa in doc.see_also %}
-   |     {{bold(render_inner(sa.name.__class__.__name__, sa.name))}}
-   |         {{render_paragraph(sa.descriptions) | wordwrap | indent(width=8)}}
+   |     {{bold(render_II(sa.name))}}
+   |        {{render_paragraph(sa.descriptions) | wordwrap |trim| indent(width=8)}}
    |{% endfor %}
 {% endif %}