From b89e66d37636fbb7e9d643c3f4104a558e2a5e63 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Fri, 6 Oct 2023 07:38:13 -0500 Subject: [PATCH] more progress --- python/docs/_base.py | 26 --- python/docs/_quartodoc.old.yml | 1 + python/docs/_renderer.py | 283 +++++++++++++--------------- python/docs/autosummary.py | 34 +++- python/docs/styles.css | 11 +- python/pysrc/kaskada/_timestream.py | 2 +- python/pysrc/kaskada/results.py | 44 +++-- 7 files changed, 203 insertions(+), 198 deletions(-) diff --git a/python/docs/_base.py b/python/docs/_base.py index 7f7430625..f2063e0d1 100644 --- a/python/docs/_base.py +++ b/python/docs/_base.py @@ -3,32 +3,6 @@ from plum import dispatch -# utils ----------------------------------------------------------------------- - - -def escape(val: str): - return f"`{val}`" - - -def sanitize(val: str, allow_markdown=False): - # sanitize common tokens that break tables - res = val.replace("\n", " ").replace("|", "\\|") - - # sanitize elements that can get interpreted as markdown links - # or citations - if not allow_markdown: - return res.replace("[", "\\[").replace("]", "\\]") - - return res - - -def convert_rst_link_to_md(rst): - expr = ( - r"((:external(\+[a-zA-Z\._]+))?(:[a-zA-Z\._]+)?:[a-zA-Z\._]+:`~?[a-zA-Z\._]+`)" - ) - - return re.sub(expr, r"[](\1)", rst, flags=re.MULTILINE) - # render ----------------------------------------------------------------------- diff --git a/python/docs/_quartodoc.old.yml b/python/docs/_quartodoc.old.yml index 41ea1e556..946146763 100644 --- a/python/docs/_quartodoc.old.yml +++ b/python/docs/_quartodoc.old.yml @@ -9,6 +9,7 @@ quartodoc: style: _renderer.py options: children: separate + include_empty: true dynamic: true render_interlinks: true diff --git a/python/docs/_renderer.py b/python/docs/_renderer.py index 43fd21c1e..8f75fda0d 100644 --- a/python/docs/_renderer.py +++ b/python/docs/_renderer.py @@ -3,15 +3,14 @@ import quartodoc.ast as qast from contextlib import contextmanager -from functools import wraps from griffe.docstrings import dataclasses as ds from griffe import dataclasses as dc from tabulate import tabulate from plum import dispatch -from typing import Tuple, Union, Optional +from typing import Union, Optional from quartodoc import layout -from _base import BaseRenderer, escape, sanitize, convert_rst_link_to_md +from _base import BaseRenderer try: @@ -28,62 +27,40 @@ def _has_attr_section(el: dc.Docstring | None): return any([isinstance(x, ds.DocstringSectionAttributes) for x in el.parsed]) +def escape(val: str): + return f"`{val}`" + + +def sanitize(val: str, allow_markdown=False): + # sanitize common tokens that break tables + res = val.replace("\n", " ").replace("|", "\\|") + + # sanitize elements that can get interpreted as markdown links + # or citations + if not allow_markdown: + return res.replace("[", "\\[").replace("]", "\\]") + + return res + + class Renderer(BaseRenderer): """Render docstrings to markdown. Parameters ---------- - header_level: int - The level of the header (e.g. 1 is the biggest). - show_signature: bool - Whether to show the function signature. show_signature_annotations: bool Whether to show annotations in the function signature. - display_name: str - The default name shown for documented functions. Either "name", "relative", - "full", or "canonical". These options range from just the function name, to its - full path relative to its package, to including the package name, to its - the its full path relative to its .__module__. - - Examples - -------- - - >>> from quartodoc import MdRenderer, get_object - >>> renderer = MdRenderer(header_level=2) - >>> f = get_object("quartodoc", "get_object") - >>> print(renderer.render(f)[:81]) - ## get_object - `get_object(module: str, object_name: str, parser: str = 'numpy')` - """ style = "markdown" def __init__( self, - header_level: int = 1, - show_signature: bool = True, show_signature_annotations: bool = False, - display_name: str = "relative", hook_pre=None, - render_interlinks=False, ): - self.header_level = header_level - self.show_signature = show_signature self.show_signature_annotations = show_signature_annotations - self.display_name = display_name self.hook_pre = hook_pre - self.render_interlinks = render_interlinks - - self.crnt_header_level = self.header_level - - @contextmanager - def _increment_header(self, n=1): - self.crnt_header_level += n - try: - yield - finally: - self.crnt_header_level -= n def _fetch_object_dispname(self, el: "dc.Alias | dc.Object"): parts = el.path.split(".")[1:] @@ -110,14 +87,18 @@ def _fetch_method_parameters(self, el: dc.Function): return el.parameters - def _render_definition_list(self, title: str, items: [str]) -> str: + def _render_definition_list(self, title: str, items: [str], title_class: Optional[str] = None) -> str: + # title = f'{{.{title_class}}}{title_text}' if title_class else title_text rows = [title] for item in items: if len(rows) == 1: rows.append(f'~ {item}') else: rows.append(f' {item}') - return "\n".join(rows) + if title_class: + rows.insert(0, f':::{{.{title_class}}}') + rows.append(':::') + return "\n" + "\n".join(rows) def _render_header(self, title: str, order: Optional[str] = None) -> str: text = ["---"] @@ -154,14 +135,7 @@ def render_annotation(self, el: expr.Expression) -> str: @dispatch def signature(self, el: layout.Doc): - orig = self.display_name - - # set signature path, generate signature, then set back - self.display_name = el.signature_name - res = self.signature(el.obj) - self.display_name = orig - - return res + return self.signature(el.obj) @dispatch def signature(self, el: dc.Alias, source: Optional[dc.Alias] = None): @@ -172,7 +146,7 @@ def signature(self, el: dc.Alias, source: Optional[dc.Alias] = None): def signature(self, el: dc.Function, source: Optional[dc.Alias] = None) -> str: name = self._fetch_object_dispname(source or el) pars = self.render(self._fetch_method_parameters(el)) - return f"{name}(*** {pars} ***)" + return f"{name}([{pars}]{{.bold-italic}})" @dispatch def signature(self, el: dc.Class, source: Optional[dc.Alias] = None) -> str: @@ -219,79 +193,38 @@ def render(self, el: layout.Doc): raise NotImplementedError(f"Unsupported Doc type: {type(el)}") @dispatch - def render(self, el: Union[layout.DocClass, layout.DocModule], single_page: bool = False): + def render(self, el: Union[layout.DocClass, layout.DocModule], single_page: bool = False) -> str: title = self._render_header(el.name) - attr_docs = [] - meth_docs = [] - class_docs = [] + sig = self.signature(el) + body_rows = self.render(el.obj).split("\n") if el.members: - sub_header = "#" * (self.crnt_header_level + 1) + # add attributes + # skip if docstring has an attributes section raw_attrs = [x for x in el.members if x.obj.is_attribute] - raw_meths = [x for x in el.members if x.obj.is_function] - raw_classes = [x for x in el.members if x.obj.is_class] - - header = "| Name | Description |\n| --- | --- |" - - # attribute summary table ---- - # docstrings can define an attributes section. If that exists on - # then we skip summarizing each attribute into a table. - # TODO: for now, we skip making an attribute table on classes, unless - # they contain an attributes section in the docstring - if ( - raw_attrs - and not _has_attr_section(el.obj.docstring) - # TODO: what should backwards compat be? - # and not isinstance(el, layout.DocClass) - ): - - _attrs_table = "\n".join(map(self.summarize, raw_attrs)) - attrs = f"{sub_header} Attributes\n\n{header}\n{_attrs_table}" - attr_docs.append(attrs) - - # classes summary table ---- - if raw_classes: - _summary_table = "\n".join(map(self.summarize, raw_classes)) - section_name = "Classes" - objs = f"{sub_header} {section_name}\n\n{header}\n{_summary_table}" - class_docs.append(objs) - - n_incr = 1 if el.flat else 2 - with self._increment_header(n_incr): - class_docs.extend( - [ - self.render(x) - for x in raw_classes - if isinstance(x, layout.Doc) - ] - ) - - # method summary table ---- - if raw_meths: - # _summary_table = "\n".join(map(self.summarize, raw_meths)) - # section_name = ( - # "Methods" if isinstance(el, layout.DocClass) else "Functions" - # ) - # objs = f"{sub_header} {section_name}\n\n{header}\n{_summary_table}" - # meth_docs.append(objs) - - # TODO use context manager, or context variable? - n_incr = 1 if el.flat else 2 - with self._increment_header(n_incr): - meth_docs.extend( - [self.render(x, single_page=True) - for x in raw_meths if isinstance(x, layout.Doc)] - ) + if raw_attrs and not _has_attr_section(el.obj.docstring): + attr_rows = map(self.render, raw_attrs) + attr_text = self._render_definition_list( + "Attributes:", attr_rows, title_class="highlight") + body_rows.extend(attr_text.split("\n")) + + # add classes + for raw_class in el.members: + if raw_class.obj.is_class and isinstance(raw_class, layout.Doc): + body_rows.extend(self.render(raw_class, single_page=True).split("\n")) + + # add methods + for raw_method in el.members: + if raw_method.obj.is_function and isinstance(raw_method, layout.Doc): + body_rows.extend(self.render(raw_method, single_page=True).split("\n")) - sig = self.signature(el) - body_rows = self.render(el.obj).split("\n") text = self._render_definition_list(sig, body_rows) - return "\n\n".join([title, text, *attr_docs, *class_docs, *meth_docs]) + return "\n\n".join([title, text]) @dispatch - def render(self, el: Union[layout.DocFunction, layout.DocAttribute], single_page: bool = False): + def render(self, el: layout.DocFunction, single_page: bool = False): title = "" if single_page else self._render_header(el.name) sig = self.signature(el) @@ -300,6 +233,13 @@ def render(self, el: Union[layout.DocFunction, layout.DocAttribute], single_page return "\n\n".join([title, text]) + @dispatch + def render(self, el: layout.DocAttribute, single_page: bool = False): + link = f"[{el.name}](#{el.anchor})" + description = self.summarize(el.obj) + + return " -- ".join([link, description]) + # render griffe objects =================================================== @dispatch @@ -361,6 +301,7 @@ def render(self, el: dc.Parameters): @dispatch def render(self, el: dc.Parameter): + print(f'Parameter: {el}') # TODO: missing annotation splats = {dc.ParameterKind.var_keyword, dc.ParameterKind.var_positional} has_default = el.default and el.kind not in splats @@ -406,33 +347,41 @@ def render(self, el: ds.DocstringSectionText): def render(self, el: ds.DocstringSectionParameters): # if more than one param, render as un-ordered list prefix = "* " if len(el.value) > 1 else "" + follow = " " if len(el.value) > 1 else "" rows = [] for param in el.value: + name = sanitize(param.name) anno = self.render_annotation(param.annotation) - desc = sanitize(param.description, allow_markdown=True) default = f', default: {escape(param.default)}' if param.default else "" - rows.append(f'{prefix}**{param.name}** ({anno}{default}) -- {desc}') + rows.append(f'{prefix}**{name}** ({anno}{default})') + rows.append("") + for row in param.description.split("\n"): + rows.append(f'{follow}{row}') + rows.append("") - return self._render_definition_list("Parameters:", rows) + return self._render_definition_list("Parameters:", rows, title_class="highlight") # attributes ---- @dispatch def render(self, el: ds.DocstringSectionAttributes): - header = ["Name", "Type", "Description"] - rows = list(map(self.render, el.value)) - return self._render_table(rows, header) + # if more than one param, render as un-ordered list + prefix = "* " if len(el.value) > 1 else "" + follow = " " if len(el.value) > 1 else "" - @dispatch - def render(self, el: ds.DocstringAttribute): - row = [ - sanitize(el.name), - self.render_annotation(el.annotation), - sanitize(el.description or "", allow_markdown=True), - ] - return row + rows = [] + for attr in el.value: + name = sanitize(attr.name) + anno = self.render_annotation(attr.annotation) + rows.append(f'{prefix}**{name}** ({anno})') + rows.append("") + for row in attr.description.split("\n"): + rows.append(f'{follow}{row}') + rows.append("") + + return self._render_definition_list("Attributes:", rows, title_class="highlight") # warnings ---- @@ -474,27 +423,67 @@ def render(self, el: qast.ExampleText): # returns ---- @dispatch - def render(self, el: Union[ds.DocstringSectionReturns, ds.DocstringSectionRaises]): - # render as a definition list - rows = list(map(self.render, el.value)) - return "\n".join(rows) + def render(self, el: ds.DocstringSectionReturns): + # if more than one param, render as un-ordered list + prefix = "* " if len(el.value) > 1 else "" + follow = " " if len(el.value) > 1 else "" - @dispatch - def render(self, el: ds.DocstringReturn) -> str: - returns = [] - return_type = self.render_annotation(el.annotation) - if return_type: - returns.append(return_type) + rows = [] + for item in el.value: + title = prefix + name = sanitize(item.name) + if name: + title += f'**{name}**' - return_desc = sanitize(el.description, allow_markdown=True) - if return_desc: - returns.append(return_desc) + return_type = self.render_annotation(item.annotation) + if return_type: + title += return_type - returns_text = " -- ".join(returns) - if returns_text: - return self._render_definition_list("Returns:", [returns_text]) - else: - return "" + if title != prefix: + rows.append(title) + + if item.description: + rows.append("") + for row in item.description.split("\n"): + rows.append(f'{follow}{row}') + rows.append("") + + return self._render_definition_list("Returns:", rows, title_class="highlight") + + @dispatch + def render(self, el: ds.DocstringSectionRaises): + # if more than one param, render as un-ordered list + prefix = "* " if len(el.value) > 1 else "" + follow = " " if len(el.value) > 1 else "" + + rows = [] + for item in el.value: + # name = sanitize(item.name) + anno = self.render_annotation(item.annotation) + rows.append(f'{prefix}{anno}') + rows.append("") + for row in item.description.split("\n"): + rows.append(f'{follow}{row}') + rows.append("") + + return self._render_definition_list("Raises:", rows, title_class="highlight") + + # @dispatch + # def render(self, el: ds.DocstringReturn) -> str: + # returns = [] + # return_type = self.render_annotation(el.annotation) + # if return_type: + # returns.append(return_type) + + # return_desc = sanitize(el.description, allow_markdown=True) + # if return_desc: + # returns.append(return_desc) + + # returns_text = " -- ".join(returns) + # if returns_text: + # return self._render_definition_list("Returns:", [returns_text], title_class="highlight") + # else: + # return "" @dispatch def render(self, el: ds.DocstringRaise): diff --git a/python/docs/autosummary.py b/python/docs/autosummary.py index 2eee714c5..8311e9c28 100644 --- a/python/docs/autosummary.py +++ b/python/docs/autosummary.py @@ -217,6 +217,34 @@ def build(self, filter: str = "*"): self.write_doc_pages(pages, filter, hierarchy) # inventory ---- + print(hierarchy) + + # update paths in items + for item in items: + uri = item.uri + uri = uri.removeprefix(f'{self.dir}/') + + path = uri.split('#')[0] if "#" in uri else uri + fragment = uri.split('#')[1] if "#" in uri else "" + + location = None + uri_lookup = f'kaskada.{path.removesuffix(".html")}' + prefix_lookup = f".".join(item.name.split(".")[:-1]) + if uri_lookup in hierarchy: + location = hierarchy[uri_lookup] + elif prefix_lookup in hierarchy: + location = hierarchy[prefix_lookup] + elif item.name in hierarchy: + location = hierarchy[item.name] + else: + print( + f'Lost Item: {item}, uri_lookup: {uri_lookup}, prefix_lookup: {prefix_lookup}') + + if location: + item.uri = f'{self.dir}/{location}/{path}#{fragment}' + + for i in range(10): + print(f'Item: {items[i]}') _log.info("Creating inventory file") inv = self.create_inventory(items) @@ -248,8 +276,8 @@ def gen_hierarchy(self, blueprint: layout.Layout) -> {str: str}: hierarchy = {} for section in blueprint.sections: - #preview(section, max_depth=4) - #print() + # preview(section, max_depth=4) + # print() if section.title: last_title = section.title @@ -259,7 +287,7 @@ def gen_hierarchy(self, blueprint: layout.Layout) -> {str: str}: for item in section.contents: hierarchy[f'{self.get_package(section)}.{item.path}'] = location - #print(hierarchy) + # print(hierarchy) return hierarchy def header(self, title: str, order: Optional[str] = None) -> str: diff --git a/python/docs/styles.css b/python/docs/styles.css index 2c7e74f3e..2b9bf8b30 100644 --- a/python/docs/styles.css +++ b/python/docs/styles.css @@ -66,17 +66,22 @@ dd { color: #e83e8c } -dl dl dt { +.highlight dt { padding-bottom: 4px; padding-left: 8px; padding-right: 8px; padding-top: 2px; } -body.quarto-light dl dl dt { +body.quarto-light .highlight dt { background-color: #f5f5f5; } -body.quarto-dark dl dl dt { +body.quarto-dark .highlight dt { background-color: #2f2f2f; +} + +.bold-italic { + font-weight: bolder; + font-style: italic; } \ No newline at end of file diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 6dfa10868..7aabea57c 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -746,7 +746,7 @@ def coalesce(self, arg: Arg, *args: Arg) -> Timestream: Returns: Timestream containing the first non-null value from each point. - If all values are null, then returns null. + If all values are null, then returns null. """ return Timestream._call("coalesce", self, arg, *args, input=self) diff --git a/python/pysrc/kaskada/results.py b/python/pysrc/kaskada/results.py index a661ad744..4d6286965 100644 --- a/python/pysrc/kaskada/results.py +++ b/python/pysrc/kaskada/results.py @@ -7,32 +7,40 @@ @dataclass class History: - """Execution options for queries producing all historic points.""" + """Execution options for queries producing all historic points. + + Args: + since: If set, only returns points after this time. + + Setting this allows incremental execution to use a checkpoint + from a time before the `since` time. + + until: Only return points less than or equal to this time. + If not set, the current time will be used. + """ - #: If set, only returns points after this time. - #: - #: Setting this allows incremental execution to use a checkpoint - #: from a time before the `since` time. since: Optional[datetime] = None - #: Only return points less than or equal to this time. - #: If not set, the current time will be used. until: Optional[datetime] = None @dataclass class Snapshot: - """Execution options for queries producing snapshots at a specific time.""" - - #: If set, only includes entities that changed after this time. - #: - #: Snapshot queries support incremental execution even when this isn't set. - #: However, every snapshot will include every entity unless this is set. - #: When writing results to an external store that already has values - #: from an earlier snapshot, this can be used to reduce the amount of - #: data to be written. + """Execution options for queries producing snapshots at a specific time. + + Args: + changed_since: If set, only includes entities that changed after this time. + + Snapshot queries support incremental execution even when this isn't set. + However, every snapshot will include every entity unless this is set. + When writing results to an external store that already has values + from an earlier snapshot, this can be used to reduce the amount of + data to be written. + + at: If set, produces the snapshot at the given time. + If not set, the current time will be used. + """ + changed_since: Optional[datetime] = None - #: If set, produces the snapshot at the given time. - #: If not set, the current time will be used. at: Optional[datetime] = None