diff --git a/python/docs/_quartodoc.old.yml b/python/docs/_quartodoc.old.yml index 9153ef351..41ea1e556 100644 --- a/python/docs/_quartodoc.old.yml +++ b/python/docs/_quartodoc.old.yml @@ -9,21 +9,21 @@ quartodoc: style: _renderer.py options: children: separate + dynamic: true render_interlinks: true - # write sidebar data to this file - sidebar: reference/_sidebar.yml - sections: - title: Timestream options: children: embedded - package: null contents: - - name: kaskada.Timestream - members: [data_type] - - kaskada.Arg - - kaskada.LiteralValue + # maybe make this a single page with the 3 items? + - name: Timestream + members: + - data_type + - _timestream.LiteralValue + - _timestream.Arg + - subtitle: Aggregation desc: | The User Guide has details on [aggregations in general](../../../guide/aggregation.qmd). diff --git a/python/docs/_renderer.py b/python/docs/_renderer.py index c8f191676..43fd21c1e 100644 --- a/python/docs/_renderer.py +++ b/python/docs/_renderer.py @@ -92,13 +92,10 @@ def _fetch_object_dispname(self, el: "dc.Alias | dc.Object"): dispname = f"**{prefix}.**[**{name}**]{{.red}}" if isinstance(el, dc.Object): - print(f'Object: {el.labels}') if 'staticmethod' in el.labels: dispname = "***static*** " + dispname - print(dispname) text = [dispname] - print(dispname) # if isinstance(el, dc.Object) and el.kind == dc.Kind.CLASS: # text.append(f"Bases: []({el.parent.name})") @@ -113,6 +110,23 @@ def _fetch_method_parameters(self, el: dc.Function): return el.parameters + def _render_definition_list(self, title: str, items: [str]) -> str: + rows = [title] + for item in items: + if len(rows) == 1: + rows.append(f'~ {item}') + else: + rows.append(f' {item}') + return "\n".join(rows) + + def _render_header(self, title: str, order: Optional[str] = None) -> str: + text = ["---"] + text.append(f'title: {title}') + if order: + text.append(f'order: {order}') + text.append("---") + return "\n".join(text) + def _render_table(self, rows, headers): table = tabulate(rows, headers=headers, tablefmt="github") @@ -122,12 +136,6 @@ def _render_table(self, rows, headers): @dispatch def render_annotation(self, el: str) -> str: - """Special hook for rendering a type annotation. - Parameters - ---------- - el: - An object representing a type annotation. - """ return sanitize(el) @dispatch @@ -136,12 +144,7 @@ def render_annotation(self, el: None) -> str: @dispatch def render_annotation(self, el: expr.Name) -> str: - # TODO: maybe there is a way to get tabulate to handle this? - # unescaped pipes screw up table formatting - if self.render_interlinks: - return f"[{sanitize(el.source)}](`{el.full}`)" - - return sanitize(el.source) + return f"[{sanitize(el.source)}](`{el.full}`)" @dispatch def render_annotation(self, el: expr.Expression) -> str: @@ -154,9 +157,6 @@ def signature(self, el: layout.Doc): orig = self.display_name # set signature path, generate signature, then set back - # TODO: this is for backwards compatibility with the old approach - # of only defining signature over griffe objects, which projects - # like shiny currently extend self.display_name = el.signature_name res = self.signature(el.obj) self.display_name = orig @@ -172,8 +172,7 @@ def signature(self, el: dc.Alias, source: Optional[dc.Alias] = None): def signature(self, el: dc.Function, source: Optional[dc.Alias] = None) -> str: name = self._fetch_object_dispname(source or el) pars = self.render(self._fetch_method_parameters(el)) - - return f"{name}(***{pars}***)" + return f"{name}(*** {pars} ***)" @dispatch def signature(self, el: dc.Class, source: Optional[dc.Alias] = None) -> str: @@ -187,11 +186,6 @@ def signature( name = self._fetch_object_dispname(source or el) return f"`{name}`" - @dispatch - def render_header(self, el: layout.Doc): - - return self.header(el.name) - # render method ----------------------------------------------------------- @dispatch @@ -206,24 +200,19 @@ def render(self, el: str): # render layouts ========================================================== - def header(self, title: str, order: Optional[str] = None) -> str: - text = ["---"] - text.append(f'title: {title}') - if order: - text.append(f'order: {order}') - text.append("---") - return "\n".join(text) - @dispatch def render(self, el: layout.Page): + rows = [] if el.summary: - sum_ = el.summary - header = [f"{self.header(sum_.name)}\n\n{sum_.desc}"] - else: - header = [] + if el.summary.name: + rows.append(self._render_header(el.summary.name)) + if el.summary.desc: + rows.append(sanitize(el.summary.desc, allow_markdown=True)) - result = map(self.render, el.contents) - return "\n\n".join([*header, *result]) + for item in el.contents: + rows.append(self.render(item)) + + return "\n\n".join(rows) @dispatch def render(self, el: layout.Doc): @@ -231,8 +220,7 @@ def render(self, el: layout.Doc): @dispatch def render(self, el: Union[layout.DocClass, layout.DocModule], single_page: bool = False): - print("boop") - title = self.render_header(el) + title = self._render_header(el.name) attr_docs = [] meth_docs = [] @@ -296,22 +284,21 @@ def render(self, el: Union[layout.DocClass, layout.DocModule], single_page: bool for x in raw_meths if isinstance(x, layout.Doc)] ) - str_sig = self.signature(el) - sig_part = [str_sig] if self.show_signature else [] - - body = self.render(el.obj) + sig = self.signature(el) + body_rows = self.render(el.obj).split("\n") + text = self._render_definition_list(sig, body_rows) - return "\n\n".join([title, *sig_part, body, *attr_docs, *class_docs, *meth_docs]) + return "\n\n".join([title, text, *attr_docs, *class_docs, *meth_docs]) @dispatch def render(self, el: Union[layout.DocFunction, layout.DocAttribute], single_page: bool = False): - title = "" if single_page else self.render_header(el) - - sig = self.signature(el) if self.show_signature else "" + title = "" if single_page else self._render_header(el.name) + sig = self.signature(el) body_rows = self.render(el.obj).split("\n") + text = self._render_definition_list(sig, body_rows) - return "\n\n".join([title, sig, self.get_definition_list(body_rows)]) + return "\n\n".join([title, text]) # render griffe objects =================================================== @@ -415,32 +402,20 @@ def render(self, el: ds.DocstringSectionText): # parameters ---- - def get_definition_list(self, items: [str]) -> str: - rows = [] - for item in items: - if len(rows) == 0: - rows.append(f'~ {item}') - else: - rows.append(f' {item}') - return "\n".join(rows) - @dispatch def render(self, el: ds.DocstringSectionParameters): - rows = list(map(self.render, el.value)) - return f'Parameters:\n{self.get_definition_list(rows)}' + # if more than one param, render as un-ordered list + prefix = "* " if len(el.value) > 1 else "" - @dispatch - def render(self, el: ds.DocstringParameter) -> str: - # render as an un-ordered list item - name = el.name - annotation = self.render_annotation(el.annotation) - clean_desc = sanitize(el.description, allow_markdown=True) + rows = [] + for param in el.value: + anno = self.render_annotation(param.annotation) + desc = sanitize(param.description, allow_markdown=True) + default = f', default: {escape(param.default)}' if param.default else "" - if el.default is None: - return f'* **{name}** ({annotation}) -- {clean_desc}' - else: - default = escape(el.default) - return f'* **{name}** ({annotation}, default: {default}) -- {clean_desc}' + rows.append(f'{prefix}**{param.name}** ({anno}{default}) -- {desc}') + + return self._render_definition_list("Parameters:", rows) # attributes ---- @@ -505,21 +480,21 @@ def render(self, el: Union[ds.DocstringSectionReturns, ds.DocstringSectionRaises return "\n".join(rows) @dispatch - def render(self, el: ds.DocstringReturn): - # similar to DocstringParameter, but no name or default - text = [] - - returns = sanitize(el.description, allow_markdown=True) - if returns: - text.append('Returns:') - text.append(f'~ {returns}') - + def render(self, el: ds.DocstringReturn) -> str: + returns = [] return_type = self.render_annotation(el.annotation) if return_type: - text.append('Return type:') - text.append(f'~ {return_type}') + returns.append(return_type) - return "\n\n".join(text) + return_desc = sanitize(el.description, allow_markdown=True) + if return_desc: + returns.append(return_desc) + + returns_text = " -- ".join(returns) + if returns_text: + return self._render_definition_list("Returns:", [returns_text]) + else: + return "" @dispatch def render(self, el: ds.DocstringRaise): diff --git a/python/docs/autosummary.py b/python/docs/autosummary.py index 6b387f793..2eee714c5 100644 --- a/python/docs/autosummary.py +++ b/python/docs/autosummary.py @@ -248,8 +248,8 @@ def gen_hierarchy(self, blueprint: layout.Layout) -> {str: str}: hierarchy = {} for section in blueprint.sections: - preview(section, max_depth=4) - print() + #preview(section, max_depth=4) + #print() if section.title: last_title = section.title @@ -259,7 +259,7 @@ def gen_hierarchy(self, blueprint: layout.Layout) -> {str: str}: for item in section.contents: hierarchy[f'{self.get_package(section)}.{item.path}'] = location - print(hierarchy) + #print(hierarchy) return hierarchy def header(self, title: str, order: Optional[str] = None) -> str: diff --git a/python/pysrc/kaskada/_timestream.py b/python/pysrc/kaskada/_timestream.py index 1d32f7bf3..6dfa10868 100644 --- a/python/pysrc/kaskada/_timestream.py +++ b/python/pysrc/kaskada/_timestream.py @@ -20,9 +20,6 @@ ) import kaskada -from kaskada.windows import Window, Since, Sliding, Trailing, Tumbling -from kaskada.results import History, Snapshot -from kaskada.destinations import Destination import kaskada._ffi as _ffi import pandas as pd import pyarrow as pa @@ -649,7 +646,7 @@ def collect( *, max: Optional[int], min: Optional[int] = 0, - window: Optional[Window] = None, + window: Optional[kaskada.windows.Window] = None, ) -> Timestream: """Return a Timestream collecting up to the last `max` values in the `window`. @@ -807,7 +804,7 @@ def shift_until(self, predicate: Arg) -> Timestream: """ return Timestream._call("shift_until", predicate, self, input=self) - def sum(self, *, window: Optional[Window] = None) -> Timestream: + def sum(self, *, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream summing the values in the `window`. Computes the sum for each key separately. @@ -817,7 +814,7 @@ def sum(self, *, window: Optional[Window] = None) -> Timestream: """ return _aggregation("sum", self, window) - def first(self, *, window: Optional[Window] = None) -> Timestream: + def first(self, *, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the first value in the `window`. Computed for each key separately. @@ -827,7 +824,7 @@ def first(self, *, window: Optional[Window] = None) -> Timestream: """ return _aggregation("first", self, window) - def last(self, window: Optional[Window] = None) -> Timestream: + def last(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the last value in the `window`. Computed for each key separately. @@ -837,7 +834,7 @@ def last(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("last", self, window) - def count(self, window: Optional[Window] = None) -> Timestream: + def count(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the count value in the `window`. Computed for each key separately. @@ -847,7 +844,7 @@ def count(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("count", self, window) - def count_if(self, window: Optional[Window] = None) -> Timestream: + def count_if(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the count of `true` values in `window`. Computed for each key separately. @@ -857,7 +854,7 @@ def count_if(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("count_if", self, window) - def max(self, window: Optional[Window] = None) -> Timestream: + def max(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the max value in the `window`. Computed for each key separately. @@ -872,7 +869,7 @@ def max(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("max", self, window) - def min(self, window: Optional[Window] = None) -> Timestream: + def min(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the min value in the `window`. Computed for each key separately. @@ -887,7 +884,7 @@ def min(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("min", self, window) - def mean(self, window: Optional[Window] = None) -> Timestream: + def mean(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the mean value in the `window`. Computed for each key separately. @@ -897,7 +894,7 @@ def mean(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("mean", self, window) - def stddev(self, window: Optional[Window] = None) -> Timestream: + def stddev(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the standard deviation in the `window`. Computed for each key separately. @@ -907,7 +904,7 @@ def stddev(self, window: Optional[Window] = None) -> Timestream: """ return _aggregation("stddev", self, window) - def variance(self, window: Optional[Window] = None) -> Timestream: + def variance(self, window: Optional[kaskada.windows.Window] = None) -> Timestream: """Return a Timestream containing the variance in the `window`. Computed for each key separately. @@ -1062,7 +1059,7 @@ def least(self, rhs: Arg) -> Timestream: def preview( self, limit: int = 10, - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, ) -> pd.DataFrame: """Preview the points in this TimeStream as a DataFrame. @@ -1074,7 +1071,7 @@ def preview( def to_pandas( self, - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, *, row_limit: Optional[int] = None, ) -> pd.DataFrame: @@ -1098,9 +1095,9 @@ def to_pandas( def write( self, - destination: Destination, + destination: kaskada.destinations.Destination, mode: Literal["once", "live"] = "once", - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, ) -> Execution: """Execute the TimeStream writing to the given destination. @@ -1124,7 +1121,7 @@ def run_iter( kind: Literal["pandas"] = "pandas", *, mode: Literal["once", "live"] = "once", - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, row_limit: Optional[int] = None, max_batch_size: Optional[int] = None, ) -> ResultIterator[pd.DataFrame]: @@ -1136,7 +1133,7 @@ def run_iter( kind: Literal["pyarrow"], *, mode: Literal["once", "live"] = "once", - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, row_limit: Optional[int] = None, max_batch_size: Optional[int] = None, ) -> ResultIterator[pa.RecordBatch]: @@ -1148,7 +1145,7 @@ def run_iter( kind: Literal["row"], *, mode: Literal["once", "live"] = "once", - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, row_limit: Optional[int] = None, max_batch_size: Optional[int] = None, ) -> ResultIterator[dict]: @@ -1159,7 +1156,7 @@ def run_iter( kind: Literal["pandas", "pyarrow", "row"] = "pandas", *, mode: Literal["once", "live"] = "once", - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, row_limit: Optional[int] = None, max_batch_size: Optional[int] = None, ) -> Union[ @@ -1202,7 +1199,7 @@ def run_iter( def explain( self, kind: Literal["initial_dfg", "final_dfg", "final_plan"] = "final_plan", - results: Optional[Union[History, Snapshot]] = None, + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]] = None, mode: Literal["once", "live"] = "once", ) -> "graphviz.Source": """Return an explanation of this Timestream will be executed. @@ -1246,7 +1243,7 @@ def explain( def _execute( self, - results: Optional[Union[History, Snapshot]], + results: Optional[Union[kaskada.results.History, kaskada.results.Snapshot]], *, row_limit: Optional[int] = None, max_batch_size: Optional[int] = None, @@ -1265,7 +1262,7 @@ def _execute( def _aggregation( op: str, input: Timestream, - window: Optional[Window], + window: Optional[kaskada.windows.Window], *args: Union[Timestream, LiteralValue], ) -> Timestream: """Return the aggregation `op` with the given `input`, `window` and `args`. @@ -1281,12 +1278,12 @@ def _aggregation( """ if window is None: return Timestream._call(op, input, *args, None, None) - elif isinstance(window, Since): + elif isinstance(window, kaskada.windows.Since): predicate = window.predicate if callable(predicate): predicate = predicate(input) return Timestream._call(op, input, *args, predicate, None) - elif isinstance(window, Sliding): + elif isinstance(window, kaskada.windows.Since): predicate = window.predicate if callable(predicate): predicate = predicate(input) @@ -1294,7 +1291,7 @@ def _aggregation( return Timestream._call(op, input, *args, predicate, window.duration).filter( predicate ) - elif isinstance(window, Trailing): + elif isinstance(window, kaskada.windows.Trailing): if op != "collect": raise NotImplementedError( f"Aggregation '{op} does not support trailing windows" @@ -1316,7 +1313,7 @@ def _aggregation( # `duration` has passed with no "real" inputs. merged_input = record({"input": input, "shift": input_shift}).col("input") return Timestream._call("collect", merged_input, *args, None, trailing_ns) - elif isinstance(window, Tumbling): + elif isinstance(window, kaskada.windows.Tumbling): # Tumbling windows are analogous to Since windows, aside from output behavior. # Tumbling windows only emit once per window. However, this behavior is not implemented # in Sparrow yet, so we hack this by using a Since window with a filter applied afterwards