feat: First pass at doc site for Python (#605)

This uses Sphinx and the [sphinx-book-theme](https://sphinx-book-theme.readthedocs.io/en/stable/) to create organized documentation of the available methods for creating and executing timestreams. Sphinx makes it easy to include syntax-highlighted examples from the documentation, as well as embedding the **rendered** notebook output of cells (embedding them in documentation we write) and even including notebooks as part of the documentation. With a few tweaks, we can make every notebook cell executable as well. See for instance https://jupyterbook.org/en/stable/interactive/thebe.html.
kaskada-ai · Aug 7, 2023 · d31cb43 · d31cb43
1 parent 98dec21
commit d31cb43
Show file tree

Hide file tree

Showing 31 changed files with 1,504 additions and 139 deletions.
diff --git a/sparrow-py/docs/.gitignore b/sparrow-py/docs/.gitignore
@@ -1 +1,4 @@
 _build
+.jupyter_cache
+jupyter_execute
+source/reference/**/api
diff --git a/sparrow-py/docs/quickstart.md b/sparrow-py/docs/quickstart.md
diff --git a/sparrow-py/docs/reference/execution.md b/sparrow-py/docs/reference/execution.md
diff --git a/sparrow-py/docs/reference/sources.md b/sparrow-py/docs/reference/sources.md
diff --git a/sparrow-py/docs/reference/timestream.md b/sparrow-py/docs/reference/timestream.md
diff --git a/sparrow-py/docs/source/_static/favicon.png b/sparrow-py/docs/source/_static/favicon.png
diff --git a/sparrow-py/docs/source/_templates/autosummary/class.rst b/sparrow-py/docs/source/_templates/autosummary/class.rst
@@ -0,0 +1,5 @@
+{{ objname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. auto{{ objtype }}:: {{ objname }}
diff --git a/sparrow-py/docs/source/_templates/autosummary/function.rst b/sparrow-py/docs/source/_templates/autosummary/function.rst
@@ -0,0 +1,5 @@
+{{ objname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. auto{{ objtype }}:: {{ objname }}
diff --git a/sparrow-py/docs/source/_templates/autosummary/method.rst b/sparrow-py/docs/source/_templates/autosummary/method.rst
@@ -0,0 +1,5 @@
+{{ objname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. auto{{ objtype }}:: {{ objname }}
diff --git a/sparrow-py/docs/source/concepts.md b/sparrow-py/docs/source/concepts.md
@@ -0,0 +1 @@
+# Concepts
diff --git a/sparrow-py/docs/conf.py → sparrow-py/docs/source/conf.py b/sparrow-py/docs/conf.py → sparrow-py/docs/source/conf.py
@@ -7,40 +7,68 @@
 copyright = "2023, Kaskada Contributors"
 extensions = [
     "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
     "sphinx.ext.napoleon",
     "sphinx.ext.intersphinx",
     "sphinx.ext.todo",
-    "myst_parser",
+    # "myst_parser",
+    "myst_nb",
+    "sphinx_copybutton",
 ]
 autodoc_typehints = "description"
-html_theme = "furo"
-html_title = "Kaskada"
 language = "en"
 
-intersphinx_mapping: Dict[str, Any] = {
-    'python': ('http://docs.python.org/3', None),
-    'pandas': ('http://pandas.pydata.org/docs', None),
-    'pyarrow': ('https://arrow.apache.org/docs', None),
-}
+html_theme = "sphinx_book_theme"
+html_favicon = "_static/favicon.png"
+html_logo = "_static/kaskada.svg"
+html_title = "Kaskada Timestreams"
 
 html_theme_options: Dict[str, Any] = {
-    "footer_icons": [
+    "repository_url": "https://github.com/kaskada-ai/kaskada",
+    "use_repository_button": True,
+    "use_source_button": True,
+    "use_edit_page_button": True,
+    "use_issues_button": True,
+    "repository_branch": "main",
+    "path_to_docs": "sparrow-py/docs/",
+    "icon_links": [
         {
             "name": "GitHub",
-            "url": "https://github.com/kaskada-ai/kaskada",
-            "html": """
-                <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
-                    <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>
-                </svg>
-            """,
-            "class": "",
+            "url": "https://github.com/kaskada-ai/kaskada",  # required
+            "icon": "fa-brands fa-square-github",
+            "type": "fontawesome",
+        },
+        {
+            "name": "Slack",
+            "url": "https://join.slack.com/t/kaskada-hq/shared_invite/zt-1t1lms085-bqs2jtGO2TYr9kuuam~c9w",
+            "icon": "fa-brands fa-slack",
         },
     ],
-    "source_repository": "https://github.com/kaskada-ai/kaskada/",
-    "source_branch": "main",
-    "source_directory": "sparrow-py/docs/",
+    "show_nav_level": 3,
+    "show_toc_level": 2,
+}
+
+templates_path = ["_templates"]
+html_static_path = ["_static"]
+
+html_context = {
+    "github_user": "kaskada-ai",
+    "github_repo": "kaskada",
+    "github_version": "main",
+    "doc_path": "sparrow-py/docs/",
 }
 
+intersphinx_mapping: Dict[str, Any] = {
+    'python': ('http://docs.python.org/3', None),
+    'pandas': ('http://pandas.pydata.org/docs', None),
+    'pyarrow': ('https://arrow.apache.org/docs', None),
+}
+
+# adds useful copy functionality to all the examples; also
+# strips the '>>>' and '...' prompt/continuation prefixes.
+copybutton_prompt_text = r">>> |\.\.\. "
+copybutton_prompt_is_regexp = True
+
 # Options for Todos
 todo_include_todos = True
 
@@ -54,7 +82,6 @@
 ]
 myst_heading_anchors = 3
 
-# Suggested options from Furo theme
 # -- Options for autodoc ----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
 
@@ -66,3 +93,5 @@
 autodoc_class_signature = "separated"
 
 autodoc_type_aliases = { 'Arg': 'sparrow_py.Arg' }
+
+autosummary_generate = True
diff --git a/sparrow-py/docs/source/examples/index.md b/sparrow-py/docs/source/examples/index.md
@@ -0,0 +1 @@
+# Examples
diff --git a/sparrow-py/docs/index.md → sparrow-py/docs/source/index.md b/sparrow-py/docs/index.md → sparrow-py/docs/source/index.md
@@ -26,7 +26,7 @@ data.count().run().to_pandas()
 ```
 
 ## What are "Timestreams"?
-A [Timestream](reference/timestream) describes how a value changes over time. In the same way that SQL
+A [Timestream](reference/timestream/index) describes how a value changes over time. In the same way that SQL
 queries transform tables and graph queries transform nodes and edges,
 Kaskada queries transform Timestreams.
 
@@ -39,13 +39,17 @@ be defined at arbitrary times corresponding to when the events occur.
 :hidden:
 
 quickstart
+concepts
+examples/index
 ```
 
 ```{toctree}
 :caption: Reference
 :hidden:
+:maxdepth: 3
 
-reference/timestream
+reference/timestream/index
+reference/windows
 reference/sources
-reference/execution
+reference/results
 ```
diff --git a/sparrow-py/docs/source/quickstart.md b/sparrow-py/docs/source/quickstart.md
@@ -0,0 +1,33 @@
+---
+file_format: mystnb
+kernelspec:
+  name: python3
+  display_name: Python 3
+mystnb:
+  execution_mode: cache
+---
+
+# Quick Start
+
+```{todo}
+
+Write the quick start.
+```
+
+```{code-cell}
+import sparrow_py as kt
+kt.init_session()
+content = "\n".join(
+    [
+        "time,key,m,n",
+        "1996-12-19T16:39:57-08:00,A,5,10",
+        "1996-12-19T16:39:58-08:00,B,24,3",
+        "1996-12-19T16:39:59-08:00,A,17,6",
+        "1996-12-19T16:40:00-08:00,A,,9",
+        "1996-12-19T16:40:01-08:00,A,12,",
+        "1996-12-19T16:40:02-08:00,A,,",
+    ]
+)
+source = kt.sources.CsvSource("time", "key", content)
+source.run().to_pandas()
+```
diff --git a/sparrow-py/docs/source/reference/results.md b/sparrow-py/docs/source/reference/results.md
@@ -0,0 +1,10 @@
+# Results
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autosummary::
+   :toctree: api/
+
+    Result
+```
diff --git a/sparrow-py/docs/source/reference/sources.md b/sparrow-py/docs/source/reference/sources.md
@@ -0,0 +1,11 @@
+# Sources
+
+```{eval-rst}
+.. currentmodule:: sparrow_py.sources
+
+.. autosummary::
+   :toctree: api/
+
+    ArrowSource
+    CsvSource
+```
diff --git a/sparrow-py/docs/source/reference/timestream/aggregation.md b/sparrow-py/docs/source/reference/timestream/aggregation.md
@@ -0,0 +1,22 @@
+# Aggregation
+
+Timestream aggregations are:
+
+Cumulative:
+    They reflect all values up to and including the current time.
+Grouped:
+    They reflect the values for each entity separately.
+Windowed:
+    They reflect the values within a specific [window](../windows.md).
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autosummary::
+   :toctree: api/
+
+    Timestream.sum
+    Timestream.first
+    Timestream.last
+    Timestream.collect
+```
diff --git a/sparrow-py/docs/source/reference/timestream/arithmetic.md b/sparrow-py/docs/source/reference/timestream/arithmetic.md
@@ -0,0 +1,18 @@
+# Arithmetic
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autosummary::
+   :toctree: api/
+
+    Timestream.__add__
+    Timestream.__radd__
+    Timestream.__sub__
+    Timestream.__rsub__
+    Timestream.__mul__
+    Timestream.__rmul__
+    Timestream.__truediv__
+    Timestream.__rtruediv__
+    Timestream.neg
+```
diff --git a/sparrow-py/docs/source/reference/timestream/comparison.md b/sparrow-py/docs/source/reference/timestream/comparison.md
@@ -0,0 +1,17 @@
+# Comparison
+
+Comparison operations produce boolean Timestreams.
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autosummary::
+   :toctree: api/
+
+    Timestream.eq
+    Timestream.ne
+    Timestream.__gt__
+    Timestream.__ge__
+    Timestream.__lt__
+    Timestream.__le__
+```
diff --git a/sparrow-py/docs/source/reference/timestream/execution.md b/sparrow-py/docs/source/reference/timestream/execution.md
@@ -0,0 +1,11 @@
+# Execution
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autosummary::
+   :toctree: api/
+
+    Timestream.preview
+    Timestream.run
+```
diff --git a/sparrow-py/docs/source/reference/timestream/index.md b/sparrow-py/docs/source/reference/timestream/index.md
@@ -0,0 +1,28 @@
+# Timestream
+
+```{todo}
+- [ ] Expand the `Arg` type alias in timestreams accordingly.
+```
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autoclass:: Timestream
+    :members:
+    :noindex:
+    :autosummary:
+    :autosummary-nosignatures:
+```
+
+```{toctree}
+:hidden:
+:maxdepth: 3
+
+aggregation
+arithmetic
+comparison
+execution
+logical
+misc
+records
+```
diff --git a/sparrow-py/docs/source/reference/timestream/logical.md b/sparrow-py/docs/source/reference/timestream/logical.md
@@ -0,0 +1,12 @@
+# Logical
+
+```{eval-rst}
+.. currentmodule:: sparrow_py
+
+.. autosummary::
+   :toctree: api/
+
+    Timestream.and_
+    Timestream.or_
+    Timestream.not_
+```