Merge branch 'master' into agent-sync-plugin

.github/workflows/single-binary.yml

@@ -6,14 +6,6 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - .github/workflows/single-binary.yml
-      - charts/flyte-binary/**
-      - charts/flyte-sandbox/**
-      - cmd/**
-      - docker/sandbox-bundled/**
-      - Dockerfile
-      - go.*
   push:
     branches:
       - master

docs/.readthedocs.yaml

@@ -8,15 +8,16 @@ build:
   os: "ubuntu-22.04"
   tools:
     python: "mambaforge-22.9"
-  commands:
-    - cat monodocs-environment.lock.yaml
-    - mamba install -c conda-forge conda-lock
-    - conda-lock install -p /home/docs/monodocs-env monodocs-environment.lock.yaml
-    - conda info
-    - conda env list
-    - cat docs/conf.py
-    - cd docs && /home/docs/monodocs-env/bin/python -m sphinx -T -E -b html -d docs/_build/doctrees -D language=en . $READTHEDOCS_OUTPUT/html
+  jobs:
+    post_install:
+      - conda-lock install --name $READTHEDOCS_VERSION monodocs-environment.lock.yaml
+      - conda info
+      - conda env list
+      - conda list
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
+
+conda:
+  environment: docs/build-environment.yaml

docs/_ext/import_projects.py

@@ -1,3 +1,4 @@
+import inspect
 import os
 import re
 import shutil
@@ -8,9 +9,13 @@
 from pathlib import Path
 from typing import Optional, List, Union
 
+import git
 from git import Repo
+from docutils import nodes
+from docutils.statemachine import StringList, string2lines
 from sphinx.application import Sphinx
 from sphinx.config import Config
+from sphinx.util.docutils import SphinxDirective
 
 __version__ = "0.0.0"
 
@@ -20,10 +25,12 @@ class ImportProjectsConfig:
     clone_dir: str
     flytekit_api_dir: str
     source_regex_mapping: dict = field(default_factory=dict)
+    list_table_toc: List[str] = field(default_factory=list)
 
 
 @dataclass
 class Project:
+    name: str
     source: str
     dest: str
     local: bool = False
@@ -32,6 +39,46 @@ class Project:
     refresh: bool = False
 
 
+TOC_TEMPLATE = """
+```{{toctree}}
+:maxdepth: 1
+:hidden:
+{toc}
+```
+"""
+
+TABLE_TEMPLATE = """
+```{{list-table}}
+{content}
+```
+"""
+
+
+class ListTableToc(SphinxDirective):
+    """Custom directive to convert list-table into both list-table and toctree."""
+
+    has_content = True
+
+    def run(self) -> list:
+        return [self.parse()]
+
+    def parse(self):
+        """Parses the list-table and adds a toctree"""
+        toc = ""
+
+        # finds all doc references in the form <doc/path>
+        for doc in re.findall(r"<(.+)>", self.block_text):
+            toc += f"\n{doc}"
+
+        container = nodes.container("")
+        toc = inspect.cleandoc(TOC_TEMPLATE.format(toc=toc))
+        table = inspect.cleandoc(TABLE_TEMPLATE.format(content=self.block_text))
+        content = f"{toc}\n\n{table}"
+
+        self.state.nested_parse(StringList(string2lines(content)), 0, container)
+        return container
+
+
 def update_sys_path_for_flytekit(import_project_config: ImportProjectsConfig):
     # create flytekit/_version.py file manually
     with open(f"{import_project_config.flytekit_api_dir}/flytekit/_version.py", "w") as f:
@@ -54,6 +101,16 @@ def update_sys_path_for_flytekit(import_project_config: ImportProjectsConfig):
             sys.path.insert(0, dir_path)
 
 
+def update_html_context(project: Project, tag: str, commit: str, config: Config):
+    tag_url = f"{project.source}/releases/tag/{tag}"
+    commit_url = f"{project.source}/tree/{commit}"
+
+    config.html_context[f"{project.name}_tag"] = tag
+    config.html_context[f"{project.name}_tag_url"] = tag_url
+    config.html_context[f"{project.name}_commit"] = commit
+    config.html_context[f"{project.name}_commit_url"] = commit_url
+
+
 def import_projects(app: Sphinx, config: Config):
     """Clone projects from git or copy from local directory."""
     projects = [Project(**p) for p in config.import_projects]
@@ -67,17 +124,42 @@ def import_projects(app: Sphinx, config: Config):
     ):
         (srcdir / _dir).mkdir(parents=True, exist_ok=True)
 
+    if not hasattr(config, "html_context"):
+        config.html_context = {}
+
+    show_repo_tags = False
     for project in projects:
         if project.local:
             local_dir = srcdir / project.source
+            try:
+                repo = Repo(local_dir)
+                show_repo_tags = True
+            except git.InvalidGitRepositoryError:
+                repo = None
         else:
             local_dir = srcdir / import_projects_config.clone_dir / project.dest
             shutil.rmtree(local_dir, ignore_errors=True)
-            Repo.clone_from(project.source, local_dir)
+            repo = Repo.clone_from(project.source, local_dir)
+            show_repo_tags = True
 
         local_docs_path = local_dir / project.docs_path
         dest_docs_dir = srcdir / project.dest
 
+        if repo:
+            tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+            if not tags:
+                # If tags don't exist just use the current commit. This occurs
+                # when the git repo is a shallow clone.
+                tag_str = "dev"
+                commit = str(repo.head.commit)[:7]
+            else:
+                tag = tags[-1]
+                tag_str = str(tag)
+                commit = str(tag.commit)[:7]
+                repo.git.checkout(tag_str)
+
+            update_html_context(project, tag_str, commit, config)
+
         if project.refresh or not dest_docs_dir.exists():
             shutil.rmtree(dest_docs_dir, ignore_errors=True)
             shutil.copytree(local_docs_path, dest_docs_dir, dirs_exist_ok=True)
@@ -89,6 +171,8 @@ def import_projects(app: Sphinx, config: Config):
             else:
                 subprocess.run(project.cmd)
 
+    config.html_context["show_repo_tags"] = show_repo_tags
+
     # remove cloned directories
     shutil.rmtree(import_projects_config.clone_dir)
 
@@ -132,10 +216,24 @@ def replace_refs_in_docstrings(
         lines[i] = text
 
 
+def add_list_table_toc(app: Sphinx, docname: str, source: List[str]):
+    """This replaces list-table directives in specific documents with list-table-toc.
+
+    This is important for automatically generating a toctree and list-table from
+    a list-table.
+    """
+    if docname in app.config.import_projects_config["list_table_toc"]:
+        text = source[0]
+        text = re.sub(r"{list-table}", r"{list-table-toc}", text)
+        source[0] = text
+
+
 def setup(app: Sphinx) -> dict:
     app.add_config_value("import_projects_config", None, False)
     app.add_config_value("import_projects", None, False)
+    app.add_directive("list-table-toc", ListTableToc)
     app.connect("config-inited", import_projects, priority=0)
+    app.connect("source-read", add_list_table_toc, priority=0)
 
     return {
         "version": __version__,