Add a remark plugin for generating a TOC menu

As we reorganize the documentation, it becomes cumbersome to manually
change lists of links to pages within the documentation, e.g., in table
of contents pages for subsections of the docs. This change adds a remark
plugin for generating a list of links to pages in the current directory.

The plugin works similarly to `remark-includes`, and accesses the local
filesystem during the docs build. It replaces any lines consisting of
`(!toc!)` with a list of links to pages in the current directory. The
assumption is that a category page within a directory can use this to
list contents.

docs-contributors/ui-reference.md

@@ -338,6 +338,23 @@ Here is an image:
 When including the partial, the docs engine will rewrite the link path to load
 the image in `docs/img/screenshot.png`.
 
+## Tables of Contents
+
+You can add a list of links to pages in the current directory by adding the
+following line to a docs page:
+
+```
+(!toc!)
+```
+
+The docs engine replaces this line with a list of links to pages in the current
+directory, using the title and description of each page to populate the link:
+
+```
+- [Page 1](page1.mdx): This is a description of Page 1.
+- [Page 2](page2.mdx): This is a description of Page 2.
+```
+
 ## Tabs
 
 To insert a tabs block like the one above, use this syntax:

server/fixtures/toc/database-access/database-access.mdx

@@ -0,0 +1,4 @@
+---
+title: Protect Databases with Teleport
+description: Guides to protecting databases with Teleport.
+---

server/fixtures/toc/database-access/mysql.mdx

@@ -0,0 +1,4 @@
+---
+title: Protect MySQL with Teleport
+description: How to enroll your MySQL database with Teleport
+---

server/fixtures/toc/database-access/postgres.mdx

@@ -0,0 +1,4 @@
+---
+title: Protect Postgres with Teleport
+description: How to enroll Postgres with your Teleport cluster
+---

server/fixtures/toc/database-access/source.mdx

@@ -0,0 +1,5 @@
+## Header
+
+Here is an intro.
+
+(!toc!)

server/fixtures/toc/expected.mdx

@@ -0,0 +1,7 @@
+## Header
+
+Here is an intro.
+
+*   [Protect Databases with Teleport](database-access.mdx): Guides to protecting databases with Teleport.
+*   [Protect MySQL with Teleport](mysql.mdx): How to enroll your MySQL database with Teleport
+*   [Protect Postgres with Teleport](postgres.mdx): How to enroll Postgres with your Teleport cluster

server/markdown-config.ts

@@ -16,6 +16,7 @@ import remarkVariables from "./remark-variables";
 import remarkCodeSnippet from "./remark-code-snippet";
 import remarkParse from "remark-parse";
 import remarkRehype from "remark-rehype";
+import remarkTOC from "./remark-toc";
 import remarkCopyLinkedFiles from "remark-copy-linked-files";
 import rehypeImages from "./rehype-images";
 import { getVersion, getVersionRootPath } from "./docs-helpers";
@@ -42,6 +43,8 @@ export const transformToAST = async (value: string, vfile: VFile) => {
 
   // run() will apply plugins and return modified AST
   const AST = await unified()
+    // Resolves (!toc dir/path!) syntax
+    .use(remarkTOC)
     .use(remarkIncludes, {
       rootDir: getVersionRootPath(vfile.path),
     }) // Resolves (!include.ext!) syntax

server/remark-toc.ts

@@ -0,0 +1,120 @@
+import * as nodeFS from "fs";
+import path from "path";
+import matter from "gray-matter";
+import { visitParents } from "unist-util-visit-parents";
+import { fromMarkdown } from "mdast-util-from-markdown";
+import type { Parent } from "unist";
+import type { VFile } from "vfile";
+import type { Content } from "mdast";
+import type { Transformer } from "unified";
+
+// relativePathToFile takes a filepath and returns a path we can use in links
+// to the file in a table of contents page. The link path is a relative path
+// to the directory where we are placing the table of contents page.
+// @param root {string} - the directory path to the table of contents page.
+// @param filepath {string} - the path from which to generate a link path.
+const relativePathToFile = (root: string, filepath: string) => {
+  // Return the filepath without the first segment, removing the first
+  // slash. This is because the TOC file we are generating is located at
+  // root.
+  return filepath.slice(root.length).replace(/^\//, "");
+};
+
+// getTOC generates a list of links to all files in the same directory as
+// filePath except for filePath. The return value is an object with two
+// properties:
+// - result: a string containing the resulting list of links.
+// - error: an error message encountered during processing
+export const getTOC = (filePath: string, fs = nodeFS) => {
+  const dirPath = path.dirname(filePath);
+  if (!fs.existsSync(dirPath)) {
+    return {
+      error: `Cannot generate a table of contents for nonexistent directory at ${dirPath}`,
+    };
+  }
+
+  const { name } = path.parse(filePath);
+
+  const files = fs.readdirSync(dirPath, "utf8");
+  let mdxFiles = new Set();
+  const dirs = files.reduce((accum, current) => {
+    // Don't add a TOC entry for the current file.
+    if (name == path.parse(current).name) {
+      return accum;
+    }
+    const stats = fs.statSync(path.join(dirPath, current));
+    if (!stats.isDirectory() && current.endsWith(".mdx")) {
+      mdxFiles.add(path.join(dirPath, current));
+      return accum;
+    }
+    accum.add(path.join(dirPath, current));
+    return accum;
+  }, new Set());
+
+  // Add rows to the menu page for non-menu pages.
+  let entries = [];
+  mdxFiles.forEach((f: string, idx: number) => {
+    const text = fs.readFileSync(f, "utf8");
+    let relPath = relativePathToFile(dirPath, f);
+    const { data } = matter(text);
+    entries.push(`- [${data.title}](${relPath}): ${data.description}`);
+  });
+
+  // Add rows to the menu page for first-level child menu pages
+  dirs.forEach((f: string, idx: number) => {
+    const menuPath = path.join(f, path.parse(f).base + ".mdx");
+    if (!fs.existsSync(menuPath)) {
+      return {
+        error: `there must be a page called ${menuPath} that introduces ${f}`,
+      };
+    }
+    const text = fs.readFileSync(menuPath, "utf8");
+    let relPath = relativePathToFile(dirPath, menuPath);
+    const { data } = matter(text);
+
+    entries.push(`- [${data.title}](${relPath}): ${data.description}`);
+  });
+  entries.sort();
+  return { result: entries.join("\n") };
+};
+
+const tocRegexpPattern = "^\\(!toc!\\)$";
+
+// remarkTOC replaces (!toc!) syntax in a page with a list of docs pages at a
+// given directory location.
+export default function remarkTOC(): Transformer {
+  return (root: Content, vfile: VFile) => {
+    const lastErrorIndex = vfile.messages.length;
+
+    visitParents(root, (node, ancestors: Parent[]) => {
+      if (node.type !== "text") {
+        return;
+      }
+      const parent = ancestors[ancestors.length - 1];
+
+      if (parent.type !== "paragraph") {
+        return;
+      }
+      if (!parent.children || parent.children.length !== 1) {
+        return;
+      }
+
+      const tocExpr = node.value.trim().match(tocRegexpPattern);
+      if (!tocExpr) {
+        return;
+      }
+
+      const { result, error } = getTOC(vfile.path);
+      if (!!error) {
+        vfile.message(error, node);
+        return;
+      }
+      const tree = fromMarkdown(result, {});
+
+      const grandParent = ancestors[ancestors.length - 2] as Parent;
+      const parentIndex = grandParent.children.indexOf(parent);
+
+      grandParent.children.splice(parentIndex, 1, ...tree.children);
+    });
+  };
+}

uvu-tests/remark-toc.test.ts

@@ -0,0 +1,185 @@
+import { Volume, createFsFromVolume } from "memfs";
+import { default as remarkTOC, getTOC } from "../server/remark-toc";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { suite } from "uvu";
+import * as assert from "uvu/assert";
+import { VFile, VFileOptions } from "vfile";
+import remarkMdx from "remark-mdx";
+import remarkGFM from "remark-gfm";
+import { remark } from "remark";
+
+const Suite = suite("server/remark-toc");
+
+const testFilesTwoSections = {
+  "/docs/docs.mdx": `---
+title: "Documentation Home"
+description: "Guides to setting up the product."
+---
+
+Guides to setting up the product.
+
+`,
+  "/docs/database-access/database-access.mdx": `---
+title: "Database Access"
+description: Guides related to Database Access.
+---
+
+Guides related to Database Access.
+
+`,
+  "/docs/database-access/page1.mdx": `---
+title: "Database Access Page 1"
+description: "Protecting DB 1 with Teleport"
+---`,
+  "/docs/database-access/page2.mdx": `---
+title: "Database Access Page 2"
+description: "Protecting DB 2 with Teleport"
+---`,
+  "/docs/application-access/application-access.mdx": `---
+title: "Application Access"
+description: "Guides related to Application Access"
+---
+
+Guides related to Application Access.
+
+`,
+  "/docs/application-access/page1.mdx": `---
+title: "Application Access Page 1"
+description: "Protecting App 1 with Teleport"
+---`,
+  "/docs/application-access/page2.mdx": `---
+title: "Application Access Page 2"
+description: "Protecting App 2 with Teleport"
+---`,
+};
+
+Suite("getTOC with one link to a directory", () => {
+  const expected = `- [Application Access](application-access/application-access.mdx): Guides related to Application Access`;
+
+  const vol = Volume.fromJSON({
+    "/docs/docs.mdx": `---
+title: Documentation Home
+description: Guides for setting up the product.
+---
+
+Guides for setting up the product.
+
+`,
+    "/docs/application-access/application-access.mdx": `---
+title: "Application Access"
+description: "Guides related to Application Access"
+---
+
+`,
+    "/docs/application-access/page1.mdx": `---
+title: "Application Access Page 1"
+description: "Protecting App 1 with Teleport"
+---`,
+    "/docs/application-access/page2.mdx": `---
+title: "Application Access Page 2"
+description: "Protecting App 2 with Teleport"
+---`,
+  });
+  const fs = createFsFromVolume(vol);
+  const actual = getTOC("/docs/docs.mdx", fs);
+  assert.equal(actual.result, expected);
+});
+
+Suite("getTOC with multiple links to directories", () => {
+  const expected = `- [Application Access](application-access/application-access.mdx): Guides related to Application Access
+- [Database Access](database-access/database-access.mdx): Guides related to Database Access.`;
+
+  const vol = Volume.fromJSON(testFilesTwoSections);
+  const fs = createFsFromVolume(vol);
+  const actual = getTOC("/docs/docs.mdx", fs);
+  assert.equal(actual.result, expected);
+});
+
+Suite("getTOC orders sections correctly", () => {
+  const expected = `- [API Usage](api.mdx): Using the API.
+- [Application Access](application-access/application-access.mdx): Guides related to Application Access
+- [Desktop Access](desktop-access/desktop-access.mdx): Guides related to Desktop Access
+- [Initial Setup](initial-setup.mdx): How to set up the product for the first time.
+- [Kubernetes](kubernetes.mdx): A guide related to Kubernetes.`;
+
+  const vol = Volume.fromJSON({
+    "/docs/docs.mdx": `---
+title: Documentation Home
+description: Guides to setting up the product.
+---
+
+Guides to setting up the product.
+
+`,
+    "/docs/desktop-access/desktop-access.mdx": `---
+title: "Desktop Access"
+description: "Guides related to Desktop Access"
+---
+
+`,
+
+    "/docs/application-access/application-access.mdx": `---
+title: "Application Access"
+description: "Guides related to Application Access"
+---
+
+`,
+    "/docs/desktop-access/get-started.mdx": `---
+title: "Get Started"
+description: "Get started with desktop access."
+---`,
+    "/docs/application-access/page1.mdx": `---
+title: "Application Access Page 1"
+description: "Protecting App 1 with Teleport"
+---`,
+    "/docs/kubernetes.mdx": `---
+title: "Kubernetes"
+description: "A guide related to Kubernetes."
+---`,
+
+    "/docs/initial-setup.mdx": `---
+title: "Initial Setup"
+description: "How to set up the product for the first time."
+---`,
+    "/docs/api.mdx": `---
+title: "API Usage"
+description: "Using the API."
+---`,
+  });
+  const fs = createFsFromVolume(vol);
+  const actual = getTOC("/docs/docs.mdx", fs);
+  assert.equal(actual.result, expected);
+});
+
+const transformer = (vfileOptions: VFileOptions) => {
+  const file = new VFile(vfileOptions);
+
+  return remark()
+    .use(remarkMdx)
+    .use(remarkGFM)
+    .use(remarkTOC)
+    .processSync(file);
+};
+
+Suite("replaces inclusion expressions", () => {
+  const sourcePath = "server/fixtures/toc/database-access/source.mdx";
+  const value = readFileSync(resolve(sourcePath), "utf-8");
+
+  const result = transformer({
+    value,
+    path: sourcePath,
+  });
+
+  const actual = result.toString();
+
+  const expected = readFileSync(
+    resolve("server/fixtures/toc/expected.mdx"),
+    "utf-8"
+  );
+
+  assert.equal(result.messages, []);
+  assert.equal(actual, expected);
+});
+
+Suite.run();