feat(playground): Implement codemirror based template string editor (#…

…4943)

* feat(playground): Implement codemirror based template string editor

* Implement mustacheLike Language grammar

* Fix top level language definitions

* feat(playground): Support template escaping and nesting in fstring template lang

* feat(playground): Improve mustache grammar and implement mustache format fn

* fix(playground): Allow Enter key within codemirror editor

* refactor(playground): Improve comments for mustache like lang

* refactor(playground): Refactor variable extraction and lang format funcs into shared utils

* test(playground): Test FStringTemplate and MustacheTemplate languages

* refactor(playground): Remove debug logging

* fix(playground): Correctly parse empty templates, escape slashes, triple braces

* fix(playground): Apply parsing fixes to fstring as well

* refactor(playground): Remove debug

* docs(playground): Add comments to lexer grammars

* docs(playground): Add comments to debug fns

* docs(playground): Improve comment formatting

* refactor(playground): Use named object arguments for variable extractor

* refactor(playground): rename lang directories

* fix(playground): Add a lightmode theme to template editor

* refactor(playground): More detailed typing to variable format record

app/package.json

@@ -13,14 +13,18 @@
     "@codemirror/lang-javascript": "^6.2.2",
     "@codemirror/lang-json": "6.0.1",
     "@codemirror/lang-python": "6.1.3",
+    "@codemirror/language": "^6.10.3",
     "@codemirror/lint": "^6.8.1",
     "@codemirror/view": "^6.28.5",
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/sortable": "^8.0.0",
     "@dnd-kit/utilities": "^3.2.2",
+    "@lezer/highlight": "^1.2.1",
+    "@lezer/lr": "^1.4.2",
     "@react-three/drei": "^9.108.4",
     "@react-three/fiber": "8.0.12",
     "@tanstack/react-table": "^8.19.3",
+    "@uiw/codemirror-theme-github": "^4.23.5",
     "@uiw/codemirror-theme-nord": "^4.23.0",
     "@uiw/react-codemirror": "^4.23.0",
     "copy-to-clipboard": "^3.3.3",
@@ -55,6 +59,7 @@
   },
   "devDependencies": {
     "@emotion/react": "^11.11.4",
+    "@lezer/generator": "^1.7.1",
     "@playwright/test": "^1.48.0",
     "@types/d3-format": "^3.0.4",
     "@types/d3-scale-chromatic": "^3.0.3",

app/src/components/templateEditor/TemplateEditor.tsx

@@ -0,0 +1,64 @@
+import React, { useMemo } from "react";
+import { githubLight } from "@uiw/codemirror-theme-github";
+import { nord } from "@uiw/codemirror-theme-nord";
+import CodeMirror, {
+  BasicSetupOptions,
+  ReactCodeMirrorProps,
+} from "@uiw/react-codemirror";
+
+import { useTheme } from "@phoenix/contexts";
+import { assertUnreachable } from "@phoenix/typeUtils";
+
+import { FStringTemplating } from "./language/fString";
+import { MustacheLikeTemplating } from "./language/mustacheLike";
+
+export const TemplateLanguages = {
+  FString: "f-string", // {variable}
+  Mustache: "mustache", // {{variable}}
+} as const;
+
+type TemplateLanguage =
+  (typeof TemplateLanguages)[keyof typeof TemplateLanguages];
+
+type TemplateEditorProps = ReactCodeMirrorProps & {
+  templateLanguage: TemplateLanguage;
+};
+
+const basicSetupOptions: BasicSetupOptions = {
+  lineNumbers: false,
+  highlightActiveLine: false,
+  foldGutter: false,
+  highlightActiveLineGutter: false,
+  bracketMatching: false,
+};
+
+export const TemplateEditor = ({
+  templateLanguage,
+  ...props
+}: TemplateEditorProps) => {
+  const { theme } = useTheme();
+  const codeMirrorTheme = theme === "light" ? githubLight : nord;
+  const extensions = useMemo(() => {
+    const ext: TemplateEditorProps["extensions"] = [];
+    switch (templateLanguage) {
+      case TemplateLanguages.FString:
+        ext.push(FStringTemplating());
+        break;
+      case TemplateLanguages.Mustache:
+        ext.push(MustacheLikeTemplating());
+        break;
+      default:
+        assertUnreachable(templateLanguage);
+    }
+    return ext;
+  }, [templateLanguage]);
+
+  return (
+    <CodeMirror
+      theme={codeMirrorTheme}
+      extensions={extensions}
+      basicSetup={basicSetupOptions}
+      {...props}
+    />
+  );
+};

app/src/components/templateEditor/index.tsx

@@ -0,0 +1 @@
+export * from "./TemplateEditor";

app/src/components/templateEditor/language/__tests__/languageUtils.test.ts

@@ -0,0 +1,182 @@
+import { formatFString, FStringTemplatingLanguage } from "../fString";
+import { extractVariables } from "../languageUtils";
+import {
+  formatMustacheLike,
+  MustacheLikeTemplatingLanguage,
+} from "../mustacheLike";
+
+describe("language utils", () => {
+  it("should extract variable names from a mustache-like template", () => {
+    const tests = [
+      { input: "{{name}}", expected: ["name"] },
+      // TODO: add support for triple mustache escaping or at least use the inner most mustache as value
+      // { input: "{{name}} {{{age}}}", expected: ["name"] },
+      {
+        input:
+          "Hi I'm {{name}} and I'm {{age}} years old and I live in {{city}}",
+        expected: ["name", "age", "city"],
+      },
+      {
+        input: `
+hi there {{name}}
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": {{age}} }`,
+        expected: ["name", "age"],
+      },
+      {
+        input: `{"name": "{{name}}", "age": {{age}}}`,
+        expected: ["name", "age"],
+      },
+      {
+        input: `{"name": "\\{{name}}", "age": \\{{age}}}`,
+        expected: [],
+      },
+      {
+        input: `{"name": "{{{name}}}"}`,
+        expected: ["{name}"],
+      },
+    ] as const;
+    tests.forEach(({ input, expected }) => {
+      expect(
+        extractVariables({
+          parser: MustacheLikeTemplatingLanguage.parser,
+          text: input,
+        })
+      ).toEqual(expected);
+    });
+  });
+
+  it("should extract variable names from a f-string template", () => {
+    const tests = [
+      { input: "{name}", expected: ["name"] },
+      { input: "{name} {age}", expected: ["name", "age"] },
+      { input: "{name} {{age}}", expected: ["name"] },
+      {
+        input: "Hi I'm {name} and I'm {age} years old and I live in {city}",
+        expected: ["name", "age", "city"],
+      },
+      {
+        input: `
+hi there {name}
+how are you?
+
+can you help with this json?
+
+{{ "name": "John", "age": {age} }}`,
+        expected: ["name", "age"],
+      },
+      { input: "\\{test}", expected: [] },
+    ] as const;
+    tests.forEach(({ input, expected }) => {
+      expect(
+        extractVariables({
+          parser: FStringTemplatingLanguage.parser,
+          text: input,
+        })
+      ).toEqual(expected);
+    });
+  });
+
+  it("should format a mustache-like template", () => {
+    const tests = [
+      {
+        input: "{{name}}",
+        variables: { name: "John" },
+        expected: "John",
+      },
+      {
+        input: "Hi {{name}}, this is bad syntax {{}}",
+        variables: { name: "John", age: 30 },
+        expected: "Hi John, this is bad syntax {{}}",
+      },
+      {
+        input: "{{name}} {{age}}",
+        variables: { name: "John", age: 30 },
+        expected: "John 30",
+      },
+      {
+        input: "{{name}} {age} {{city}}",
+        variables: { name: "John", city: "New York" },
+        expected: "John {age} New York",
+      },
+      {
+        input: `
+hi there {{name}}
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": {{age}} }`,
+        variables: { name: "John", age: 30 },
+        expected: `
+hi there John
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": 30 }`,
+      },
+      {
+        input: `{"name": "{{name}}", "age": {{age}}}`,
+        variables: { name: "John", age: 30 },
+        expected: `{"name": "John", "age": 30}`,
+      },
+      {
+        input: `{"name": "\\{{name}}", "age": "{{age\\}}"}`,
+        variables: { name: "John", age: 30 },
+        expected: `{"name": "{{name}}", "age": "{{age\\}}"}`,
+      },
+    ] as const;
+    tests.forEach(({ input, variables, expected }) => {
+      expect(formatMustacheLike({ text: input, variables })).toEqual(expected);
+    });
+  });
+
+  it("should format a f-string template", () => {
+    const tests = [
+      {
+        input: "{name}",
+        variables: { name: "John" },
+        expected: "John",
+      },
+      {
+        input: "{name} {age}",
+        variables: { name: "John", age: 30 },
+        expected: "John 30",
+      },
+      {
+        input: "{name} {{age}}",
+        variables: { name: "John", age: 30 },
+        expected: "John {age}",
+      },
+      {
+        input: `
+hi there {name}
+how are you?
+
+can you help with this json?
+
+{{ "name": "John", "age": {age} }}`,
+        variables: { name: "John", age: 30 },
+        expected: `
+hi there John
+how are you?
+
+can you help with this json?
+
+{ "name": "John", "age": 30 }`,
+      },
+      {
+        input: "\\{test\\}",
+        variables: { test: "value" },
+        expected: "{test\\}",
+      },
+    ] as const;
+    tests.forEach(({ input, variables, expected }) => {
+      expect(formatFString({ text: input, variables })).toEqual(expected);
+    });
+  });
+});

app/src/components/templateEditor/language/fString/fStringTemplating.syntax.grammar

@@ -0,0 +1,35 @@
+// https://lezer.codemirror.net/docs/guide/
+
+// the top level rule is the entry point for the parser
+// these are the tokens that can appear in the top level of the tree
+@top FStringTemplate {(Template | char | emptyTemplate | lEscape | sym )*}
+
+@skip {} {
+  // https://lezer.codemirror.net/docs/guide/#local-token-groups
+  // this rule uses local tokens so it must be defined
+  // inside of a skip block
+  Template { LBrace Variable+ RBrace }
+}
+
+//https://lezer.codemirror.net/docs/guide/#tokens
+// lowercase tokens are consumed by the parser but not included in the tree
+// uppercase tokens are included in the tree
+@tokens {
+  LBrace { "{" }
+  emptyTemplate { "{}" }
+  lEscape { "\\" "{" | "{{" }
+  sym { "{{" | "}}" | "\"" | "'" }
+  char { $[\n\r\t\u{20}\u{21}\u{23}-\u{5b}\u{5d}-\u{10ffff}] | "\\" esc }
+  esc  { $["\\\/bfnrt] | "u" hex hex hex hex }
+  hex  { $[0-9a-fA-F] }
+  @precedence { lEscape, LBrace, char, sym }
+}
+
+// https://lezer.codemirror.net/docs/guide/#local-token-groups
+// tokens that only exist in the context that they are used
+// they only apply while inside the Template scope in this case
+@local tokens {
+  RBrace { "}" }
+  Variable { ("\\" "}") | (![}])+ }
+  @else else
+}

app/src/components/templateEditor/language/fString/fStringTemplating.syntax.grammar.d.ts

@@ -0,0 +1,3 @@
+import { LRParser } from "@lezer/lr";
+
+export declare const parser: LRParser;

app/src/components/templateEditor/language/fString/fStringTemplating.ts

@@ -0,0 +1,73 @@
+import { LanguageSupport, LRLanguage } from "@codemirror/language";
+import { styleTags, tags as t } from "@lezer/highlight";
+
+import { format } from "../languageUtils";
+
+import { parser } from "./fStringTemplating.syntax.grammar";
+
+/**
+ * Define the language for the FString templating system
+ *
+ * @see https://codemirror.net/examples/lang-package/
+ *
+ * @example
+ * ```
+ * {question}
+ *
+ * {{
+ *   "answer": {answer}
+ * }}
+ * ```
+ * In this example, the variables are `question` and `answer`.
+ * Double braces are not considered as variables, and will be converted to a single brace on format.
+ */
+export const FStringTemplatingLanguage = LRLanguage.define({
+  parser: parser.configure({
+    props: [
+      // https://lezer.codemirror.net/docs/ref/#highlight.styleTags
+      styleTags({
+        // style the opening brace of a template, not floating braces
+        "Template/LBrace": t.quote,
+        // style the closing brace of a template, not floating braces
+        "Template/RBrace": t.quote,
+        // style variables (stuff inside {})
+        "Template/Variable": t.variableName,
+        // style invalid stuff, undefined tokens will be highlighted
+        "⚠": t.invalid,
+      }),
+    ],
+  }),
+  languageData: {},
+});
+
+/**
+ * Generates a string representation of the parse tree of the given text
+ *
+ * Useful for debugging the parser
+ */
+export const debugParser = (text: string) => {
+  const tree = FStringTemplatingLanguage.parser.parse(text);
+  return tree.toString();
+};
+
+/**
+ * Formats an FString template with the given variables.
+ */
+export const formatFString = ({
+  text,
+  variables,
+}: Omit<Parameters<typeof format>[0], "parser" | "postFormat">) =>
+  format({
+    parser: FStringTemplatingLanguage.parser,
+    text,
+    variables,
+    postFormat: (text) =>
+      text.replaceAll("\\{", "{").replaceAll("{{", "{").replaceAll("}}", "}"),
+  });
+
+/**
+ * Creates a CodeMirror extension for the FString templating system
+ */
+export function FStringTemplating() {
+  return new LanguageSupport(FStringTemplatingLanguage);
+}

app/src/components/templateEditor/language/fString/index.ts

@@ -0,0 +1 @@
+export * from "./fStringTemplating";