✍️ Parse Markdown and LaTeX outputs into AST

.changeset/ten-bats-warn.md

@@ -0,0 +1,10 @@
+---
+"mystmd": minor
+"myst-directives": patch
+"myst-transforms": patch
+"myst-spec-ext": patch
+"myst-execute": patch
+"myst-cli": patch
+---
+
+Add support for new Outputs node

.changeset/weak-bananas-cover.md

@@ -0,0 +1,5 @@
+---
+"myst-cli": patch
+---
+
+Move kernel execution transform earlier in pipeline

packages/myst-cli/src/process/mdast.ts

@@ -67,6 +67,7 @@ import {
   transformFilterOutputStreams,
   transformLiftCodeBlocksInJupytext,
   transformMystXRefs,
+  liftOutputs,
 } from '../transforms/index.js';
 import type { ImageExtensions } from '../utils/resolveExtension.js';
 import { logMessagesFromVFile } from '../utils/logging.js';
@@ -194,6 +195,12 @@ export async function transformMdast(
       log: session.log,
     });
   }
+  transformRenderInlineExpressions(mdast, vfile);
+  await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });
+  liftOutputs(session, mdast, vfile, {
+    parseMyst: (content: string) => parseMyst(session, content, file),
+  });
+  transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
 
   const pipe = unified()
     .use(reconstructHtmlPlugin) // We need to group and link the HTML first
@@ -238,9 +245,6 @@ export async function transformMdast(
   // Combine file-specific citation renderers with project renderers from bib files
   const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles);
 
-  transformRenderInlineExpressions(mdast, vfile);
-  await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });
-  transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
   transformCitations(session, file, mdast, fileCitationRenderer, references);
   await unified()
     .use(codePlugin, { lang: frontmatter?.kernelspec?.language })

packages/myst-cli/src/process/notebook.ts

@@ -84,6 +84,21 @@ export async function processNotebook(
   return mdast;
 }
 
+/**
+ * Embed the Jupyter output data for a user expression into the AST
+ */
+function embedInlineExpressions(
+  userExpressions: IUserExpressionMetadata[] | undefined,
+  block: GenericNode,
+) {
+  const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[];
+  inlineNodes.forEach((inlineExpression) => {
+    const data = findExpression(userExpressions, inlineExpression.value);
+    if (!data) return;
+    inlineExpression.result = data.result as unknown as Record<string, unknown>;
+  });
+}
+
 export async function processNotebookFull(
   session: ISession,
   file: string,
@@ -136,17 +151,7 @@ export async function processNotebookFull(
           return acc.concat(...cellMdast.children);
         }
         const block = blockParent(cell, cellMdast.children) as GenericNode;
-
-        // Embed expression results into expression
-        const userExpressions = block.data?.[metadataSection] as
-          | IUserExpressionMetadata[]
-          | undefined;
-        const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[];
-        inlineNodes.forEach((inlineExpression) => {
-          const data = findExpression(userExpressions, inlineExpression.value);
-          if (!data) return;
-          inlineExpression.result = data.result as unknown as Record<string, unknown>;
-        });
+        embedInlineExpressions(block.data?.[metadataSection], block);
         return acc.concat(block);
       }
       if (cell.cell_type === CELL_TYPES.raw) {
@@ -165,17 +170,16 @@ export async function processNotebookFull(
           value: ensureString(cell.source),
         };
 
-        // Embed outputs in an output block
-        const output: { type: 'output'; id: string; data: IOutput[] } = {
-          type: 'output',
+        const outputs = {
+          type: 'outputs',
           id: nanoid(),
-          data: [],
+          children: (cell.outputs as IOutput[]).map((output) => ({
+            type: 'output',
+            jupyter_data: output,
+            children: [],
+          })),
         };
-
-        if (cell.outputs && (cell.outputs as IOutput[]).length > 0) {
-          output.data = cell.outputs as IOutput[];
-        }
-        return acc.concat(blockParent(cell, [code, output]));
+        return acc.concat(blockParent(cell, [code, outputs]));
       }
       return acc;
     },

packages/myst-cli/src/transforms/code.spec.ts

@@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) {
     ],
   };
   if (has_output) {
-    mdast.children[0].children.push({ type: 'output' });
+    mdast.children[0].children.push({
+      type: 'outputs',
+      children: [{ type: 'output', children: [] }],
+    });
   }
   return mdast;
 }
@@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => {
           const mdast = build_mdast([tag], has_output);
           propagateBlockDataToCode(new Session(), new VFile(), mdast);
           let result = '';
-          const outputNode = mdast.children[0].children[1];
+          const outputsNode = mdast.children[0].children[1];
           switch (target) {
             case 'cell':
               result = mdast.children[0].visibility;
@@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => {
               result = mdast.children[0].children[0].visibility;
               break;
             case 'output':
-              if (!has_output && target == 'output') {
-                expect(outputNode).toEqual(undefined);
+              if (!has_output) {
+                expect(outputsNode).toEqual(undefined);
                 continue;
               }
-              result = outputNode.visibility;
+              result = outputsNode.visibility;
               break;
+            default:
+              throw new Error();
           }
           expect(result).toEqual(action);
         }
@@ -290,18 +295,18 @@ describe('propagateBlockDataToCode', () => {
         propagateBlockDataToCode(new Session(), new VFile(), mdast);
         const blockNode = mdast.children[0];
         const codeNode = mdast.children[0].children[0];
-        const outputNode = mdast.children[0].children[1];
+        const outputsNode = mdast.children[0].children[1];
         expect(blockNode.visibility).toEqual(action);
         expect(codeNode.visibility).toEqual(action);
         if (has_output) {
-          expect(outputNode.visibility).toEqual(action);
+          expect(outputsNode.visibility).toEqual(action);
         } else {
-          expect(outputNode).toEqual(undefined);
+          expect(outputsNode).toEqual(undefined);
         }
       }
     }
   });
-  it('placeholder creates image node child of output', async () => {
+  it('placeholder creates image node child of outputs', async () => {
     const mdast: any = {
       type: 'root',
       children: [
@@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => {
               executable: true,
             },
             {
-              type: 'output',
+              type: 'outputs',
+              children: [],
             },
           ],
           data: {
@@ -323,12 +329,12 @@ describe('propagateBlockDataToCode', () => {
       ],
     };
     propagateBlockDataToCode(new Session(), new VFile(), mdast);
-    const outputNode = mdast.children[0].children[1];
-    expect(outputNode.children?.length).toEqual(1);
-    expect(outputNode.children[0].type).toEqual('image');
-    expect(outputNode.children[0].placeholder).toBeTruthy();
+    const outputsNode = mdast.children[0].children[1];
+    expect(outputsNode.children?.length).toEqual(1);
+    expect(outputsNode.children[0].type).toEqual('image');
+    expect(outputsNode.children[0].placeholder).toBeTruthy();
   });
-  it('placeholder passes with no output', async () => {
+  it('placeholder passes with no outputs', async () => {
     const mdast: any = {
       type: 'root',
       children: [

packages/myst-cli/src/transforms/code.ts

@@ -1,6 +1,6 @@
 import type { GenericNode, GenericParent } from 'myst-common';
 import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common';
-import type { Image, Output } from 'myst-spec-ext';
+import type { Image, Outputs } from 'myst-spec-ext';
 import { select, selectAll } from 'unist-util-select';
 import yaml from 'js-yaml';
 import type { VFile } from 'vfile';
@@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast:
   const blocks = selectAll('block', mdast) as GenericNode[];
   blocks.forEach((block) => {
     if (!block.data) return;
-    const outputNode = select('output', block) as Output | null;
-    if (block.data.placeholder && outputNode) {
-      if (!outputNode.children) outputNode.children = [];
-      outputNode.children.push({
+    const outputsNode = select('outputs', block) as Outputs | null;
+    if (block.data.placeholder && outputsNode) {
+      outputsNode.children.push({
         type: 'image',
         placeholder: true,
         url: block.data.placeholder as string,
@@ -195,18 +194,18 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast:
           if (codeNode) codeNode.visibility = 'remove';
           break;
         case NotebookCellTags.hideOutput:
-          if (outputNode) outputNode.visibility = 'hide';
+          if (outputsNode) outputsNode.visibility = 'hide';
           break;
         case NotebookCellTags.removeOutput:
-          if (outputNode) outputNode.visibility = 'remove';
+          if (outputsNode) outputsNode.visibility = 'remove';
           break;
         default:
           session.log.debug(`tag '${tag}' is not valid in code-cell tags'`);
       }
     });
     if (!block.visibility) block.visibility = 'show';
     if (codeNode && !codeNode.visibility) codeNode.visibility = 'show';
-    if (outputNode && !outputNode.visibility) outputNode.visibility = 'show';
+    if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show';
   });
 }
 
@@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) {
         child.type === 'block' &&
         child.children?.length === 2 &&
         child.children?.[0].type === 'code' &&
-        child.children?.[1].type === 'output'
+        child.children?.[1].type === 'outputs'
       ) {
         newBlocks.push(child as GenericParent);
         newBlocks.push({ type: 'block', children: [] });