From 8d571cbf620637a1644bfe3ebf35eb31842c7ba4 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 19 Aug 2024 12:07:33 +0100 Subject: [PATCH 1/5] feat: run mmdc --- packages/myst-cli/src/build/tex/single.ts | 2 + packages/myst-cli/src/process/mdast.ts | 6 ++ packages/myst-common/src/ruleids.ts | 1 + packages/myst-transforms/package.json | 4 +- packages/myst-transforms/src/index.ts | 1 + packages/myst-transforms/src/mermaid.ts | 107 ++++++++++++++++++++++ 6 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 packages/myst-transforms/src/mermaid.ts diff --git a/packages/myst-cli/src/build/tex/single.ts b/packages/myst-cli/src/build/tex/single.ts index 24d456eee..963c320c5 100644 --- a/packages/myst-cli/src/build/tex/single.ts +++ b/packages/myst-cli/src/build/tex/single.ts @@ -153,6 +153,7 @@ export async function localArticleToTexRaw( imageAltOutputFolder: 'files/', imageExtensions: TEX_IMAGE_EXTENSIONS, simplifyFigures: true, + mermaidAsImage: true, }); return mdastToTex(session, mdast, references, frontmatter, null, false); }), @@ -252,6 +253,7 @@ export async function localArticleToTexTemplated( imageAltOutputFolder: 'files/', imageExtensions: TEX_IMAGE_EXTENSIONS, simplifyFigures: true, + mermaidAsImage: true, }); partDefinitions.forEach((def) => { diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index da6e7779a..2e89092ee 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -32,6 +32,7 @@ import { inlineMathSimplificationPlugin, checkLinkTextTransform, indexIdentifierPlugin, + mermaidToImageTransform, } from 'myst-transforms'; import { unified } from 'unified'; import { select, selectAll } from 'unist-util-select'; @@ -341,6 +342,7 @@ export async function finalizeMdast( file: string, { imageWriteFolder, + mermaidAsImage, useExistingImages, imageAltOutputFolder, imageExtensions, @@ -350,6 +352,7 @@ export async function finalizeMdast( maxSizeWebp, }: { imageWriteFolder: string; + mermaidAsImage?: boolean; useExistingImages?: boolean; imageAltOutputFolder?: string; imageExtensions?: ImageExtensions[]; @@ -372,6 +375,9 @@ export async function finalizeMdast( vfile, }); if (!useExistingImages) { + if (mermaidAsImage) { + await mermaidToImageTransform(session, mdast, imageWriteFolder, vfile); + } await transformImagesToDisk(session, mdast, file, imageWriteFolder, { altOutputFolder: imageAltOutputFolder, imageExtensions, diff --git a/packages/myst-common/src/ruleids.ts b/packages/myst-common/src/ruleids.ts index d833a0dd0..afc9fc045 100644 --- a/packages/myst-common/src/ruleids.ts +++ b/packages/myst-common/src/ruleids.ts @@ -48,6 +48,7 @@ export enum RuleId { imageFormatConverts = 'image-format-converts', imageCopied = 'image-copied', imageFormatOptimizes = 'image-format-optimizes', + mermaidDiagramConverted = 'mermaid-diagram-converted', // Math rules mathLabelLifted = 'math-label-lifted', mathEquationEnvRemoved = 'math-equation-env-removed', diff --git a/packages/myst-transforms/package.json b/packages/myst-transforms/package.json index ff8e47970..62e119c16 100644 --- a/packages/myst-transforms/package.json +++ b/packages/myst-transforms/package.json @@ -21,10 +21,10 @@ "dependencies": { "doi-utils": "^2.0.0", "hast-util-from-html": "^2.0.1", + "hast-util-to-mdast": "^8.3.1", "intersphinx": "^1.0.2", "js-yaml": "^4.1.0", "katex": "^0.15.2", - "hast-util-to-mdast": "^8.3.1", "mdast-util-find-and-replace": "^2.1.0", "myst-common": "^1.5.3", "myst-frontmatter": "^1.5.3", @@ -36,8 +36,8 @@ "unified": "^10.0.0", "unist-builder": "^3.0.0", "unist-util-find-after": "^4.0.0", - "unist-util-modify-children": "^3.1.0", "unist-util-map": "^3.0.0", + "unist-util-modify-children": "^3.1.0", "unist-util-remove": "^3.1.0", "unist-util-select": "^4.0.3", "unist-util-visit": "^4.1.0", diff --git a/packages/myst-transforms/src/index.ts b/packages/myst-transforms/src/index.ts index 0dc78a0db..6d6d3626d 100644 --- a/packages/myst-transforms/src/index.ts +++ b/packages/myst-transforms/src/index.ts @@ -41,6 +41,7 @@ export { } from './code.js'; export { blockquotePlugin, blockquoteTransform } from './blockquote.js'; export { imageAltTextPlugin, imageAltTextTransform } from './images.js'; +export { mermaidToImageTransform } from './mermaid.js'; export { buildIndexTransform, indexIdentifierPlugin, indexIdentifierTransform } from './indices.js'; export { liftMystDirectivesAndRolesPlugin, diff --git a/packages/myst-transforms/src/mermaid.ts b/packages/myst-transforms/src/mermaid.ts new file mode 100644 index 000000000..2239b512f --- /dev/null +++ b/packages/myst-transforms/src/mermaid.ts @@ -0,0 +1,107 @@ +import type { Plugin } from 'unified'; +import { selectAll } from 'unist-util-select'; +import type { GenericParent, GenericNode } from 'myst-common'; +import { RuleId } from 'myst-common'; +import which from 'which'; +import type { VFile } from 'vfile'; +import type { LoggerDE } from 'myst-cli-utils'; +import type { ISession } from 'myst-cli'; +import { makeExecutable, tic } from 'myst-cli-utils'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import { join } from 'node:path'; +import { createTempFolder, addWarningForFile } from 'myst-cli'; + +function isMMDCCommandAvailable() { + return !!which.sync('mmdc', { nothrow: true }); +} +type Literal = { + type: string; + value: string; +}; + +function createMMDCLogger(session: ISession): LoggerDE { + const logger = { + debug(data: string) { + const line = data.trim(); + if (!line) return; + session.log.debug(data); + }, + error(data: string) { + const line = data.trim(); + if (!line) return; + session.log.error(data); + }, + }; + return logger; +} + +async function convertMermaidToSVG( + session: ISession, + writeFolder: string, + data: string, + vfile: VFile, +) { + const hash = createHash('md5').update(data).digest('hex'); + const tempFolder = createTempFolder(session); + + const srcPath = join(tempFolder, `${hash}.mmd`); + await fs.writeFile(srcPath, data); + + await fs.mkdir(writeFolder, { recursive: true }); + const dstPath = join(writeFolder, `${hash}.pdf`); + + const executable = `mmdc -i ${srcPath} -o ${dstPath}`; + const exec = makeExecutable(executable, createMMDCLogger(session)); + try { + await exec(); + } catch (err) { + addWarningForFile( + session, + vfile.path, + `Could not convert Mermaid diagram to svg: ${err}`, + 'error', + { + ruleId: RuleId.mermaidDiagramConverted, + }, + ); + return null; + } + + return dstPath; +} + +/** + * Ensure caption content is nested in a paragraph. + * + * This function is idempotent. + */ +export async function mermaidToImageTransform( + session: ISession, + tree: GenericParent, + writeFolder: string, + vfile: VFile, +) { + if (!isMMDCCommandAvailable()) { + addWarningForFile( + session, + vfile.path, + `Could not find mmdc, required for conversion of Mermaid diagrams\n`, + 'warn', + { ruleId: RuleId.mermaidDiagramConverted }, + ); + return null; + } + + const nodes = selectAll('mermaid', tree) as Literal[]; + await Promise.all( + nodes.map(async (node) => { + const dst = await convertMermaidToSVG(session, writeFolder, node.value, vfile); + if (dst) { + const newNode = node as GenericNode; + newNode.type = 'image'; + newNode.url = dst ?? ''; + } + }), + ); +} From 3882c7acb32c4ea2f8d59d6dacfe1e7608051d53 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 12 Aug 2024 09:46:44 +0100 Subject: [PATCH 2/5] chore: add changeset --- .changeset/silver-carrots-allow.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/silver-carrots-allow.md diff --git a/.changeset/silver-carrots-allow.md b/.changeset/silver-carrots-allow.md new file mode 100644 index 000000000..bbced4811 --- /dev/null +++ b/.changeset/silver-carrots-allow.md @@ -0,0 +1,7 @@ +--- +"myst-transforms": minor +"myst-common": patch +"myst-cli": patch +--- + +Add new mermaid-conversion transform From ba0dc40807a00bb758a6fd63585c726b05ad773d Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 12 Aug 2024 09:46:57 +0100 Subject: [PATCH 3/5] chore: run linter --- .changeset/silver-carrots-allow.md | 6 +++--- packages/myst-cli/src/build/tex/single.ts | 4 ++-- packages/myst-cli/src/process/mdast.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.changeset/silver-carrots-allow.md b/.changeset/silver-carrots-allow.md index bbced4811..77f0e40ec 100644 --- a/.changeset/silver-carrots-allow.md +++ b/.changeset/silver-carrots-allow.md @@ -1,7 +1,7 @@ --- -"myst-transforms": minor -"myst-common": patch -"myst-cli": patch +'myst-transforms': minor +'myst-common': patch +'myst-cli': patch --- Add new mermaid-conversion transform diff --git a/packages/myst-cli/src/build/tex/single.ts b/packages/myst-cli/src/build/tex/single.ts index 963c320c5..a058eb90e 100644 --- a/packages/myst-cli/src/build/tex/single.ts +++ b/packages/myst-cli/src/build/tex/single.ts @@ -153,7 +153,7 @@ export async function localArticleToTexRaw( imageAltOutputFolder: 'files/', imageExtensions: TEX_IMAGE_EXTENSIONS, simplifyFigures: true, - mermaidAsImage: true, + mermaidAsImage: true, }); return mdastToTex(session, mdast, references, frontmatter, null, false); }), @@ -253,7 +253,7 @@ export async function localArticleToTexTemplated( imageAltOutputFolder: 'files/', imageExtensions: TEX_IMAGE_EXTENSIONS, simplifyFigures: true, - mermaidAsImage: true, + mermaidAsImage: true, }); partDefinitions.forEach((def) => { diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 2e89092ee..bbc0ab72f 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -376,7 +376,7 @@ export async function finalizeMdast( }); if (!useExistingImages) { if (mermaidAsImage) { - await mermaidToImageTransform(session, mdast, imageWriteFolder, vfile); + await mermaidToImageTransform(session, mdast, imageWriteFolder, vfile); } await transformImagesToDisk(session, mdast, file, imageWriteFolder, { altOutputFolder: imageAltOutputFolder, From 22b97bf4b392a61cded19ff0617ecdb3fbe0614f Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 19 Aug 2024 12:16:24 +0100 Subject: [PATCH 4/5] chore: appease linter --- packages/myst-transforms/package.json | 3 ++- packages/myst-transforms/src/mermaid.ts | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/myst-transforms/package.json b/packages/myst-transforms/package.json index 62e119c16..8b12953ea 100644 --- a/packages/myst-transforms/package.json +++ b/packages/myst-transforms/package.json @@ -42,7 +42,8 @@ "unist-util-select": "^4.0.3", "unist-util-visit": "^4.1.0", "vfile": "^5.0.0", - "vfile-message": "^3.1.2" + "vfile-message": "^3.1.2", + "which": "^3.0.1" }, "devDependencies": { "@types/katex": "^0.14.0" diff --git a/packages/myst-transforms/src/mermaid.ts b/packages/myst-transforms/src/mermaid.ts index 2239b512f..8c1caaec2 100644 --- a/packages/myst-transforms/src/mermaid.ts +++ b/packages/myst-transforms/src/mermaid.ts @@ -1,4 +1,3 @@ -import type { Plugin } from 'unified'; import { selectAll } from 'unist-util-select'; import type { GenericParent, GenericNode } from 'myst-common'; import { RuleId } from 'myst-common'; @@ -6,7 +5,7 @@ import which from 'which'; import type { VFile } from 'vfile'; import type { LoggerDE } from 'myst-cli-utils'; import type { ISession } from 'myst-cli'; -import { makeExecutable, tic } from 'myst-cli-utils'; +import { makeExecutable } from 'myst-cli-utils'; import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import { join } from 'node:path'; From f4cbafd204ddc7b7bde831ebd652e9b63de0e011 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 19 Aug 2024 12:19:58 +0100 Subject: [PATCH 5/5] chore: add package lock --- package-lock.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 454c1cb54..8e5d4d58b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15963,7 +15963,8 @@ "unist-util-select": "^4.0.3", "unist-util-visit": "^4.1.0", "vfile": "^5.0.0", - "vfile-message": "^3.1.2" + "vfile-message": "^3.1.2", + "which": "^3.0.1" }, "devDependencies": { "@types/katex": "^0.14.0"