diff --git a/.changeset/nervous-plants-refuse.md b/.changeset/nervous-plants-refuse.md new file mode 100644 index 000000000..174799577 --- /dev/null +++ b/.changeset/nervous-plants-refuse.md @@ -0,0 +1,6 @@ +--- +'myst-cli': patch +'myst-spec-ext': patch +--- + +support jupyter cell meta tags diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 2a3232974..60cf6019e 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -48,6 +48,7 @@ import { transformThumbnail, StaticFileTransformer, inlineExpressionsPlugin, + propagateBlockDataToCode, } from '../transforms/index.js'; import type { ImageExtensions } from '../utils/index.js'; import { logMessagesFromVFile } from '../utils/index.js'; @@ -144,6 +145,9 @@ export async function transformMdast( .use(joinGatesPlugin) .run(mdast, vfile); + // This needs to come after basic transformations since meta tags are added there + propagateBlockDataToCode(session, vfile, mdast); + // Run the link transformations that can be done without knowledge of other files const intersphinx = projectPath ? await loadIntersphinx(session, { projectPath }) : []; const transformers = [ diff --git a/packages/myst-cli/src/transforms/code.spec.ts b/packages/myst-cli/src/transforms/code.spec.ts index aca3a689f..858deab44 100644 --- a/packages/myst-cli/src/transforms/code.spec.ts +++ b/packages/myst-cli/src/transforms/code.spec.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; +import { VFile } from 'vfile'; import { Session } from '../session'; -import { liftCodeMetadataToBlock, metadataFromCode } from './code'; +import { + liftCodeMetadataToBlock, + metadataFromCode, + propagateBlockDataToCode, + checkMetaTags, +} from './code'; describe('metadataFromCode', () => { it('empty code returns self', async () => { @@ -115,3 +121,163 @@ describe('liftCodeMetadataToBlock', () => { expect(mdast.children[0].children[1].value).toEqual('print("hello world2")'); }); }); + +function build_mdast(tags: string[], has_output: boolean) { + const mdast: any = { + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'code', + executable: true, + }, + ], + data: { + tags: tags, + }, + }, + ], + }; + if (has_output) { + mdast.children[0].children.push({ type: 'output' }); + } + return mdast; +} + +describe('checkMetaTags', () => { + it('filter tags', async () => { + const tags = ['hide-cell', 'remove-input', 'tag-1', 'tag-2']; + for (const filter of [true, false]) { + const mdast = build_mdast(tags, true); + const vfile = new VFile(); + checkMetaTags(vfile, {} as any, tags, filter); + const result = mdast.children[0].data.tags; + if (filter) { + expect(result).toEqual(['tag-1', 'tag-2']); + } else { + expect(result).toEqual(tags); + } + } + }); + it('validate tags with duplicate', async () => { + for (const action of ['hide', 'remove']) { + const tags: string[] = []; + for (const target of ['cell', 'input', 'output']) { + const tag = `${action}-${target}`; + tags.push(tag); + tags.push(tag); + } + const validMetatags = checkMetaTags(new VFile(), {} as any, tags, true); + const expected: string[] = []; + for (const target of ['cell', 'input', 'output']) { + expected.push(`${action}-${target}`); + } + expect(validMetatags).toEqual(expected); + } + }); + it('validate tags with conflict', async () => { + const tags: string[] = []; + for (const action of ['hide', 'remove']) { + for (const target of ['cell', 'input', 'output']) { + tags.push(`${action}-${target}`); + } + } + const validMetatags = checkMetaTags(new VFile(), {} as any, tags, true); + const expected: string[] = []; + for (const target of ['cell', 'input', 'output']) { + expected.push(`remove-${target}`); + } + expect(validMetatags).toEqual(expected); + }); + it('validate tags with duplicate, conflict and filter', async () => { + for (const filter of [true, false]) { + const tags = ['tag-1', 'tag-2']; + for (const action of ['hide', 'remove']) { + for (const target of ['cell', 'input', 'output']) { + tags.push(`${action}-${target}`); + tags.push(`${action}-${target}`); + } + } + const validMetatags = checkMetaTags(new VFile(), {} as any, tags, filter); + const expected: string[] = []; + for (const target of ['cell', 'input', 'output']) { + expected.push(`remove-${target}`); + } + expect(validMetatags).toEqual(expected); + } + }); + it('duplicate tag warn', async () => { + for (const action of ['hide', 'remove']) { + for (const target of ['cell', 'input', 'output']) { + const tag = `${action}-${target}`; + const mdast = build_mdast([tag, tag], true); + const vfile = new VFile(); + propagateBlockDataToCode(new Session(), vfile, mdast); + expect(vfile.messages[0].message).toBe(`tag '${tag}' is duplicated`); + } + } + }); + it('tag conflict warn', async () => { + for (const target of ['cell', 'input', 'output']) { + const tags = [`hide-${target}`, `remove-${target}`]; + const mdast = build_mdast(tags, true); + const vfile = new VFile(); + propagateBlockDataToCode(new Session(), vfile, mdast); + const message = `'hide-${target}' and 'remove-${target}' both exist`; + expect(vfile.messages[0].message).toBe(message); + } + }); +}); + +describe('propagateBlockDataToCode', () => { + it('single tag propagation', async () => { + for (const action of ['hide', 'remove']) { + for (const target of ['cell', 'input', 'output']) { + for (const has_output of [true, false]) { + const tag = `${action}-${target}`; + const mdast = build_mdast([tag], has_output); + propagateBlockDataToCode(new Session(), new VFile(), mdast); + let result = ''; + const outputNode = mdast.children[0].children[1]; + switch (target) { + case 'cell': + result = mdast.children[0].visibility; + break; + case 'input': + result = mdast.children[0].children[0].visibility; + break; + case 'output': + if (!has_output && target == 'output') { + expect(outputNode).toEqual(undefined); + continue; + } + result = outputNode.visibility; + break; + } + expect(result).toEqual(action); + } + } + } + }); + it('multi tags propagation', async () => { + for (const action of [`hide`, `remove`]) { + for (const has_output of [true, false]) { + const tags = [`${action}-cell`, `${action}-input`, `${action}-output`]; + const mdast = build_mdast(tags, has_output); + propagateBlockDataToCode(new Session(), new VFile(), mdast); + const blockNode = mdast.children[0]; + const codeNode = mdast.children[0].children[0]; + const outputNode = mdast.children[0].children[1]; + expect(blockNode.visibility).toEqual(action); + expect(codeNode.visibility).toEqual(action); + if (has_output) { + expect(outputNode.visibility).toEqual(action); + } else { + expect(outputNode).toEqual(undefined); + } + } + } + }); +}); diff --git a/packages/myst-cli/src/transforms/code.ts b/packages/myst-cli/src/transforms/code.ts index 5e88bb9c6..499967d43 100644 --- a/packages/myst-cli/src/transforms/code.ts +++ b/packages/myst-cli/src/transforms/code.ts @@ -1,8 +1,10 @@ import type { Root } from 'mdast'; import type { GenericNode } from 'myst-common'; -import { selectAll } from 'unist-util-select'; +import { fileError, fileWarn } from 'myst-common'; +import { select, selectAll } from 'unist-util-select'; import yaml from 'js-yaml'; import type { ISession } from '../session/types.js'; +import type { VFile } from 'vfile'; const CELL_OPTION_PREFIX = '#| '; @@ -83,3 +85,96 @@ export function liftCodeMetadataToBlock(session: ISession, filename: string, mda } }); } + +/** + * Check duplicated meta tags and conflict meta tags. + * Separate the meta tags from tag if filter is true, otherwise just go through and process. + */ +export function checkMetaTags( + vfile: VFile, + node: GenericNode, + tags: string[], + filter: boolean, +): string[] { + const metaTagsCounter = new Map(); + for (const action of ['hide', 'remove']) { + for (const target of ['cell', 'input', 'output']) { + metaTagsCounter.set(`${action}-${target}`, 0); + } + } + const check = (tag: string) => { + const isMetaTag = metaTagsCounter.has(tag); + if (isMetaTag) { + metaTagsCounter.set(tag, metaTagsCounter.get(tag) + 1); + } + return !isMetaTag; + }; + if (filter) { + tags.splice(0, tags.length, ...tags.filter(check)); + } else { + tags.forEach(check); + } + const validMetatags = []; + metaTagsCounter.forEach((value, key) => { + if (value >= 2) { + fileWarn(vfile, `tag '${key}' is duplicated`, { node }); + } + }); + for (const target of ['cell', 'input', 'output']) { + const hide = metaTagsCounter.get(`hide-${target}`) > 0; + const remove = metaTagsCounter.get(`remove-${target}`) > 0; + if (hide && remove) { + fileWarn(vfile, `'hide-${target}' and 'remove-${target}' both exist`, { node }); + validMetatags.push(`remove-${target}`); + } else if (hide) { + validMetatags.push(`hide-${target}`); + } else if (remove) { + validMetatags.push(`remove-${target}`); + } + } + return validMetatags; +} + +/** + * Traverse mdast, propagate block tags to code and output + */ +export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: Root) { + const blocks = selectAll('block', mdast) as GenericNode[]; + blocks.forEach((block) => { + if (!block.data || !block.data.tags) return; + if (!Array.isArray(block.data.tags)) { + fileError(vfile, `tags in code-cell directive must be a list of strings`, { node: block }); + } + const validMetatags = checkMetaTags(vfile, block, block.data.tags, true); + const codeNode = select('code[executable=true]', block) as GenericNode | null; + const outputNode = select('output', block) as GenericNode | null; + validMetatags.forEach((tag: string) => { + switch (tag) { + // should we raise when hide and remove both exist? + case 'hide-cell': + block.visibility = 'hide'; + break; + case 'remove-cell': + block.visibility = 'remove'; + break; + case 'hide-input': + if (codeNode) codeNode.visibility = 'hide'; + break; + case 'remove-input': + if (codeNode) codeNode.visibility = 'remove'; + break; + case 'hide-output': + if (outputNode) outputNode.visibility = 'hide'; + break; + case 'remove-output': + if (outputNode) outputNode.visibility = 'remove'; + break; + default: + session.log.debug(`tag '${tag}' is not valid in code-cell tags'`); + } + }); + if (!block.visibility) block.visibility = 'show'; + if (codeNode && !codeNode.visibility) codeNode.visibility = 'show'; + if (outputNode && !outputNode.visibility) outputNode.visibility = 'show'; + }); +} diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index 8b3ba4c8f..808b28d15 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -70,6 +70,7 @@ export type Admonition = SpecAdmonition & { export type Code = SpecCode & { executable?: boolean; + visibility?: 'show' | 'hide' | 'remove'; }; export type ListItem = SpecListItem & {