From 465e0e4f7e76a805a551c8368720c167a0432214 Mon Sep 17 00:00:00 2001 From: Fuxing Loh <4266087+fuxingloh@users.noreply.github.com> Date: Tue, 3 Oct 2023 21:22:25 +0800 Subject: [PATCH] feat(contented): ability to customize unified plugins (#629) #### What this PR does / why we need it: Make it easier to customize content processor with subset of plugins. --- .../contented-example/docs/04-markdown.md | 10 +++--- .../src/MarkdownPipeline.ts | 21 +++++++++-- .../src/MarkdownPipeline.unit.ts | 31 ++++++++++++++++ .../src/UnifiedProcessor.ts | 35 +++++++++++++++++-- .../src/plugins/RemarkFrontmatter.ts | 6 ++-- 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/packages/contented-example/docs/04-markdown.md b/packages/contented-example/docs/04-markdown.md index 641ad747..c5235208 100644 --- a/packages/contented-example/docs/04-markdown.md +++ b/packages/contented-example/docs/04-markdown.md @@ -15,10 +15,12 @@ processor .use(remarkParse) .use(remarkLink) .use(remarkDirective) + .use(remarkDirectiveRehypeCodeblockHeader) + .use(remarkDirectiveRehypeCodeblockGroup) .use(remarkDirectiveRehype) - .use(collectFields) - .use(resolveFields) - .use(validateFields) + .use(remarkFrontmatterCollect) + .use(remarkFrontmatterResolve) + .use(remarkFrontmatterValidate) .use(options.remarks) .use(remarkRehype) .use(options.rehypes) @@ -28,7 +30,7 @@ processor .use(rehypeToc) .use(rehypeHeading) .use(rehypeMermaid) - .use(rehypeShiki, { highlighter }) + .use(rehypeShiki) .use(rehypeStringify) .use(options.after); ``` diff --git a/packages/contented-pipeline-md/src/MarkdownPipeline.ts b/packages/contented-pipeline-md/src/MarkdownPipeline.ts index fd1c6b6a..cd58f7ac 100644 --- a/packages/contented-pipeline-md/src/MarkdownPipeline.ts +++ b/packages/contented-pipeline-md/src/MarkdownPipeline.ts @@ -1,7 +1,7 @@ import console from 'node:console'; import { join } from 'node:path'; -import { ContentedPipeline, FileContent, FileIndex } from '@contentedjs/contented-pipeline'; +import { ContentedPipeline, FileContent, FileIndex, Pipeline } from '@contentedjs/contented-pipeline'; import { read } from 'to-vfile'; import { Processor, unified } from 'unified'; import { VFile } from 'vfile'; @@ -13,7 +13,7 @@ export class MarkdownPipeline extends ContentedPipeline { protected readonly processor: Processor = unified(); async init() { - await initProcessor(this.processor); + initProcessor(this.processor); } protected override async processFileIndex( @@ -56,6 +56,23 @@ export class MarkdownPipeline extends ContentedPipeline { fields: contented.fields, }; } + + /** + * Create a new MarkdownPipeline with a custom processor. + * This is useful for only using a subset of the plugins that you need. + * @param processor is a function that takes a Processor and adds plugins to it. + */ + static withProcessor( + processor: (processor: Processor) => void, + ): new (rootPath: string, pipeline: Pipeline) => ContentedPipeline { + class WithProcessor extends MarkdownPipeline { + async init(): Promise { + processor(this.processor); + } + } + + return WithProcessor; + } } export class MarkdownVFile extends VFile { diff --git a/packages/contented-pipeline-md/src/MarkdownPipeline.unit.ts b/packages/contented-pipeline-md/src/MarkdownPipeline.unit.ts index 2ac15ba5..781e8429 100644 --- a/packages/contented-pipeline-md/src/MarkdownPipeline.unit.ts +++ b/packages/contented-pipeline-md/src/MarkdownPipeline.unit.ts @@ -1,6 +1,7 @@ import { join } from 'node:path'; import { MarkdownPipeline } from './MarkdownPipeline'; +import { rehypeStringify, remarkParse, remarkRehype } from './UnifiedProcessor'; const rootPath = join(__dirname, '../fixtures'); @@ -95,3 +96,33 @@ describe('Without Config', () => { ]); }); }); + +describe('Custom Pipeline', () => { + const pipeline = new (MarkdownPipeline.withProcessor((processor) => { + processor.use(remarkParse).use(remarkRehype).use(rehypeStringify); + }))(__dirname, { + type: 'Markdown', + pattern: '/See.Nothing.md', + processor: 'md', + }); + + beforeAll(async () => { + await pipeline.init(); + }); + + it('should process See.Nothing.md', async () => { + const content = await pipeline.process(rootPath, 'See.Nothing.md'); + expect(content).toStrictEqual([ + { + type: 'Markdown', + fields: {}, + headings: [], + path: '/see-nothing', + sections: [], + fileId: expect.stringMatching(/[0-f]{64}/), + modifiedDate: expect.any(Number), + html: '

Nothing To See

\n

Markdown for testing MarkdownPipeline.unit.ts.

', + }, + ]); + }); +}); diff --git a/packages/contented-pipeline-md/src/UnifiedProcessor.ts b/packages/contented-pipeline-md/src/UnifiedProcessor.ts index 93a014f6..68dd976b 100644 --- a/packages/contented-pipeline-md/src/UnifiedProcessor.ts +++ b/packages/contented-pipeline-md/src/UnifiedProcessor.ts @@ -18,7 +18,11 @@ import { remarkDirectiveRehypeCodeblockHeader, } from './plugins/RemarkCodeblock.js'; import { remarkDirectiveRehype } from './plugins/RemarkDirectiveRehype.js'; -import { collectFields, resolveFields, validateFields } from './plugins/RemarkFrontmatter.js'; +import { + remarkFrontmatterCollect, + remarkFrontmatterResolve, + remarkFrontmatterValidate, +} from './plugins/RemarkFrontmatter.js'; import { remarkLink } from './plugins/RemarkLink.js'; export interface UnifiedOptions { @@ -28,7 +32,7 @@ export interface UnifiedOptions { after?: Plugin[]; } -export async function initProcessor(processor: Processor, options?: UnifiedOptions) { +export function initProcessor(processor: Processor, options?: UnifiedOptions): Processor { options?.before?.forEach((plugin) => { processor.use(plugin); }); @@ -43,7 +47,7 @@ export async function initProcessor(processor: Processor, options?: UnifiedOptio .use(remarkDirectiveRehypeCodeblockGroup) .use(remarkDirectiveRehype); - processor.use(collectFields).use(resolveFields).use(validateFields); + processor.use(remarkFrontmatterCollect).use(remarkFrontmatterResolve).use(remarkFrontmatterValidate); options?.remarks?.forEach((plugin) => { processor.use(plugin); @@ -69,4 +73,29 @@ export async function initProcessor(processor: Processor, options?: UnifiedOptio options?.after?.forEach((plugin) => { processor.use(plugin); }); + + return processor; } + +export { + rehypeAutolinkHeadings, + rehypeExternalLinks, + rehypeHeading, + rehypeMermaid, + rehypeShiki, + rehypeSlug, + rehypeStringify, + rehypeToc, + remarkDirective, + remarkDirectiveRehype, + remarkDirectiveRehypeCodeblockGroup, + remarkDirectiveRehypeCodeblockHeader, + remarkFrontmatter, + remarkFrontmatterCollect, + remarkFrontmatterResolve, + remarkFrontmatterValidate, + remarkGfm, + remarkLink, + remarkParse, + remarkRehype, +}; diff --git a/packages/contented-pipeline-md/src/plugins/RemarkFrontmatter.ts b/packages/contented-pipeline-md/src/plugins/RemarkFrontmatter.ts index b65deb56..7e8cbdcb 100644 --- a/packages/contented-pipeline-md/src/plugins/RemarkFrontmatter.ts +++ b/packages/contented-pipeline-md/src/plugins/RemarkFrontmatter.ts @@ -9,7 +9,7 @@ import { VFile } from 'vfile'; import { UnifiedContented } from './Plugin.js'; -export function collectFields(): Transformer { +export function remarkFrontmatterCollect(): Transformer { return (tree: Parent, file) => { const node = tree.children?.[0]; if (node?.type === 'yaml') { @@ -49,7 +49,7 @@ function visitDescription(file: VFile): (node: Paragraph) => void { }; } -export function resolveFields(): Transformer { +export function remarkFrontmatterResolve(): Transformer { return async (tree: Parent, file) => { const contented = file.data.contented as UnifiedContented; const entries = Object.entries(contented.pipeline.fields ?? {}); @@ -70,7 +70,7 @@ export function resolveFields(): Transformer { }; } -export function validateFields(): Transformer { +export function remarkFrontmatterValidate(): Transformer { return async (tree: Parent, file) => { const contented = file.data.contented as UnifiedContented; const entries = Object.entries(contented.pipeline.fields ?? {});