From 88a9270632591f3fc48a41f0ed5036b178725009 Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Fri, 20 Sep 2024 10:39:11 +0100 Subject: [PATCH 1/5] feat(shiki): add support for `langAlias` --- .changeset/dirty-socks-sip.md | 32 +++++++++++++++++++++ packages/astro/src/core/config/schema.ts | 4 +++ packages/markdown/remark/src/index.ts | 1 + packages/markdown/remark/src/shiki.ts | 7 +++++ packages/markdown/remark/src/types.ts | 4 ++- packages/markdown/remark/test/shiki.test.js | 28 ++++++++++++++++++ 6 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 .changeset/dirty-socks-sip.md diff --git a/.changeset/dirty-socks-sip.md b/.changeset/dirty-socks-sip.md new file mode 100644 index 000000000000..b75d515cb6ea --- /dev/null +++ b/.changeset/dirty-socks-sip.md @@ -0,0 +1,32 @@ +--- +'@astrojs/markdown-remark': minor +'astro': minor +--- + +Adds a configuration called https://shiki.style/guide/load-lang#custom-language-aliases, that allows a non-supported code language to a known language. + +The below example will tell shiki to highlight the code blocks `cjs` using the `javascript` syntax highlighting. + +```js +import { defineConfig } from "astro/config"; + +export default defineConfig({ + markdown: { + shikiConfig: { + langAlias: { + cjs: "javascript" + } + } + } +}) +``` + +``````md +```cjs +"use strict" + +function commonJs() { + return "I am a commonjs file" +} +``` +`````` diff --git a/packages/astro/src/core/config/schema.ts b/packages/astro/src/core/config/schema.ts index ad10f725ac6c..3ec4ee581aa9 100644 --- a/packages/astro/src/core/config/schema.ts +++ b/packages/astro/src/core/config/schema.ts @@ -311,6 +311,10 @@ export const AstroConfigSchema = z.object({ return langs; }) .default([]), + langAlias: z + .record(z.string(), z.string()) + .optional() + .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.langAlias!), theme: z .enum(Object.keys(bundledThemes) as [BuiltinTheme, ...BuiltinTheme[]]) .or(z.custom()) diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 66cf00043ff4..706851b12825 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -37,6 +37,7 @@ export const markdownConfigDefaults: Required = { themes: {}, wrap: false, transformers: [], + langAlias: {}, }, remarkPlugins: [], rehypePlugins: [], diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts index 0028ec927f1d..c8f02ce9397e 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/markdown/remark/src/shiki.ts @@ -45,11 +45,18 @@ export async function createShikiHighlighter({ defaultColor, wrap = false, transformers = [], + langAlias = {}, }: ShikiConfig = {}): Promise { theme = theme === 'css-variables' ? cssVariablesTheme() : theme; + const aliasKeys = []; + for (const [fromLang, toLang] of Object.entries(langAlias)) { + langs?.push(toLang); + aliasKeys.push(fromLang); + } const highlighter = await getHighlighter({ langs: ['plaintext', ...langs], + langAlias, themes: Object.values(themes).length ? Object.values(themes) : [theme], }); diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index aa7b62c9a780..152b6ed8bab3 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -3,6 +3,7 @@ import type * as mdast from 'mdast'; import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { BuiltinTheme, + HighlighterCoreOptions, LanguageRegistration, ShikiTransformer, ThemeRegistration, @@ -36,7 +37,8 @@ export type RemarkRehype = RemarkRehypeOptions; export type ThemePresets = BuiltinTheme | 'css-variables'; export interface ShikiConfig { - langs?: LanguageRegistration[]; + langs?: (LanguageRegistration | string)[]; + langAlias?: HighlighterCoreOptions['langAlias']; theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; themes?: Record; defaultColor?: 'light' | 'dark' | string | false; diff --git a/packages/markdown/remark/test/shiki.test.js b/packages/markdown/remark/test/shiki.test.js index c3cb813702db..ca17ab1d8e27 100644 --- a/packages/markdown/remark/test/shiki.test.js +++ b/packages/markdown/remark/test/shiki.test.js @@ -101,4 +101,32 @@ describe('shiki syntax highlighting', () => { // Doesn't have `color` or `background-color` properties. assert.doesNotMatch(code, /color:/); }); + + it('the highlighter supports lang alias', async () => { + const highlighter = await createShikiHighlighter({ + langAlias: { + cjs: 'javascript', + }, + }); + + const html = await highlighter.highlight(`let test = "some string"`, 'cjs', { + attributes: { 'data-foo': 'bar', autofocus: true }, + }); + + assert.match(html, /data-language="cjs"/); + }); + + it('the markdown processsor support lang alias', async () => { + const processor = await createMarkdownProcessor({ + shikiConfig: { + langAlias: { + cjs: 'javascript', + }, + }, + }); + + const { code } = await processor.render('```cjs\nlet foo = "bar"\n```'); + + assert.match(code, /data-language="cjs"/); + }); }); From cb61df0766140ad98f3afc7c4cfd962f451ada0b Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Tue, 1 Oct 2024 09:37:20 +0100 Subject: [PATCH 2/5] chore: apply feedback --- .changeset/dirty-socks-sip.md | 16 +++++++++++++--- packages/markdown/remark/src/shiki.ts | 5 ----- packages/markdown/remark/test/shiki.test.js | 2 ++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.changeset/dirty-socks-sip.md b/.changeset/dirty-socks-sip.md index b75d515cb6ea..56503e0aca1c 100644 --- a/.changeset/dirty-socks-sip.md +++ b/.changeset/dirty-socks-sip.md @@ -4,8 +4,10 @@ --- Adds a configuration called https://shiki.style/guide/load-lang#custom-language-aliases, that allows a non-supported code language to a known language. - -The below example will tell shiki to highlight the code blocks `cjs` using the `javascript` syntax highlighting. + +This option requires `langs` to be defined with the correct values. The option will tell shiki which language to load when mapping the alias. + +The below example will tell shiki to highlight the code blocks `cjs` using the `javascript` syntax highlighting, The `langs` list will contain the `javascript` language. ```js import { defineConfig } from "astro/config"; @@ -15,7 +17,8 @@ export default defineConfig({ shikiConfig: { langAlias: { cjs: "javascript" - } + }, + langs: ['javascript'] } } }) @@ -30,3 +33,10 @@ function commonJs() { } ``` `````` + +Failing to define `langs` will result in an error: + +``` +Error [ShikiError]: Failed to parse Markdown file "undefined": +Language `cjs` not found, you may need to load it first +``` diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts index c8f02ce9397e..69f958dbed19 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/markdown/remark/src/shiki.ts @@ -48,12 +48,7 @@ export async function createShikiHighlighter({ langAlias = {}, }: ShikiConfig = {}): Promise { theme = theme === 'css-variables' ? cssVariablesTheme() : theme; - const aliasKeys = []; - for (const [fromLang, toLang] of Object.entries(langAlias)) { - langs?.push(toLang); - aliasKeys.push(fromLang); - } const highlighter = await getHighlighter({ langs: ['plaintext', ...langs], langAlias, diff --git a/packages/markdown/remark/test/shiki.test.js b/packages/markdown/remark/test/shiki.test.js index ca17ab1d8e27..ab6ab0fdd9d5 100644 --- a/packages/markdown/remark/test/shiki.test.js +++ b/packages/markdown/remark/test/shiki.test.js @@ -107,6 +107,7 @@ describe('shiki syntax highlighting', () => { langAlias: { cjs: 'javascript', }, + langs: ['javascript'] }); const html = await highlighter.highlight(`let test = "some string"`, 'cjs', { @@ -122,6 +123,7 @@ describe('shiki syntax highlighting', () => { langAlias: { cjs: 'javascript', }, + langs: ['javascript'] }, }); From 04019cac4f779fe24925497a21346d10a7489b62 Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Tue, 1 Oct 2024 10:18:44 +0100 Subject: [PATCH 3/5] Update packages/markdown/remark/src/types.ts Co-authored-by: Bjorn Lu --- packages/markdown/remark/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index 152b6ed8bab3..06735a38f0a1 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -37,7 +37,7 @@ export type RemarkRehype = RemarkRehypeOptions; export type ThemePresets = BuiltinTheme | 'css-variables'; export interface ShikiConfig { - langs?: (LanguageRegistration | string)[]; + langs?: (LanguageRegistration | BuiltinLanguage | SpecialLanguage)[]; langAlias?: HighlighterCoreOptions['langAlias']; theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; themes?: Record; From 128ca85b30b0eb75d7daf849afe4f0f75ca80e4d Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Tue, 1 Oct 2024 10:21:36 +0100 Subject: [PATCH 4/5] fix build --- packages/markdown/remark/src/types.ts | 2 ++ packages/markdown/remark/test/shiki.test.js | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index 06735a38f0a1..3dafe15aff19 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -2,10 +2,12 @@ import type * as hast from 'hast'; import type * as mdast from 'mdast'; import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { + BuiltinLanguage, BuiltinTheme, HighlighterCoreOptions, LanguageRegistration, ShikiTransformer, + SpecialLanguage, ThemeRegistration, ThemeRegistrationRaw, } from 'shiki'; diff --git a/packages/markdown/remark/test/shiki.test.js b/packages/markdown/remark/test/shiki.test.js index ab6ab0fdd9d5..276396e8ab94 100644 --- a/packages/markdown/remark/test/shiki.test.js +++ b/packages/markdown/remark/test/shiki.test.js @@ -107,7 +107,7 @@ describe('shiki syntax highlighting', () => { langAlias: { cjs: 'javascript', }, - langs: ['javascript'] + langs: ['javascript'], }); const html = await highlighter.highlight(`let test = "some string"`, 'cjs', { @@ -123,7 +123,7 @@ describe('shiki syntax highlighting', () => { langAlias: { cjs: 'javascript', }, - langs: ['javascript'] + langs: ['javascript'], }, }); From a6a6995d7c4bee14d3791b7d944fae5e84b42383 Mon Sep 17 00:00:00 2001 From: bluwy Date: Sat, 5 Oct 2024 16:45:04 +0800 Subject: [PATCH 5/5] Fix bug --- .changeset/dirty-socks-sip.md | 32 +++++++-------------- packages/markdown/remark/src/shiki.ts | 11 ++++--- packages/markdown/remark/src/types.ts | 4 +-- packages/markdown/remark/test/shiki.test.js | 2 -- 4 files changed, 19 insertions(+), 30 deletions(-) diff --git a/.changeset/dirty-socks-sip.md b/.changeset/dirty-socks-sip.md index 56503e0aca1c..52d23cd76a32 100644 --- a/.changeset/dirty-socks-sip.md +++ b/.changeset/dirty-socks-sip.md @@ -3,40 +3,30 @@ 'astro': minor --- -Adds a configuration called https://shiki.style/guide/load-lang#custom-language-aliases, that allows a non-supported code language to a known language. +Adds a [`markdown.shikiConfig.langAlias` option](https://shiki.style/guide/load-lang#custom-language-aliases) that allows aliasing a non-supported code language to a known language. -This option requires `langs` to be defined with the correct values. The option will tell shiki which language to load when mapping the alias. - -The below example will tell shiki to highlight the code blocks `cjs` using the `javascript` syntax highlighting, The `langs` list will contain the `javascript` language. +For example, the below configuration tells shiki to highlight `cjs` code blocks using the `javascript` syntax highlighter: ```js -import { defineConfig } from "astro/config"; +import { defineConfig } from 'astro/config'; export default defineConfig({ markdown: { shikiConfig: { langAlias: { - cjs: "javascript" + cjs: 'javascript', }, - langs: ['javascript'] - } - } -}) + }, + }, +}); ``` -``````md +````md ```cjs -"use strict" +'use strict'; function commonJs() { - return "I am a commonjs file" + return 'I am a commonjs file'; } ``` -`````` - -Failing to define `langs` will result in an error: - -``` -Error [ShikiError]: Failed to parse Markdown file "undefined": -Language `cjs` not found, you may need to load it first -``` +```` diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts index 69f958dbed19..7a584fbedfa5 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/markdown/remark/src/shiki.ts @@ -57,15 +57,18 @@ export async function createShikiHighlighter({ return { async highlight(code, lang = 'plaintext', options) { + const resolvedLang = langAlias[lang] ?? lang; const loadedLanguages = highlighter.getLoadedLanguages(); - if (!isSpecialLang(lang) && !loadedLanguages.includes(lang)) { + if (!isSpecialLang(lang) && !loadedLanguages.includes(resolvedLang)) { try { - await highlighter.loadLanguage(lang as BundledLanguage); + await highlighter.loadLanguage(resolvedLang as BundledLanguage); } catch (_err) { + const langStr = + lang === resolvedLang ? `"${lang}"` : `"${lang}" (aliased to "${resolvedLang}")`; // eslint-disable-next-line no-console console.warn( - `[Shiki] The language "${lang}" doesn't exist, falling back to "plaintext".`, + `[Shiki] The language ${langStr} doesn't exist, falling back to "plaintext".`, ); lang = 'plaintext'; } @@ -123,7 +126,7 @@ export async function createShikiHighlighter({ // Add "user-select: none;" for "+"/"-" diff symbols. // Transform `+ something // into `+ something` - if (lang === 'diff') { + if (resolvedLang === 'diff') { const innerSpanNode = node.children[0]; const innerSpanTextNode = innerSpanNode?.type === 'element' && innerSpanNode.children?.[0]; diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index 3dafe15aff19..d95676b55b63 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -2,12 +2,10 @@ import type * as hast from 'hast'; import type * as mdast from 'mdast'; import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { - BuiltinLanguage, BuiltinTheme, HighlighterCoreOptions, LanguageRegistration, ShikiTransformer, - SpecialLanguage, ThemeRegistration, ThemeRegistrationRaw, } from 'shiki'; @@ -39,7 +37,7 @@ export type RemarkRehype = RemarkRehypeOptions; export type ThemePresets = BuiltinTheme | 'css-variables'; export interface ShikiConfig { - langs?: (LanguageRegistration | BuiltinLanguage | SpecialLanguage)[]; + langs?: LanguageRegistration[]; langAlias?: HighlighterCoreOptions['langAlias']; theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; themes?: Record; diff --git a/packages/markdown/remark/test/shiki.test.js b/packages/markdown/remark/test/shiki.test.js index 276396e8ab94..ca17ab1d8e27 100644 --- a/packages/markdown/remark/test/shiki.test.js +++ b/packages/markdown/remark/test/shiki.test.js @@ -107,7 +107,6 @@ describe('shiki syntax highlighting', () => { langAlias: { cjs: 'javascript', }, - langs: ['javascript'], }); const html = await highlighter.highlight(`let test = "some string"`, 'cjs', { @@ -123,7 +122,6 @@ describe('shiki syntax highlighting', () => { langAlias: { cjs: 'javascript', }, - langs: ['javascript'], }, });