diff --git a/.changeset/olive-melons-sleep.md b/.changeset/olive-melons-sleep.md new file mode 100644 index 000000000..0050319cd --- /dev/null +++ b/.changeset/olive-melons-sleep.md @@ -0,0 +1,7 @@ +--- +"@astrojs/compiler": minor +--- + +Adds two new options to `convertToTSX`: `includeScripts` and `includeStyles`. These options allow you to optionally remove scripts and styles from the output TSX file. + +Additionally this PR makes it so scripts and styles metadata are now included in the `metaRanges` property of the result of `convertToTSX`. This is notably useful in order to extract scripts and styles from the output TSX file into separate files for language servers. diff --git a/cmd/astro-wasm/astro-wasm.go b/cmd/astro-wasm/astro-wasm.go index 2930d0ad7..b8346a1a4 100644 --- a/cmd/astro-wasm/astro-wasm.go +++ b/cmd/astro-wasm/astro-wasm.go @@ -37,6 +37,13 @@ func jsString(j js.Value) string { return j.String() } +func jsBoolOptional(j js.Value, defaultValue bool) bool { + if j.Equal(js.Undefined()) || j.Equal(js.Null()) { + return defaultValue + } + return j.Bool() +} + func jsBool(j js.Value) bool { if j.Equal(js.Undefined()) || j.Equal(js.Null()) { return false @@ -148,6 +155,16 @@ func makeTransformOptions(options js.Value) transform.TransformOptions { } } +func makeTSXOptions(options js.Value) printer.TSXOptions { + includeScripts := jsBoolOptional(options.Get("includeScripts"), true) + includeStyles := jsBoolOptional(options.Get("includeStyles"), true) + + return printer.TSXOptions{ + IncludeScripts: includeScripts, + IncludeStyles: includeStyles, + } +} + type RawSourceMap struct { File string `js:"file"` Mappings string `js:"mappings"` @@ -260,7 +277,10 @@ func ConvertToTSX() any { if err != nil { h.AppendError(err) } - result := printer.PrintToTSX(source, doc, transformOptions, h) + + tsxOptions := makeTSXOptions(js.Value(args[1])) + + result := printer.PrintToTSX(source, doc, tsxOptions, transformOptions, h) // AFTER printing, exec transformations to pickup any errors/warnings transform.Transform(doc, transformOptions, h) diff --git a/internal/printer/print-to-tsx.go b/internal/printer/print-to-tsx.go index cde378991..1db3bcf5a 100644 --- a/internal/printer/print-to-tsx.go +++ b/internal/printer/print-to-tsx.go @@ -20,14 +20,20 @@ func getTSXPrefix() string { return "/* @jsxImportSource astro */\n\n" } -func PrintToTSX(sourcetext string, n *Node, opts transform.TransformOptions, h *handler.Handler) PrintResult { +type TSXOptions struct { + IncludeScripts bool + IncludeStyles bool +} + +func PrintToTSX(sourcetext string, n *Node, opts TSXOptions, transformOpts transform.TransformOptions, h *handler.Handler) PrintResult { p := &printer{ sourcetext: sourcetext, - opts: opts, + opts: transformOpts, builder: sourcemap.MakeChunkBuilder(nil, sourcemap.GenerateLineOffsetTables(sourcetext, len(strings.Split(sourcetext, "\n")))), } p.print(getTSXPrefix()) - renderTsx(p, n) + renderTsx(p, n, &opts) + return PrintResult{ Output: p.output, SourceMapChunk: p.builder.GenerateChunk(p.output), @@ -36,14 +42,147 @@ func PrintToTSX(sourcetext string, n *Node, opts transform.TransformOptions, h * } type TSXRanges struct { - Frontmatter loc.TSXRange `js:"frontmatter"` - Body loc.TSXRange `js:"body"` + Frontmatter loc.TSXRange `js:"frontmatter"` + Body loc.TSXRange `js:"body"` + Scripts []TSXExtractedTag `js:"scripts"` + Styles []TSXExtractedTag `js:"styles"` +} + +var htmlEvents = map[string]bool{ + "onabort": true, + "onafterprint": true, + "onauxclick": true, + "onbeforematch": true, + "onbeforeprint": true, + "onbeforeunload": true, + "onblur": true, + "oncancel": true, + "oncanplay": true, + "oncanplaythrough": true, + "onchange": true, + "onclick": true, + "onclose": true, + "oncontextlost": true, + "oncontextmenu": true, + "oncontextrestored": true, + "oncopy": true, + "oncuechange": true, + "oncut": true, + "ondblclick": true, + "ondrag": true, + "ondragend": true, + "ondragenter": true, + "ondragleave": true, + "ondragover": true, + "ondragstart": true, + "ondrop": true, + "ondurationchange": true, + "onemptied": true, + "onended": true, + "onerror": true, + "onfocus": true, + "onformdata": true, + "onhashchange": true, + "oninput": true, + "oninvalid": true, + "onkeydown": true, + "onkeypress": true, + "onkeyup": true, + "onlanguagechange": true, + "onload": true, + "onloadeddata": true, + "onloadedmetadata": true, + "onloadstart": true, + "onmessage": true, + "onmessageerror": true, + "onmousedown": true, + "onmouseenter": true, + "onmouseleave": true, + "onmousemove": true, + "onmouseout": true, + "onmouseover": true, + "onmouseup": true, + "onoffline": true, + "ononline": true, + "onpagehide": true, + "onpageshow": true, + "onpaste": true, + "onpause": true, + "onplay": true, + "onplaying": true, + "onpopstate": true, + "onprogress": true, + "onratechange": true, + "onrejectionhandled": true, + "onreset": true, + "onresize": true, + "onscroll": true, + "onscrollend": true, + "onsecuritypolicyviolation": true, + "onseeked": true, + "onseeking": true, + "onselect": true, + "onslotchange": true, + "onstalled": true, + "onstorage": true, + "onsubmit": true, + "onsuspend": true, + "ontimeupdate": true, + "ontoggle": true, + "onunhandledrejection": true, + "onunload": true, + "onvolumechange": true, + "onwaiting": true, + "onwheel": true, +} + +func getScriptTypeForNode(n Node) string { + if n.Attr == nil || len(n.Attr) == 0 { + return "processed-module" + } + + // If the script tag has `type="module"`, it's not processed, but it's still a module + for _, attr := range n.Attr { + if attr.Key == "type" { + if strings.Contains(attr.Val, "module") { + return "module" + } + + if ScriptJSONMimeTypes[strings.ToLower(attr.Val)] { + return "json" + } + } + + } + + // Otherwise, it's an inline script + return "inline" +} + +type TSXExtractedTag struct { + Loc loc.TSXRange `js:"position"` + Type string `js:"type"` + Content string `js:"content"` } func isScript(p *astro.Node) bool { return p.DataAtom == atom.Script } +func isStyle(p *astro.Node) bool { + return p.DataAtom == atom.Style +} + +// Has is:raw attribute +func isRawText(p *astro.Node) bool { + for _, a := range p.Attr { + if a.Key == "is:raw" { + return true + } + } + return false +} + var ScriptMimeTypes map[string]bool = map[string]bool{ "module": true, "text/typescript": true, @@ -52,6 +191,13 @@ var ScriptMimeTypes map[string]bool = map[string]bool{ "application/node": true, } +var ScriptJSONMimeTypes map[string]bool = map[string]bool{ + "application/json": true, + "application/ld+json": true, + "importmap": true, + "speculationrules": true, +} + // This is not perfect (as in, you wouldn't use this to make a spec compliant parser), but it's good enough // for the real world. Thankfully, JSX is also a bit more lax than JavaScript, so we can spare some work. func isValidTSXAttribute(a Attribute) bool { @@ -95,20 +241,35 @@ type TextType uint32 const ( RawText TextType = iota + Text ScriptText + JsonScriptText + StyleText ) func getTextType(n *astro.Node) TextType { if script := n.Closest(isScript); script != nil { attr := astro.GetAttribute(script, "type") - if attr == nil || (attr != nil && ScriptMimeTypes[strings.ToLower(attr.Val)]) { + if attr == nil || ScriptMimeTypes[strings.ToLower(attr.Val)] { return ScriptText } + + if attr != nil && ScriptJSONMimeTypes[strings.ToLower(attr.Val)] { + return JsonScriptText + } + } + if style := n.Closest(isStyle); style != nil { + return StyleText + } + + if n.Closest(isRawText) != nil { + return RawText } - return RawText + + return Text } -func renderTsx(p *printer, n *Node) { +func renderTsx(p *printer, n *Node, o *TSXOptions) { // Root of the document, print all children if n.Type == DocumentNode { source := []byte(p.sourcetext) @@ -147,7 +308,7 @@ func renderTsx(p *printer, n *Node) { hasChildren = true } - renderTsx(p, c) + renderTsx(p, c, o) } p.addSourceMapping(loc.Loc{Start: len(p.sourcetext)}) p.print("\n") @@ -206,7 +367,7 @@ declare const Astro: Readonly {") - p.printTextWithSourcemap(n.Data, n.Loc[0]) + textType := getTextType(n) + if textType == ScriptText { p.addNilSourceMapping() - p.print("}}\n") + if o.IncludeScripts { + p.print("\n{() => {") + p.printTextWithSourcemap(n.Data, n.Loc[0]) + p.addNilSourceMapping() + p.print("}}\n") + } p.addSourceMapping(loc.Loc{Start: n.Loc[0].Start + len(n.Data)}) - return - } else if strings.ContainsAny(n.Data, "{}<>'\"") && n.Data[0] != '<' { - p.addNilSourceMapping() - p.print("{`") - p.printTextWithSourcemap(escapeText(n.Data), n.Loc[0]) + } else if textType == StyleText || textType == JsonScriptText || textType == RawText { p.addNilSourceMapping() - p.print("`}") + if (textType == StyleText && o.IncludeStyles) || textType == JsonScriptText || textType == RawText { + p.print("{`") + p.printTextWithSourcemap(escapeText(n.Data), n.Loc[0]) + p.addNilSourceMapping() + p.print("`}") + } + p.addSourceMapping(loc.Loc{Start: n.Loc[0].Start + len(n.Data)}) } else { - p.printTextWithSourcemap(n.Data, n.Loc[0]) + p.printEscapedJSXTextWithSourcemap(n.Data, n.Loc[0]) } return case ElementNode: @@ -284,7 +450,7 @@ declare const Astro: Readonly`) } - renderTsx(p, c) + renderTsx(p, c, o) if c.NextSibling == nil || c.NextSibling.Type == TextNode { p.addNilSourceMapping() p.print(``) @@ -310,7 +476,7 @@ declare const Astro: Readonly") + startTagEnd := endLoc - p.bytesToSkip + // Render any child nodes for c := n.FirstChild; c != nil; c = c.NextSibling { - renderTsx(p, c) + renderTsx(p, c, o) if len(c.Loc) > 1 { endLoc = c.Loc[1].Start + len(c.Data) + 1 } else if len(c.Loc) == 1 { endLoc = c.Loc[0].Start + len(c.Data) } } + + if n.FirstChild != nil && (n.DataAtom == atom.Script || n.DataAtom == atom.Style) { + if n.DataAtom == atom.Script { + p.addTSXScript(startTagEnd, endLoc-p.bytesToSkip, n.FirstChild.Data, getScriptTypeForNode(*n)) + } + if n.DataAtom == atom.Style { + p.addTSXStyle(startTagEnd, endLoc-p.bytesToSkip, n.FirstChild.Data, "tag") + } + } + // Special case because of trailing expression close in scripts if n.DataAtom == atom.Script { p.printf("", n.Data) diff --git a/internal/printer/printer.go b/internal/printer/printer.go index f227cd9e7..c5b83405d 100644 --- a/internal/printer/printer.go +++ b/internal/printer/printer.go @@ -36,6 +36,9 @@ type printer struct { // Optional, used only for TSX output ranges TSXRanges + // Keep track of how many multi-byte characters we've printed so that they can be skipped whenever we need a character-based index + // This could be directly in the token / node information, however this would require a fairly large refactor + bytesToSkip int } var TEMPLATE_TAG = "$$render" @@ -82,6 +85,28 @@ func (p *printer) setTSXBodyRange(componentRange loc.TSXRange) { p.ranges.Body = componentRange } +func (p *printer) addTSXScript(start int, end int, content string, scriptType string) { + p.ranges.Scripts = append(p.ranges.Scripts, TSXExtractedTag{ + Loc: loc.TSXRange{ + Start: start, + End: end, + }, + Content: content, + Type: scriptType, + }) +} + +func (p *printer) addTSXStyle(start int, end int, content string, styleType string) { + p.ranges.Styles = append(p.ranges.Styles, TSXExtractedTag{ + Loc: loc.TSXRange{ + Start: start, + End: end, + }, + Content: content, + Type: styleType, + }) +} + func (p *printer) printTextWithSourcemap(text string, l loc.Loc) { start := l.Start for pos, c := range text { @@ -93,6 +118,40 @@ func (p *printer) printTextWithSourcemap(text string, l loc.Loc) { continue } _, nextCharByteSize := utf8.DecodeRuneInString(text[pos:]) + if nextCharByteSize > 1 { + p.bytesToSkip += nextCharByteSize - 1 + } + p.addSourceMapping(loc.Loc{Start: start}) + p.print(string(c)) + start += nextCharByteSize + } +} + +func (p *printer) printEscapedJSXTextWithSourcemap(text string, l loc.Loc) { + start := l.Start + for pos, c := range text { + // Handle Windows-specific "\r\n" newlines + if c == '\r' && len(text[pos:]) > 1 && text[pos+1] == '\n' { + // tiny optimization: avoid calling `utf8.DecodeRuneInString` + // if we know the next char is `\n` + start++ + continue + } + + // If we encounter characters invalid in JSX, escape them by putting them in a JS expression + // No need to map, since it's just text. We also don't need to handle tags, since this is only for text nodes. + if c == '>' || c == '}' { + p.print("{`") + p.print(string(c)) + p.print("`}") + start++ + continue + } + + _, nextCharByteSize := utf8.DecodeRuneInString(text[pos:]) + if nextCharByteSize > 1 { + p.bytesToSkip += nextCharByteSize - 1 + } p.addSourceMapping(loc.Loc{Start: start}) p.print(string(c)) start += nextCharByteSize diff --git a/packages/compiler/src/shared/types.ts b/packages/compiler/src/shared/types.ts index d7e60cad6..505063e8b 100644 --- a/packages/compiler/src/shared/types.ts +++ b/packages/compiler/src/shared/types.ts @@ -70,7 +70,16 @@ export interface TransformOptions { export type ConvertToTSXOptions = Pick< TransformOptions, 'filename' | 'normalizedFilename' | 'sourcemap' ->; +> & { + /** If set to true, script tags content will be included in the generated TSX + * Scripts will be wrapped in an arrow function to be compatible with JSX's spec + */ + includeScripts?: boolean; + /** If set to true, style tags content will be included in the generated TSX + * Styles will be wrapped in a template literal to be compatible with JSX's spec + */ + includeStyles?: boolean; +}; export type HoistedScript = { type: string } & ( | { @@ -113,19 +122,33 @@ export interface SourceMap { version: number; } +export interface TSXLocation { + start: number; + end: number; +} + +export interface TSXExtractedTag { + position: TSXLocation; + content: string; +} + +export interface TSXExtractedScript extends TSXExtractedTag { + type: 'processed-module' | 'module' | 'inline' | 'event-attribute' | 'json' | 'unknown'; +} + +export interface TSXExtractedStyle extends TSXExtractedTag { + type: 'tag' | 'style-attribute'; +} + export interface TSXResult { code: string; map: SourceMap; diagnostics: DiagnosticMessage[]; metaRanges: { - frontmatter: { - start: number; - end: number; - }; - body: { - start: number; - end: number; - }; + frontmatter: TSXLocation; + body: TSXLocation; + scripts?: TSXExtractedScript[]; + styles?: TSXExtractedStyle[]; }; } diff --git a/packages/compiler/test/tsx/basic.ts b/packages/compiler/test/tsx/basic.ts index 402dde873..5e111b33e 100644 --- a/packages/compiler/test/tsx/basic.ts +++ b/packages/compiler/test/tsx/basic.ts @@ -259,36 +259,4 @@ export default function __AstroComponent_(_props: Record): any {}\n assert.snapshot(code, output, 'expected code to match snapshot'); }); -test('return ranges', async () => { - const input = `---\nconsole.log("Hello!")\n---\n\n
`; - const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' }); - - assert.equal(metaRanges, { - frontmatter: { - start: 30, - end: 54, - }, - body: { - start: 68, - end: 80, - }, - }); -}); - -test('return ranges - no frontmatter', async () => { - const input = '
'; - const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' }); - - assert.equal(metaRanges, { - frontmatter: { - start: 30, - end: 30, - }, - body: { - start: 41, - end: 53, - }, - }); -}); - test.run(); diff --git a/packages/compiler/test/tsx/escape.ts b/packages/compiler/test/tsx/escape.ts index c9303cce5..2d186b336 100644 --- a/packages/compiler/test/tsx/escape.ts +++ b/packages/compiler/test/tsx/escape.ts @@ -43,4 +43,41 @@ export default function __AstroComponent_(_props: Record): any {}\n assert.snapshot(code, output, 'expected code to match snapshot'); }); +test('does not escape tag opening unnecessarily', async () => { + const input = `
+ +
+
+export default function __AstroComponent_(_props: Record): any {}\n`; + const { code } = await convertToTSX(input, { sourcemap: 'external' }); + assert.snapshot(code, output, 'expected code to match snapshot'); +}); + +test('does not escape tag opening unnecessarily II', async () => { + const input = `
+
+`; + const output = `${TSXPrefix} +
+
+
+ +export default function __AstroComponent_(_props: Record): any {}\n`; + const { code } = await convertToTSX(input, { sourcemap: 'external' }); + assert.snapshot(code, output, 'expected code to match snapshot'); +}); + +test('does not escape tag opening unnecessarily III', async () => { + const input = '
{[].map((something) =>
)}
'; + const output = `${TSXPrefix} +
{[].map((something) =>
)
}
+ +export default function __AstroComponent_(_props: Record): any {}\n`; + const { code } = await convertToTSX(input, { sourcemap: 'external' }); + assert.snapshot(code, output, 'expected code to match snapshot'); +}); + test.run(); diff --git a/packages/compiler/test/tsx/meta.ts b/packages/compiler/test/tsx/meta.ts new file mode 100644 index 000000000..6289a373d --- /dev/null +++ b/packages/compiler/test/tsx/meta.ts @@ -0,0 +1,121 @@ +import { convertToTSX } from '@astrojs/compiler'; +import { test } from 'uvu'; +import * as assert from 'uvu/assert'; + +test('return ranges', async () => { + const input = `---\nconsole.log("Hello!")\n---\n\n
`; + const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' }); + + assert.equal(metaRanges, { + frontmatter: { + start: 30, + end: 54, + }, + body: { + start: 68, + end: 80, + }, + scripts: null, + styles: null, + }); +}); + +test('return ranges - no frontmatter', async () => { + const input = '
'; + const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' }); + + assert.equal(metaRanges, { + frontmatter: { + start: 30, + end: 30, + }, + body: { + start: 41, + end: 53, + }, + scripts: null, + styles: null, + }); +}); + +test('extract scripts', async () => { + const input = `
`; + + const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' }); + assert.equal( + metaRanges.scripts, + [ + { + position: { + start: 22, + end: 87, + }, + type: 'module', + content: 'console.log({ test: `literal` })', + }, + { + position: { + start: 93, + end: 158, + }, + type: 'inline', + content: 'console.log({ test: `literal` })', + }, + { + position: { + start: 169, + end: 188, + }, + type: 'json', + content: '{"a":"b"}', + }, + { + position: { + start: 205, + end: 246, + }, + type: 'inline', + content: 'console.log("hello")', + }, + { + position: { + start: 247, + end: 266, + }, + type: 'event-attribute', + content: "console.log('hey')", + }, + ], + 'expected metaRanges.scripts to match snapshot' + ); +}); + +test('extract styles', async () => { + const input = `
`; + + const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' }); + assert.equal( + metaRanges.styles, + [ + { + position: { + start: 7, + end: 48, + }, + type: 'tag', + content: 'body { color: red; }', + }, + { + position: { + start: 47, + end: 60, + }, + type: 'style-attribute', + content: 'color: blue;', + }, + ], + 'expected metaRanges.styles to match snapshot' + ); +}); + +test.run();