feat: handle simple mdx rendering

packages/astro/src/content/consts.ts

@@ -3,11 +3,21 @@ export const CONTENT_RENDER_FLAG = 'astroRenderContent';
 export const CONTENT_FLAG = 'astroContentCollectionEntry';
 export const DATA_FLAG = 'astroDataCollectionEntry';
 export const CONTENT_IMAGE_FLAG = 'astroContentImageFlag';
+export const CONTENT_MODULE_FLAG = 'astroContentModuleFlag';
 
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
 export const DATA_STORE_VIRTUAL_ID = 'astro:data-layer-content';
 export const RESOLVED_DATA_STORE_VIRTUAL_ID = '\0' + DATA_STORE_VIRTUAL_ID;
+
+// Used by the content layer to create a virtual module that loads the `modules.mjs`, a file created by the content layer
+// to map modules that are renderer at runtime
+export const MODULES_MJS_ID = 'astro:content-module-imports';
+export const MODULES_MJS_VIRTUAL_ID = '\0' + MODULES_MJS_ID;
+
+export const DEFERRED_MODULE = 'astro:content-layer-deferred-module';
+
+// Used by the content layer to create a virtual module that loads the `assets.mjs`
 export const ASSET_IMPORTS_VIRTUAL_ID = 'astro:asset-imports';
 export const ASSET_IMPORTS_RESOLVED_STUB_ID = '\0' + ASSET_IMPORTS_VIRTUAL_ID;
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
@@ -21,11 +31,13 @@ export const CONTENT_FLAGS = [
 	DATA_FLAG,
 	PROPAGATED_ASSET_FLAG,
 	CONTENT_IMAGE_FLAG,
+	CONTENT_MODULE_FLAG,
 ] as const;
 
 export const CONTENT_TYPES_FILE = 'types.d.ts';
 
 export const DATA_STORE_FILE = 'data-store.json';
 export const ASSET_IMPORTS_FILE = 'assets.mjs';
+export const MODULES_IMPORTS_FILE = 'modules.mjs';
 
 export const CONTENT_LAYER_TYPE = 'experimental_content';

packages/astro/src/content/content-layer.ts

@@ -5,7 +5,12 @@ import type { FSWatcher } from 'vite';
 import xxhash from 'xxhash-wasm';
 import type { AstroSettings } from '../@types/astro.js';
 import type { Logger } from '../core/logger/core.js';
-import { ASSET_IMPORTS_FILE, CONTENT_LAYER_TYPE, DATA_STORE_FILE } from './consts.js';
+import {
+	ASSET_IMPORTS_FILE,
+	CONTENT_LAYER_TYPE,
+	DATA_STORE_FILE,
+	MODULES_IMPORTS_FILE,
+} from './consts.js';
 import type { DataStore } from './data-store.js';
 import type { LoaderContext } from './loaders/types.js';
 import { getEntryDataAndImages, globalContentConfigObserver, posixRelative } from './utils.js';
@@ -208,6 +213,8 @@ export class ContentLayer {
 		}
 		const assetImportsFile = new URL(ASSET_IMPORTS_FILE, this.#settings.dotAstroDir);
 		await this.#store.writeAssetImports(assetImportsFile);
+		const modulesImportsFile = new URL(MODULES_IMPORTS_FILE, this.#settings.dotAstroDir);
+		await this.#store.writeModuleImports(modulesImportsFile);
 		logger.info('Synced content');
 	}
 }

packages/astro/src/content/data-store.ts

@@ -3,6 +3,8 @@ import type { MarkdownHeading } from '@astrojs/markdown-remark';
 import * as devalue from 'devalue';
 import { imageSrcToImportId, importIdToSymbolName } from '../assets/utils/resolveImports.js';
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
+import {CONTENT_MODULE_FLAG, DEFERRED_MODULE} from './consts.js';
+
 const SAVE_DEBOUNCE_MS = 500;
 
 export interface RenderedContent {
@@ -33,6 +35,10 @@ export interface DataEntry<TData extends Record<string, unknown> = Record<string
 	digest?: number | string;
 	/** The rendered content of the entry, if applicable. */
 	rendered?: RenderedContent;
+	/**
+	 * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase when calling `renderEntry`.
+	 */
+	deferredRender?: boolean;
 }
 
 export class DataStore {
@@ -41,46 +47,57 @@ export class DataStore {
 	#file?: PathLike;
 
 	#assetsFile?: PathLike;
+	#modulesFile?: PathLike;
 
 	#saveTimeout: NodeJS.Timeout | undefined;
 	#assetsSaveTimeout: NodeJS.Timeout | undefined;
+	#modulesSaveTimeout: NodeJS.Timeout | undefined;
 
 	#dirty = false;
 	#assetsDirty = false;
+	#modulesDirty = false;
 
 	#assetImports = new Set<string>();
+	#moduleImports = new Map<string, string>();
 
 	constructor() {
 		this.#collections = new Map();
 	}
+
 	get<T = unknown>(collectionName: string, key: string): T | undefined {
 		return this.#collections.get(collectionName)?.get(String(key));
 	}
+
 	entries<T = unknown>(collectionName: string): Array<[id: string, T]> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.entries()];
 	}
+
 	values<T = unknown>(collectionName: string): Array<T> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.values()];
 	}
+
 	keys(collectionName: string): Array<string> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.keys()];
 	}
+
 	set(collectionName: string, key: string, value: unknown) {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		collection.set(String(key), value);
 		this.#collections.set(collectionName, collection);
 		this.#saveToDiskDebounced();
 	}
+
 	delete(collectionName: string, key: string) {
 		const collection = this.#collections.get(collectionName);
 		if (collection) {
 			collection.delete(String(key));
 			this.#saveToDiskDebounced();
 		}
 	}
+
 	clear(collectionName: string) {
 		this.#collections.delete(collectionName);
 		this.#saveToDiskDebounced();
@@ -122,6 +139,17 @@ export class DataStore {
 		assets.forEach((asset) => this.addAssetImport(asset, filePath));
 	}
 
+	addModuleImport(fileName: string) {
+		const id = contentModuleToId(fileName);
+		if (id) {
+			this.#moduleImports.set(fileName, id);
+			// We debounce the writes to disk because addAssetImport is called for every image in every file,
+			// and can be called many times in quick succession by a filesystem watcher. We only want to write
+			// the file once, after all the imports have been added.
+			this.#writeModulesImportsDebounced();
+		}
+	}
+
 	async writeAssetImports(filePath: PathLike) {
 		this.#assetsFile = filePath;
 
@@ -164,6 +192,45 @@ export default new Map([${exports.join(', ')}]);
 		this.#assetsDirty = false;
 	}
 
+	async writeModuleImports(filePath: PathLike) {
+		this.#modulesFile = filePath;
+
+		if (this.#moduleImports.size === 0) {
+			try {
+				await fs.writeFile(filePath, 'export default new Map();');
+			} catch (err) {
+				throw new AstroError({
+					message: (err as Error).message,
+					...AstroErrorData.ContentLayerWriteError,
+				});
+			}
+		}
+
+		if (!this.#modulesDirty && existsSync(filePath)) {
+			return;
+		}
+
+		// Import the assets, with a symbol name that is unique to the import id. The import
+		// for each asset is an object with path, format and dimensions.
+		// We then export them all, mapped by the import id, so we can find them again in the build.
+		const lines: Array<string> = [];
+		for (const [fileName, specifier] of this.#moduleImports) {
+			lines.push(`['${fileName}', () => import('${specifier}')]`);
+		}
+		const code = /* js */ `
+export default new Map([\n${lines.join(',\n')}]);
+		`;
+		try {
+			await fs.writeFile(filePath, code);
+		} catch (err) {
+			throw new AstroError({
+				message: (err as Error).message,
+				...AstroErrorData.ContentLayerWriteError,
+			});
+		}
+		this.#modulesDirty = false;
+	}
+
 	#writeAssetsImportsDebounced() {
 		this.#assetsDirty = true;
 		if (this.#assetsFile) {
@@ -177,6 +244,19 @@ export default new Map([${exports.join(', ')}]);
 		}
 	}
 
+	#writeModulesImportsDebounced() {
+		this.#modulesDirty = true;
+		if (this.#modulesFile) {
+			if (this.#modulesSaveTimeout) {
+				clearTimeout(this.#modulesSaveTimeout);
+			}
+			this.#modulesSaveTimeout = setTimeout(() => {
+				this.#modulesSaveTimeout = undefined;
+				this.writeModuleImports(this.#modulesFile!);
+			}, SAVE_DEBOUNCE_MS);
+		}
+	}
+
 	#saveToDiskDebounced() {
 		this.#dirty = true;
 		// Only save to disk if it has already been saved once
@@ -198,7 +278,7 @@ export default new Map([${exports.join(', ')}]);
 			entries: () => this.entries(collectionName),
 			values: () => this.values(collectionName),
 			keys: () => this.keys(collectionName),
-			set: ({ id: key, data, body, filePath, digest, rendered }) => {
+			set: ({ id: key, data, body, filePath, deferredRender, digest, rendered }) => {
 				if (!key) {
 					throw new Error(`ID must be a non-empty string`);
 				}
@@ -230,7 +310,12 @@ export default new Map([${exports.join(', ')}]);
 				if (rendered) {
 					entry.rendered = rendered;
 				}
-
+				if (deferredRender) {
+					entry.deferredRender = deferredRender;
+					if (filePath) {
+						this.addModuleImport(filePath)
+					}
+				}
 				this.set(collectionName, id, entry);
 				return true;
 			},
@@ -241,6 +326,8 @@ export default new Map([${exports.join(', ')}]);
 				this.addAssetImport(assetImport, fileName),
 			addAssetImports: (assets: Array<string>, fileName: string) =>
 				this.addAssetImports(assets, fileName),
+			addModuleImport: (fileName: string) =>
+				this.addModuleImport(fileName),
 		};
 	}
 	/**
@@ -275,6 +362,7 @@ export default new Map([${exports.join(', ')}]);
 			});
 		}
 	}
+
 	/**
 	 * Attempts to load a DataStore from the virtual module.
 	 * This only works in Vite.
@@ -329,6 +417,10 @@ export interface ScopedDataStore {
 		digest?: number | string;
 		/** The rendered content, if applicable. */
 		rendered?: RenderedContent;
+		/**
+		 * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase.
+		 */
+		deferredRender?: boolean;
 	}) => boolean;
 	values: () => Array<DataEntry>;
 	keys: () => Array<string>;
@@ -343,6 +435,14 @@ export interface ScopedDataStore {
 	 * @internal Adds an asset import to the store. This is used to track image imports for the build. This API is subject to change.
 	 */
 	addAssetImport: (assetImport: string, fileName: string) => void;
+	/**
+	 * Adds a single asset to the store. This asset will be transformed
+	 * by Vite, and the URL will be available in the final build.
+	 * @param fileName
+	 * @param specifier
+	 * @returns
+	 */
+	addModuleImport: (fileName: string) => void;
 }
 
 /**
@@ -371,5 +471,13 @@ function dataStoreSingleton() {
 	};
 }
 
+// TODO: find a better place to put this image
+export function contentModuleToId(fileName: string) {
+	const params = new URLSearchParams(DEFERRED_MODULE);
+	params.set('fileName', fileName);
+	params.set(CONTENT_MODULE_FLAG, 'true');
+	return `${DEFERRED_MODULE}?${params.toString()}`;
+}
+
 /** @internal */
 export const globalDataStore = dataStoreSingleton();

packages/astro/src/content/loaders/glob.ts

@@ -18,6 +18,7 @@ export interface GenerateIdOptions {
 	/** The parsed, unvalidated data of the entry. */
 	data: Record<string, unknown>;
 }
+
 export interface GlobOptions {
 	/** The glob pattern to match files, relative to the base directory */
 	pattern: string;
@@ -102,6 +103,10 @@ export function glob(globOptions: GlobOptions): Loader {
 				const digest = generateDigest(contents);
 
 				if (existingEntry && existingEntry.digest === digest && existingEntry.filePath) {
+					if (existingEntry.deferredRender) {
+						store.addModuleImport(existingEntry.filePath);
+					}
+
 					if (existingEntry.rendered?.metadata?.imagePaths?.length) {
 						// Add asset imports for existing entries
 						store.addAssetImports(
@@ -124,7 +129,9 @@ export function glob(globOptions: GlobOptions): Loader {
 					filePath,
 				});
 
-				if (entryType.getRenderFunction) {
+				if (entryType.extensions.includes('.mdx')) {
+					store.set({ id, data: parsedData, body, filePath: relativePath, digest, deferredRender: true });
+				} else if (entryType.getRenderFunction) {
 					let render = renderFunctionByContentType.get(entryType);
 					if (!render) {
 						render = await entryType.getRenderFunction(settings);
@@ -261,6 +268,7 @@ export function glob(globOptions: GlobOptions): Loader {
 				await syncData(entry, baseUrl, entryType);
 				logger.info(`Reloaded data from ${green(entry)}`);
 			}
+
 			watcher.on('change', onChange);
 
 			watcher.on('add', onChange);

packages/astro/src/content/runtime.ts

@@ -20,6 +20,7 @@ import {
 import { CONTENT_LAYER_TYPE, IMAGE_IMPORT_PREFIX } from './consts.js';
 import { type DataEntry, globalDataStore } from './data-store.js';
 import type { ContentLookupMap } from './utils.js';
+
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;
 type CollectionToEntryMap = Record<string, GlobResult>;
@@ -87,6 +88,7 @@ export function createGetCollection({
 				const data = rawEntry.filePath
 					? updateImageReferencesInData(rawEntry.data, rawEntry.filePath, imageAssetMap)
 					: rawEntry.data;
+
 				const entry = {
 					...rawEntry,
 					data,
@@ -107,6 +109,7 @@ export function createGetCollection({
 			);
 			return [];
 		}
+
 		const lazyImports = Object.values(
 			type === 'content'
 				? contentCollectionToEntryMap[collection]
@@ -368,7 +371,7 @@ const CONTENT_LAYER_IMAGE_REGEX = /__ASTRO_IMAGE_="([^"]+)"/g;
 async function updateImageReferencesInBody(html: string, fileName: string) {
 	// @ts-expect-error Virtual module
 	const { default: imageAssetMap } = await import('astro:asset-imports');
-
+	
 	const imageObjects = new Map<string, GetImageResult>();
 
 	// @ts-expect-error Virtual module resolved at runtime
@@ -442,6 +445,20 @@ export async function renderEntry(
 		return entry.render();
 	}
 
+	if (entry.deferredRender) {
+		try {
+			// @ts-expect-error	virtual module
+			const { default: contentModules } = await import('astro:content-module-imports');
+			const module = contentModules.get(entry.filePath);
+			return await module();
+
+		} catch (e) {
+			// eslint-disable-next-line
+			console.error(e);
+		}
+
+	}
+
 	const html =
 		entry?.rendered?.metadata?.imagePaths?.length && entry.filePath
 			? await updateImageReferencesInBody(entry.rendered.html, entry.filePath)