Merge branch 'content-layer' into feat/inject-types

.changeset/smooth-chicken-wash.md

@@ -0,0 +1,5 @@
+---
+'astro': minor
+---
+
+Implements Content Layer

benchmark/bench/memory.js

@@ -18,7 +18,7 @@ export async function run(projectDir, outputFile) {
 	const outputFilePath = fileURLToPath(outputFile);
 
 	console.log('Building and benchmarking...');
-	await execaCommand(`node --expose-gc --max_old_space_size=256 ${astroBin} build`, {
+	await execaCommand(`node --expose-gc --max_old_space_size=10000 ${astroBin} build --silent`, {
 		cwd: root,
 		stdio: 'inherit',
 		env: {

benchmark/make-project/markdown-cc1.js

@@ -0,0 +1,63 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir));
+
+	const promises = [];
+
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+
+`;
+		promises.push(
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.slug }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

benchmark/make-project/markdown-cc2.js

@@ -0,0 +1,80 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./data/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./image.jpg', projectDir));
+
+	const promises = [];
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+`;
+		promises.push(
+			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+	await fs.writeFile(
+		new URL(`./src/content/config.ts`, projectDir),
+		/*ts */ `
+		import { defineCollection, z } from 'astro:content';
+		import { glob } from 'astro/loaders';
+
+		const blog = defineCollection({
+			loader: glob({ pattern: '*', base: './data/blog' }),
+		});
+		
+		export const collections = { blog }
+
+		`
+	);
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection, render } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.id }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await render(entry);
+
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+		experimental: {
+			contentLayer: true
+		}
+});`,
+		'utf-8'
+	);
+}

benchmark/make-project/mdx-cc1.js

@@ -0,0 +1,66 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir));
+
+	const promises = [];
+
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+
+`;
+		promises.push(
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.mdx`, projectDir), content, 'utf-8')
+		);
+	}
+
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.slug }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+import mdx from '@astrojs/mdx';
+
+export default defineConfig({
+  integrations: [mdx()],
+});`,
+		'utf-8'
+	);
+}

benchmark/make-project/mdx-cc2.js

@@ -0,0 +1,83 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./data/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./image.jpg', projectDir));
+
+	const promises = [];
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+`;
+		promises.push(
+			fs.writeFile(new URL(`./data/blog/article-${i}.mdx`, projectDir), content, 'utf-8')
+		);
+	}
+
+	await fs.writeFile(
+		new URL(`./src/content/config.ts`, projectDir),
+		/*ts */ `
+		import { defineCollection, z } from 'astro:content';
+		import { glob } from 'astro/loaders';
+
+		const blog = defineCollection({
+			loader: glob({ pattern: '*', base: './data/blog' }),
+		});
+		
+		export const collections = { blog }
+
+		`
+	);
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection, render } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.id }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await render(entry);
+
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+import mdx from '@astrojs/mdx';
+
+export default defineConfig({
+  integrations: [mdx()],
+		experimental: {
+			contentLayer: true
+		}
+});`,
+		'utf-8'
+	);
+}

benchmark/package.json

@@ -16,6 +16,7 @@
     "markdown-table": "^3.0.3",
     "mri": "^1.2.0",
     "port-authority": "^2.0.1",
-    "pretty-bytes": "^6.1.1"
+    "pretty-bytes": "^6.1.1",
+    "sharp": "^0.33.3"
   }
 }

examples/container-with-vitest/package.json

@@ -19,7 +19,7 @@
     "vitest": "^2.0.5"
   },
   "devDependencies": {
-    "@types/react-dom": "^18.3.0",
-    "@types/react": "^18.3.3"
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0"
   }
 }

packages/astro/package.json

@@ -68,6 +68,7 @@
     "./assets/services/sharp": "./dist/assets/services/sharp.js",
     "./assets/services/squoosh": "./dist/assets/services/squoosh.js",
     "./assets/services/noop": "./dist/assets/services/noop.js",
+    "./loaders": "./dist/content/loaders/index.js",
     "./content/runtime": "./dist/content/runtime.js",
     "./content/runtime-assets": "./dist/content/runtime-assets.js",
     "./debug": "./components/Debug.astro",
@@ -132,6 +133,7 @@
     "@babel/plugin-transform-react-jsx": "^7.25.2",
     "@babel/traverse": "^7.25.3",
     "@babel/types": "^7.25.2",
+    "@rollup/pluginutils": "^5.1.0",
     "@types/babel__core": "^7.20.5",
     "@types/cookie": "^0.6.0",
     "acorn": "^8.12.1",
@@ -162,7 +164,9 @@
     "js-yaml": "^4.1.0",
     "kleur": "^4.1.5",
     "magic-string": "^0.30.11",
+    "micromatch": "^4.0.7",
     "mrmime": "^2.0.0",
+    "neotraverse": "^0.6.9",
     "ora": "^8.0.1",
     "p-limit": "^6.1.0",
     "p-queue": "^8.0.1",
@@ -180,9 +184,11 @@
     "vite": "^5.3.5",
     "vitefu": "^0.2.5",
     "which-pm": "^3.0.0",
+    "xxhash-wasm": "^1.0.2",
     "yargs-parser": "^21.1.1",
     "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.2"
+    "zod-to-json-schema": "^3.23.2",
+    "zod-to-ts": "^1.2.0"
   },
   "optionalDependencies": {
     "sharp": "^0.33.3"
@@ -203,6 +209,7 @@
     "@types/html-escaper": "^3.0.2",
     "@types/http-cache-semantics": "^4.0.4",
     "@types/js-yaml": "^4.0.9",
+    "@types/micromatch": "^4.0.9",
     "@types/prompts": "^2.4.9",
     "@types/semver": "^7.5.8",
     "@types/yargs-parser": "^21.0.3",
@@ -232,4 +239,4 @@
   "publishConfig": {
     "provenance": true
   }
-}
+}