From eb28c152ec8d793bea31820a3fb90504473c752e Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 10 Jul 2024 15:31:25 +0100 Subject: [PATCH] feat: add glob loader (#11398) * feat: add glob loader * Enable watching and fix paths * Store the full entry object, not just data * Add generateId support * Fix test * Rename loaders to sync * Refacctor imports * Use getEntry * Format * Fix import * Remove type from output * Windows path * Add test for absolute path * Update lockfile * Debugging windows * Allow file URL for base dir * Reset time limit --- packages/astro/package.json | 5 +- packages/astro/src/@types/astro.ts | 2 +- packages/astro/src/content/data-store.ts | 54 +++++- .../astro/src/content/{ => loaders}/file.ts | 9 +- packages/astro/src/content/loaders/glob.ts | 176 ++++++++++++++++++ packages/astro/src/content/loaders/index.ts | 3 + packages/astro/src/content/loaders/types.ts | 46 +++++ packages/astro/src/content/runtime.ts | 47 ++++- .../astro/src/content/{loaders.ts => sync.ts} | 58 +----- packages/astro/src/content/types-generator.ts | 2 +- .../vite-plugin-content-virtual-mod.ts | 2 +- packages/astro/src/core/build/index.ts | 4 +- packages/astro/src/core/dev/dev.ts | 2 +- packages/astro/src/core/sync/index.ts | 2 +- packages/astro/templates/content/module.mjs | 2 +- packages/astro/test/content-layer.test.js | 30 ++- .../content-outside-src/columbia-copy.md | 15 ++ .../content-outside-src/columbia.md | 15 ++ .../content-outside-src/endeavour.md | 14 ++ .../content-outside-src/enterprise.md | 14 ++ .../content-outside-src/index.md | 15 ++ .../content-layer/src/content/config.ts | 97 ++++++---- .../src/{content/_data => data}/dogs.json | 0 .../src/data/glob-data/index.json | 3 + .../content-layer/src/data/glob-data/one.json | 3 + .../src/data/glob-data/three.json | 3 + .../content-layer/src/data/glob-data/two.json | 3 + .../content-layer/src/loaders/post-loader.ts | 5 +- .../src/pages/collections.json.js | 6 +- .../src/pages/spacecraft/[slug].astro | 27 +++ packages/astro/types/content.d.ts | 56 ++---- pnpm-lock.yaml | 47 ++++- 32 files changed, 590 insertions(+), 177 deletions(-) rename packages/astro/src/content/{ => loaders}/file.ts (92%) create mode 100644 packages/astro/src/content/loaders/glob.ts create mode 100644 packages/astro/src/content/loaders/index.ts create mode 100644 packages/astro/src/content/loaders/types.ts rename packages/astro/src/content/{loaders.ts => sync.ts} (68%) create mode 100644 packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md create mode 100644 packages/astro/test/fixtures/content-layer/content-outside-src/columbia.md create mode 100644 packages/astro/test/fixtures/content-layer/content-outside-src/endeavour.md create mode 100644 packages/astro/test/fixtures/content-layer/content-outside-src/enterprise.md create mode 100644 packages/astro/test/fixtures/content-layer/content-outside-src/index.md rename packages/astro/test/fixtures/content-layer/src/{content/_data => data}/dogs.json (100%) create mode 100644 packages/astro/test/fixtures/content-layer/src/data/glob-data/index.json create mode 100644 packages/astro/test/fixtures/content-layer/src/data/glob-data/one.json create mode 100644 packages/astro/test/fixtures/content-layer/src/data/glob-data/three.json create mode 100644 packages/astro/test/fixtures/content-layer/src/data/glob-data/two.json create mode 100644 packages/astro/test/fixtures/content-layer/src/pages/spacecraft/[slug].astro diff --git a/packages/astro/package.json b/packages/astro/package.json index df095373b316..21fd52eb980f 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -67,6 +67,7 @@ "./assets/services/sharp": "./dist/assets/services/sharp.js", "./assets/services/squoosh": "./dist/assets/services/squoosh.js", "./assets/services/noop": "./dist/assets/services/noop.js", + "./loaders": "./dist/content/loaders/index.js", "./content/runtime": "./dist/content/runtime.js", "./content/runtime-assets": "./dist/content/runtime-assets.js", "./debug": "./components/Debug.astro", @@ -165,6 +166,7 @@ "js-yaml": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.10", + "micromatch": "^4.0.7", "mrmime": "^2.0.0", "ora": "^8.0.1", "p-limit": "^5.0.0", @@ -208,6 +210,7 @@ "@types/html-escaper": "^3.0.2", "@types/http-cache-semantics": "^4.0.4", "@types/js-yaml": "^4.0.9", + "@types/micromatch": "^4.0.9", "@types/probe-image-size": "^7.2.4", "@types/prompts": "^2.4.9", "@types/semver": "^7.5.8", @@ -240,4 +243,4 @@ "publishConfig": { "provenance": true } -} +} \ No newline at end of file diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 3668aa9de33f..d8ba590297ee 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -46,7 +46,7 @@ import type { TransitionBeforePreparationEvent, TransitionBeforeSwapEvent, } from '../transitions/events.js'; -import type { DeepPartial, OmitIndexSignature, Simplify, WithRequired } from '../type-utils.js'; +import type { DeepPartial, OmitIndexSignature, Simplify } from '../type-utils.js'; import type { SUPPORTED_MARKDOWN_FILE_EXTENSIONS } from './../core/constants.js'; export type { AstroIntegrationLogger, ToolbarServerHelpers }; diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index f8c24ac184aa..acd159c25ffc 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -1,6 +1,14 @@ import { promises as fs, type PathLike, existsSync } from 'fs'; const SAVE_DEBOUNCE_MS = 500; + +export interface DataEntry { + id: string; + data: Record; + filePath?: string; + body?: string; +} + export class DataStore { #collections = new Map>(); @@ -13,14 +21,14 @@ export class DataStore { constructor() { this.#collections = new Map(); } - get(collectionName: string, key: string) { + get(collectionName: string, key: string): T | undefined { return this.#collections.get(collectionName)?.get(String(key)); } - entries(collectionName: string): Array<[id: string, any]> { + entries(collectionName: string): Array<[id: string, T]> { const collection = this.#collections.get(collectionName) ?? new Map(); return [...collection.entries()]; } - values(collectionName: string): Array { + values(collectionName: string): Array { const collection = this.#collections.get(collectionName) ?? new Map(); return [...collection.values()]; } @@ -78,11 +86,30 @@ export class DataStore { scopedStore(collectionName: string): ScopedDataStore { return { - get: (key: string) => this.get(collectionName, key), + get: (key: string) => this.get(collectionName, key), entries: () => this.entries(collectionName), values: () => this.values(collectionName), keys: () => this.keys(collectionName), - set: (key: string, value: any) => this.set(collectionName, key, value), + set: (key, data, body, filePath) => { + if (!key) { + throw new Error(`Key must be a non-empty string`); + } + const id = String(key); + const entry: DataEntry = { + id, + data, + }; + // We do it like this so we don't waste space stringifying + // the body and filePath if they are not set + if (body) { + entry.body = body; + } + if (filePath) { + entry.filePath = filePath; + } + + this.set(collectionName, id, entry); + }, delete: (key: string) => this.delete(collectionName, key), clear: () => this.clear(collectionName), has: (key: string) => this.has(collectionName, key), @@ -90,7 +117,13 @@ export class DataStore { } metaStore(collectionName: string): MetaStore { - return this.scopedStore(`meta:${collectionName}`) as MetaStore; + const collectionKey = `meta:${collectionName}`; + return { + get: (key: string) => this.get(collectionKey, key), + set: (key: string, data: string) => this.set(collectionKey, key, data), + delete: (key: string) => this.delete(collectionKey, key), + has: (key: string) => this.has(collectionKey, key), + }; } toString() { @@ -148,10 +181,10 @@ export class DataStore { } export interface ScopedDataStore { - get: (key: string) => unknown; - entries: () => Array<[id: string, unknown]>; - set: (key: string, value: unknown) => void; - values: () => Array; + get: (key: string) => DataEntry | undefined; + entries: () => Array<[id: string, DataEntry]>; + set: (key: string, data: Record, body?: string, filePath?: string) => void; + values: () => Array; keys: () => Array; delete: (key: string) => void; clear: () => void; @@ -166,6 +199,7 @@ export interface MetaStore { get: (key: string) => string | undefined; set: (key: string, value: string) => void; has: (key: string) => boolean; + delete: (key: string) => void; } function dataStoreSingleton() { diff --git a/packages/astro/src/content/file.ts b/packages/astro/src/content/loaders/file.ts similarity index 92% rename from packages/astro/src/content/file.ts rename to packages/astro/src/content/loaders/file.ts index 091f1e5ea4b3..adbd1bf8a291 100644 --- a/packages/astro/src/content/file.ts +++ b/packages/astro/src/content/loaders/file.ts @@ -1,6 +1,6 @@ -import { fileURLToPath } from 'url'; -import type { Loader, LoaderContext } from './loaders.js'; import { promises as fs, existsSync } from 'fs'; +import { fileURLToPath } from 'url'; +import type { Loader, LoaderContext } from './types.js'; /** * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys. @@ -38,7 +38,7 @@ export function file(fileName: string): Loader { continue; } const item = await parseData({ id, data: rawItem, filePath }); - store.set(id, item); + store.set(id, item, undefined, filePath); } } else if (typeof json === 'object') { const entries = Object.entries>(json); @@ -57,9 +57,8 @@ export function file(fileName: string): Loader { name: 'file-loader', load: async (options) => { const { settings, logger, watcher } = options; - const contentDir = new URL('./content/', settings.config.srcDir); logger.debug(`Loading data from ${fileName}`); - const url = new URL(fileName, contentDir); + const url = new URL(fileName, settings.config.root); if (!existsSync(url)) { logger.error(`File not found: ${fileName}`); return; diff --git a/packages/astro/src/content/loaders/glob.ts b/packages/astro/src/content/loaders/glob.ts new file mode 100644 index 000000000000..03d384c7aa35 --- /dev/null +++ b/packages/astro/src/content/loaders/glob.ts @@ -0,0 +1,176 @@ +import { promises as fs } from 'fs'; +import { fileURLToPath, pathToFileURL } from 'url'; +import fastGlob from 'fast-glob'; +import { green } from 'kleur/colors'; +import micromatch from 'micromatch'; +import pLimit from 'p-limit'; +import { relative } from 'path/posix'; +import type { ContentEntryType } from '../../@types/astro.js'; +import { getContentEntryIdAndSlug, getEntryConfigByExtMap } from '../utils.js'; +import type { Loader, LoaderContext } from './types.js'; + +export interface GenerateIdOptions { + /** The path to the entry file, relative to the base directory. */ + entry: string; + + /** The base directory URL. */ + base: URL; + /** The parsed, unvalidated data of the entry. */ + data: Record; +} +export interface GlobOptions { + /** The glob pattern to match files, relative to the base directory */ + pattern: string; + /** The base directory to resolve the glob pattern from. Relative to the root directory, or an absolute file URL. Defaults to `.` */ + base?: string | URL; + /** + * Function that generates an ID for an entry. Default implementation generates a slug from the entry path. + * @returns The ID of the entry. Must be unique per collection. + **/ + generateId?: (options: GenerateIdOptions) => string; +} + +function generateIdDefault({ entry, base, data }: GenerateIdOptions): string { + if (data.slug) { + return data.slug as string; + } + const entryURL = new URL(entry, base); + const { slug } = getContentEntryIdAndSlug({ + entry: entryURL, + contentDir: base, + collection: '', + }); + return slug; +} + +/** + * Loads multiple entries, using a glob pattern to match files. + * @param pattern A glob pattern to match files, relative to the content directory. + */ +export function glob(globOptions: GlobOptions): Loader { + if (globOptions.pattern.startsWith('../')) { + throw new Error( + 'Glob patterns cannot start with `../`. Set the `base` option to a parent directory instead.' + ); + } + if (globOptions.pattern.startsWith('/')) { + throw new Error( + 'Glob patterns cannot start with `/`. Set the `base` option to a parent directory or use a relative path instead.' + ); + } + + const generateId = globOptions?.generateId ?? generateIdDefault; + + const fileToIdMap = new Map(); + + async function syncData( + entry: string, + base: URL, + { logger, parseData, store }: LoaderContext, + entryType?: ContentEntryType + ) { + if (!entryType) { + logger.warn(`No entry type found for ${entry}`); + return; + } + const fileUrl = new URL(entry, base); + const { body, data } = await entryType.getEntryInfo({ + contents: await fs.readFile(fileUrl, 'utf-8'), + fileUrl, + }); + + const id = generateId({ entry, base, data }); + + const filePath = fileURLToPath(fileUrl); + + const parsedData = await parseData({ + id, + data, + filePath, + }); + + store.set(id, parsedData, body, filePath); + + fileToIdMap.set(filePath, id); + } + + return { + name: 'glob-loader', + load: async (options) => { + const { settings, logger, watcher } = options; + + const entryConfigByExt = getEntryConfigByExtMap([ + ...settings.contentEntryTypes, + ...settings.dataEntryTypes, + ] as Array); + + const baseDir = globOptions.base + ? new URL(globOptions.base, settings.config.root) + : settings.config.root; + + if (!baseDir.pathname.endsWith('/')) { + baseDir.pathname = `${baseDir.pathname}/`; + } + + const files = await fastGlob(globOptions.pattern, { + cwd: fileURLToPath(baseDir), + }); + + function configForFile(file: string) { + const ext = file.split('.').at(-1); + if (!ext) { + logger.warn(`No extension found for ${file}`); + return; + } + return entryConfigByExt.get(`.${ext}`); + } + + const limit = pLimit(10); + options.store.clear(); + await Promise.all( + files.map((entry) => + limit(async () => { + const entryType = configForFile(entry); + await syncData(entry, baseDir, options, entryType); + }) + ) + ); + + if (!watcher) { + return; + } + + const matcher: RegExp = micromatch.makeRe(globOptions.pattern); + + const matchesGlob = (entry: string) => !entry.startsWith('../') && matcher.test(entry); + + const basePath = fileURLToPath(baseDir); + + async function onChange(changedPath: string) { + const entry = relative(basePath, changedPath); + if (!matchesGlob(entry)) { + return; + } + const entryType = configForFile(changedPath); + const baseUrl = pathToFileURL(basePath); + await syncData(entry, baseUrl, options, entryType); + logger.info(`Reloaded data from ${green(entry)}`); + } + watcher.on('change', onChange); + + watcher.on('add', onChange); + + watcher.on('unlink', async (deletedPath) => { + const entry = relative(basePath, deletedPath); + if (!matchesGlob(entry)) { + return; + } + const id = fileToIdMap.get(deletedPath); + if (id) { + options.store.delete(id); + fileToIdMap.delete(deletedPath); + } + }); + }, + }; +} diff --git a/packages/astro/src/content/loaders/index.ts b/packages/astro/src/content/loaders/index.ts new file mode 100644 index 000000000000..30b4bfbe5334 --- /dev/null +++ b/packages/astro/src/content/loaders/index.ts @@ -0,0 +1,3 @@ +export { file } from './file.js'; +export { glob } from './glob.js'; +export * from './types.js'; diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts new file mode 100644 index 000000000000..865e6724e544 --- /dev/null +++ b/packages/astro/src/content/loaders/types.ts @@ -0,0 +1,46 @@ +import type { FSWatcher } from 'vite'; +import type { ZodSchema } from 'zod'; +import type { AstroIntegrationLogger, AstroSettings } from '../../@types/astro.js'; +import type { MetaStore, ScopedDataStore } from '../data-store.js'; + +export interface ParseDataOptions { + /** The ID of the entry. Unique per collection */ + id: string; + /** The raw, unvalidated data of the entry */ + data: Record; + /** An optional file path, where the entry represents a local file. */ + filePath?: string; +} + +export type DataWithId = { + id: string; + [key: string]: unknown; +}; + +export interface LoaderContext { + /** The unique name of the collection */ + collection: string; + /** A database abstraction to store the actual data */ + store: ScopedDataStore; + /** A simple KV store, designed for things like sync tokens */ + meta: MetaStore; + logger: AstroIntegrationLogger; + + settings: AstroSettings; + + /** Validates and parses the data according to the collection schema */ + parseData(props: ParseDataOptions): Promise; + + /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ + watcher?: FSWatcher; +} + +export interface Loader { + /** Unique name of the loader, e.g. the npm package name */ + name: string; + /** Do the actual loading of the data */ + load: (context: LoaderContext) => Promise; + /** Optionally, define the schema of the data. Will be overridden by user-defined schema */ + schema?: ZodSchema | Promise | (() => ZodSchema | Promise); + render?: (entry: any) => any; +} diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts index a22bd291947e..7b5d2b6f48af 100644 --- a/packages/astro/src/content/runtime.ts +++ b/packages/astro/src/content/runtime.ts @@ -13,10 +13,8 @@ import { renderUniqueStylesheet, unescapeHTML, } from '../runtime/server/index.js'; +import { type DataEntry, globalDataStore } from './data-store.js'; import type { ContentLookupMap } from './utils.js'; -import { globalDataStore } from './data-store.js'; -export { file } from './file.js'; - type LazyImport = () => Promise; type GlobResult = Record; type CollectionToEntryMap = Record; @@ -65,11 +63,9 @@ export function createGetCollection({ } else if (collection in dataCollectionToEntryMap) { type = 'data'; } else if (store.hasCollection(collection)) { - return [...store.entries(collection)].map(([id, data]) => ({ - id, + return [...store.values(collection)].map((entry) => ({ + ...entry, collection, - data, - type: 'experimental_data', })); } else { // eslint-disable-next-line no-console @@ -139,6 +135,22 @@ export function createGetEntryBySlug({ getRenderEntryImport: GetEntryImport; }) { return async function getEntryBySlug(collection: string, slug: string) { + const store = await globalDataStore.get(); + + if (store.hasCollection(collection)) { + const data = store.get(collection, slug); + if (!data) { + throw new Error(`Entry ${collection} → ${slug} was not found.`); + } + + const entry = store.get(collection, slug); + + return { + ...entry, + collection, + }; + } + const entryImport = await getEntryImport(collection, slug); if (typeof entryImport !== 'function') return undefined; @@ -171,10 +183,11 @@ export function createGetDataEntryById({ getEntryImport }: { getEntryImport: Get throw new Error(`Entry ${collection} → ${id} was not found.`); } + const entry = store.get(collection, id); + return { - id, + ...entry, collection, - data: store.get(collection, id), }; } @@ -241,6 +254,22 @@ export function createGetEntry({ : collectionOrLookupObject.slug; } + const store = await globalDataStore.get(); + + if (store.hasCollection(collection)) { + const data = store.get(collection, lookupId); + if (!data) { + throw new Error(`Entry ${collection} → ${lookupId} was not found.`); + } + + const entry = store.get(collection, lookupId); + + return { + ...entry, + collection, + } as DataEntryResult | ContentEntryResult; + } + const entryImport = await getEntryImport(collection, lookupId); if (typeof entryImport !== 'function') return undefined; diff --git a/packages/astro/src/content/loaders.ts b/packages/astro/src/content/sync.ts similarity index 68% rename from packages/astro/src/content/loaders.ts rename to packages/astro/src/content/sync.ts index f2cfee3962a5..819c8915cc25 100644 --- a/packages/astro/src/content/loaders.ts +++ b/packages/astro/src/content/sync.ts @@ -1,54 +1,11 @@ -import type { ZodSchema } from 'zod'; -import type { AstroSettings } from '../@types/astro.js'; -import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js'; -import { DataStore, globalDataStore, type MetaStore, type ScopedDataStore } from './data-store.js'; -import { getEntryData, globalContentConfigObserver } from './utils.js'; import { promises as fs, existsSync } from 'fs'; -import { DATA_STORE_FILE } from './consts.js'; import type { FSWatcher } from 'vite'; - -export interface ParseDataOptions { - /** The ID of the entry. Unique per collection */ - id: string; - /** The raw, unvalidated data of the entry */ - data: Record; - /** An optional file path, where the entry represents a local file. */ - filePath?: string; -} - -export type DataWithId = { - id: string; - [key: string]: unknown; -}; - -export interface LoaderContext { - /** The unique name of the collection */ - collection: string; - /** A database abstraction to store the actual data */ - store: ScopedDataStore; - /** A simple KV store, designed for things like sync tokens */ - meta: MetaStore; - logger: AstroIntegrationLogger; - - settings: AstroSettings; - - /** Validates and parses the data according to the collection schema */ - parseData(props: ParseDataOptions): Promise; - - /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ - watcher?: FSWatcher; -} - -export interface Loader { - /** Unique name of the loader, e.g. the npm package name */ - name: string; - /** Do the actual loading of the data */ - load: (context: LoaderContext) => Promise; - /** Optionally, define the schema of the data. Will be overridden by user-defined schema */ - schema?: ZodSchema | Promise | (() => ZodSchema | Promise); - render?: (entry: any) => any; -} - +import type { AstroSettings } from '../@types/astro.js'; +import type { Logger } from '../core/logger/core.js'; +import { DATA_STORE_FILE } from './consts.js'; +import { DataStore, globalDataStore } from './data-store.js'; +import type { DataWithId, LoaderContext } from './loaders/types.js'; +import { getEntryData, globalContentConfigObserver } from './utils.js'; export interface SyncContentLayerOptions { store?: DataStore; settings: AstroSettings; @@ -67,6 +24,9 @@ export async function syncContentLayer({ store, watcher, }: SyncContentLayerOptions) { + // The default max listeners is 10, which can be exceeded when using a lot of loaders + watcher?.setMaxListeners(50); + const logger = globalLogger.forkIntegrationLogger('content'); logger.info('Syncing content'); if (!store) { diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index 53133dc02809..0ad52eedfc81 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -5,8 +5,8 @@ import glob from 'fast-glob'; import { bold, cyan } from 'kleur/colors'; import { type ViteDevServer, normalizePath } from 'vite'; import { z } from 'zod'; -import { zodToTs, printNode } from 'zod-to-ts'; import { zodToJsonSchema } from 'zod-to-json-schema'; +import { printNode, zodToTs } from 'zod-to-ts'; import type { AstroSettings, ContentEntryType } from '../@types/astro.js'; import { AstroError } from '../core/errors/errors.js'; import { AstroErrorData } from '../core/errors/index.js'; diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 2273c478176d..fbd0880db5dc 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -1,10 +1,10 @@ import nodeFs from 'node:fs'; import { extname } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { dataToEsm } from '@rollup/pluginutils'; import glob from 'fast-glob'; import pLimit from 'p-limit'; import type { Plugin } from 'vite'; -import { dataToEsm } from '@rollup/pluginutils'; import type { AstroSettings } from '../@types/astro.js'; import { encodeName } from '../core/build/util.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts index b318ffb5b4b2..854a3fbd0914 100644 --- a/packages/astro/src/core/build/index.ts +++ b/packages/astro/src/core/build/index.ts @@ -11,7 +11,8 @@ import type { RuntimeMode, } from '../../@types/astro.js'; import { injectImageEndpoint } from '../../assets/endpoint/config.js'; -import { syncContentLayer } from '../../content/loaders.js'; +import { DataStore, globalDataStore } from '../../content/data-store.js'; +import { syncContentLayer } from '../../content/sync.js'; import { telemetry } from '../../events/index.js'; import { eventCliSession } from '../../events/session.js'; import { @@ -33,7 +34,6 @@ import { collectPagesData } from './page-data.js'; import { staticBuild, viteBuild } from './static-build.js'; import type { StaticBuildOptions } from './types.js'; import { getTimeStat } from './util.js'; -import { globalDataStore, DataStore } from '../../content/data-store.js'; export interface BuildOptions { /** * Teardown the compiler WASM instance after build. This can improve performance when diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts index c920e40822d7..ce7c6606d542 100644 --- a/packages/astro/src/core/dev/dev.ts +++ b/packages/astro/src/core/dev/dev.ts @@ -7,7 +7,7 @@ import { gt, major, minor, patch } from 'semver'; import type * as vite from 'vite'; import type { AstroInlineConfig } from '../../@types/astro.js'; import { attachContentServerListeners } from '../../content/index.js'; -import { syncContentLayer } from '../../content/loaders.js'; +import { syncContentLayer } from '../../content/sync.js'; import { telemetry } from '../../events/index.js'; import * as msg from '../messages.js'; import { ensureProcessNodeEnv } from '../util.js'; diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index e399ee516a02..d1b80f00ac4f 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -6,7 +6,7 @@ import { type HMRPayload, createServer } from 'vite'; import type { AstroConfig, AstroInlineConfig, AstroSettings } from '../../@types/astro.js'; import { getPackage } from '../../cli/install-package.js'; import { createContentTypesGenerator } from '../../content/index.js'; -import { syncContentLayer } from '../../content/loaders.js'; +import { syncContentLayer } from '../../content/sync.js'; import { globalContentConfigObserver } from '../../content/utils.js'; import { syncAstroEnv } from '../../env/sync.js'; import { telemetry } from '../../events/index.js'; diff --git a/packages/astro/templates/content/module.mjs b/packages/astro/templates/content/module.mjs index 92ae22b31119..f246678a25ec 100644 --- a/packages/astro/templates/content/module.mjs +++ b/packages/astro/templates/content/module.mjs @@ -9,7 +9,7 @@ import { createReference, } from 'astro/content/runtime'; -export { defineCollection, file } from 'astro/content/runtime'; +export { defineCollection } from 'astro/content/runtime'; export { z } from 'astro/zod'; const contentDir = '@@CONTENT_DIR@@'; diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 04e49f6a8c93..4e1f2d7c3a66 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; import { after, before, describe, it } from 'node:test'; import { loadFixture } from './test-utils.js'; +import { sep } from 'node:path'; +import { sep as posixSep } from 'node:path/posix'; describe('Content Layer', () => { /** @type {import("./test-utils.js").Fixture} */ @@ -33,7 +35,6 @@ describe('Content Layer', () => { title: 'sunt aut facere repellat provident occaecati excepturi optio reprehenderit', body: 'quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto', }, - type: 'experimental_data', }); }); @@ -72,8 +73,15 @@ describe('Content Layer', () => { }); it('Returns data entry by id', async () => { - assert.ok(json.hasOwnProperty('dataEntryById')); - assert.deepEqual(json.dataEntryById, { + assert.ok(json.hasOwnProperty('dataEntry')); + assert.ok( + json.dataEntry.filePath + ?.split(sep) + .join(posixSep) + .endsWith('packages/astro/test/fixtures/content-layer/src/data/dogs.json') + ); + delete json.dataEntry.filePath; + assert.deepEqual(json.dataEntry, { id: 'beagle', collection: 'dogs', data: { @@ -92,7 +100,6 @@ describe('Content Layer', () => { assert.ok(Array.isArray(json.simpleLoader)); const item = json.simpleLoader[0]; - assert.equal(json.simpleLoader.length, 4); assert.deepEqual(item, { id: 'siamese', collection: 'cats', @@ -104,7 +111,6 @@ describe('Content Layer', () => { lifespan: '15 years', temperament: ['Active', 'Affectionate', 'Social', 'Playful'], }, - type: 'experimental_data', }); }); }); @@ -137,7 +143,6 @@ describe('Content Layer', () => { title: 'sunt aut facere repellat provident occaecati excepturi optio reprehenderit', body: 'quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto', }, - type: 'experimental_data', }); }); @@ -176,8 +181,15 @@ describe('Content Layer', () => { }); it('Returns data entry by id', async () => { - assert.ok(json.hasOwnProperty('dataEntryById')); - assert.deepEqual(json.dataEntryById, { + assert.ok(json.hasOwnProperty('dataEntry')); + assert.ok( + json.dataEntry.filePath + ?.split(sep) + .join(posixSep) + .endsWith('packages/astro/test/fixtures/content-layer/src/data/dogs.json') + ); + delete json.dataEntry.filePath; + assert.deepEqual(json.dataEntry, { id: 'beagle', collection: 'dogs', data: { @@ -196,7 +208,7 @@ describe('Content Layer', () => { const initialJson = await rawJsonResponse.json(); assert.equal(initialJson.fileLoader[0].data.temperament.includes('Bouncy'), false); - await fixture.editFile('/src/content/_data/dogs.json', (prev) => { + await fixture.editFile('/src/data/dogs.json', (prev) => { const data = JSON.parse(prev); data[0].temperament.push('Bouncy'); return JSON.stringify(data, null, 2); diff --git a/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md b/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md new file mode 100644 index 000000000000..4971108e36ed --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md @@ -0,0 +1,15 @@ +--- +title: Columbia +description: 'Learn about the Columbia NASA space shuttle.' +publishedDate: 'Sat May 21 2022 00:00:00 GMT-0400 (Eastern Daylight Time)' +tags: [space, 90s] +--- + +**Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Endeavour) + +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. + +The United States Congress approved the construction of Endeavour in 1987 to replace the Space Shuttle Challenger, which was destroyed in 1986. + +NASA chose, on cost grounds, to build much of Endeavour from spare parts rather than refitting the Space Shuttle Enterprise, and used structural spares built during the construction of Discovery and Atlantis in its assembly. +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. diff --git a/packages/astro/test/fixtures/content-layer/content-outside-src/columbia.md b/packages/astro/test/fixtures/content-layer/content-outside-src/columbia.md new file mode 100644 index 000000000000..4971108e36ed --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/content-outside-src/columbia.md @@ -0,0 +1,15 @@ +--- +title: Columbia +description: 'Learn about the Columbia NASA space shuttle.' +publishedDate: 'Sat May 21 2022 00:00:00 GMT-0400 (Eastern Daylight Time)' +tags: [space, 90s] +--- + +**Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Endeavour) + +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. + +The United States Congress approved the construction of Endeavour in 1987 to replace the Space Shuttle Challenger, which was destroyed in 1986. + +NASA chose, on cost grounds, to build much of Endeavour from spare parts rather than refitting the Space Shuttle Enterprise, and used structural spares built during the construction of Discovery and Atlantis in its assembly. +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. diff --git a/packages/astro/test/fixtures/content-layer/content-outside-src/endeavour.md b/packages/astro/test/fixtures/content-layer/content-outside-src/endeavour.md new file mode 100644 index 000000000000..51d6e8c42178 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/content-outside-src/endeavour.md @@ -0,0 +1,14 @@ +--- +title: Endeavour +description: 'Learn about the Endeavour NASA space shuttle.' +publishedDate: 'Sun Jul 11 2021 00:00:00 GMT-0400 (Eastern Daylight Time)' +tags: [space, 90s] +--- + +**Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Endeavour) + +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. + +The United States Congress approved the construction of Endeavour in 1987 to replace the Space Shuttle Challenger, which was destroyed in 1986. + +NASA chose, on cost grounds, to build much of Endeavour from spare parts rather than refitting the Space Shuttle Enterprise, and used structural spares built during the construction of Discovery and Atlantis in its assembly. diff --git a/packages/astro/test/fixtures/content-layer/content-outside-src/enterprise.md b/packages/astro/test/fixtures/content-layer/content-outside-src/enterprise.md new file mode 100644 index 000000000000..3131e6d5df3a --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/content-outside-src/enterprise.md @@ -0,0 +1,14 @@ +--- +title: 'Enterprise' +description: 'Learn about the Enterprise NASA space shuttle.' +publishedDate: 'Tue Jun 08 2021 00:00:00 GMT-0400 (Eastern Daylight Time)' +tags: [space, 70s] +--- + +**Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Enterprise) + +Space Shuttle Enterprise (Orbiter Vehicle Designation: OV-101) was the first orbiter of the Space Shuttle system. Rolled out on September 17, 1976, it was built for NASA as part of the Space Shuttle program to perform atmospheric test flights after being launched from a modified Boeing 747. It was constructed without engines or a functional heat shield. As a result, it was not capable of spaceflight. + +Originally, Enterprise had been intended to be refitted for orbital flight to become the second space-rated orbiter in service. However, during the construction of Space Shuttle Columbia, details of the final design changed, making it simpler and less costly to build Challenger around a body frame that had been built as a test article. Similarly, Enterprise was considered for refit to replace Challenger after the latter was destroyed, but Endeavour was built from structural spares instead. + +Enterprise was restored and placed on display in 2003 at the Smithsonian's new Steven F. Udvar-Hazy Center in Virginia. Following the retirement of the Space Shuttle fleet, Discovery replaced Enterprise at the Udvar-Hazy Center, and Enterprise was transferred to the Intrepid Sea, Air & Space Museum in New York City, where it has been on display since July 2012. diff --git a/packages/astro/test/fixtures/content-layer/content-outside-src/index.md b/packages/astro/test/fixtures/content-layer/content-outside-src/index.md new file mode 100644 index 000000000000..4971108e36ed --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/content-outside-src/index.md @@ -0,0 +1,15 @@ +--- +title: Columbia +description: 'Learn about the Columbia NASA space shuttle.' +publishedDate: 'Sat May 21 2022 00:00:00 GMT-0400 (Eastern Daylight Time)' +tags: [space, 90s] +--- + +**Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Endeavour) + +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. + +The United States Congress approved the construction of Endeavour in 1987 to replace the Space Shuttle Challenger, which was destroyed in 1986. + +NASA chose, on cost grounds, to build much of Endeavour from spare parts rather than refitting the Space Shuttle Enterprise, and used structural spares built during the construction of Discovery and Atlantis in its assembly. +Space Shuttle Endeavour (Orbiter Vehicle Designation: OV-105) is a retired orbiter from NASA's Space Shuttle program and the fifth and final operational Shuttle built. It embarked on its first mission, STS-49, in May 1992 and its 25th and final mission, STS-134, in May 2011. STS-134 was expected to be the final mission of the Space Shuttle program, but with the authorization of STS-135, Atlantis became the last shuttle to fly. diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 78b6858111f5..49d9d6f4a2b5 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -1,59 +1,62 @@ -import { defineCollection, file, z } from 'astro:content'; +import { defineCollection, z } from 'astro:content'; +import { file, glob } from 'astro/loaders'; import { loader } from '../loaders/post-loader.js'; +import { fileURLToPath } from 'node:url'; const blog = defineCollection({ - type: "experimental_data", - loader: loader({ url: "https://jsonplaceholder.typicode.com/posts" }), + type: 'experimental_data', + loader: loader({ url: 'https://jsonplaceholder.typicode.com/posts' }), }); const dogs = defineCollection({ - type: "experimental_data", - loader: file("_data/dogs.json"), + type: 'experimental_data', + loader: file('src/data/dogs.json'), schema: z.object({ breed: z.string(), id: z.string(), size: z.string(), origin: z.string(), lifespan: z.string(), - temperament: z.array(z.string()) + temperament: z.array(z.string()), }), -}) +}); const cats = defineCollection({ - type: "experimental_data", - loader: async function() { - return [{ - "breed": "Siamese", - "id": "siamese", - "size": "Medium", - "origin": "Thailand", - "lifespan": "15 years", - "temperament": ["Active", "Affectionate", "Social", "Playful"] + type: 'experimental_data', + loader: async function () { + return [ + { + breed: 'Siamese', + id: 'siamese', + size: 'Medium', + origin: 'Thailand', + lifespan: '15 years', + temperament: ['Active', 'Affectionate', 'Social', 'Playful'], }, - { - "breed": "Persian", - "id": "persian", - "size": "Medium", - "origin": "Iran", - "lifespan": "15 years", - "temperament": ["Calm", "Affectionate", "Social"] + { + breed: 'Persian', + id: 'persian', + size: 'Medium', + origin: 'Iran', + lifespan: '15 years', + temperament: ['Calm', 'Affectionate', 'Social'], }, - { - "breed": "Tabby", - "id": "tabby", - "size": "Medium", - "origin": "Egypt", - "lifespan": "15 years", - "temperament": ["Curious", "Playful", "Independent"] + { + breed: 'Tabby', + id: 'tabby', + size: 'Medium', + origin: 'Egypt', + lifespan: '15 years', + temperament: ['Curious', 'Playful', 'Independent'], }, { - "breed": "Ragdoll", - "id": "ragdoll", - "size": "Medium", - "origin": "United States", - "lifespan": "15 years", - "temperament": ["Calm", "Affectionate", "Social"] - } + breed: 'Ragdoll', + id: 'ragdoll', + size: 'Medium', + origin: 'United States', + lifespan: '15 years', + temperament: ['Calm', 'Affectionate', 'Social'], + }, ]; }, schema: z.object({ @@ -62,8 +65,22 @@ const cats = defineCollection({ size: z.string(), origin: z.string(), lifespan: z.string(), - temperament: z.array(z.string()) + temperament: z.array(z.string()), }), -}) +}); + + +// Absolute paths should also work +const absoluteRoot = new URL('../../content-outside-src', import.meta.url); + +const spacecraft = defineCollection({ + type: 'experimental_data', + loader: glob({ pattern: '*', base: absoluteRoot }), +}); + +const numbers = defineCollection({ + type: 'experimental_data', + loader: glob({ pattern: 'src/data/glob-data/*', base: '.' }), +}); -export const collections = { blog, dogs, cats }; +export const collections = { blog, dogs, cats, numbers, spacecraft }; diff --git a/packages/astro/test/fixtures/content-layer/src/content/_data/dogs.json b/packages/astro/test/fixtures/content-layer/src/data/dogs.json similarity index 100% rename from packages/astro/test/fixtures/content-layer/src/content/_data/dogs.json rename to packages/astro/test/fixtures/content-layer/src/data/dogs.json diff --git a/packages/astro/test/fixtures/content-layer/src/data/glob-data/index.json b/packages/astro/test/fixtures/content-layer/src/data/glob-data/index.json new file mode 100644 index 000000000000..efc60137d68e --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/glob-data/index.json @@ -0,0 +1,3 @@ +{ + "title": "One" +} diff --git a/packages/astro/test/fixtures/content-layer/src/data/glob-data/one.json b/packages/astro/test/fixtures/content-layer/src/data/glob-data/one.json new file mode 100644 index 000000000000..efc60137d68e --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/glob-data/one.json @@ -0,0 +1,3 @@ +{ + "title": "One" +} diff --git a/packages/astro/test/fixtures/content-layer/src/data/glob-data/three.json b/packages/astro/test/fixtures/content-layer/src/data/glob-data/three.json new file mode 100644 index 000000000000..7d028e937a71 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/glob-data/three.json @@ -0,0 +1,3 @@ +{ + "title": "Three" +} diff --git a/packages/astro/test/fixtures/content-layer/src/data/glob-data/two.json b/packages/astro/test/fixtures/content-layer/src/data/glob-data/two.json new file mode 100644 index 000000000000..1a8215509b0a --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/glob-data/two.json @@ -0,0 +1,3 @@ +{ + "title": "Two" +} diff --git a/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts b/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts index 38e4e28b5816..1d7aa0df1843 100644 --- a/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts +++ b/packages/astro/test/fixtures/content-layer/src/loaders/post-loader.ts @@ -1,4 +1,5 @@ -import { type Loader, z } from 'astro:content'; +import { z } from 'astro:content'; +import type { Loader } from "astro/loaders" export interface PostLoaderConfig { url: string; @@ -25,7 +26,7 @@ export function loader(config:PostLoaderConfig): Loader { store.clear(); - for (const post of posts) { + for (const post of posts.slice(0, 10)) { store.set(post.id, post); } meta.set('lastSynced', String(Date.now())); diff --git a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js index 821d8dc2a964..5f98a51285b9 100644 --- a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js +++ b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js @@ -1,14 +1,14 @@ -import { getCollection, getDataEntryById } from 'astro:content'; +import { getCollection, getEntry } from 'astro:content'; export async function GET() { const customLoader = (await getCollection('blog')).slice(0, 10); const fileLoader = await getCollection('dogs'); - const dataEntryById = await getDataEntryById('dogs', 'beagle'); + const dataEntry= await getEntry('dogs', 'beagle'); const simpleLoader = await getCollection('cats'); return new Response( - JSON.stringify({ customLoader, fileLoader, dataEntryById, simpleLoader }), + JSON.stringify({ customLoader, fileLoader, dataEntry, simpleLoader }), ); } diff --git a/packages/astro/test/fixtures/content-layer/src/pages/spacecraft/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/spacecraft/[slug].astro new file mode 100644 index 000000000000..ceff0b361e71 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/pages/spacecraft/[slug].astro @@ -0,0 +1,27 @@ +--- +import type { GetStaticPaths } from "astro"; +import { getCollection } from "astro:content" +export const getStaticPaths = (async () => { + const collection = await getCollection("spacecraft"); + if(!collection) return [] + return collection.map((craft) => ({ + params: { + slug: `/${craft.id}` + }, + props: { + craft + } + })); +}) satisfies GetStaticPaths; + + + + +const { craft } = Astro.props as any +--- + +

{craft.data.title}

+
+	{craft.body}
+
+ diff --git a/packages/astro/types/content.d.ts b/packages/astro/types/content.d.ts index 2c5ff792936f..d66a8a64d14c 100644 --- a/packages/astro/types/content.d.ts +++ b/packages/astro/types/content.d.ts @@ -20,55 +20,31 @@ declare module 'astro:content' { >; }>; + export interface DataEntry { + id: string; + data: Record; + filePath?: string; + body?: string; + } + export interface DataStore { - get: (key: string) => any; - entries: () => IterableIterator<[id: string, any]>; - set: (key: string, value: any) => void; + get: (key: string) => DataEntry; + entries: () => Array<[id: string, DataEntry]>; + set: (key: string, data: Record, body?: string, filePath?: string) => void; + values: () => Array; + keys: () => Array; delete: (key: string) => void; clear: () => void; has: (key: string) => boolean; } + export interface MetaStore { get: (key: string) => string | undefined; set: (key: string, value: string) => void; + delete: (key: string) => void; has: (key: string) => boolean; } - export interface ParseDataOptions { - /** The ID of the entry. Unique per collection */ - id: string; - /** The raw, unvalidated data of the entry */ - data: Record; - /** An optional file path, where the entry represents a local file */ - filePath?: string; - } - export interface LoaderContext { - collection: string; - /** A database abstraction to store the actual data */ - store: DataStore; - /** A simple KV store, designed for things like sync tokens */ - meta: MetaStore; - logger: import('astro').AstroIntegrationLogger; - settings: import('astro').AstroSettings; - /** Validates and parses the data according to the schema */ - parseData = Record>( - props: ParseDataOptions - ): T; - /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ - watcher?: import('vite').FSWatcher; - } - export interface Loader { - /** Unique name of the loader, e.g. the npm package name */ - name: string; - /** Do the actual loading of the data */ - load: (context: LoaderContext) => Promise; - /** Optionally, define the schema of the data. Will be overridden by user-defined schema */ - schema?: BaseSchema | Promise | (() => BaseSchema | Promise); - render?: (entry: any) => any; - } - - export function file(filePath: string): Loader; - type BaseSchemaWithoutEffects = | import('astro/zod').AnyZodObject | import('astro/zod').ZodUnion<[BaseSchemaWithoutEffects, ...BaseSchemaWithoutEffects[]]> @@ -89,7 +65,9 @@ declare module 'astro:content' { type ContentCollectionV2Config = { type: 'experimental_data'; schema?: S | ((context: SchemaContext) => S); - loader: Loader | (() => Array | Promise>); + loader: + | import('astro/loader/types').Loader + | (() => Array | Promise>); }; type DataCollectionConfig = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3b3cbce48dc9..0bc02855e9cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -654,6 +654,9 @@ importers: magic-string: specifier: ^0.30.10 version: 0.30.10 + micromatch: + specifier: ^4.0.7 + version: 4.0.7 mrmime: specifier: ^2.0.0 version: 2.0.0 @@ -773,6 +776,9 @@ importers: '@types/js-yaml': specifier: ^4.0.9 version: 4.0.9 + '@types/micromatch': + specifier: ^4.0.9 + version: 4.0.9 '@types/probe-image-size': specifier: ^7.2.4 version: 7.2.4 @@ -7274,6 +7280,9 @@ packages: '@types/body-parser@1.19.5': resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + '@types/braces@3.0.4': + resolution: {integrity: sha512-0WR3b8eaISjEW7RpZnclONaLFDf7buaowRHdqLp4vLj54AsSAYWfh3DRbfiYJY9XDxMgx1B4sE1Afw2PGpuHOA==} + '@types/canvas-confetti@1.6.4': resolution: {integrity: sha512-fNyZ/Fdw/Y92X0vv7B+BD6ysHL4xVU5dJcgzgxLdGbn8O3PezZNIJpml44lKM0nsGur+o/6+NZbZeNTt00U1uA==} @@ -7367,6 +7376,9 @@ packages: '@types/mdx@2.0.13': resolution: {integrity: sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==} + '@types/micromatch@4.0.9': + resolution: {integrity: sha512-7V+8ncr22h4UoYRLnLXSpTxjQrNUXtWHGeMPRJt1nULXI57G9bIcpyrHlmrQ7QK24EyyuXvYcSSWAM8GA9nqCg==} + '@types/mime@1.3.5': resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} @@ -7938,6 +7950,10 @@ packages: resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} engines: {node: '>=8'} + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + browserslist@4.23.0: resolution: {integrity: sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -8753,6 +8769,10 @@ packages: resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} engines: {node: '>=8'} + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + finalhandler@1.2.0: resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} engines: {node: '>= 0.8'} @@ -9699,6 +9719,10 @@ packages: resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} engines: {node: '>=8.6'} + micromatch@4.0.7: + resolution: {integrity: sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==} + engines: {node: '>=8.6'} + mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -13271,6 +13295,8 @@ snapshots: '@types/connect': 3.4.38 '@types/node': 18.19.31 + '@types/braces@3.0.4': {} + '@types/canvas-confetti@1.6.4': {} '@types/chai@4.3.16': {} @@ -13367,6 +13393,10 @@ snapshots: '@types/mdx@2.0.13': {} + '@types/micromatch@4.0.9': + dependencies: + '@types/braces': 3.0.4 + '@types/mime@1.3.5': {} '@types/ms@0.7.34': {} @@ -14122,6 +14152,10 @@ snapshots: dependencies: fill-range: 7.0.1 + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + browserslist@4.23.0: dependencies: caniuse-lite: 1.0.30001610 @@ -14862,7 +14896,7 @@ snapshots: '@nodelib/fs.walk': 1.2.8 glob-parent: 5.1.2 merge2: 1.4.1 - micromatch: 4.0.5 + micromatch: 4.0.7 fast-json-stable-stringify@2.1.0: {} @@ -14893,6 +14927,10 @@ snapshots: dependencies: to-regex-range: 5.0.1 + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + finalhandler@1.2.0: dependencies: debug: 2.6.9 @@ -14917,7 +14955,7 @@ snapshots: find-yarn-workspace-root2@1.2.16: dependencies: - micromatch: 4.0.5 + micromatch: 4.0.7 pkg-dir: 4.2.0 flat-cache@4.0.1: @@ -16261,6 +16299,11 @@ snapshots: braces: 3.0.2 picomatch: 2.3.1 + micromatch@4.0.7: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + mime-db@1.52.0: {} mime-types@2.1.35: