From d1bd1a11f7aca4d2141d1c4665f2db0440393d03 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 29 Aug 2024 15:22:40 +0100 Subject: [PATCH] Moves content layer sync to a queue and support selective sync (#11767) * wip: allow integrations to refresh contel layer * Use queue for sync jobs * Remove integration-specific code * Fix type * changeset --- .changeset/sixty-masks-lie.md | 5 ++ packages/astro/package.json | 1 + packages/astro/src/@types/astro.ts | 5 ++ packages/astro/src/content/content-layer.ts | 66 ++++++++++----------- packages/astro/src/content/loaders/types.ts | 1 + packages/astro/src/core/dev/restart.ts | 4 +- pnpm-lock.yaml | 3 + 7 files changed, 49 insertions(+), 36 deletions(-) create mode 100644 .changeset/sixty-masks-lie.md diff --git a/.changeset/sixty-masks-lie.md b/.changeset/sixty-masks-lie.md new file mode 100644 index 000000000000..d9b8d7a5208c --- /dev/null +++ b/.changeset/sixty-masks-lie.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Refactors content layer sync to use a queue diff --git a/packages/astro/package.json b/packages/astro/package.json index faf4dfb5ec29..44c302709190 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -152,6 +152,7 @@ "esbuild": "^0.21.5", "estree-walker": "^3.0.3", "fast-glob": "^3.3.2", + "fastq": "^1.17.1", "flattie": "^1.1.1", "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 5aeb6d112c40..87ddb65e2699 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -3242,6 +3242,11 @@ export interface SSRLoadedRenderer extends Pick; + context?: Record; +} + export type HookParameters< Hook extends keyof AstroIntegration['hooks'], Fn = AstroIntegration['hooks'][Hook], diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index 66dfdd2e75e3..606754eac85b 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -1,9 +1,10 @@ import { promises as fs, existsSync } from 'node:fs'; import { isAbsolute } from 'node:path'; import { fileURLToPath } from 'node:url'; +import * as fastq from 'fastq'; import type { FSWatcher } from 'vite'; import xxhash from 'xxhash-wasm'; -import type { AstroSettings, ContentEntryType } from '../@types/astro.js'; +import type { AstroSettings, ContentEntryType, RefreshContentOptions } from '../@types/astro.js'; import { AstroUserError } from '../core/errors/errors.js'; import type { Logger } from '../core/logger/core.js'; import { @@ -38,7 +39,8 @@ export class ContentLayer { #generateDigest?: (data: Record | string) => string; - #loading = false; + #queue: fastq.queueAsPromised; + constructor({ settings, logger, store, watcher }: ContentLayerOptions) { // The default max listeners is 10, which can be exceeded when using a lot of loaders watcher?.setMaxListeners(50); @@ -47,13 +49,14 @@ export class ContentLayer { this.#store = store; this.#settings = settings; this.#watcher = watcher; + this.#queue = fastq.promise(this.#doSync.bind(this), 1); } /** * Whether the content layer is currently loading content */ get loading() { - return this.#loading; + return !this.#queue.idle(); } /** @@ -62,11 +65,7 @@ export class ContentLayer { watchContentConfig() { this.#unsubscribe?.(); this.#unsubscribe = globalContentConfigObserver.subscribe(async (ctx) => { - if ( - !this.#loading && - ctx.status === 'loaded' && - ctx.config.digest !== this.#lastConfigDigest - ) { + if (ctx.status === 'loaded' && ctx.config.digest !== this.#lastConfigDigest) { this.sync(); } }); @@ -76,23 +75,6 @@ export class ContentLayer { this.#unsubscribe?.(); } - /** - * Run the `load()` method of each collection's loader, which will load the data and save it in the data store. - * The loader itself is responsible for deciding whether this will clear and reload the full collection, or - * perform an incremental update. After the data is loaded, the data store is written to disk. - */ - async sync() { - if (this.#loading) { - return; - } - this.#loading = true; - try { - await this.#doSync(); - } finally { - this.#loading = false; - } - } - async #getGenerateDigest() { if (this.#generateDigest) { return this.#generateDigest; @@ -113,10 +95,12 @@ export class ContentLayer { collectionName, loaderName = 'content', parseData, + refreshContextData, }: { collectionName: string; loaderName: string; parseData: LoaderContext['parseData']; + refreshContextData?: Record; }): Promise { return { collection: collectionName, @@ -127,6 +111,7 @@ export class ContentLayer { parseData, generateDigest: await this.#getGenerateDigest(), watcher: this.#watcher, + refreshContextData, entryTypes: getEntryConfigByExtMap([ ...this.#settings.contentEntryTypes, ...this.#settings.dataEntryTypes, @@ -134,7 +119,18 @@ export class ContentLayer { }; } - async #doSync() { + /** + * Enqueues a sync job that runs the `load()` method of each collection's loader, which will load the data and save it in the data store. + * The loader itself is responsible for deciding whether this will clear and reload the full collection, or + * perform an incremental update. After the data is loaded, the data store is written to disk. Jobs are queued, + * so that only one sync can run at a time. The function returns a promise that resolves when this sync job is complete. + */ + + sync(options: RefreshContentOptions = {}): Promise { + return this.#queue.push(options); + } + + async #doSync(options: RefreshContentOptions) { const contentConfig = globalContentConfigObserver.get(); const logger = this.#logger.forkIntegrationLogger('content'); if (contentConfig?.status !== 'loaded') { @@ -180,6 +176,15 @@ export class ContentLayer { } } + // If loaders are specified, only sync the specified loaders + if ( + options?.loaders && + (typeof collection.loader !== 'object' || + !options.loaders.includes(collection.loader.name)) + ) { + return; + } + const collectionWithResolvedSchema = { ...collection, schema }; const parseData: LoaderContext['parseData'] = async ({ id, data, filePath = '' }) => { @@ -213,6 +218,7 @@ export class ContentLayer { collectionName: name, parseData, loaderName: collection.loader.name, + refreshContextData: options?.context, }); if (typeof collection.loader === 'function') { @@ -293,18 +299,12 @@ export async function simpleLoader( function contentLayerSingleton() { let instance: ContentLayer | null = null; return { - initialized: () => Boolean(instance), init: (options: ContentLayerOptions) => { instance?.unwatchContentConfig(); instance = new ContentLayer(options); return instance; }, - get: () => { - if (!instance) { - throw new Error('Content layer not initialized'); - } - return instance; - }, + get: () => instance, dispose: () => { instance?.unwatchContentConfig(); instance = null; diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts index 67e7e13f0825..cb687c22e451 100644 --- a/packages/astro/src/content/loaders/types.ts +++ b/packages/astro/src/content/loaders/types.ts @@ -31,6 +31,7 @@ export interface LoaderContext { /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ watcher?: FSWatcher; + refreshContextData?: Record; entryTypes: Map; } diff --git a/packages/astro/src/core/dev/restart.ts b/packages/astro/src/core/dev/restart.ts index ee0ba995c446..30821362c744 100644 --- a/packages/astro/src/core/dev/restart.ts +++ b/packages/astro/src/core/dev/restart.ts @@ -185,9 +185,7 @@ export async function createContainerWithAutomaticRestart({ key: 's', description: 'sync content layer', action: () => { - if (globalContentLayer.initialized()) { - globalContentLayer.get().sync(); - } + globalContentLayer.get()?.sync(); }, }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef9e2f2c26c6..f478b152ccfb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -645,6 +645,9 @@ importers: fast-glob: specifier: ^3.3.2 version: 3.3.2 + fastq: + specifier: ^1.17.1 + version: 1.17.1 flattie: specifier: ^1.1.1 version: 1.1.1