Skip to content

Commit

Permalink
fix: keep data store outside of node_modules during dev (#11902)
Browse files Browse the repository at this point in the history
* fix: don't keep data store in node_modules during dev

* Lint

* Fix test

* Wait for data store

* Use helper for data store file

* Fix data store file helper

* Lint

* Handle case where Vite already knows about save
  • Loading branch information
ascorbic authored Sep 3, 2024
1 parent 5705200 commit d63bc50
Show file tree
Hide file tree
Showing 9 changed files with 84 additions and 36 deletions.
5 changes: 5 additions & 0 deletions .changeset/forty-spies-train.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'astro': patch
---

Fixes case where content layer did not update during clean dev builds on Linux and Windows
11 changes: 10 additions & 1 deletion packages/astro/src/content/content-layer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ export class ContentLayer {
if (!existsSync(this.#settings.config.cacheDir)) {
await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
}
const cacheFile = new URL(DATA_STORE_FILE, this.#settings.config.cacheDir);
const cacheFile = getDataStoreFile(this.#settings);
await this.#store.writeToDisk(cacheFile);
if (!existsSync(this.#settings.dotAstroDir)) {
await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
Expand Down Expand Up @@ -283,6 +283,15 @@ export async function simpleLoader<TData extends { id: string }>(
context.store.set({ id: raw.id, data: item });
}
}
/**
* Get the path to the data store file.
* During development, this is in the `.astro` directory so that the Vite watcher can see it.
* In production, it's in the cache directory so that it's preserved between builds.
*/
export function getDataStoreFile(settings: AstroSettings, isDev?: boolean) {
isDev ??= process?.env.NODE_ENV === 'development';
return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
}

function contentLayerSingleton() {
let instance: ContentLayer | null = null;
Expand Down
3 changes: 3 additions & 0 deletions packages/astro/src/content/data-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ export class DataStore {
try {
// @ts-expect-error - this is a virtual module
const data = await import('astro:data-layer-content');
if (data.default instanceof Map) {
return DataStore.fromMap(data.default);
}
const map = devalue.unflatten(data.default);
return DataStore.fromMap(map);
} catch {}
Expand Down
41 changes: 24 additions & 17 deletions packages/astro/src/content/vite-plugin-content-virtual-mod.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import {
CONTENT_FLAG,
CONTENT_RENDER_FLAG,
DATA_FLAG,
DATA_STORE_FILE,
DATA_STORE_VIRTUAL_ID,
MODULES_IMPORTS_FILE,
MODULES_MJS_ID,
Expand All @@ -29,6 +28,7 @@ import {
RESOLVED_VIRTUAL_MODULE_ID,
VIRTUAL_MODULE_ID,
} from './consts.js';
import { getDataStoreFile } from './content-layer.js';
import {
type ContentLookupMap,
getContentEntryIdAndSlug,
Expand All @@ -54,12 +54,13 @@ export function astroContentVirtualModPlugin({
}: AstroContentVirtualModPluginParams): Plugin {
let IS_DEV = false;
const IS_SERVER = isServerLikeOutput(settings.config);
const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
let dataStoreFile: URL;
return {
name: 'astro-content-virtual-mod-plugin',
enforce: 'pre',
configResolved(config) {
IS_DEV = config.mode === 'development';
dataStoreFile = getDataStoreFile(settings, IS_DEV);
},
async resolveId(id) {
if (id === VIRTUAL_MODULE_ID) {
Expand Down Expand Up @@ -180,25 +181,31 @@ export function astroContentVirtualModPlugin({

configureServer(server) {
const dataStorePath = fileURLToPath(dataStoreFile);
// Watch for changes to the data store file
if (Array.isArray(server.watcher.options.ignored)) {
// The data store file is in node_modules, so is ignored by default,
// so we need to un-ignore it.
server.watcher.options.ignored.push(`!${dataStorePath}`);
}

server.watcher.add(dataStorePath);

function invalidateDataStore() {
const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
if (module) {
server.moduleGraph.invalidateModule(module);
}
server.ws.send({
type: 'full-reload',
path: '*',
});
}

// If the datastore file changes, invalidate the virtual module

server.watcher.on('add', (addedPath) => {
if (addedPath === dataStorePath) {
invalidateDataStore();
}
});

server.watcher.on('change', (changedPath) => {
// If the datastore file changes, invalidate the virtual module
if (changedPath === dataStorePath) {
const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
if (module) {
server.moduleGraph.invalidateModule(module);
}
server.ws.send({
type: 'full-reload',
path: '*',
});
invalidateDataStore();
}
});
},
Expand Down
7 changes: 4 additions & 3 deletions packages/astro/src/core/build/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ export default async function build(
const logger = createNodeLogger(inlineConfig);
const { userConfig, astroConfig } = await resolveConfig(inlineConfig, 'build');
telemetry.record(eventCliSession('build', userConfig));

const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));

if (inlineConfig.force) {
if (astroConfig.experimental.contentCollectionCache) {
const contentCacheDir = new URL('./content/', astroConfig.cacheDir);
Expand All @@ -70,11 +73,9 @@ export default async function build(
logger.warn('content', 'content cache cleared (force)');
}
}
await clearContentLayerCache({ astroConfig, logger, fs });
await clearContentLayerCache({ settings, logger, fs });
}

const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));

const builder = new AstroBuilder(settings, {
...options,
logger,
Expand Down
5 changes: 2 additions & 3 deletions packages/astro/src/core/dev/dev.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ import { green } from 'kleur/colors';
import { gt, major, minor, patch } from 'semver';
import type * as vite from 'vite';
import type { AstroInlineConfig } from '../../@types/astro.js';
import { DATA_STORE_FILE } from '../../content/consts.js';
import { globalContentLayer } from '../../content/content-layer.js';
import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
import { attachContentServerListeners } from '../../content/index.js';
import { MutableDataStore } from '../../content/mutable-data-store.js';
import { globalContentConfigObserver } from '../../content/utils.js';
Expand Down Expand Up @@ -108,7 +107,7 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS

let store: MutableDataStore | undefined;
try {
const dataStoreFile = new URL(DATA_STORE_FILE, restart.container.settings.config.cacheDir);
const dataStoreFile = getDataStoreFile(restart.container.settings);
if (existsSync(dataStoreFile)) {
store = await MutableDataStore.fromFile(dataStoreFile);
}
Expand Down
16 changes: 8 additions & 8 deletions packages/astro/src/core/sync/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ import { performance } from 'node:perf_hooks';
import { fileURLToPath } from 'node:url';
import { dim } from 'kleur/colors';
import { type HMRPayload, createServer } from 'vite';
import type { AstroConfig, AstroInlineConfig, AstroSettings } from '../../@types/astro.js';
import { CONTENT_TYPES_FILE, DATA_STORE_FILE } from '../../content/consts.js';
import { globalContentLayer } from '../../content/content-layer.js';
import type { AstroInlineConfig, AstroSettings } from '../../@types/astro.js';
import { CONTENT_TYPES_FILE } from '../../content/consts.js';
import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
import { createContentTypesGenerator } from '../../content/index.js';
import { MutableDataStore } from '../../content/mutable-data-store.js';
import { getContentPaths, globalContentConfigObserver } from '../../content/utils.js';
Expand Down Expand Up @@ -69,11 +69,11 @@ export default async function sync(
* Clears the content layer and content collection cache, forcing a full rebuild.
*/
export async function clearContentLayerCache({
astroConfig,
settings,
logger,
fs = fsMod,
}: { astroConfig: AstroConfig; logger: Logger; fs?: typeof fsMod }) {
const dataStore = new URL(DATA_STORE_FILE, astroConfig.cacheDir);
}: { settings: AstroSettings; logger: Logger; fs?: typeof fsMod }) {
const dataStore = getDataStoreFile(settings);
if (fs.existsSync(dataStore)) {
logger.debug('content', 'clearing data store');
await fs.promises.rm(dataStore, { force: true });
Expand All @@ -95,7 +95,7 @@ export async function syncInternal({
force,
}: SyncOptions): Promise<void> {
if (force) {
await clearContentLayerCache({ astroConfig: settings.config, logger, fs });
await clearContentLayerCache({ settings, logger, fs });
}

const timerStart = performance.now();
Expand All @@ -106,7 +106,7 @@ export async function syncInternal({
settings.timer.start('Sync content layer');
let store: MutableDataStore | undefined;
try {
const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
const dataStoreFile = getDataStoreFile(settings);
if (existsSync(dataStoreFile)) {
store = await MutableDataStore.fromFile(dataStoreFile);
}
Expand Down
10 changes: 6 additions & 4 deletions packages/astro/test/content-layer.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,11 @@ describe('Content Layer', () => {
let devServer;
let json;
before(async () => {
devServer = await fixture.startDevServer();
devServer = await fixture.startDevServer({ force: true });
// Vite may not have noticed the saved data store yet. Wait a little just in case.
await fixture.onNextDataStoreChange(1000).catch(() => {
// Ignore timeout, because it may have saved before we get here.
})
const rawJsonResponse = await fixture.fetch('/collections.json');
const rawJson = await rawJsonResponse.text();
json = devalue.parse(rawJson);
Expand Down Expand Up @@ -286,9 +290,7 @@ describe('Content Layer', () => {
return JSON.stringify(data, null, 2);
});

// Writes are debounced to 500ms
await new Promise((r) => setTimeout(r, 700));

await fixture.onNextDataStoreChange();
const updatedJsonResponse = await fixture.fetch('/collections.json');
const updated = devalue.parse(await updatedJsonResponse.text());
assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy'));
Expand Down
22 changes: 22 additions & 0 deletions packages/astro/test/test-utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ process.env.ASTRO_TELEMETRY_DISABLED = true;
* @property {() => Promise<App>} loadTestAdapterApp
* @property {() => Promise<(req: NodeRequest, res: NodeResponse) => void>} loadNodeAdapterHandler
* @property {() => Promise<void>} onNextChange
* @property {(timeout?: number) => Promise<void>} onNextDataStoreChange
* @property {typeof check} check
* @property {typeof sync} sync
* @property {AstroConfig} config
Expand Down Expand Up @@ -183,6 +184,27 @@ export async function loadFixture(inlineConfig) {
config.server.port = devServer.address.port; // update port
return devServer;
},
onNextDataStoreChange: (timeout = 5000) => {
if (!devServer) {
return Promise.reject(new Error('No dev server running'));
}

const dataStoreFile = path.join(root, '.astro', 'data-store.json');

return new Promise((resolve, reject) => {
const changeHandler = (fileName) => {
if (fileName === dataStoreFile) {
devServer.watcher.removeListener('change', changeHandler);
resolve();
}
};
devServer.watcher.on('change', changeHandler);
setTimeout(() => {
devServer.watcher.removeListener('change', changeHandler);
reject(new Error('Data store did not update within timeout'));
}, timeout);
});
},
config,
resolveUrl,
fetch: async (url, init) => {
Expand Down

0 comments on commit d63bc50

Please sign in to comment.