From fe9b6325f83f48dfb86e7ebd43a18af9fec3f381 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 6 Jan 2025 12:08:07 +0000 Subject: [PATCH 1/6] wip: add shim for downgrading data --- packages/myst-cli/src/transforms/crossReferences.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index 470c1a956..b957b482e 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -32,6 +32,10 @@ export type MystData = { references?: References; }; +async function maybeDowngradeMystData(rawData: MystData): Promise { + return rawData; +} + async function fetchMystData( session: ISession, dataUrl: string | undefined, @@ -48,7 +52,8 @@ async function fetchMystData( try { const resp = await session.fetch(dataUrl); if (resp.ok) { - const data = (await resp.json()) as MystData; + const rawData = (await resp.json()) as MystData; + const data = await maybeDowngradeMystData(rawData); writeToCache(session, filename, JSON.stringify(data)); return data; } From 20786e59b2764426d333ce337043444687ecc217 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 6 Jan 2025 14:17:28 +0000 Subject: [PATCH 2/6] wip: --- .../src/transforms/crossReferences.ts | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index b957b482e..12bfb116a 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -32,10 +32,6 @@ export type MystData = { references?: References; }; -async function maybeDowngradeMystData(rawData: MystData): Promise { - return rawData; -} - async function fetchMystData( session: ISession, dataUrl: string | undefined, @@ -52,8 +48,7 @@ async function fetchMystData( try { const resp = await session.fetch(dataUrl); if (resp.ok) { - const rawData = (await resp.json()) as MystData; - const data = await maybeDowngradeMystData(rawData); + const data = (await resp.json()) as MystData; writeToCache(session, filename, JSON.stringify(data)); return data; } @@ -84,7 +79,18 @@ export async function fetchMystXRefData(session: ISession, node: CrossReference, if (node.remoteBaseUrl && node.dataUrl) { dataUrl = `${node.remoteBaseUrl}${node.dataUrl}`; } - return fetchMystData(session, dataUrl, node.urlSource, vfile); + const rawData = await fetchMystData(session, dataUrl, node.urlSource, vfile); + let data: MystData | undefined; + if (node.remoteBaseUrl) { + // TODO + } else { + fileWarn( + vfile, + `Unable to determine XRef AST version for external MyST reference: ${node.urlSource}`, + ); + data = rawData; + } + return data; } export function nodesFromMystXRefData( From e9aad2d68627d06a0e04541fc29405834be423ba Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 6 Jan 2025 14:33:11 +0000 Subject: [PATCH 3/6] wip: cache inv --- .../myst-cli/src/process/loadReferences.ts | 23 ++++++------------- .../src/transforms/crossReferences.ts | 6 ++++- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/packages/myst-cli/src/process/loadReferences.ts b/packages/myst-cli/src/process/loadReferences.ts index e347d99ee..177586353 100644 --- a/packages/myst-cli/src/process/loadReferences.ts +++ b/packages/myst-cli/src/process/loadReferences.ts @@ -15,13 +15,11 @@ import { } from '../session/cache.js'; import type { ISession } from '../session/types.js'; import { selectors } from '../store/index.js'; -import { XREF_MAX_AGE } from '../transforms/crossReferences.js'; +import { XREF_MAX_AGE, mystXRefsCacheFilename } from '../transforms/crossReferences.js'; import { dirname } from 'node:path'; -function inventoryCacheFilename(refKind: 'intersphinx' | 'myst', id: string, path: string) { - const hashcontent = `${id}${path}`; - const ext = refKind === 'intersphinx' ? 'inv' : 'json'; - return `xrefs-${refKind}-${computeHash(hashcontent)}.${ext}`; +function sphinxInventoryCacheFilename(path: string) { + return `xrefs-intersphinx-${computeHash(path)}.inv`; } async function preloadReference(session: ISession, key: string, reference: ExternalReference) { @@ -31,11 +29,11 @@ async function preloadReference(session: ISession, key: string, reference: Exter kind: reference.kind, }; const toc = tic(); - const mystXRefFilename = inventoryCacheFilename('myst', key, reference.url); + const mystXRefFilename = mystXRefsCacheFilename(reference.url); const mystXRefData = loadFromCache(session, mystXRefFilename, { maxAge: XREF_MAX_AGE, }); - const intersphinxFilename = inventoryCacheFilename('intersphinx', key, reference.url); + const intersphinxFilename = sphinxInventoryCacheFilename(reference.url); if ((!ref.kind || ref.kind === 'myst') && !!mystXRefData) { session.log.debug(`Loading cached inventory file for ${reference.url}: ${mystXRefFilename}`); const xrefs = JSON.parse(mystXRefData); @@ -93,11 +91,7 @@ async function loadReference( reference.kind = 'myst'; const mystXRefs = (await mystXRefsResp?.json()) as MystXRefs; session.log.debug(`Saving remote myst xref file to cache: ${reference.url}`); - writeToCache( - session, - inventoryCacheFilename('myst', reference.key, reference.url), - JSON.stringify(mystXRefs), - ); + writeToCache(session, mystXRefsCacheFilename(reference.url), JSON.stringify(mystXRefs)); reference.value = mystXRefs; session.log.info( toc(`🏫 Read ${mystXRefs.references.length} myst references for "${reference.key}" in %s.`), @@ -132,10 +126,7 @@ async function loadReference( reference.kind = 'intersphinx'; reference.value = inventory; if (inventory.id && inventory.path && isUrl(inventory.path)) { - const intersphinxPath = cachePath( - session, - inventoryCacheFilename('intersphinx', inventory.id, inventory.path), - ); + const intersphinxPath = cachePath(session, sphinxInventoryCacheFilename(inventory.path)); if (!fs.existsSync(intersphinxPath)) { session.log.debug(`Saving remote inventory file to cache: ${inventory.path}`); fs.mkdirSync(dirname(intersphinxPath), { recursive: true }); diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index 12bfb116a..fc53cfb9d 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -7,7 +7,7 @@ import { addChildrenFromTargetNode } from 'myst-transforms'; import type { PageFrontmatter } from 'myst-frontmatter'; import type { CrossReference, Dependency, Link, SourceFileKind } from 'myst-spec-ext'; import type { ISession } from '../session/types.js'; -import { loadFromCache, writeToCache } from '../session/cache.js'; +import { loadFromCache, writeToCache, checkCache } from '../session/cache.js'; import type { SiteAction, SiteExport } from 'myst-config'; export const XREF_MAX_AGE = 1; // in days @@ -16,6 +16,10 @@ function mystDataFilename(dataUrl: string) { return `myst-${computeHash(dataUrl)}.json`; } +export function mystXRefsCacheFilename(url: string) { + return `xrefs-myst-${computeHash(url)}.json`; +} + export type MystData = { kind?: SourceFileKind; sha256?: string; From a422919a3c5ca61b49bdb8d85a65735dcb345191 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 6 Jan 2025 14:43:48 +0000 Subject: [PATCH 4/6] wip: scaffold for upgrading --- packages/myst-cli/src/transforms/crossReferences.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index fc53cfb9d..4ce10c0c7 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -86,7 +86,17 @@ export async function fetchMystXRefData(session: ISession, node: CrossReference, const rawData = await fetchMystData(session, dataUrl, node.urlSource, vfile); let data: MystData | undefined; if (node.remoteBaseUrl) { - // TODO + const cachePath = mystXRefsCacheFilename(node.remoteBaseUrl); + const mystXRefData = loadFromCache(session, cachePath, { + maxAge: XREF_MAX_AGE, + }); + if (!mystXRefData) { + fileWarn(vfile, `Unable to load external MyST reference data: ${node.remoteBaseUrl}`); + } else { + const { version } = JSON.parse(mystXRefData) as { version: string }; + console.log(`Loading xref ${node.urlSource} with version ${version}`); + } + data = rawData; } else { fileWarn( vfile, From 048ffe3a354124f39cb13364e3d7bdd5dc8acf89 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 6 Jan 2025 15:02:27 +0000 Subject: [PATCH 5/6] wip: downgrading --- .../src/transforms/crossReferences.ts | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index 4ce10c0c7..90fa0ee19 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs'; import type { VFile } from 'vfile'; import { selectAll } from 'unist-util-select'; import type { FrontmatterParts, GenericNode, GenericParent, References } from 'myst-common'; @@ -78,6 +79,42 @@ export async function fetchMystLinkData(session: ISession, node: Link, vfile: VF return fetchMystData(session, node.dataUrl, node.urlSource, vfile); } +const MYST_SPEC_VERSION = '0'; + +function upgradeMystData(version: string, data: any): MystData { + return data; +} + +async function stepwiseDowngrade( + session: ISession, + fromVersion: string, + toVersion: string, + data: any, +): Promise { + const downgradeCachePath = `myst-downgrade-${fromVersion}-${toVersion}.mjs`; + if ( + !checkCache(session, downgradeCachePath, { + maxAge: 7, // days + }) + ) { + const response = await fetch(`http://localhost:9000/${downgradeCachePath}`); + const body = await response.text(); + fs.writeFileSync(downgradeCachePath, body); + } + + const module = await import(downgradeCachePath); + return module(data); +} + +async function downgradeMystData(session: ISession, version: string, data: any): Promise { + const fromVersion = parseInt(version); + const toVersion = parseInt(MYST_SPEC_VERSION); + for (let stepVersion = fromVersion; stepVersion !== toVersion; stepVersion--) { + data = await stepwiseDowngrade(session, `${version}`, `${stepVersion - 1}`, data); + } + return data; +} + export async function fetchMystXRefData(session: ISession, node: CrossReference, vfile: VFile) { let dataUrl: string | undefined; if (node.remoteBaseUrl && node.dataUrl) { @@ -85,16 +122,26 @@ export async function fetchMystXRefData(session: ISession, node: CrossReference, } const rawData = await fetchMystData(session, dataUrl, node.urlSource, vfile); let data: MystData | undefined; - if (node.remoteBaseUrl) { + if (node.remoteBaseUrl && !!rawData) { + // Retrieve the external xref information to determine the spec version const cachePath = mystXRefsCacheFilename(node.remoteBaseUrl); const mystXRefData = loadFromCache(session, cachePath, { maxAge: XREF_MAX_AGE, }); if (!mystXRefData) { fileWarn(vfile, `Unable to load external MyST reference data: ${node.remoteBaseUrl}`); - } else { + } + // Bring potentially incompatible schema into-alignment + else { const { version } = JSON.parse(mystXRefData) as { version: string }; - console.log(`Loading xref ${node.urlSource} with version ${version}`); + if (version === MYST_SPEC_VERSION) { + data = rawData; + } else if (parseInt(version) < parseInt(MYST_SPEC_VERSION)) { + data = upgradeMystData(version, rawData); + } else { + console.log(`Upgrading xref ${node.urlSource} with version ${version}`); + data = await downgradeMystData(session, version, rawData); + } } data = rawData; } else { From 35af04e78b47956664fce064a57df520cf68297a Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 6 Jan 2025 17:36:21 +0000 Subject: [PATCH 6/6] wip: fetch downgrader --- .../src/transforms/crossReferences.ts | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index 90fa0ee19..e2909dd4d 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -8,7 +8,7 @@ import { addChildrenFromTargetNode } from 'myst-transforms'; import type { PageFrontmatter } from 'myst-frontmatter'; import type { CrossReference, Dependency, Link, SourceFileKind } from 'myst-spec-ext'; import type { ISession } from '../session/types.js'; -import { loadFromCache, writeToCache, checkCache } from '../session/cache.js'; +import { loadFromCache, writeToCache, checkCache, cachePath } from '../session/cache.js'; import type { SiteAction, SiteExport } from 'myst-config'; export const XREF_MAX_AGE = 1; // in days @@ -89,6 +89,7 @@ async function stepwiseDowngrade( session: ISession, fromVersion: string, toVersion: string, + vfile: VFile, data: any, ): Promise { const downgradeCachePath = `myst-downgrade-${fromVersion}-${toVersion}.mjs`; @@ -97,20 +98,33 @@ async function stepwiseDowngrade( maxAge: 7, // days }) ) { - const response = await fetch(`http://localhost:9000/${downgradeCachePath}`); - const body = await response.text(); - fs.writeFileSync(downgradeCachePath, body); + try { + const response = await fetch(`http://localhost:9000/${downgradeCachePath}`); + const body = await response.text(); + fs.writeFileSync(cachePath(session, downgradeCachePath), body); + } catch (err) { + fileWarn( + vfile, + `Unable to load utility for downgrading XRef from ${fromVersion} to ${toVersion}`, + ); + return data; + } } - const module = await import(downgradeCachePath); - return module(data); + const module = await import(cachePath(session, downgradeCachePath)); + return module.default(data); } -async function downgradeMystData(session: ISession, version: string, data: any): Promise { +async function downgradeMystData( + session: ISession, + version: string, + vfile: VFile, + data: any, +): Promise { const fromVersion = parseInt(version); const toVersion = parseInt(MYST_SPEC_VERSION); for (let stepVersion = fromVersion; stepVersion !== toVersion; stepVersion--) { - data = await stepwiseDowngrade(session, `${version}`, `${stepVersion - 1}`, data); + data = await stepwiseDowngrade(session, `${version}`, `${stepVersion - 1}`, vfile, data); } return data; } @@ -140,7 +154,7 @@ export async function fetchMystXRefData(session: ISession, node: CrossReference, data = upgradeMystData(version, rawData); } else { console.log(`Upgrading xref ${node.urlSource} with version ${version}`); - data = await downgradeMystData(session, version, rawData); + data = await downgradeMystData(session, version, vfile, rawData); } } data = rawData;