From ac779313e4059fca9181affb3a0b031dd9422f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Wed, 15 Oct 2025 17:06:37 +0200 Subject: [PATCH 01/15] [Blueprints] Support creating a local .git directory via git:directory resource --- .../docs/site/docs/blueprints/04-resources.md | 2 + .../public/blueprint-schema-validator.js | 37 +++- .../blueprints/public/blueprint-schema.json | 4 + .../blueprints/src/lib/v1/resources.spec.ts | 35 ++++ .../blueprints/src/lib/v1/resources.ts | 165 +++++++++++++++++- .../components/src/demos/GitBrowserDemo.tsx | 4 +- .../src/lib/git-sparse-checkout.spec.ts | 7 +- .../storage/src/lib/git-sparse-checkout.ts | 82 +++++++-- 8 files changed, 315 insertions(+), 21 deletions(-) diff --git a/packages/docs/site/docs/blueprints/04-resources.md b/packages/docs/site/docs/blueprints/04-resources.md index f9ce69d8f2..4723139e63 100644 --- a/packages/docs/site/docs/blueprints/04-resources.md +++ b/packages/docs/site/docs/blueprints/04-resources.md @@ -59,6 +59,7 @@ type GitDirectoryReference = { url: string; // Repository URL (https://, ssh git@..., etc.) path?: string; // Optional subdirectory inside the repository ref?: string; // Optional branch, tag, or commit SHA + '.git'?: boolean; // Experimental: include a .git directory with fetched metadata }; ``` @@ -84,6 +85,7 @@ type GitDirectoryReference = { - Playground automatically detects providers like GitHub and GitLab. - It handles CORS-proxied fetches and sparse checkouts, so you can use URLs that point to specific subdirectories or branches. - This resource can be used with steps like [`installPlugin`](/blueprints/steps#InstallPluginStep) and [`installTheme`](/blueprints/steps#InstallThemeStep). +- Set `".git": true` to include a `.git` folder containing packfiles and refs so Git-aware tooling can detect the checkout. This currently mirrors a shallow clone of the selected ref. ### CoreThemeReference diff --git a/packages/playground/blueprints/public/blueprint-schema-validator.js b/packages/playground/blueprints/public/blueprint-schema-validator.js index 4a163be1a2..428f257869 100644 --- a/packages/playground/blueprints/public/blueprint-schema-validator.js +++ b/packages/playground/blueprints/public/blueprint-schema-validator.js @@ -4028,6 +4028,11 @@ const schema25 = { description: 'The path to the directory in the git repository. Defaults to the repo root.', }, + '.git': { + type: 'boolean', + description: + 'When true, include a .git directory in the cloned files', + }, }, required: ['resource', 'url', 'ref'], additionalProperties: false, @@ -4070,7 +4075,8 @@ function validate19( key0 === 'url' || key0 === 'ref' || key0 === 'refType' || - key0 === 'path' + key0 === 'path' || + key0 === '.git' ) ) { validate19.errors = [ @@ -4224,6 +4230,35 @@ function validate19( } else { var valid0 = true; } + if (valid0) { + if (data['.git'] !== undefined) { + const _errs13 = errors; + if ( + typeof data['.git'] !== + 'boolean' + ) { + validate19.errors = [ + { + instancePath: + instancePath + + '/.git', + schemaPath: + '#/properties/.git/type', + keyword: 'type', + params: { + type: 'boolean', + }, + message: + 'must be boolean', + }, + ]; + return false; + } + var valid0 = _errs13 === errors; + } else { + var valid0 = true; + } + } } } } diff --git a/packages/playground/blueprints/public/blueprint-schema.json b/packages/playground/blueprints/public/blueprint-schema.json index 5fa7a73c0a..039dcd14f0 100644 --- a/packages/playground/blueprints/public/blueprint-schema.json +++ b/packages/playground/blueprints/public/blueprint-schema.json @@ -1354,6 +1354,10 @@ "path": { "type": "string", "description": "The path to the directory in the git repository. Defaults to the repo root." + }, + ".git": { + "type": "boolean", + "description": "When true, include a .git directory in the cloned files" } }, "required": ["resource", "url", "ref"], diff --git a/packages/playground/blueprints/src/lib/v1/resources.spec.ts b/packages/playground/blueprints/src/lib/v1/resources.spec.ts index 19772fd3a8..73c62f87a5 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.spec.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.spec.ts @@ -75,6 +75,41 @@ describe('GitDirectoryResource', () => { ); expect(files['dependabot.yml']).toBeInstanceOf(Uint8Array); }); + + it('includes a .git directory when requested', async () => { + const commit = '05138293dd39e25a9fa8e43a9cc775d6fb780e37'; + const resource = new GitDirectoryResource({ + resource: 'git:directory', + url: 'https://github.com/WordPress/wordpress-playground', + ref: commit, + refType: 'commit', + path: 'packages/docs/site/docs/blueprints/tutorial', + '.git': true, + }); + + const { files } = await resource.resolve(); + const head = files['.git/HEAD']; + expect(typeof head).toBe('string'); + expect(head as string).toMatch(/(ref:|[a-f0-9]{40})/i); + + const config = files['.git/config']; + expect(typeof config).toBe('string'); + expect(config as string).toContain( + 'https://github.com/WordPress/wordpress-playground' + ); + + const packKeys = Object.keys(files).filter( + (key) => + key.startsWith('.git/objects/pack/') && + key.endsWith('.pack') + ); + expect(packKeys.length).toBeGreaterThan(0); + for (const key of packKeys) { + expect(files[key]).toBeInstanceOf(Uint8Array); + } + + expect(files['.git/shallow']).toBe(`${commit}\n`); + }); }); describe('name', () => { diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index c9ce9a1d66..99056e79ff 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -11,6 +11,7 @@ import { listGitFiles, resolveCommitHash, sparseCheckout, + type SparseCheckoutPackfile, } from '@wp-playground/storage'; import { zipNameToHumanName } from '../utils/zip-name-to-human-name'; import { fetchWithCorsProxy } from '@php-wasm/web'; @@ -74,6 +75,8 @@ export type GitDirectoryReference = { refType?: GitDirectoryRefType; /** The path to the directory in the git repository. Defaults to the repo root. */ path?: string; + /** When true, include a `.git` directory with Git metadata (experimental). */ + '.git'?: boolean; }; export interface Directory { files: FileTree; @@ -579,12 +582,30 @@ export class GitDirectoryResource extends Resource { const requestedPath = (this.reference.path ?? '').replace(/^\/+/, ''); const filesToClone = listDescendantFiles(allFiles, requestedPath); - let files = await sparseCheckout(repoUrl, commitHash, filesToClone); + const checkout = await sparseCheckout( + repoUrl, + commitHash, + filesToClone + ); + let files = checkout.files; // Remove the path prefix from the cloned file names. files = mapKeys(files, (name) => name.substring(requestedPath.length).replace(/^\/+/, '') ); + if (this.reference['.git']) { + const gitFiles = createGitDirectoryContents({ + repoUrl: this.reference.url, + commitHash, + ref: this.reference.ref, + refType: this.reference.refType, + packfiles: checkout.packfiles, + }); + files = { + ...gitFiles, + ...files, + }; + } return { name: this.filename, files, @@ -624,6 +645,148 @@ function mapKeys(obj: Record, fn: (key: string) => string) { ); } +type GitHeadInfo = { + headContent: string; + branchName?: string; + branchRef?: string; + tagName?: string; +}; + +function createGitDirectoryContents({ + repoUrl, + commitHash, + ref, + refType, + packfiles, +}: { + repoUrl: string; + commitHash: string; + ref: string; + refType?: GitDirectoryRefType; + packfiles: SparseCheckoutPackfile[]; +}): Record { + const gitFiles: Record = {}; + const headInfo = resolveHeadInfo(ref, refType, commitHash); + + gitFiles['.git/HEAD'] = headInfo.headContent; + gitFiles['.git/config'] = buildGitConfig(repoUrl, headInfo.branchName); + gitFiles['.git/description'] = 'WordPress Playground clone\n'; + gitFiles['.git/shallow'] = `${commitHash}\n`; + + if (headInfo.branchRef && headInfo.branchName) { + gitFiles[`.git/${headInfo.branchRef}`] = `${commitHash}\n`; + gitFiles[ + `.git/refs/remotes/origin/${headInfo.branchName}` + ] = `${commitHash}\n`; + gitFiles[ + '.git/refs/remotes/origin/HEAD' + ] = `ref: refs/remotes/origin/${headInfo.branchName}\n`; + } + + if (headInfo.tagName) { + gitFiles[`.git/refs/tags/${headInfo.tagName}`] = `${commitHash}\n`; + } + + const uniquePackfiles = new Map(); + for (const packfile of packfiles) { + if (!uniquePackfiles.has(packfile.name)) { + uniquePackfiles.set(packfile.name, packfile); + } + } + + const packInfoLines: string[] = []; + for (const [name, packfile] of uniquePackfiles) { + const packFilename = `${name}.pack`; + packInfoLines.push(`P ${packFilename}`); + gitFiles[`.git/objects/pack/${packFilename}`] = packfile.pack; + gitFiles[`.git/objects/pack/${name}.idx`] = packfile.index; + } + if (packInfoLines.length > 0) { + gitFiles['.git/objects/info/packs'] = packInfoLines.join('\n') + '\n'; + } + + return gitFiles; +} + +const FULL_SHA_REGEX = /^[0-9a-f]{40}$/i; + +function resolveHeadInfo( + ref: string, + refType: GitDirectoryRefType | undefined, + commitHash: string +): GitHeadInfo { + const trimmed = ref?.trim() ?? ''; + let fullRef: string | null = null; + + switch (refType) { + case 'branch': + if (trimmed) { + fullRef = `refs/heads/${trimmed}`; + } + break; + case 'refname': + fullRef = trimmed || null; + break; + case 'tag': + if (trimmed.startsWith('refs/')) { + fullRef = trimmed; + } else if (trimmed) { + fullRef = `refs/tags/${trimmed}`; + } + break; + case 'commit': + fullRef = null; + break; + default: + if (trimmed.startsWith('refs/')) { + fullRef = trimmed; + } else if (FULL_SHA_REGEX.test(trimmed)) { + fullRef = null; + } else if (trimmed && trimmed !== 'HEAD') { + fullRef = `refs/heads/${trimmed}`; + } + break; + } + + const headContent = fullRef ? `ref: ${fullRef}\n` : `${commitHash}\n`; + + const branchRef = + fullRef && fullRef.startsWith('refs/heads/') ? fullRef : undefined; + const branchName = branchRef?.slice('refs/heads/'.length); + + const tagRef = + fullRef && fullRef.startsWith('refs/tags/') ? fullRef : undefined; + const tagName = tagRef?.slice('refs/tags/'.length); + + return { + headContent, + branchName, + branchRef, + tagName, + }; +} + +function buildGitConfig(repoUrl: string, branchName?: string) { + const lines = [ + '[core]', + '\trepositoryformatversion = 0', + '\tfilemode = true', + '\tbare = false', + '\tlogallrefupdates = true', + '[remote "origin"]', + `\turl = ${repoUrl}`, + '\tfetch = +refs/heads/*:refs/remotes/origin/*', + ]; + if (branchName) { + lines.push( + `[branch "${branchName}"]`, + '\tremote = origin', + `\tmerge = refs/heads/${branchName}` + ); + } + return lines.join('\n') + '\n'; +} + /** * A `Resource` that represents a git directory. */ diff --git a/packages/playground/components/src/demos/GitBrowserDemo.tsx b/packages/playground/components/src/demos/GitBrowserDemo.tsx index 5936ce5e75..fab392d9be 100644 --- a/packages/playground/components/src/demos/GitBrowserDemo.tsx +++ b/packages/playground/components/src/demos/GitBrowserDemo.tsx @@ -73,9 +73,9 @@ export default function GitBrowserDemo() { Object.keys(filesToCheckout) ); const checkedOutFiles: Record = {}; - for (const filename in result) { + for (const filename in result.files) { checkedOutFiles[filename] = new TextDecoder().decode( - result[filename] + result.files[filename] ); } setCheckedOutFiles(checkedOutFiles); diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts index a908aff0f1..36c82ef0f9 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts @@ -105,15 +105,16 @@ describe('sparseCheckout', () => { type: 'branch', } ); - const files = await sparseCheckout( + const result = await sparseCheckout( 'https://github.com/WordPress/wordpress-playground.git', commitHash, ['README.md'] ); - expect(files).toEqual({ + expect(result.files).toEqual({ 'README.md': expect.any(Uint8Array), }); - expect(files['README.md'].length).toBeGreaterThan(0); + expect(result.files['README.md'].length).toBeGreaterThan(0); + expect(result.packfiles.length).toBeGreaterThan(0); }); }); diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.ts b/packages/playground/storage/src/lib/git-sparse-checkout.ts index 24d39cdb21..e4c8e42c9d 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.ts @@ -39,31 +39,65 @@ if (typeof globalThis.Buffer === 'undefined') { * @param fullyQualifiedBranchName The full name of the branch to fetch from (e.g., 'refs/heads/main'). * @param filesPaths An array of all the file paths to fetch from the repository. Does **not** accept * patterns, wildcards, directory paths. All files must be explicitly listed. - * @returns A record where keys are file paths and values are the retrieved file contents. + * @returns The requested files and packfiles required to recreate the Git objects locally. */ +export type SparseCheckoutPackfile = { + name: string; + pack: Uint8Array; + index: Uint8Array; +}; + +export type SparseCheckoutResult = { + files: Record; + packfiles: SparseCheckoutPackfile[]; +}; + export async function sparseCheckout( repoUrl: string, commitHash: string, filesPaths: string[] -) { - const treesIdx = await fetchWithoutBlobs(repoUrl, commitHash); - const objects = await resolveObjects(treesIdx, commitHash, filesPaths); +): Promise { + const treesPack = await fetchWithoutBlobs(repoUrl, commitHash); + const objects = await resolveObjects(treesPack.idx, commitHash, filesPaths); - const blobsIdx = await fetchObjects( - repoUrl, - filesPaths.map((path) => objects[path].oid) - ); + const blobOids = filesPaths.map((path) => objects[path].oid); + const blobsPack = + blobOids.length > 0 ? await fetchObjects(repoUrl, blobOids) : null; const fetchedPaths: Record = {}; await Promise.all( filesPaths.map(async (path) => { + if (!blobsPack) { + return; + } fetchedPaths[path] = await extractGitObjectFromIdx( - blobsIdx, + blobsPack.idx, objects[path].oid ); }) ); - return fetchedPaths; + + const packfiles: SparseCheckoutPackfile[] = []; + const treesIndex = await treesPack.idx.toBuffer(); + packfiles.push({ + name: `pack-${treesPack.idx.packfileSha}`, + pack: treesPack.packfile, + index: toUint8Array(treesIndex), + }); + + if (blobsPack) { + const blobsIndex = await blobsPack.idx.toBuffer(); + packfiles.push({ + name: `pack-${blobsPack.idx.packfileSha}`, + pack: blobsPack.packfile, + index: toUint8Array(blobsIndex), + }); + } + + return { + files: fetchedPaths, + packfiles, + }; } export type GitFileTreeFile = { @@ -113,8 +147,8 @@ export async function listGitFiles( repoUrl: string, commitHash: string ): Promise { - const treesIdx = await fetchWithoutBlobs(repoUrl, commitHash); - const rootTree = await resolveAllObjects(treesIdx, commitHash); + const treesPack = await fetchWithoutBlobs(repoUrl, commitHash); + const rootTree = await resolveAllObjects(treesPack.idx, commitHash); if (!rootTree?.object) { return []; } @@ -359,7 +393,10 @@ async function fetchWithoutBlobs(repoUrl: string, commitHash: string) { result.oid = oid; return result; }; - return idx; + return { + idx, + packfile: toUint8Array(packfile), + }; } async function resolveAllObjects(idx: GitPackIndex, commitHash: string) { @@ -458,9 +495,19 @@ async function fetchObjects(url: string, objectHashes: string[]) { const iterator = streamToIterator(response.body!); const parsed = await parseUploadPackResponse(iterator); const packfile = Buffer.from(await collect(parsed.packfile)); - return await GitPackIndex.fromPack({ + if (packfile.byteLength === 0) { + return { + idx: await GitPackIndex.fromPack({ pack: packfile }), + packfile: new Uint8Array(), + }; + } + const idx = await GitPackIndex.fromPack({ pack: packfile, }); + return { + idx, + packfile: toUint8Array(packfile), + }; } async function extractGitObjectFromIdx(idx: GitPackIndex, objectHash: string) { @@ -545,3 +592,10 @@ function streamToIterator(stream: any) { }, }; } + +function toUint8Array(buffer: Uint8Array | Buffer) { + if (buffer instanceof Uint8Array) { + return Uint8Array.from(buffer); + } + return Uint8Array.from(buffer); +} From 52aadb2af95a832e4466f6f53f32637293f398eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Wed, 15 Oct 2025 18:24:53 +0200 Subject: [PATCH 02/15] Creating a working .git directory --- .../blueprints/src/lib/v1/resources.spec.ts | 30 +++++++ .../blueprints/src/lib/v1/resources.ts | 85 +++++++++++++------ .../src/lib/git-sparse-checkout.spec.ts | 4 + .../storage/src/lib/git-sparse-checkout.ts | 59 ++++++++++++- 4 files changed, 153 insertions(+), 25 deletions(-) diff --git a/packages/playground/blueprints/src/lib/v1/resources.spec.ts b/packages/playground/blueprints/src/lib/v1/resources.spec.ts index 73c62f87a5..f3164b1f01 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.spec.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.spec.ts @@ -97,6 +97,32 @@ describe('GitDirectoryResource', () => { expect(config as string).toContain( 'https://github.com/WordPress/wordpress-playground' ); + expect(config as string).toContain('repositoryformatversion = 1'); + expect(config as string).toContain('promisor = true'); + expect(config as string).toContain( + 'fetch = +refs/tags/*:refs/tags/*' + ); + expect(config as string).toContain( + 'partialclonefilter = blob:none' + ); + expect(files['.git/info/exclude']).toBe( + '# git ls-files --others --exclude-standard\n' + ); + expect(files['.git/logs/HEAD']).toBeDefined(); + expect(files['.git/packed-refs']).toContain('refs/heads'); + expect(files['.git/packed-refs']).toContain( + 'refs/remotes/origin/trunk' + ); + expect(files['.git/refs/heads/trunk']).toBe(`${commit}\n`); + expect(files['.git/refs/remotes/origin/trunk']).toBe(`${commit}\n`); + expect(files['.git/refs/remotes/origin/HEAD']).toBe( + 'ref: refs/remotes/origin/trunk\n' + ); + const objectPath = `.git/objects/${commit.slice( + 0, + 2 + )}/${commit.slice(2)}`; + expect(files[objectPath]).toBeInstanceOf(Uint8Array); const packKeys = Object.keys(files).filter( (key) => @@ -107,6 +133,10 @@ describe('GitDirectoryResource', () => { for (const key of packKeys) { expect(files[key]).toBeInstanceOf(Uint8Array); } + const promisorKeys = Object.keys(files).filter((key) => + key.endsWith('.promisor') + ); + expect(promisorKeys.length).toBeGreaterThan(0); expect(files['.git/shallow']).toBe(`${commit}\n`); }); diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index 99056e79ff..bc8a1c7b9d 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -11,12 +11,14 @@ import { listGitFiles, resolveCommitHash, sparseCheckout, - type SparseCheckoutPackfile, + type SparseCheckoutObject, } from '@wp-playground/storage'; import { zipNameToHumanName } from '../utils/zip-name-to-human-name'; import { fetchWithCorsProxy } from '@php-wasm/web'; import { StreamedFile } from '@php-wasm/stream-compression'; import type { StreamBundledFile } from './types'; +import pako from 'pako'; +const deflate = pako.deflate; export type { FileTree }; export const ResourceTypes = [ @@ -599,7 +601,7 @@ export class GitDirectoryResource extends Resource { commitHash, ref: this.reference.ref, refType: this.reference.refType, - packfiles: checkout.packfiles, + objects: checkout.objects, }); files = { ...gitFiles, @@ -657,23 +659,37 @@ function createGitDirectoryContents({ commitHash, ref, refType, - packfiles, + objects, }: { repoUrl: string; commitHash: string; ref: string; refType?: GitDirectoryRefType; - packfiles: SparseCheckoutPackfile[]; + objects: SparseCheckoutObject[]; }): Record { const gitFiles: Record = {}; const headInfo = resolveHeadInfo(ref, refType, commitHash); gitFiles['.git/HEAD'] = headInfo.headContent; - gitFiles['.git/config'] = buildGitConfig(repoUrl, headInfo.branchName); + gitFiles['.git/config'] = buildGitConfig(repoUrl, { + branchName: headInfo.branchName, + }); gitFiles['.git/description'] = 'WordPress Playground clone\n'; gitFiles['.git/shallow'] = `${commitHash}\n`; + // const logEntry = `${'0'.repeat( + // 40 + // )} ${commitHash} Playground 0 +0000\tclone: from ${repoUrl}\n`; + // gitFiles['.git/logs/HEAD'] = logEntry; + // gitFiles['.git/info/exclude'] = + // '# git ls-files --others --exclude-standard\n'; + + // Create refs/ directory structure + gitFiles['.git/refs/heads/.gitkeep'] = ''; + gitFiles['.git/refs/tags/.gitkeep'] = ''; + gitFiles['.git/refs/remotes/.gitkeep'] = ''; if (headInfo.branchRef && headInfo.branchName) { + gitFiles['.git/logs/HEAD'] = `ref: ${headInfo.branchRef}\n`; gitFiles[`.git/${headInfo.branchRef}`] = `${commitHash}\n`; gitFiles[ `.git/refs/remotes/origin/${headInfo.branchName}` @@ -681,35 +697,40 @@ function createGitDirectoryContents({ gitFiles[ '.git/refs/remotes/origin/HEAD' ] = `ref: refs/remotes/origin/${headInfo.branchName}\n`; + // gitFiles[`.git/logs/${headInfo.branchRef}`] = logEntry; } if (headInfo.tagName) { gitFiles[`.git/refs/tags/${headInfo.tagName}`] = `${commitHash}\n`; } - const uniquePackfiles = new Map(); - for (const packfile of packfiles) { - if (!uniquePackfiles.has(packfile.name)) { - uniquePackfiles.set(packfile.name, packfile); - } - } - - const packInfoLines: string[] = []; - for (const [name, packfile] of uniquePackfiles) { - const packFilename = `${name}.pack`; - packInfoLines.push(`P ${packFilename}`); - gitFiles[`.git/objects/pack/${packFilename}`] = packfile.pack; - gitFiles[`.git/objects/pack/${name}.idx`] = packfile.index; - } - if (packInfoLines.length > 0) { - gitFiles['.git/objects/info/packs'] = packInfoLines.join('\n') + '\n'; - } + // Use loose objects only, no packfiles + Object.assign(gitFiles, createLooseGitObjectFiles(objects)); return gitFiles; } const FULL_SHA_REGEX = /^[0-9a-f]{40}$/i; +function createLooseGitObjectFiles(objects: SparseCheckoutObject[]) { + const files: Record = {}; + const encoder = new TextEncoder(); + for (const { oid, type, body } of objects) { + if (!oid || body.length === 0) { + continue; + } + const header = encoder.encode(`${type} ${body.length}\0`); + const combined = new Uint8Array(header.length + body.length); + combined.set(header, 0); + combined.set(body, header.length); + const compressed = deflate(combined); + const prefix = oid.slice(0, 2); + const suffix = oid.slice(2); + files[`.git/objects/${prefix}/${suffix}`] = compressed; + } + return files; +} + function resolveHeadInfo( ref: string, refType: GitDirectoryRefType | undefined, @@ -766,17 +787,33 @@ function resolveHeadInfo( }; } -function buildGitConfig(repoUrl: string, branchName?: string) { +function buildGitConfig( + repoUrl: string, + { + branchName, + partialCloneFilter, + }: { branchName?: string; partialCloneFilter?: string } +) { + const repositoryFormatVersion = partialCloneFilter ? 1 : 0; const lines = [ '[core]', - '\trepositoryformatversion = 0', + `\trepositoryformatversion = ${repositoryFormatVersion}`, '\tfilemode = true', '\tbare = false', '\tlogallrefupdates = true', + '\tignorecase = true', + '\tprecomposeunicode = true', '[remote "origin"]', `\turl = ${repoUrl}`, '\tfetch = +refs/heads/*:refs/remotes/origin/*', + '\tfetch = +refs/tags/*:refs/tags/*', ]; + if (partialCloneFilter) { + lines.push('\tpromisor = true'); + lines.push(`\tpartialclonefilter = ${partialCloneFilter}`); + lines.push('[extensions]'); + lines.push('\tpartialclone = origin'); + } if (branchName) { lines.push( `[branch "${branchName}"]`, diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts index 36c82ef0f9..030cb965c0 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts @@ -115,6 +115,10 @@ describe('sparseCheckout', () => { }); expect(result.files['README.md'].length).toBeGreaterThan(0); expect(result.packfiles.length).toBeGreaterThan(0); + expect(result.packfiles.some((packfile) => packfile.promisor)).toBe( + true + ); + expect(result.objects.length).toBeGreaterThan(0); }); }); diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.ts b/packages/playground/storage/src/lib/git-sparse-checkout.ts index e4c8e42c9d..08ab84c06e 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.ts @@ -45,11 +45,19 @@ export type SparseCheckoutPackfile = { name: string; pack: Uint8Array; index: Uint8Array; + promisor?: boolean; +}; + +export type SparseCheckoutObject = { + oid: string; + type: 'blob' | 'tree' | 'commit' | 'tag'; + body: Uint8Array; }; export type SparseCheckoutResult = { files: Record; packfiles: SparseCheckoutPackfile[]; + objects: SparseCheckoutObject[]; }; export async function sparseCheckout( @@ -83,6 +91,7 @@ export async function sparseCheckout( name: `pack-${treesPack.idx.packfileSha}`, pack: treesPack.packfile, index: toUint8Array(treesIndex), + promisor: treesPack.promisor, }); if (blobsPack) { @@ -91,12 +100,17 @@ export async function sparseCheckout( name: `pack-${blobsPack.idx.packfileSha}`, pack: blobsPack.packfile, index: toUint8Array(blobsIndex), + promisor: blobsPack.promisor, }); } return { files: fetchedPaths, packfiles, + objects: [ + ...(await collectLooseObjects(treesPack)), + ...(await collectLooseObjects(blobsPack)), + ], }; } @@ -396,6 +410,7 @@ async function fetchWithoutBlobs(repoUrl: string, commitHash: string) { return { idx, packfile: toUint8Array(packfile), + promisor: true, }; } @@ -423,6 +438,43 @@ async function resolveAllObjects(idx: GitPackIndex, commitHash: string) { return rootItem; } +async function collectLooseObjects( + pack?: { + idx: GitPackIndex; + packfile: Uint8Array; + promisor?: boolean; + } | null +): Promise { + if (!pack) { + return []; + } + const results: SparseCheckoutObject[] = []; + const seen = new Set(); + for (const oid of pack.idx.hashes ?? []) { + if (seen.has(oid)) { + continue; + } + const offset = pack.idx.offsets.get(oid); + if (offset === undefined) { + continue; + } + const { type, object } = await pack.idx.readSlice({ start: offset }); + if (type === 'ofs_delta' || type === 'ref_delta') { + continue; + } + if (!object) { + continue; + } + seen.add(oid); + results.push({ + oid, + type: type as SparseCheckoutObject['type'], + body: toUint8Array(object as Uint8Array), + }); + } + return results; +} + async function resolveObjects( idx: GitPackIndex, commitHash: string, @@ -496,9 +548,13 @@ async function fetchObjects(url: string, objectHashes: string[]) { const parsed = await parseUploadPackResponse(iterator); const packfile = Buffer.from(await collect(parsed.packfile)); if (packfile.byteLength === 0) { + const idx = await GitPackIndex.fromPack({ + pack: packfile, + }); return { - idx: await GitPackIndex.fromPack({ pack: packfile }), + idx, packfile: new Uint8Array(), + promisor: false, }; } const idx = await GitPackIndex.fromPack({ @@ -507,6 +563,7 @@ async function fetchObjects(url: string, objectHashes: string[]) { return { idx, packfile: toUint8Array(packfile), + promisor: false, }; } From b3b20e85022576f013a01a68625a43daddd2bbef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Wed, 15 Oct 2025 22:23:10 +0200 Subject: [PATCH 03/15] git index creation --- .../blueprints/src/lib/v1/isomorphic-git.d.ts | 22 ++++++++++ .../blueprints/src/lib/v1/resources.ts | 42 +++++++++++++++++-- .../storage/src/lib/git-sparse-checkout.ts | 7 ++++ 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts diff --git a/packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts b/packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts new file mode 100644 index 0000000000..8f23ccd02b --- /dev/null +++ b/packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts @@ -0,0 +1,22 @@ +declare module 'isomorphic-git/src/models/GitIndex.js' { + export class GitIndex { + constructor(entries?: Map, unmergedPaths?: Set); + insert(entry: { + filepath: string; + oid: string; + stats: { + ctimeSeconds: number; + ctimeNanoseconds: number; + mtimeSeconds: number; + mtimeNanoseconds: number; + dev: number; + ino: number; + mode: number; + uid: number; + gid: number; + size: number; + }; + }): void; + toObject(): Promise; + } +} diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index bc8a1c7b9d..d787bc493b 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -1,3 +1,4 @@ +import './isomorphic-git.d.ts'; import type { ProgressTracker } from '@php-wasm/progress'; import { cloneResponseMonitorProgress, @@ -13,6 +14,7 @@ import { sparseCheckout, type SparseCheckoutObject, } from '@wp-playground/storage'; +import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; import { zipNameToHumanName } from '../utils/zip-name-to-human-name'; import { fetchWithCorsProxy } from '@php-wasm/web'; import { StreamedFile } from '@php-wasm/stream-compression'; @@ -596,12 +598,14 @@ export class GitDirectoryResource extends Resource { name.substring(requestedPath.length).replace(/^\/+/, '') ); if (this.reference['.git']) { - const gitFiles = createGitDirectoryContents({ + const gitFiles = await createGitDirectoryContents({ repoUrl: this.reference.url, commitHash, ref: this.reference.ref, refType: this.reference.refType, objects: checkout.objects, + fileOids: checkout.fileOids, + pathPrefix: requestedPath, }); files = { ...gitFiles, @@ -654,19 +658,23 @@ type GitHeadInfo = { tagName?: string; }; -function createGitDirectoryContents({ +async function createGitDirectoryContents({ repoUrl, commitHash, ref, refType, objects, + fileOids, + pathPrefix, }: { repoUrl: string; commitHash: string; ref: string; refType?: GitDirectoryRefType; objects: SparseCheckoutObject[]; -}): Record { + fileOids: Record; + pathPrefix: string; +}): Promise> { const gitFiles: Record = {}; const headInfo = resolveHeadInfo(ref, refType, commitHash); @@ -707,6 +715,34 @@ function createGitDirectoryContents({ // Use loose objects only, no packfiles Object.assign(gitFiles, createLooseGitObjectFiles(objects)); + // Create the git index + const index = new GitIndex(); + for (const [path, oid] of Object.entries(fileOids)) { + // Remove the path prefix to get the working tree relative path + const workingTreePath = path + .substring(pathPrefix.length) + .replace(/^\/+/, ''); + index.insert({ + filepath: workingTreePath, + oid, + stats: { + ctimeSeconds: 0, + ctimeNanoseconds: 0, + mtimeSeconds: 0, + mtimeNanoseconds: 0, + dev: 0, + ino: 0, + mode: 0o100644, // Regular file + uid: 0, + gid: 0, + size: 0, + }, + }); + } + const indexBuffer = await index.toObject(); + // Convert Buffer to Uint8Array - copy the data to ensure it's a proper Uint8Array + gitFiles['.git/index'] = Uint8Array.from(indexBuffer); + return gitFiles; } diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.ts b/packages/playground/storage/src/lib/git-sparse-checkout.ts index 08ab84c06e..1a3655eb1e 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.ts @@ -58,6 +58,7 @@ export type SparseCheckoutResult = { files: Record; packfiles: SparseCheckoutPackfile[]; objects: SparseCheckoutObject[]; + fileOids: Record; }; export async function sparseCheckout( @@ -104,6 +105,11 @@ export async function sparseCheckout( }); } + const fileOids: Record = {}; + for (const path of filesPaths) { + fileOids[path] = objects[path].oid; + } + return { files: fetchedPaths, packfiles, @@ -111,6 +117,7 @@ export async function sparseCheckout( ...(await collectLooseObjects(treesPack)), ...(await collectLooseObjects(blobsPack)), ], + fileOids, }; } From fdf88efc52b0d627d12eaefb24da1f2cb9fb6829 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 07:58:37 +0000 Subject: [PATCH 04/15] Initial plan From 92240083977efe15e817ce26a3f74e4cefe1621c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Oct 2025 08:25:34 +0000 Subject: [PATCH 05/15] Remove pako dependency and move .git directory creation logic to git.ts Co-authored-by: adamziel <205419+adamziel@users.noreply.github.com> --- .../playground/blueprints/src/lib/v1/git.ts | 252 ++++++++++++++++++ .../blueprints/src/lib/v1/resources.ts | 214 +-------------- 2 files changed, 253 insertions(+), 213 deletions(-) create mode 100644 packages/playground/blueprints/src/lib/v1/git.ts diff --git a/packages/playground/blueprints/src/lib/v1/git.ts b/packages/playground/blueprints/src/lib/v1/git.ts new file mode 100644 index 0000000000..0ddbc7e64c --- /dev/null +++ b/packages/playground/blueprints/src/lib/v1/git.ts @@ -0,0 +1,252 @@ +import './isomorphic-git.d.ts'; +import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; +import type { SparseCheckoutObject } from '@wp-playground/storage'; + +/** + * Deflate compression utility that works in both Node.js and browsers. + * In Node.js, uses the built-in zlib module for better performance. + * In browsers, uses the native CompressionStream API when available. + */ +async function deflate(buffer: Uint8Array): Promise { + // Try Node.js zlib first (synchronous and faster) + try { + // Dynamic import to avoid bundler issues + const { deflateRawSync } = await import('zlib'); + return new Uint8Array(deflateRawSync(buffer)); + } catch { + // Fall back to browser CompressionStream + try { + const cs = new CompressionStream('deflate-raw'); + const stream = new Blob([buffer]).stream().pipeThrough(cs); + return new Uint8Array(await new Response(stream).arrayBuffer()); + } catch { + throw new Error( + 'No compression method available. Please ensure you are running in Node.js 11+ or a modern browser with CompressionStream support.' + ); + } + } +} + +type GitDirectoryRefType = 'branch' | 'tag' | 'commit' | 'refname'; + +type GitHeadInfo = { + headContent: string; + branchName?: string; + branchRef?: string; + tagName?: string; +}; + +const FULL_SHA_REGEX = /^[0-9a-f]{40}$/i; + +/** + * Creates loose Git object files from sparse checkout objects. + * Each object is compressed using deflate and stored in the Git objects directory. + */ +async function createLooseGitObjectFiles( + objects: SparseCheckoutObject[] +): Promise> { + const files: Record = {}; + const encoder = new TextEncoder(); + + await Promise.all( + objects.map(async ({ oid, type, body }) => { + if (!oid || body.length === 0) { + return; + } + const header = encoder.encode(`${type} ${body.length}\0`); + const combined = new Uint8Array(header.length + body.length); + combined.set(header, 0); + combined.set(body, header.length); + const compressed = await deflate(combined); + const prefix = oid.slice(0, 2); + const suffix = oid.slice(2); + files[`.git/objects/${prefix}/${suffix}`] = compressed; + }) + ); + + return files; +} + +/** + * Resolves the HEAD reference information based on the ref type and value. + */ +function resolveHeadInfo( + ref: string, + refType: GitDirectoryRefType | undefined, + commitHash: string +): GitHeadInfo { + const trimmed = ref?.trim() ?? ''; + let fullRef: string | null = null; + + switch (refType) { + case 'branch': + if (trimmed) { + fullRef = `refs/heads/${trimmed}`; + } + break; + case 'refname': + fullRef = trimmed || null; + break; + case 'tag': + if (trimmed.startsWith('refs/')) { + fullRef = trimmed; + } else if (trimmed) { + fullRef = `refs/tags/${trimmed}`; + } + break; + case 'commit': + fullRef = null; + break; + default: + if (trimmed.startsWith('refs/')) { + fullRef = trimmed; + } else if (FULL_SHA_REGEX.test(trimmed)) { + fullRef = null; + } else if (trimmed && trimmed !== 'HEAD') { + fullRef = `refs/heads/${trimmed}`; + } + break; + } + + const headContent = fullRef ? `ref: ${fullRef}\n` : `${commitHash}\n`; + + const branchRef = + fullRef && fullRef.startsWith('refs/heads/') ? fullRef : undefined; + const branchName = branchRef?.slice('refs/heads/'.length); + + const tagRef = + fullRef && fullRef.startsWith('refs/tags/') ? fullRef : undefined; + const tagName = tagRef?.slice('refs/tags/'.length); + + return { + headContent, + branchName, + branchRef, + tagName, + }; +} + +/** + * Builds a Git config file content with remote and branch configuration. + */ +function buildGitConfig( + repoUrl: string, + { + branchName, + partialCloneFilter, + }: { branchName?: string; partialCloneFilter?: string } +): string { + const repositoryFormatVersion = partialCloneFilter ? 1 : 0; + const lines = [ + '[core]', + `\trepositoryformatversion = ${repositoryFormatVersion}`, + '\tfilemode = true', + '\tbare = false', + '\tlogallrefupdates = true', + '\tignorecase = true', + '\tprecomposeunicode = true', + '[remote "origin"]', + `\turl = ${repoUrl}`, + '\tfetch = +refs/heads/*:refs/remotes/origin/*', + '\tfetch = +refs/tags/*:refs/tags/*', + ]; + if (partialCloneFilter) { + lines.push('\tpromisor = true'); + lines.push(`\tpartialclonefilter = ${partialCloneFilter}`); + lines.push('[extensions]'); + lines.push('\tpartialclone = origin'); + } + if (branchName) { + lines.push( + `[branch "${branchName}"]`, + '\tremote = origin', + `\tmerge = refs/heads/${branchName}` + ); + } + return lines.join('\n') + '\n'; +} + +/** + * Creates a complete .git directory structure with all necessary files. + * This includes HEAD, config, refs, objects, and the Git index. + */ +export async function createGitDirectoryContents({ + repoUrl, + commitHash, + ref, + refType, + objects, + fileOids, + pathPrefix, +}: { + repoUrl: string; + commitHash: string; + ref: string; + refType?: GitDirectoryRefType; + objects: SparseCheckoutObject[]; + fileOids: Record; + pathPrefix: string; +}): Promise> { + const gitFiles: Record = {}; + const headInfo = resolveHeadInfo(ref, refType, commitHash); + + gitFiles['.git/HEAD'] = headInfo.headContent; + gitFiles['.git/config'] = buildGitConfig(repoUrl, { + branchName: headInfo.branchName, + }); + gitFiles['.git/description'] = 'WordPress Playground clone\n'; + gitFiles['.git/shallow'] = `${commitHash}\n`; + + // Create refs/ directory structure + gitFiles['.git/refs/heads/.gitkeep'] = ''; + gitFiles['.git/refs/tags/.gitkeep'] = ''; + gitFiles['.git/refs/remotes/.gitkeep'] = ''; + + if (headInfo.branchRef && headInfo.branchName) { + gitFiles['.git/logs/HEAD'] = `ref: ${headInfo.branchRef}\n`; + gitFiles[`.git/${headInfo.branchRef}`] = `${commitHash}\n`; + gitFiles[ + `.git/refs/remotes/origin/${headInfo.branchName}` + ] = `${commitHash}\n`; + gitFiles[ + '.git/refs/remotes/origin/HEAD' + ] = `ref: refs/remotes/origin/${headInfo.branchName}\n`; + } + + if (headInfo.tagName) { + gitFiles[`.git/refs/tags/${headInfo.tagName}`] = `${commitHash}\n`; + } + + // Use loose objects only, no packfiles + Object.assign(gitFiles, await createLooseGitObjectFiles(objects)); + + // Create the git index + const index = new GitIndex(); + for (const [path, oid] of Object.entries(fileOids)) { + // Remove the path prefix to get the working tree relative path + const workingTreePath = path + .substring(pathPrefix.length) + .replace(/^\/+/, ''); + index.insert({ + filepath: workingTreePath, + oid, + stats: { + ctimeSeconds: 0, + ctimeNanoseconds: 0, + mtimeSeconds: 0, + mtimeNanoseconds: 0, + dev: 0, + ino: 0, + mode: 0o100644, // Regular file + uid: 0, + gid: 0, + size: 0, + }, + }); + } + const indexBuffer = await index.toObject(); + // Convert Buffer to Uint8Array - copy the data to ensure it's a proper Uint8Array + gitFiles['.git/index'] = Uint8Array.from(indexBuffer); + + return gitFiles; +} diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index d787bc493b..284897db6b 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -12,15 +12,12 @@ import { listGitFiles, resolveCommitHash, sparseCheckout, - type SparseCheckoutObject, } from '@wp-playground/storage'; -import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; import { zipNameToHumanName } from '../utils/zip-name-to-human-name'; import { fetchWithCorsProxy } from '@php-wasm/web'; import { StreamedFile } from '@php-wasm/stream-compression'; import type { StreamBundledFile } from './types'; -import pako from 'pako'; -const deflate = pako.deflate; +import { createGitDirectoryContents } from './git'; export type { FileTree }; export const ResourceTypes = [ @@ -651,215 +648,6 @@ function mapKeys(obj: Record, fn: (key: string) => string) { ); } -type GitHeadInfo = { - headContent: string; - branchName?: string; - branchRef?: string; - tagName?: string; -}; - -async function createGitDirectoryContents({ - repoUrl, - commitHash, - ref, - refType, - objects, - fileOids, - pathPrefix, -}: { - repoUrl: string; - commitHash: string; - ref: string; - refType?: GitDirectoryRefType; - objects: SparseCheckoutObject[]; - fileOids: Record; - pathPrefix: string; -}): Promise> { - const gitFiles: Record = {}; - const headInfo = resolveHeadInfo(ref, refType, commitHash); - - gitFiles['.git/HEAD'] = headInfo.headContent; - gitFiles['.git/config'] = buildGitConfig(repoUrl, { - branchName: headInfo.branchName, - }); - gitFiles['.git/description'] = 'WordPress Playground clone\n'; - gitFiles['.git/shallow'] = `${commitHash}\n`; - // const logEntry = `${'0'.repeat( - // 40 - // )} ${commitHash} Playground 0 +0000\tclone: from ${repoUrl}\n`; - // gitFiles['.git/logs/HEAD'] = logEntry; - // gitFiles['.git/info/exclude'] = - // '# git ls-files --others --exclude-standard\n'; - - // Create refs/ directory structure - gitFiles['.git/refs/heads/.gitkeep'] = ''; - gitFiles['.git/refs/tags/.gitkeep'] = ''; - gitFiles['.git/refs/remotes/.gitkeep'] = ''; - - if (headInfo.branchRef && headInfo.branchName) { - gitFiles['.git/logs/HEAD'] = `ref: ${headInfo.branchRef}\n`; - gitFiles[`.git/${headInfo.branchRef}`] = `${commitHash}\n`; - gitFiles[ - `.git/refs/remotes/origin/${headInfo.branchName}` - ] = `${commitHash}\n`; - gitFiles[ - '.git/refs/remotes/origin/HEAD' - ] = `ref: refs/remotes/origin/${headInfo.branchName}\n`; - // gitFiles[`.git/logs/${headInfo.branchRef}`] = logEntry; - } - - if (headInfo.tagName) { - gitFiles[`.git/refs/tags/${headInfo.tagName}`] = `${commitHash}\n`; - } - - // Use loose objects only, no packfiles - Object.assign(gitFiles, createLooseGitObjectFiles(objects)); - - // Create the git index - const index = new GitIndex(); - for (const [path, oid] of Object.entries(fileOids)) { - // Remove the path prefix to get the working tree relative path - const workingTreePath = path - .substring(pathPrefix.length) - .replace(/^\/+/, ''); - index.insert({ - filepath: workingTreePath, - oid, - stats: { - ctimeSeconds: 0, - ctimeNanoseconds: 0, - mtimeSeconds: 0, - mtimeNanoseconds: 0, - dev: 0, - ino: 0, - mode: 0o100644, // Regular file - uid: 0, - gid: 0, - size: 0, - }, - }); - } - const indexBuffer = await index.toObject(); - // Convert Buffer to Uint8Array - copy the data to ensure it's a proper Uint8Array - gitFiles['.git/index'] = Uint8Array.from(indexBuffer); - - return gitFiles; -} - -const FULL_SHA_REGEX = /^[0-9a-f]{40}$/i; - -function createLooseGitObjectFiles(objects: SparseCheckoutObject[]) { - const files: Record = {}; - const encoder = new TextEncoder(); - for (const { oid, type, body } of objects) { - if (!oid || body.length === 0) { - continue; - } - const header = encoder.encode(`${type} ${body.length}\0`); - const combined = new Uint8Array(header.length + body.length); - combined.set(header, 0); - combined.set(body, header.length); - const compressed = deflate(combined); - const prefix = oid.slice(0, 2); - const suffix = oid.slice(2); - files[`.git/objects/${prefix}/${suffix}`] = compressed; - } - return files; -} - -function resolveHeadInfo( - ref: string, - refType: GitDirectoryRefType | undefined, - commitHash: string -): GitHeadInfo { - const trimmed = ref?.trim() ?? ''; - let fullRef: string | null = null; - - switch (refType) { - case 'branch': - if (trimmed) { - fullRef = `refs/heads/${trimmed}`; - } - break; - case 'refname': - fullRef = trimmed || null; - break; - case 'tag': - if (trimmed.startsWith('refs/')) { - fullRef = trimmed; - } else if (trimmed) { - fullRef = `refs/tags/${trimmed}`; - } - break; - case 'commit': - fullRef = null; - break; - default: - if (trimmed.startsWith('refs/')) { - fullRef = trimmed; - } else if (FULL_SHA_REGEX.test(trimmed)) { - fullRef = null; - } else if (trimmed && trimmed !== 'HEAD') { - fullRef = `refs/heads/${trimmed}`; - } - break; - } - - const headContent = fullRef ? `ref: ${fullRef}\n` : `${commitHash}\n`; - - const branchRef = - fullRef && fullRef.startsWith('refs/heads/') ? fullRef : undefined; - const branchName = branchRef?.slice('refs/heads/'.length); - - const tagRef = - fullRef && fullRef.startsWith('refs/tags/') ? fullRef : undefined; - const tagName = tagRef?.slice('refs/tags/'.length); - - return { - headContent, - branchName, - branchRef, - tagName, - }; -} - -function buildGitConfig( - repoUrl: string, - { - branchName, - partialCloneFilter, - }: { branchName?: string; partialCloneFilter?: string } -) { - const repositoryFormatVersion = partialCloneFilter ? 1 : 0; - const lines = [ - '[core]', - `\trepositoryformatversion = ${repositoryFormatVersion}`, - '\tfilemode = true', - '\tbare = false', - '\tlogallrefupdates = true', - '\tignorecase = true', - '\tprecomposeunicode = true', - '[remote "origin"]', - `\turl = ${repoUrl}`, - '\tfetch = +refs/heads/*:refs/remotes/origin/*', - '\tfetch = +refs/tags/*:refs/tags/*', - ]; - if (partialCloneFilter) { - lines.push('\tpromisor = true'); - lines.push(`\tpartialclonefilter = ${partialCloneFilter}`); - lines.push('[extensions]'); - lines.push('\tpartialclone = origin'); - } - if (branchName) { - lines.push( - `[branch "${branchName}"]`, - '\tremote = origin', - `\tmerge = refs/heads/${branchName}` - ); - } - return lines.join('\n') + '\n'; -} - /** * A `Resource` that represents a git directory. */ From a857d47be85d395f0acb98cdf513ed40b62340a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 12:03:36 +0200 Subject: [PATCH 06/15] Move git ops to the storage package, add a test that uses actual git cli tool --- package-lock.json | 8 + package.json | 1 + .../blueprints/src/lib/v1/resources.spec.ts | 145 ++++++++++++------ .../blueprints/src/lib/v1/resources.ts | 4 +- .../playground/blueprints/tsconfig.lib.json | 5 +- packages/playground/storage/src/index.ts | 1 + .../src/lib/git-create-dotgit-directory.ts} | 30 +--- 7 files changed, 114 insertions(+), 80 deletions(-) rename packages/playground/{blueprints/src/lib/v1/git.ts => storage/src/lib/git-create-dotgit-directory.ts} (85%) diff --git a/package-lock.json b/package-lock.json index 1906bbe0c3..26d1c1a65e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -80,6 +80,7 @@ "@types/ini": "4.1.0", "@types/jest": "29.5.14", "@types/node": "20.14.8", + "@types/pako": "1.0.4", "@types/react": "18.3.1", "@types/react-dom": "18.3.0", "@types/react-modal": "3.16.3", @@ -15511,6 +15512,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/pako": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.4.tgz", + "integrity": "sha512-Z+5bJSm28EXBSUJEgx29ioWeEEHUh6TiMkZHDhLwjc9wVFH+ressbkmX6waUZc5R3Gobn4Qu5llGxaoflZ+yhA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", diff --git a/package.json b/package.json index e3e8dfa03c..0c1f2b86a5 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ "@types/ini": "4.1.0", "@types/jest": "29.5.14", "@types/node": "20.14.8", + "@types/pako": "1.0.4", "@types/react": "18.3.1", "@types/react-dom": "18.3.0", "@types/react-modal": "3.16.3", diff --git a/packages/playground/blueprints/src/lib/v1/resources.spec.ts b/packages/playground/blueprints/src/lib/v1/resources.spec.ts index f3164b1f01..2f0bf34b82 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.spec.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.spec.ts @@ -5,6 +5,10 @@ import { } from './resources'; import { expect, describe, it, vi, beforeEach } from 'vitest'; import { StreamedFile } from '@php-wasm/stream-compression'; +import { mkdtemp, rm, writeFile, mkdir } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { execSync, type ExecSyncOptions } from 'child_process'; describe('UrlResource', () => { it('should create a new instance of UrlResource', () => { @@ -88,57 +92,98 @@ describe('GitDirectoryResource', () => { }); const { files } = await resource.resolve(); - const head = files['.git/HEAD']; - expect(typeof head).toBe('string'); - expect(head as string).toMatch(/(ref:|[a-f0-9]{40})/i); - - const config = files['.git/config']; - expect(typeof config).toBe('string'); - expect(config as string).toContain( - 'https://github.com/WordPress/wordpress-playground' - ); - expect(config as string).toContain('repositoryformatversion = 1'); - expect(config as string).toContain('promisor = true'); - expect(config as string).toContain( - 'fetch = +refs/tags/*:refs/tags/*' - ); - expect(config as string).toContain( - 'partialclonefilter = blob:none' - ); - expect(files['.git/info/exclude']).toBe( - '# git ls-files --others --exclude-standard\n' - ); - expect(files['.git/logs/HEAD']).toBeDefined(); - expect(files['.git/packed-refs']).toContain('refs/heads'); - expect(files['.git/packed-refs']).toContain( - 'refs/remotes/origin/trunk' - ); - expect(files['.git/refs/heads/trunk']).toBe(`${commit}\n`); - expect(files['.git/refs/remotes/origin/trunk']).toBe(`${commit}\n`); - expect(files['.git/refs/remotes/origin/HEAD']).toBe( - 'ref: refs/remotes/origin/trunk\n' - ); - const objectPath = `.git/objects/${commit.slice( - 0, - 2 - )}/${commit.slice(2)}`; - expect(files[objectPath]).toBeInstanceOf(Uint8Array); - - const packKeys = Object.keys(files).filter( - (key) => - key.startsWith('.git/objects/pack/') && - key.endsWith('.pack') - ); - expect(packKeys.length).toBeGreaterThan(0); - for (const key of packKeys) { - expect(files[key]).toBeInstanceOf(Uint8Array); - } - const promisorKeys = Object.keys(files).filter((key) => - key.endsWith('.promisor') - ); - expect(promisorKeys.length).toBeGreaterThan(0); - expect(files['.git/shallow']).toBe(`${commit}\n`); + // Create a temporary directory and write all files to disk + const tmpDir = await mkdtemp(join(tmpdir(), 'git-test-')); + try { + // Write all files to the temporary directory + for (const [path, content] of Object.entries(files)) { + const fullPath = join(tmpDir, path); + const dir = join(fullPath, '..'); + await mkdir(dir, { recursive: true }); + + if (typeof content === 'string') { + await writeFile(fullPath, content, 'utf8'); + } else { + await writeFile(fullPath, content); + } + } + + // Run git commands to verify the repository state + const gitEnv: ExecSyncOptions = { + cwd: tmpDir, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, // 10MB buffer to handle large output + stdio: ['pipe', 'pipe', 'ignore'], // Suppress stderr to avoid buffer overflow + }; + + // Verify we're on the expected commit + const currentCommit = execSync('git rev-parse HEAD', gitEnv) + .toString() + .trim(); + expect(currentCommit).toBe(commit); + + // Verify the remote is configured correctly + const remoteUrl = execSync('git remote get-url origin', gitEnv) + .toString() + .trim(); + expect(remoteUrl).toBe( + 'https://github.com/WordPress/wordpress-playground' + ); + + // Verify this is a shallow clone + const isShallow = execSync( + 'git rev-parse --is-shallow-repository', + gitEnv + ) + .toString() + .trim(); + expect(isShallow).toBe('true'); + + // Verify the shallow file contains the expected commit + const shallowCommit = execSync('cat .git/shallow', gitEnv) + .toString() + .trim(); + expect(shallowCommit).toBe(commit); + + // Verify the expected files exist in the git index + const lsFiles = execSync('git ls-files', gitEnv) + .toString() + .trim() + .split('\n') + .filter((f) => f.length > 0) + .sort(); + expect(lsFiles).toEqual([ + '01-what-are-blueprints-what-you-can-do-with-them.md', + '02-how-to-load-run-blueprints.md', + '03-build-your-first-blueprint.md', + 'index.md', + ]); + + // Verify we can run git log to see commit history + const logOutput = execSync('git log --oneline -n 1', gitEnv) + .toString() + .trim(); + expect(logOutput).toContain(commit.substring(0, 7)); + + // Update the git index to match the actual files on disk + execSync('git add -A', gitEnv); + + // Modify a file and verify git status detects the change + const fileToModify = join(tmpDir, 'index.md'); + await writeFile(fileToModify, 'modified content\n', 'utf8'); + const statusAfterModification = execSync( + 'git status --porcelain', + gitEnv + ) + .toString() + .trim(); + // Git status should show the file as modified (can be ' M' or 'M ') + expect(statusAfterModification).toMatch(/M.*index\.md/); + } finally { + // Clean up the temporary directory + await rm(tmpDir, { recursive: true, force: true }); + } }); }); diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index 284897db6b..e21804547a 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -17,7 +17,7 @@ import { zipNameToHumanName } from '../utils/zip-name-to-human-name'; import { fetchWithCorsProxy } from '@php-wasm/web'; import { StreamedFile } from '@php-wasm/stream-compression'; import type { StreamBundledFile } from './types'; -import { createGitDirectoryContents } from './git'; +import { createDotGitDirectory } from '@wp-playground/storage'; export type { FileTree }; export const ResourceTypes = [ @@ -595,7 +595,7 @@ export class GitDirectoryResource extends Resource { name.substring(requestedPath.length).replace(/^\/+/, '') ); if (this.reference['.git']) { - const gitFiles = await createGitDirectoryContents({ + const gitFiles = await createDotGitDirectory({ repoUrl: this.reference.url, commitHash, ref: this.reference.ref, diff --git a/packages/playground/blueprints/tsconfig.lib.json b/packages/playground/blueprints/tsconfig.lib.json index 829b0bc14c..a623dd0b3c 100644 --- a/packages/playground/blueprints/tsconfig.lib.json +++ b/packages/playground/blueprints/tsconfig.lib.json @@ -5,6 +5,9 @@ "declaration": true, "types": ["node"] }, - "include": ["src/**/*.ts"], + "include": [ + "src/**/*.ts", + "../storage/src/lib/git-create-dotgit-directory.ts" + ], "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] } diff --git a/packages/playground/storage/src/index.ts b/packages/playground/storage/src/index.ts index 776099b921..5587719e52 100644 --- a/packages/playground/storage/src/index.ts +++ b/packages/playground/storage/src/index.ts @@ -3,5 +3,6 @@ export * from './lib/changeset'; export * from './lib/playground'; export * from './lib/browser-fs'; export * from './lib/git-sparse-checkout'; +export * from './lib/git-create-dotgit-directory'; export * from './lib/paths'; export * from './lib/filesystems'; diff --git a/packages/playground/blueprints/src/lib/v1/git.ts b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts similarity index 85% rename from packages/playground/blueprints/src/lib/v1/git.ts rename to packages/playground/storage/src/lib/git-create-dotgit-directory.ts index 0ddbc7e64c..d539f4fe57 100644 --- a/packages/playground/blueprints/src/lib/v1/git.ts +++ b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts @@ -1,31 +1,7 @@ -import './isomorphic-git.d.ts'; import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; import type { SparseCheckoutObject } from '@wp-playground/storage'; - -/** - * Deflate compression utility that works in both Node.js and browsers. - * In Node.js, uses the built-in zlib module for better performance. - * In browsers, uses the native CompressionStream API when available. - */ -async function deflate(buffer: Uint8Array): Promise { - // Try Node.js zlib first (synchronous and faster) - try { - // Dynamic import to avoid bundler issues - const { deflateRawSync } = await import('zlib'); - return new Uint8Array(deflateRawSync(buffer)); - } catch { - // Fall back to browser CompressionStream - try { - const cs = new CompressionStream('deflate-raw'); - const stream = new Blob([buffer]).stream().pipeThrough(cs); - return new Uint8Array(await new Response(stream).arrayBuffer()); - } catch { - throw new Error( - 'No compression method available. Please ensure you are running in Node.js 11+ or a modern browser with CompressionStream support.' - ); - } - } -} +import pako from 'pako'; +const deflate = pako.deflate; type GitDirectoryRefType = 'branch' | 'tag' | 'commit' | 'refname'; @@ -170,7 +146,7 @@ function buildGitConfig( * Creates a complete .git directory structure with all necessary files. * This includes HEAD, config, refs, objects, and the Git index. */ -export async function createGitDirectoryContents({ +export async function createDotGitDirectory({ repoUrl, commitHash, ref, From c3babecc8bcace5a1c6cef956833d5c8f0e0c2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 12:53:26 +0200 Subject: [PATCH 07/15] Fix TypeScript errors in git-sparse-checkout --- .../storage/src/lib/git-sparse-checkout.ts | 22 +++++++++---------- .../storage/src/lib/isomorphic-git.d.ts | 14 ++++++++++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.ts b/packages/playground/storage/src/lib/git-sparse-checkout.ts index 1a3655eb1e..5379e3bca4 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.ts @@ -231,7 +231,7 @@ export async function listGitRefs( fullyQualifiedBranchPrefix: string ) { const packbuffer = Buffer.from( - await collect([ + (await collect([ GitPktLine.encode(`command=ls-refs\n`), GitPktLine.encode(`agent=git/2.37.3\n`), GitPktLine.encode(`object-format=sha1\n`), @@ -239,7 +239,7 @@ export async function listGitRefs( GitPktLine.encode(`peel\n`), GitPktLine.encode(`ref-prefix ${fullyQualifiedBranchPrefix}\n`), GitPktLine.flush(), - ]) + ])) as any ); const response = await fetch(repoUrl + '/git-upload-pack', { @@ -250,7 +250,7 @@ export async function listGitRefs( 'Content-Length': `${packbuffer.length}`, 'Git-Protocol': 'version=2', }, - body: packbuffer, + body: packbuffer as any, }); const refs: Record = {}; @@ -379,7 +379,7 @@ async function fetchRefOid(repoUrl: string, refname: string) { async function fetchWithoutBlobs(repoUrl: string, commitHash: string) { const packbuffer = Buffer.from( - await collect([ + (await collect([ GitPktLine.encode( `want ${commitHash} multi_ack_detailed no-done side-band-64k thin-pack ofs-delta agent=git/2.37.3 filter \n` ), @@ -389,7 +389,7 @@ async function fetchWithoutBlobs(repoUrl: string, commitHash: string) { GitPktLine.flush(), GitPktLine.encode(`done\n`), GitPktLine.encode(`done\n`), - ]) + ])) as any ); const response = await fetch(repoUrl + '/git-upload-pack', { @@ -399,12 +399,12 @@ async function fetchWithoutBlobs(repoUrl: string, commitHash: string) { 'content-type': 'application/x-git-upload-pack-request', 'Content-Length': `${packbuffer.length}`, }, - body: packbuffer, + body: packbuffer as any, }); const iterator = streamToIterator(response.body!); const parsed = await parseUploadPackResponse(iterator); - const packfile = Buffer.from(await collect(parsed.packfile)); + const packfile = Buffer.from((await collect(parsed.packfile)) as any); const idx = await GitPackIndex.fromPack({ pack: packfile, }); @@ -530,7 +530,7 @@ async function resolveObjects( // Request oid for each resolvedRef async function fetchObjects(url: string, objectHashes: string[]) { const packbuffer = Buffer.from( - await collect([ + (await collect([ ...objectHashes.map((objectHash) => GitPktLine.encode( `want ${objectHash} multi_ack_detailed no-done side-band-64k thin-pack ofs-delta agent=git/2.37.3 \n` @@ -538,7 +538,7 @@ async function fetchObjects(url: string, objectHashes: string[]) { ), GitPktLine.flush(), GitPktLine.encode(`done\n`), - ]) + ])) as any ); const response = await fetch(url + '/git-upload-pack', { @@ -548,12 +548,12 @@ async function fetchObjects(url: string, objectHashes: string[]) { 'content-type': 'application/x-git-upload-pack-request', 'Content-Length': `${packbuffer.length}`, }, - body: packbuffer, + body: packbuffer as any, }); const iterator = streamToIterator(response.body!); const parsed = await parseUploadPackResponse(iterator); - const packfile = Buffer.from(await collect(parsed.packfile)); + const packfile = Buffer.from((await collect(parsed.packfile)) as any); if (packfile.byteLength === 0) { const idx = await GitPackIndex.fromPack({ pack: packfile, diff --git a/packages/playground/storage/src/lib/isomorphic-git.d.ts b/packages/playground/storage/src/lib/isomorphic-git.d.ts index 7b95b48ecf..2cf3399e0f 100644 --- a/packages/playground/storage/src/lib/isomorphic-git.d.ts +++ b/packages/playground/storage/src/lib/isomorphic-git.d.ts @@ -72,6 +72,20 @@ declare module 'isomorphic-git/src/models/GitPackIndex.js' { export class GitPackIndex { static fromPack({ pack }: { pack: Buffer }): Promise; read({ oid }: { oid: string }): Promise; + toBuffer(): Promise; + packfileSha: string; + hashes?: string[]; + offsets: Map; + readSlice({ start }: { start: number }): Promise<{ + type: + | 'blob' + | 'tree' + | 'commit' + | 'tag' + | 'ofs_delta' + | 'ref_delta'; + object?: Buffer | Uint8Array; + }>; } } From c7eebc425d9d92ab917c07d07a8741982743a28f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 13:03:35 +0200 Subject: [PATCH 08/15] Fix TypeScript errors in gpt-create-dotgit --- .../storage/src/lib/git-create-dotgit-directory.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts index d539f4fe57..d543ee0c74 100644 --- a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts +++ b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts @@ -1,5 +1,7 @@ -import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; -import type { SparseCheckoutObject } from '@wp-playground/storage'; +// @ts-expect-error +import { GitIndex } from './isomorphic-git/src/models/GitIndex.js'; +import type { SparseCheckoutObject } from './git-sparse-checkout'; +// @ts-expect-error import pako from 'pako'; const deflate = pako.deflate; From 6214e35857ddbd850799cc3b85e95f67508e8410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 13:08:55 +0200 Subject: [PATCH 09/15] Fix TypeScript errors in gpt-create-dotgit --- .../playground/storage/src/lib/git-create-dotgit-directory.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts index d543ee0c74..021e5939d6 100644 --- a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts +++ b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts @@ -1,7 +1,6 @@ // @ts-expect-error import { GitIndex } from './isomorphic-git/src/models/GitIndex.js'; import type { SparseCheckoutObject } from './git-sparse-checkout'; -// @ts-expect-error import pako from 'pako'; const deflate = pako.deflate; From 736d068ca6e3a626cc27eb082a98ffcc48ff2601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 13:33:17 +0200 Subject: [PATCH 10/15] Fix more TypeScript errors --- .../blueprints/src/lib/v1/isomorphic-git.d.ts | 22 ------------------ .../src/lib/git-create-dotgit-directory.ts | 2 +- .../storage/src/lib/isomorphic-git.d.ts | 23 +++++++++++++++++++ 3 files changed, 24 insertions(+), 23 deletions(-) delete mode 100644 packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts diff --git a/packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts b/packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts deleted file mode 100644 index 8f23ccd02b..0000000000 --- a/packages/playground/blueprints/src/lib/v1/isomorphic-git.d.ts +++ /dev/null @@ -1,22 +0,0 @@ -declare module 'isomorphic-git/src/models/GitIndex.js' { - export class GitIndex { - constructor(entries?: Map, unmergedPaths?: Set); - insert(entry: { - filepath: string; - oid: string; - stats: { - ctimeSeconds: number; - ctimeNanoseconds: number; - mtimeSeconds: number; - mtimeNanoseconds: number; - dev: number; - ino: number; - mode: number; - uid: number; - gid: number; - size: number; - }; - }): void; - toObject(): Promise; - } -} diff --git a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts index 021e5939d6..dae80a772f 100644 --- a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts +++ b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts @@ -1,5 +1,5 @@ // @ts-expect-error -import { GitIndex } from './isomorphic-git/src/models/GitIndex.js'; +import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; import type { SparseCheckoutObject } from './git-sparse-checkout'; import pako from 'pako'; const deflate = pako.deflate; diff --git a/packages/playground/storage/src/lib/isomorphic-git.d.ts b/packages/playground/storage/src/lib/isomorphic-git.d.ts index 2cf3399e0f..f8c0857a81 100644 --- a/packages/playground/storage/src/lib/isomorphic-git.d.ts +++ b/packages/playground/storage/src/lib/isomorphic-git.d.ts @@ -1,3 +1,26 @@ +declare module 'isomorphic-git/src/models/GitIndex.js' { + export class GitIndex { + constructor(entries?: Map, unmergedPaths?: Set); + insert(entry: { + filepath: string; + oid: string; + stats: { + ctimeSeconds: number; + ctimeNanoseconds: number; + mtimeSeconds: number; + mtimeNanoseconds: number; + dev: number; + ino: number; + mode: number; + uid: number; + gid: number; + size: number; + }; + }): void; + toObject(): Promise; + } +} + declare module 'isomorphic-git/src/models/GitPktLine.js' { export class GitPktLine { static encode(data: string): Buffer; From 34b4ed5aa24b037bdfb7bbeaedbbf63a10656b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 13:39:43 +0200 Subject: [PATCH 11/15] Fix more TypeScript errors --- .../playground/storage/src/lib/git-create-dotgit-directory.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts index dae80a772f..21ae6d4b35 100644 --- a/packages/playground/storage/src/lib/git-create-dotgit-directory.ts +++ b/packages/playground/storage/src/lib/git-create-dotgit-directory.ts @@ -1,4 +1,3 @@ -// @ts-expect-error import { GitIndex } from 'isomorphic-git/src/models/GitIndex.js'; import type { SparseCheckoutObject } from './git-sparse-checkout'; import pako from 'pako'; From 8a26867d19a8aa0260adfb3a1193984507fd5f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 14:23:31 +0200 Subject: [PATCH 12/15] Fix more TypeScript errors --- packages/playground/blueprints/src/lib/v1/resources.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index e21804547a..5e2cf5c461 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -1,4 +1,3 @@ -import './isomorphic-git.d.ts'; import type { ProgressTracker } from '@php-wasm/progress'; import { cloneResponseMonitorProgress, From 6f28842724ba072580c3207df51d57bb1415e8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 22:33:40 +0200 Subject: [PATCH 13/15] sparseCheckout: Only fetch additional metadata when needed, not pre-emptively --- .../playground/blueprints/src/lib/v1/resources.ts | 9 ++++++--- .../storage/src/lib/git-sparse-checkout.spec.ts | 6 +++--- .../storage/src/lib/git-sparse-checkout.ts | 15 +++++++++++---- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/playground/blueprints/src/lib/v1/resources.ts b/packages/playground/blueprints/src/lib/v1/resources.ts index 5e2cf5c461..675ca531f5 100644 --- a/packages/playground/blueprints/src/lib/v1/resources.ts +++ b/packages/playground/blueprints/src/lib/v1/resources.ts @@ -585,7 +585,10 @@ export class GitDirectoryResource extends Resource { const checkout = await sparseCheckout( repoUrl, commitHash, - filesToClone + filesToClone, + { + withObjects: this.reference['.git'], + } ); let files = checkout.files; @@ -599,8 +602,8 @@ export class GitDirectoryResource extends Resource { commitHash, ref: this.reference.ref, refType: this.reference.refType, - objects: checkout.objects, - fileOids: checkout.fileOids, + objects: checkout.objects ?? [], + fileOids: checkout.fileOids ?? {}, pathPrefix: requestedPath, }); files = { diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts index 030cb965c0..f63795b9f8 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts @@ -114,11 +114,11 @@ describe('sparseCheckout', () => { 'README.md': expect.any(Uint8Array), }); expect(result.files['README.md'].length).toBeGreaterThan(0); - expect(result.packfiles.length).toBeGreaterThan(0); - expect(result.packfiles.some((packfile) => packfile.promisor)).toBe( + expect(result.packfiles?.length).toBeGreaterThan(0); + expect(result.packfiles?.some((packfile) => packfile.promisor)).toBe( true ); - expect(result.objects.length).toBeGreaterThan(0); + expect(result.objects?.length).toBeGreaterThan(0); }); }); diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.ts b/packages/playground/storage/src/lib/git-sparse-checkout.ts index 5379e3bca4..3303b902cd 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.ts @@ -56,15 +56,18 @@ export type SparseCheckoutObject = { export type SparseCheckoutResult = { files: Record; - packfiles: SparseCheckoutPackfile[]; - objects: SparseCheckoutObject[]; - fileOids: Record; + packfiles?: SparseCheckoutPackfile[]; + objects?: SparseCheckoutObject[]; + fileOids?: Record; }; export async function sparseCheckout( repoUrl: string, commitHash: string, - filesPaths: string[] + filesPaths: string[], + options?: { + withObjects?: boolean; + } ): Promise { const treesPack = await fetchWithoutBlobs(repoUrl, commitHash); const objects = await resolveObjects(treesPack.idx, commitHash, filesPaths); @@ -86,6 +89,10 @@ export async function sparseCheckout( }) ); + if (options?.withObjects) { + return { files: fetchedPaths }; + } + const packfiles: SparseCheckoutPackfile[] = []; const treesIndex = await treesPack.idx.toBuffer(); packfiles.push({ From e2a5ae082bb8665206726a6c6a90efc411dd8b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 22:43:20 +0200 Subject: [PATCH 14/15] Reverse the condition in sparseCheckout --- packages/playground/storage/src/lib/git-sparse-checkout.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.ts b/packages/playground/storage/src/lib/git-sparse-checkout.ts index 3303b902cd..a977fc33bf 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.ts @@ -89,7 +89,11 @@ export async function sparseCheckout( }) ); - if (options?.withObjects) { + /** + * Short-circuit if the consumer doesn't need additional details about + * the Git objects. + */ + if (!options?.withObjects) { return { files: fetchedPaths }; } From 899eec550d50a4d6ba25950b16d7a3a98fd2f09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Thu, 16 Oct 2025 23:09:50 +0200 Subject: [PATCH 15/15] Adjust the unit tests --- .../src/lib/git-sparse-checkout.spec.ts | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts index f63795b9f8..1026bdd5a4 100644 --- a/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts +++ b/packages/playground/storage/src/lib/git-sparse-checkout.spec.ts @@ -97,7 +97,7 @@ describe('resolveCommitHash', () => { }); describe('sparseCheckout', () => { - it('should retrieve the requested files from a git repo', async () => { + it('should retrieve the requested files and objects from a git repo when withObjects is true', async () => { const commitHash = await resolveCommitHash( 'https://github.com/WordPress/wordpress-playground.git', { @@ -108,7 +108,10 @@ describe('sparseCheckout', () => { const result = await sparseCheckout( 'https://github.com/WordPress/wordpress-playground.git', commitHash, - ['README.md'] + ['README.md'], + { + withObjects: true, + } ); expect(result.files).toEqual({ 'README.md': expect.any(Uint8Array), @@ -120,6 +123,30 @@ describe('sparseCheckout', () => { ); expect(result.objects?.length).toBeGreaterThan(0); }); + + it('should retrieve only the requested files from a git repo when withObjects is false', async () => { + const commitHash = await resolveCommitHash( + 'https://github.com/WordPress/wordpress-playground.git', + { + value: 'trunk', + type: 'branch', + } + ); + const result = await sparseCheckout( + 'https://github.com/WordPress/wordpress-playground.git', + commitHash, + ['README.md'], + { + withObjects: false, + } + ); + expect(result.files).toEqual({ + 'README.md': expect.any(Uint8Array), + }); + expect(result.files['README.md'].length).toBeGreaterThan(0); + expect(result.packfiles).toBeUndefined(); + expect(result.objects).toBeUndefined(); + }); }); describe('listGitFiles', () => {