support for by-hash indexing

src/deb/cas.mts

@@ -0,0 +1,45 @@
+import * as path from 'node:path';
+import { PassThrough, Readable } from 'node:stream';
+import { buffer } from 'node:stream/consumers';
+import { createGzip } from 'node:zlib';
+import { createHash } from 'node:crypto';
+import { createWriteStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+
+import { createDir } from '../fs.mjs';
+
+export interface ContentAddress {
+	sha256: Uint8Array;
+}
+
+export interface ContentDescription {
+	size: number;
+	address: ContentAddress;
+}
+
+export async function storeGzippedStream(root: string, stream: NodeJS.ReadableStream): Promise<ContentDescription> {
+	const gzip = createGzip({ level: 9 });
+	const gzippedStream = new PassThrough();
+	await pipeline(stream, gzip, gzippedStream);
+	return storeStream(root, gzippedStream);
+}
+
+// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29
+export async function storeStream(root: string, stream: NodeJS.ReadableStream): Promise<ContentDescription> {
+	const contentBuffer = await buffer(stream);
+
+	// compute a digest
+	const sha256Hash = createHash('sha256');
+	await pipeline(Readable.from(contentBuffer), sha256Hash);
+	const sha256 = sha256Hash.read();
+
+	// store in a file
+	const sha256Dir = path.join(root, 'SHA256');
+	const fileName = path.join(sha256Dir, Buffer.from(sha256).toString('hex'));
+	createDir(sha256Dir);
+	await pipeline(Readable.from(contentBuffer), createWriteStream(fileName));
+	return {
+		size: contentBuffer.length,
+		address: { sha256 },
+	};
+}

src/deb/deb-builder.mts

@@ -1,10 +1,13 @@
-import { createGzip } from 'zlib';
+import * as path from 'node:path';
+import { appendFile, writeFile } from 'node:fs/promises';
+import { readdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { Readable } from 'node:stream';
 
-import * as fs from 'fs';
 import * as ini from 'ini';
-import * as path from 'path';
 import * as tar from 'tar';
 
+import * as cas from './cas.mjs';
 import type { Artifact, ArtifactProvider } from '../artifact-provider.mjs';
 import { createDir, execToolToFile, removeDir } from '../fs.mjs';
 import type { Deployer } from '../deployer.mjs';
@@ -23,7 +26,9 @@ const ReleaseFileTemplate =
 Label: Ubuntu/Debian
 Architecture: $ARCH
 Component: $COMPONENT
-Codename: $DISTRIBUTION\n`;
+Acquire-By-Hash: yes
+Codename: $DISTRIBUTION
+Date: $DATE\n`;
 
 function iterateComponents(repo: DebRepo, callback: (distribution: string, component: string, deb: Artifact[]) => void): void {
 	const distributions = Object.keys(repo);
@@ -94,23 +99,26 @@ export class DebBuilder implements Deployer {
 		return `${this.config.applicationName}-${version}_${arch}.deb`;
 	}
 
-	private async makeReleaseFileAndSign(distribution: string, component: string, arch: string): Promise<void> {
-		const releaseContent = ReleaseFileTemplate
+	// eslint-disable-next-line max-params
+	private async makeReleaseFileAndSign(distribution: string, component: string, arch: string, indices: BinaryPackageIndexDescription[]): Promise<void> {
+		let releaseContent = ReleaseFileTemplate
 			.replace('$ORIGIN', this.config.origin)
 			.replace('$DISTRIBUTION', distribution)
 			.replace('$ARCH', arch)
-			.replace('$COMPONENT', component);
+			.replace('$COMPONENT', component)
+			.replace('$DATE', new Date().toUTCString());
 
-		const distributionPath = path.join(this.distsPath, distribution);
-		const releaseFilePath = path.join(distributionPath, 'Release');
-		const releaseGpgFilePath = path.join(distributionPath, 'Release.gpg');
-		const inReleaseFilePath = path.join(distributionPath, 'InRelease');
+		releaseContent += 'SHA256:\n';
+		for (const { address, size, name } of indices) {
+			releaseContent += ` ${Buffer.from(address.sha256).toString('hex')}\t${size}\t${name}\n`;
+		}
 
-		await fs.promises.writeFile(releaseFilePath, releaseContent);
+		const tempReleaseFile = path.join(this.tempPath, `Release-${randomBytes(4).toString('hex')}`);
+		await writeFile(tempReleaseFile, releaseContent);
 
-		await execToolToFile('apt-ftparchive', ['release', distributionPath], releaseFilePath, true);
-		await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '-abs', '-o', releaseGpgFilePath, releaseFilePath]);
-		await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, releaseFilePath]);
+		const distributionPath = path.join(this.distsPath, distribution);
+		const inReleaseFilePath = path.join(distributionPath, 'InRelease');
+		await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, tempReleaseFile]);
 	}
 
 	private async dpkgScanpackages(): Promise<void> {
@@ -144,7 +152,7 @@ export class DebBuilder implements Deployer {
 				.pipe(tar.extract({ cwd: whereExtract, strip: 1 }, ['./control']))
 				// eslint-disable-next-line max-statements
 				.on('finish', () => {
-					const controlMetaContent = fs.readFileSync(path.join(whereExtract, 'control'), 'utf-8').replaceAll(':', '=');
+					const controlMetaContent = readFileSync(path.join(whereExtract, 'control'), 'utf-8').replaceAll(':', '=');
 					const controlMeta = ini.parse(controlMetaContent);
 					const arch = controlMeta['Architecture'];
 					const version = controlMeta['Version'];
@@ -163,7 +171,7 @@ export class DebBuilder implements Deployer {
 						`binary-${arch}`,
 						`${this.debFileName(version, arch)}.meta`);
 					createDir(path.dirname(targetMetaPath));
-					fs.renameSync(path.join(whereExtract, 'control'), targetMetaPath);
+					renameSync(path.join(whereExtract, 'control'), targetMetaPath);
 
 					removeDir(whereExtract);
 
@@ -173,7 +181,7 @@ export class DebBuilder implements Deployer {
 						this.config.applicationName,
 						distribution,
 						this.debFileName(version, arch));
-					const relativeDebPath = path.relative(this.rootPath, debPath);
+					const relativeDebPath = path.relative(this.rootPath, debPath).replace(/\\/gu, '/');
 					const debSize = controlTar.headers['content-range']?.split('/')[1];
 					const sha1 = controlTar.headers['x-checksum-sha1'];
 					const sha256 = controlTar.headers['x-checksum-sha256'];
@@ -185,7 +193,7 @@ export class DebBuilder implements Deployer {
 
 					const dataToAppend = `Filename: ${relativeDebPath}\nSize: ${debSize}\nSHA1: ${sha1}\nSHA256: ${sha256}\nMD5Sum: ${md5}\n`;
 
-					fs.promises.appendFile(targetMetaPath, dataToAppend).then(() => resolve());
+					appendFile(targetMetaPath, dataToAppend).then(() => resolve());
 
 					const createFileOperation = this.packageCreator(deb.md5, debPath);
 
@@ -201,57 +209,56 @@ export class DebBuilder implements Deployer {
 	}
 
 	private async makeRelease(): Promise<{}> {
-		const compressFile = (filePath: string): Promise<void> => new Promise<void>(resolve => {
-			const inp = fs.createReadStream(filePath);
-			const out = fs.createWriteStream(`${filePath}.gz`);
-
-			const gzip = createGzip({ level: 9 });
-
-			inp.pipe(gzip).pipe(out)
-				.on('finish', () => {
-					resolve();
-				});
-		});
-
-		const compressPromises: Promise<void>[] = [];
+		const binaryPackageIndices: BinaryPackageIndexDescription[] = [];
 
+		const componentPromises: Promise<void[]>[] = [];
 		iterateComponents(this.config.repo, (distribution, component) => {
 			const componentRoot = path.join(this.distsPath, distribution, component);
-			const componentsByArch = fs.readdirSync(componentRoot).map(dist => path.join(componentRoot, dist));
-
-			componentsByArch.forEach(dist => {
-				const targetPackagesFile = path.join(dist, 'Packages');
-				const metaFiles = fs.readdirSync(dist)
-					.filter(fileName => fileName.endsWith('.meta'))
-					.map(metaFile => path.join(dist, metaFile));
-
-				let packagesContent = '';
-
-				for (const metaFile of metaFiles) {
-					packagesContent += fs.readFileSync(metaFile);
-					packagesContent += '\n';
-					fs.unlinkSync(metaFile);
-				}
-
-				fs.writeFileSync(targetPackagesFile, packagesContent);
+			componentPromises.push(Promise.all(
+				readdirSync(componentRoot).map(async binarySubdir => {
+					const binaryPath = path.join(componentRoot, binarySubdir);
+					const metaFiles = readdirSync(binaryPath)
+						.filter(fileName => fileName.endsWith('.meta'))
+						.map(metaFile => path.join(binaryPath, metaFile));
+
+					let packagesContent = '';
+
+					for (const metaFile of metaFiles) {
+						packagesContent += readFileSync(metaFile);
+						packagesContent += '\n';
+						unlinkSync(metaFile);
+					}
 
-				compressPromises.push(compressFile(targetPackagesFile));
-			});
+					const storePackageIndex = async(
+						shortName: string,
+						store: (root: string, stream: NodeJS.ReadableStream) => Promise<cas.ContentDescription>,
+					) => {
+						const name = `${component}/${binarySubdir}/${shortName}`;
+						const { size, address } = await store(path.resolve(binaryPath, 'by-hash'), Readable.from(packagesContent));
+						binaryPackageIndices.push({ name, size, address });
+					};
+					await storePackageIndex('Packages', cas.storeStream);
+					await storePackageIndex('Packages.gz', cas.storeGzippedStream);
+				}),
+			));
 		});
-
-		await Promise.all(compressPromises);
+		await Promise.all(componentPromises);
 
 		const releasesPromises: Promise<void>[] = [];
-
 		iterateComponents(this.config.repo, (distribution, component) => {
 			const archesSet = this.archesByDistComp.get(`${distribution}/${component}`);
 			if (!archesSet) {
 				throw new Error('No arch was found for distribution');
 			}
 
-			releasesPromises.push(this.makeReleaseFileAndSign(distribution, component, [...archesSet.values()].join(' ')));
+			releasesPromises.push(this.makeReleaseFileAndSign(distribution, component, [...archesSet.values()].join(' '), binaryPackageIndices));
 		});
-
 		return Promise.all(releasesPromises);
 	}
 }
+
+interface BinaryPackageIndexDescription {
+	name: string;
+	size: number;
+	address: cas.ContentAddress;
+}