Merge pull request #20 from bdalpe/feature/lookups

feature/lookups

lib/__tests__/__snapshots__/lookups.test.ts.snap

@@ -0,0 +1,22 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Lookups > should only write the pipelines/$PIPELINE/conf.yml 1`] = `
+"id,value
+1,test
+2,goat
+"
+`;
+
+exports[`Lookups > should only write to data/lookups/{id} 1`] = `
+"id,value
+1,test
+2,goat
+"
+`;
+
+exports[`Lookups > should use the correct extension 1`] = `
+"id,value
+1,test
+2,goat
+"
+`;

lib/__tests__/lookups.test.ts

@@ -0,0 +1,59 @@
+import {it, expect, vi, beforeEach, describe} from 'vitest';
+import {vol} from 'memfs'
+import {Cribl} from "../index";
+import {Lookup} from "../objects/lookups";
+import {load} from "js-yaml";
+
+vi.mock('node:fs');
+vi.mock('node:fs/promises');
+
+let cribl: Cribl;
+
+beforeEach(() => {
+	// reset the state of in-memory fs
+	vol.reset()
+
+	cribl = new Cribl({outdir: '/tmp'});
+});
+
+const lookupFile = Buffer.from(`id,value
+1,test
+2,goat
+`)
+
+describe('Lookups', () => {
+	it('should only write to data/lookups/{id}', () => {
+		new Lookup(cribl, 'test', lookupFile);
+		cribl.synth();
+
+		const volume = vol.toJSON();
+		expect(Object.keys(volume)).toStrictEqual(expect.arrayContaining(['/tmp/data/lookups/test.yml', '/tmp/data/lookups/test.csv']));
+		expect(volume['/tmp/data/lookups/test.csv']).toMatchSnapshot();
+	});
+
+	it('should use the correct extension', () => {
+		new Lookup(cribl, 'test.csv.gz', lookupFile);
+		cribl.synth();
+
+		const volume = vol.toJSON();
+		expect(Object.keys(volume)).toStrictEqual(expect.arrayContaining(['/tmp/data/lookups/test.yml', '/tmp/data/lookups/test.csv.gz']));
+		expect(volume['/tmp/data/lookups/test.csv.gz']).toMatchSnapshot();
+	});
+
+	it('should throw an error if the file name is invalid', () => {
+		expect(() => new Lookup(cribl, 'test.fake', lookupFile)).toThrow();
+	});
+
+	it('should write the correct metadata', () => {
+		new Lookup(cribl, 'test', lookupFile);
+		cribl.synth();
+
+		const volume = vol.toJSON();
+		const meta = load(volume['/tmp/data/lookups/test.yml'].toString());
+
+		expect(meta).toEqual(expect.objectContaining({
+			rows: 4,
+			size: 23
+		}));
+	})
+});

lib/objects/lookups.ts

@@ -0,0 +1,60 @@
+import {ConfigConstruct} from "../config";
+import {File} from "../private/fs";
+import {Construct} from "constructs";
+import Registry from "../registry";
+import {pick} from "es-toolkit";
+
+export interface LookupProps {
+	/**
+	 *
+	 * Must match pattern ^\s*\w[\w -]+(?:\.csv|\.gz|\.csv\.gz|\.mmdb)?\s*$
+	 */
+	size?: number;
+	rows?: number;
+	description?: string;
+}
+
+const fileNamePattern = new RegExp(/^\s*\w[\w -]+(?:\.csv|\.gz|\.csv\.gz|\.mmdb)?\s*$/);
+
+export class Lookup extends ConfigConstruct {
+	protected lookup: Buffer;
+	protected type: string;
+
+	constructor(scope: Construct, fileName: string, file: Buffer, props?: LookupProps) {
+		if (!fileNamePattern.test(fileName)) {
+			throw new Error(`File name ${fileName} must match pattern ${fileNamePattern}`);
+		}
+
+		const _props = {
+			size: file.length,
+			rows: Buffer.from(file).toString().split('\n').length,
+			...props // override the defaults
+		}
+
+		super(scope, fileName, _props);
+
+		this.lookup = file;
+
+		// Handle the file type
+		const parts = fileName.split('.');
+		if (parts.length === 1) this.type = 'csv';
+		else this.type = fileName.substring(fileName.indexOf('.') + 1);
+	}
+
+	get kind() {
+		return 'lookups';
+	}
+
+	static dump(config: ConfigConstruct[]): undefined {
+		return undefined;
+	}
+
+	synth(): void {
+		const config = this.toYaml(pick(this.config, ['size', 'rows', 'description']));
+
+		new File(this.path('data', 'lookups', `${this.node.id.split('.').shift()}.${this.type}`)).write(this.lookup);
+		new File(this.path('data', 'lookups', `${this.node.id.split('.').shift()}.yml`)).write(config);
+	}
+}
+
+Registry.register('lookups', Lookup);