Merge branch 'main' into feature/regex

lib/__tests__/__snapshots__/parsers.test.ts.snap

@@ -0,0 +1,11 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Parsers > should be written to the file 1`] = `
+"test:
+  lib: test
+  type: test
+  fields:
+    - test1
+    - test2
+"
+`;

lib/__tests__/__snapshots__/pipelines.test.ts.snap

@@ -0,0 +1,6 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Pipelines > should only write the pipelines/$PIPELINE/conf.yml 1`] = `
+"{}
+"
+`;

lib/__tests__/breakers.test.ts

@@ -1,7 +1,7 @@
 import {describe, it, expect, vi, beforeEach} from 'vitest';
 import {vol} from 'memfs'
 import {Cribl} from "../index";
-import {EventBreakerRule, EventBreakerRuleset} from "../objects/breaker";
+import {EventBreakerRule, EventBreakerRuleset} from "../objects";
 
 vi.mock('node:fs');
 vi.mock('node:fs/promises');

lib/__tests__/certificates.test.ts

@@ -1,7 +1,7 @@
 import {it, expect, vi, beforeEach, describe} from 'vitest';
 import {vol} from 'memfs'
 import {Cribl} from "../index";
-import {Certificate} from "../objects/certificates";
+import {Certificate} from "../objects";
 
 vi.mock('node:fs');
 vi.mock('node:fs/promises');
@@ -30,7 +30,6 @@ test
 		cribl.synth();
 
 		const volume = vol.toJSON();
-		console.log(volume);
 		expect(Object.keys(volume)).toEqual(expect.arrayContaining(['/tmp/local/cribl/certificates.yml', '/tmp/local/cribl/auth/certs/goat.pem', '/tmp/local/cribl/auth/certs/goat.key']));
 		expect(volume['/tmp/local/cribl/certificates.yml']).toMatchSnapshot();
 		expect(volume['/tmp/local/cribl/auth/certs/goat.pem']).toMatchSnapshot();

lib/__tests__/parsers.test.ts

@@ -0,0 +1,27 @@
+import {it, expect, vi, beforeEach, describe} from 'vitest';
+import {vol} from 'memfs'
+import {Cribl} from "../index";
+import {Parser} from "../objects";
+
+vi.mock('node:fs');
+vi.mock('node:fs/promises');
+
+let cribl: Cribl;
+
+beforeEach(() => {
+	// reset the state of in-memory fs
+	vol.reset()
+
+	cribl = new Cribl({outdir: '/tmp'});
+});
+
+describe('Parsers', () => {
+	it('should be written to the file', () => {
+		new Parser(cribl, 'test', {lib: 'test', type: 'test', fields: ['test1', 'test2']});
+		cribl.synth();
+
+		const volume = vol.toJSON();
+		expect(Object.keys(volume)).toEqual(['/tmp/local/cribl/parsers.yml']);
+		expect(volume['/tmp/local/cribl/parsers.yml']).toMatchSnapshot();
+	});
+});

lib/__tests__/pipelines.test.ts

@@ -0,0 +1,27 @@
+import {it, expect, vi, beforeEach, describe} from 'vitest';
+import {vol} from 'memfs'
+import {Cribl} from "../index";
+import {Pipeline} from "../objects/pipeline";
+
+vi.mock('node:fs');
+vi.mock('node:fs/promises');
+
+let cribl: Cribl;
+
+beforeEach(() => {
+	// reset the state of in-memory fs
+	vol.reset()
+
+	cribl = new Cribl({outdir: '/tmp'});
+});
+
+describe('Pipelines', () => {
+	it('should only write the pipelines/$PIPELINE/conf.yml', () => {
+		new Pipeline(cribl, 'test', {});
+		cribl.synth();
+
+		const volume = vol.toJSON();
+		expect(Object.keys(volume)).toStrictEqual(['/tmp/local/cribl/pipelines/test/conf.yml']);
+		expect(volume['/tmp/local/cribl/pipelines/test/conf.yml']).toMatchSnapshot();
+	});
+});

lib/__tests__/scripts.test.ts

@@ -1,7 +1,7 @@
 import {it, expect, vi, beforeEach, describe} from 'vitest';
 import {vol} from 'memfs'
 import {Cribl} from "../index";
-import {Script} from "../objects/scripts";
+import {Script} from "../objects";
 
 vi.mock('node:fs');
 vi.mock('node:fs/promises');

lib/__tests__/vars.test.ts

@@ -1,7 +1,7 @@
 import { it, expect, vi, beforeEach } from 'vitest';
 import { vol } from 'memfs'
 import {Cribl} from "../index";
-import {Variable} from "../objects/variable";
+import {Variable} from "../objects";
 import {WorkerGroup} from "../objects/group";
 
 vi.mock('node:fs');

lib/__tests__/workergroups.test.ts

@@ -0,0 +1,58 @@
+import {describe, it, expect, vi, beforeEach} from 'vitest';
+import {Cribl, Variable} from '../index';
+import {vol} from "memfs";
+import {WorkerGroup} from "../objects/group";
+
+vi.mock('node:fs');
+vi.mock('node:fs/promises');
+
+let cribl: Cribl;
+
+beforeEach(() => {
+	// reset the state of in-memory fs
+	vol.reset()
+
+	cribl = new Cribl({outdir: '/tmp'});
+});
+
+describe('Cribl', () => {
+  it('without configs should generate nothing', () => {
+	  cribl.synth();
+
+	  const volume = vol.toJSON();
+	  expect(Object.keys(volume)).toEqual([]);
+  });
+
+  it('configs not in a worker group should be at in $CRIBL_HOME/local', () => {
+	new Variable(cribl, 'test', {value: 'test'});
+
+	cribl.synth();
+
+	const volume = vol.toJSON();
+	expect(Object.keys(volume)).toEqual(['/tmp/local/cribl/vars.yml']);
+  });
+
+  it('configs in a worker group should be at in $CRIBL_HOME/groups/$GROUP/local', () => {
+	  const group = new WorkerGroup(cribl, 'goat');
+	  new Variable(group, 'test', {value: 'test'});
+
+	  cribl.synth();
+
+	  const volume = vol.toJSON();
+	  expect(Object.keys(volume)).toEqual(['/tmp/groups/goat/local/cribl/vars.yml']);
+  });
+
+  it('should place configs in the correct scopes', () => {
+	  new Variable(cribl, 'test', {value: 'test'});
+	  const group = new WorkerGroup(cribl, 'goat');
+	  new Variable(group, 'test', {value: 'test'});
+
+	  cribl.synth();
+
+	  const volume = vol.toJSON();
+	  expect(Object.keys(volume)).toEqual(expect.arrayContaining([
+		  '/tmp/local/cribl/vars.yml',
+		  '/tmp/groups/goat/local/cribl/vars.yml'
+	  ]));
+  });
+});

lib/config.ts

@@ -20,7 +20,7 @@ export abstract class ConfigConstruct extends Construct {
 		return this._config;
 	}
 
-	static dump(config: ConfigConstruct[]): Record<string, unknown> {
+	static dump(config: ConfigConstruct[]): Record<string, unknown> | undefined {
 		return {
 			[this.prototype.kind]: config.map(c => ({[c.node.id]: c.config}))
 		}

lib/container.ts

@@ -21,11 +21,19 @@ export abstract class ContainerConstruct extends Construct {
 			if (!c || c.length === 0) continue;
 
 			for (const config of c) {
-				// This is mostly no-op, but there are some resources that write files like samples and certificates
+				/*
+				 * This is mostly no-op, but there are some resources that write
+				 * files like samples, certificates, and pipelines
+				 */
 				config.synth();
 			}
 
-			new File(this.path('local', this.package, `${type}.yml`)).write(this.toYaml(clazz.dump(c)));
+			// Handle scenario where the default config file isn't used (see pipelines)
+			const output = clazz.dump(c);
+
+			if (output) {
+				new File(this.path('local', this.package, `${type}.yml`)).write(this.toYaml(output));
+			}
 		}
 
 		const contexts = this.node.children.filter(child => child instanceof ContainerConstruct);

lib/objects/index.ts

@@ -7,3 +7,4 @@ export * from './sample';
 export * from './scripts';
 export * from './variable';
 export * from './regex'
+export * from './parsers';

lib/objects/parsers.ts

@@ -0,0 +1,50 @@
+import {ConfigConstruct} from "../config";
+import {Construct} from "constructs";
+import Registry from "../registry";
+
+export interface ParserProps {
+	/**
+	 * Parser library
+	 */
+	lib: string;
+
+	/**
+	 * Brief description of this parser.
+	 *
+	 * Optional.
+	 */
+	description?: string;
+
+	/**
+	 * One or more tags related to this parser.
+	 *
+	 * Optional.
+	 */
+	tags?: string;
+
+	/**
+	 * Parser/Formatter type to use.
+	 */
+	type: string;
+
+	/**
+	 * Fields expected to be extracted, in order. If not specified parser will auto-generate.
+	 */
+	fields: string[];
+}
+
+export class Parser extends ConfigConstruct {
+	constructor(scope: Construct, id: string, props: ParserProps) {
+		super(scope, id, props);
+	}
+
+	get kind() {
+		return 'parsers';
+	}
+
+	static dump(config: ConfigConstruct[]): Record<string, unknown> {
+		return Object.fromEntries(config.map(c => [c.node.id, c.config]));
+	}
+}
+
+Registry.register('parsers', Parser);

lib/objects/pipeline.ts

@@ -0,0 +1,30 @@
+import {ConfigConstruct} from "../config";
+import {Construct} from "constructs";
+import {File} from "../private/fs";
+import Registry from "../registry";
+
+interface PipelineProps {
+
+}
+
+export class Pipeline extends ConfigConstruct {
+	constructor(scope: Construct, id: string, props: PipelineProps) {
+		super(scope, id, props);
+	}
+
+	get kind() {
+		return 'pipelines';
+	}
+
+	synth(): void {
+		const config = this.toYaml(this.config);
+
+		new File(this.path('local', 'cribl', 'pipelines', this.node.id, 'conf.yml')).write(config);
+	}
+
+	static dump(config: ConfigConstruct[]): undefined {
+		return undefined;
+	}
+}
+
+Registry.register('pipelines', Pipeline);