Merge pull request #1274 from guardian/nt/cloudbuster3

Write cloudbuster findings to cloudquery

packages/cloudbuster/src/findings.test.ts

@@ -1,6 +1,56 @@
-import { groupFindingsByAccount } from './findings';
+import type { SecurityHubFinding } from 'common/types';
+import { findingsToGuardianFormat, groupFindingsByAccount } from './findings';
 import type { Finding, GroupedFindings } from './types';
 
+describe('findingsToGuardianFormat', () => {
+	const resource1 = {
+		Id: 'arn:instance:1',
+		Tags: { Stack: 'myStack', FakeTag: 'fake' },
+		Region: 'some-region',
+		Type: 'some-type',
+	};
+	const resource2 = {
+		...resource1,
+		Id: 'arn:instance:2',
+	};
+
+	const x: SecurityHubFinding = {
+		title: 'title',
+		aws_account_name: 'accountName',
+		remediation: { Recommendation: { Url: 'url' } },
+		severity: { Label: 'HIGH', Normalized: 75 },
+		aws_account_id: '0123456',
+		first_observed_at: new Date('2020-01-01'),
+		product_fields: { ControlId: 'S.1' },
+		resources: [resource1, resource2],
+	};
+	it('should return n elements if n resources are associated with a finding', () => {
+		const actual = findingsToGuardianFormat(x);
+		expect(actual.length).toEqual(2);
+	});
+	it('should return the relevant data in the appropriate fields', () => {
+		const actual = findingsToGuardianFormat(x);
+		const expected = {
+			severity: 'HIGH',
+			control_id: 'S.1',
+			title: 'title',
+			aws_region: 'some-region',
+			repo: null,
+			stack: 'myStack',
+			stage: null,
+			app: null,
+			first_observed_at: new Date('2020-01-01'),
+			arn: 'arn:instance:1',
+			aws_account_name: 'accountName',
+			aws_account_id: '0123456',
+			within_sla: false,
+			remediation: 'url',
+		};
+		expect(actual[0]).toEqual(expected);
+		expect(actual[1]).toEqual({ ...expected, arn: 'arn:instance:2' });
+	});
+});
+
 function mockFinding(awsAccountId: string, title: string): Finding {
 	return {
 		awsAccountId,

packages/cloudbuster/src/findings.ts

@@ -1,28 +1,56 @@
-import type { aws_securityhub_findings } from '@prisma/client';
+import type { cloudbuster_fsbp_vulnerabilities } from '@prisma/client';
 import { isWithinSlaTime, stringToSeverity } from 'common/src/functions';
-import type { Severity } from 'common/src/types';
+import type { SecurityHubFinding, Severity } from 'common/src/types';
 import type { Finding, GroupedFindings } from './types';
 
+export function findingsToGuardianFormat(
+	finding: SecurityHubFinding,
+): cloudbuster_fsbp_vulnerabilities[] {
+	const transformedFindings: cloudbuster_fsbp_vulnerabilities[] =
+		finding.resources.map((r) => {
+			const guFinding: cloudbuster_fsbp_vulnerabilities = {
+				severity: finding.severity.Label,
+				control_id: finding.product_fields.ControlId,
+				title: finding.title,
+				aws_region: r.Region,
+				repo: r.Tags?.['gu:repo'] ?? null,
+				stack: r.Tags?.['Stack'] ?? null,
+				stage: r.Tags?.Stage ?? null,
+				app: r.Tags?.App ?? null,
+				first_observed_at: finding.first_observed_at,
+				arn: r.Id,
+				aws_account_name: finding.aws_account_name,
+				aws_account_id: finding.aws_account_id,
+				within_sla: isWithinSlaTime(
+					finding.first_observed_at,
+					stringToSeverity(finding.severity.Label),
+				),
+				remediation: finding.remediation.Recommendation.Url,
+			};
+			return guFinding;
+		});
+	return transformedFindings;
+}
+
 /**
  * Transforms a SQL row into a finding
  */
-export function transformFinding(finding: aws_securityhub_findings): Finding {
+export function transformFinding(finding: SecurityHubFinding): Finding {
 	let severity: Severity = 'unknown';
 	let priority = null;
 	let remediationUrl = null;
 	let resources = null;
 
 	if (
-		finding.severity &&
 		typeof finding.severity === 'object' &&
 		'Label' in finding.severity &&
 		'Normalized' in finding.severity
 	) {
 		severity = stringToSeverity(finding.severity['Label'] as string);
-		priority = finding.severity['Normalized'] as number;
+		priority = finding.severity['Normalized'];
 	}
 
-	if (finding.remediation && typeof finding.remediation === 'object') {
+	if (typeof finding.remediation === 'object') {
 		const remediation = finding.remediation as {
 			Recommendation: {
 				Url: string | null;
@@ -38,11 +66,11 @@ export function transformFinding(finding: aws_securityhub_findings): Finding {
 		}
 	}
 
-	if (finding.resources && Array.isArray(finding.resources)) {
+	if (Array.isArray(finding.resources)) {
 		resources = finding.resources
 			.map((r) => {
-				if (r && typeof r === 'object' && 'Id' in r) {
-					return r['Id'] as string;
+				if (typeof r === 'object' && 'Id' in r) {
+					return r['Id'];
 				}
 				return null;
 			})

packages/cloudbuster/src/index.ts

@@ -1,10 +1,10 @@
 import { Anghammarad } from '@guardian/anghammarad';
 import { getFsbpFindings } from 'common/src/database-queries';
 import { getPrismaClient } from 'common/src/database-setup';
-import type { SecurityHubSeverity } from 'common/types';
+import type { SecurityHubSeverity } from 'common/src/types';
 import { getConfig } from './config';
 import { createDigestsFromFindings, sendDigest } from './digests';
-import { transformFinding } from './findings';
+import { findingsToGuardianFormat, transformFinding } from './findings';
 
 type LambdaHandlerProps = {
 	severities?: SecurityHubSeverity[];
@@ -25,9 +25,19 @@ export async function main(input: LambdaHandlerProps) {
 		`Starting Cloudbuster. Level of severities that will be scanned: ${severities.join(', ')}`,
 	);
 
-	const findings = (await getFsbpFindings(prisma, severities)).map((f) =>
-		transformFinding(f),
-	);
+	const dbResults = (await getFsbpFindings(prisma, severities)).slice(0, 5); //TODO: remove slice when ready to go live
+
+	const findings = dbResults.map((f) => transformFinding(f));
+
+	const tableContents = dbResults.flatMap(findingsToGuardianFormat);
+
+	console.table(tableContents);
+
+	await prisma.cloudbuster_fsbp_vulnerabilities.deleteMany();
+	await prisma.cloudbuster_fsbp_vulnerabilities.createMany({
+		data: tableContents,
+	});
+
 	const digests = createDigestsFromFindings(findings);
 
 	// *** NOTIFICATION SENDING ***

packages/cloudbuster/src/types.ts

@@ -8,7 +8,7 @@ export interface Finding {
 	resources: string[];
 	remediationUrl: string | null;
 	severity: Severity;
-	priority: number | null;
+	priority: number | null; //TODO remove
 	isWithinSla: boolean;
 }
 

packages/common/prisma/migrations/20240930103347_create_fsbp_table/migration.sql

@@ -0,0 +1,20 @@
+CREATE TABLE "cloudbuster_fsbp_vulnerabilities" (
+    "arn" TEXT NOT NULL,
+    "aws_account_id" TEXT NOT NULL,
+    "aws_account_name" TEXT,
+    "aws_region" TEXT NOT NULL,
+    "control_id" TEXT NOT NULL,
+    "first_observed_at" TIMESTAMP(6),
+    "repo" TEXT,
+    "stack" TEXT,
+    "stage" TEXT,
+    "app" TEXT,
+    "severity" TEXT NOT NULL,
+    "title" TEXT NOT NULL,
+    "within_sla" BOOLEAN NOT NULL,
+    "remediation" TEXT,
+
+    CONSTRAINT "cloudbuster_fsbp_vulnerabilities_pkey" PRIMARY KEY ("arn","control_id")
+);
+
+GRANT ALL ON public.cloudbuster_fsbp_vulnerabilities TO cloudbuster;

packages/common/prisma/schema.prisma

@@ -230,6 +230,25 @@ model aws_organizations_roots {
   @@ignore
 }
 
+model cloudbuster_fsbp_vulnerabilities {
+  arn               String
+  aws_account_id    String
+  aws_account_name  String?
+  aws_region        String
+  control_id        String
+  first_observed_at DateTime? @db.Timestamp(6)
+  repo              String?
+  stack             String?
+  stage             String?
+  app               String?
+  severity          String
+  title             String
+  within_sla        Boolean
+  remediation       String?
+
+  @@id([arn, control_id])
+}
+
 model github_repositories {
   cq_sync_time                   DateTime? @map("_cq_sync_time") @db.Timestamp(6)
   cq_source_name                 String?   @map("_cq_source_name")

packages/common/src/database-queries.ts

@@ -1,24 +1,26 @@
 import type { aws_securityhub_findings, PrismaClient } from '@prisma/client';
-import type { SecurityHubSeverity } from './types';
+import type { SecurityHubFinding, SecurityHubSeverity } from './types';
 /**
  * Queries the database for FSBP findings
  */
+
 export async function getFsbpFindings(
 	prisma: PrismaClient,
 	severities: SecurityHubSeverity[],
-): Promise<aws_securityhub_findings[]> {
-	const findings = await prisma.aws_securityhub_findings.findMany({
-		where: {
-			OR: severities.map((s) => ({
-				severity: { path: ['Label'], equals: s },
-			})),
-			AND: {
-				generator_id: {
-					startsWith: 'aws-foundational-security-best-practices/v/1.0.0',
+): Promise<SecurityHubFinding[]> {
+	const findings: aws_securityhub_findings[] =
+		await prisma.aws_securityhub_findings.findMany({
+			where: {
+				OR: severities.map((s) => ({
+					severity: { path: ['Label'], equals: s },
+				})),
+				AND: {
+					generator_id: {
+						startsWith: 'aws-foundational-security-best-practices/v/1.0.0',
+					},
 				},
 			},
-		},
-	});
+		});
 
-	return findings;
+	return findings as unknown as SecurityHubFinding[];
 }

packages/common/src/types.ts

@@ -1,5 +1,6 @@
 import { type StrategyOptions } from '@octokit/auth-app';
 import type {
+	aws_securityhub_findings,
 	github_repositories,
 	repocop_vulnerabilities,
 } from '@prisma/client';
@@ -106,3 +107,20 @@ export const SLAs: Record<Severity, number | undefined> = {
 };
 
 export type NonEmptyArray<T> = [T, ...T[]];
+
+type Resource = {
+	Id: string;
+	Tags: Record<string, string> | null;
+	Region: string;
+	Type: string;
+};
+
+export type SecurityHubFinding = Pick<
+	aws_securityhub_findings,
+	'first_observed_at' | 'aws_account_id' | 'aws_account_name' | 'title'
+> & {
+	remediation: { Recommendation: { Url: string } };
+	severity: { Label: SecurityHubSeverity; Normalized: number };
+	resources: Resource[];
+	product_fields: { ControlId: string };
+};

packages/obligatron/src/obligations/aws-vulnerabilities.test.ts

@@ -1,7 +1,5 @@
-import {
-	fsbpFindingsToObligatronResults,
-	type SecurityHubFinding,
-} from './aws-vulnerabilities';
+import type { SecurityHubFinding } from 'common/src/types';
+import { fsbpFindingsToObligatronResults } from './aws-vulnerabilities';
 
 describe('The dependency vulnerabilities obligation', () => {
 	const resource1 = {
@@ -18,10 +16,13 @@ describe('The dependency vulnerabilities obligation', () => {
 
 	const oneResourceFinding: SecurityHubFinding = {
 		resources: [resource1],
+		title: 'title',
+		aws_account_name: 'accountName',
+		remediation: { Recommendation: { Url: 'url' } },
 		severity: { Label: 'HIGH', Normalized: 75 },
 		aws_account_id: '0123456',
 		first_observed_at: new Date('2020-01-01'),
-		product_fields: { ControlId: 'S.1', StandardsArn: 'arn:1' },
+		product_fields: { ControlId: 'S.1' },
 	};
 
 	const twoResourceFinding: SecurityHubFinding = {

packages/obligatron/src/obligations/aws-vulnerabilities.ts

@@ -1,33 +1,13 @@
-import type { aws_securityhub_findings, PrismaClient } from '@prisma/client';
+import type { PrismaClient } from '@prisma/client';
 import { getFsbpFindings } from 'common/src/database-queries';
 import {
 	isWithinSlaTime,
 	stringToSeverity,
 	toNonEmptyArray,
 } from 'common/src/functions';
+import type { SecurityHubFinding } from 'common/src/types';
 import type { ObligationResult } from '.';
 
-type Resource = {
-	Id: string;
-	Tags: Record<string, string>;
-	Region: string;
-	Type: string;
-};
-
-type ProductFields = {
-	ControlId: string;
-	StandardsArn: string;
-};
-
-export type SecurityHubFinding = Pick<
-	aws_securityhub_findings,
-	'first_observed_at' | 'aws_account_id'
-> & {
-	severity: { Label: string; Normalized: number };
-	resources: Resource[];
-	product_fields: ProductFields;
-};
-
 type Failure = {
 	resource: string;
 	controlId: string;