From 55cda083315c6b29ba6a85ae4196735536959156 Mon Sep 17 00:00:00 2001 From: Natasha <67543397+NovemberTang@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:56:25 +0100 Subject: [PATCH] parameterise github org (#1271) * make guardian string part of repocop config * parameterise org name for github tables * make guardian org a const at infrastructure level * make gitHubOrg a prop --- .../service-catalogue.test.ts.snap | 1 + packages/cdk/lib/cloudquery/config.test.ts | 5 ++++- packages/cdk/lib/cloudquery/config.ts | 12 ++++++++---- packages/cdk/lib/cloudquery/index.ts | 5 +++++ packages/cdk/lib/repocop.ts | 2 ++ packages/cdk/lib/service-catalogue.ts | 13 ++++++++++++- packages/repocop/src/config.ts | 6 ++++++ packages/repocop/src/index.ts | 6 +++++- packages/repocop/src/query.ts | 17 ++++++++++------- .../topics/topic-monitor-production.ts | 3 +-- 10 files changed, 54 insertions(+), 16 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 51d15b952..69ec5c95f 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -23342,6 +23342,7 @@ spec: "GITHUB_APP_SECRET": { "Ref": "branchprotectorgithubappauth4E91892E", }, + "GITHUB_ORG": "guardian", "INTERACTIVES_COUNT": "3", "INTERACTIVE_MONITOR_TOPIC_ARN": { "Ref": "TopicBFC7AF6E", diff --git a/packages/cdk/lib/cloudquery/config.test.ts b/packages/cdk/lib/cloudquery/config.test.ts index 54ce6a382..03d7adcd6 100644 --- a/packages/cdk/lib/cloudquery/config.test.ts +++ b/packages/cdk/lib/cloudquery/config.test.ts @@ -147,7 +147,10 @@ spec: }); it('Should create a GitHub source configuration', () => { - const config = githubSourceConfig({ tables: ['github_repositories'] }); + const config = githubSourceConfig({ + tables: ['github_repositories'], + org: 'guardian', + }); expect(dump(config)).toMatchInlineSnapshot(` "kind: source spec: diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index f612cffd3..eb10109eb 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -15,6 +15,10 @@ interface CloudqueryTableConfig { concurrency?: number; } +interface GitHubCloudqueryTableConfig extends CloudqueryTableConfig { + org: string; +} + /** * Create a ServiceCatalogue destination configuration for Postgres. */ @@ -115,9 +119,9 @@ export function awsSourceConfigForAccount( } export function githubSourceConfig( - tableConfig: CloudqueryTableConfig, + tableConfig: GitHubCloudqueryTableConfig, ): CloudqueryConfig { - const { tables, skipTables } = tableConfig; + const { tables, skipTables, org } = tableConfig; if (!tables && !skipTables) { throw new Error('Must specify either tables or skipTables'); @@ -134,10 +138,10 @@ export function githubSourceConfig( destinations: ['postgresql'], spec: { concurrency: 1000, // TODO what's the ideal value here?! - orgs: ['guardian'], + orgs: [org], app_auth: [ { - org: 'guardian', + org, // For simplicity, read all configuration from disk. private_key_path: `${serviceCatalogueConfigDirectory}/github-private-key`, diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 54296064a..a016aa772 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -37,6 +37,7 @@ interface CloudqueryEcsClusterProps { snykCredentials: SecretsManager; loggingStreamName: string; logShippingPolicy: PolicyStatement; + gitHubOrg: string; } export function addCloudqueryEcsCluster( @@ -51,6 +52,7 @@ export function addCloudqueryEcsCluster( nonProdSchedule, loggingStreamName, logShippingPolicy, + gitHubOrg: gitHubOrgName, } = props; const riffRaffDatabaseAccessSecurityGroupParam = @@ -379,6 +381,7 @@ export function addCloudqueryEcsCluster( 'Collect GitHub repository data. Uses include RepoCop, which flags repositories that do not meet certain obligations.', schedule: nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '0' }), config: githubSourceConfig({ + org: gitHubOrgName, tables: [ 'github_repositories', 'github_repository_branches', @@ -407,6 +410,7 @@ export function addCloudqueryEcsCluster( nonProdSchedule ?? Schedule.cron({ weekDay: '1', hour: '10', minute: '0' }), config: githubSourceConfig({ + org: gitHubOrgName, tables: [ 'github_organizations', 'github_organization_members', @@ -435,6 +439,7 @@ export function addCloudqueryEcsCluster( description: 'Collect GitHub issue data (PRs and Issues)', schedule: nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '2' }), config: githubSourceConfig({ + org: gitHubOrgName, tables: ['github_issues'], }), secrets: githubSecrets, diff --git a/packages/cdk/lib/repocop.ts b/packages/cdk/lib/repocop.ts index 144d238c5..85b666bd9 100644 --- a/packages/cdk/lib/repocop.ts +++ b/packages/cdk/lib/repocop.ts @@ -29,6 +29,7 @@ export class Repocop { interactiveMonitorTopic: Topic, dbSecurityGroup: SecurityGroup, repocopGithubSecret: Secret, + gitHubOrg: string, ) { const snykIntegratorInputTopic = new Topic( guStack, @@ -65,6 +66,7 @@ export class Repocop { SNYK_INTEGRATOR_INPUT_TOPIC_ARN: snykIntegratorInputTopic.topicArn, DEPENDENCY_GRAPH_INPUT_TOPIC_ARN: dependencyGraphIntegratorInputTopic.topicArn, + GITHUB_ORG: gitHubOrg, }, vpc, securityGroups: [dbSecurityGroup], diff --git a/packages/cdk/lib/service-catalogue.ts b/packages/cdk/lib/service-catalogue.ts index e3e70b8df..7073de628 100644 --- a/packages/cdk/lib/service-catalogue.ts +++ b/packages/cdk/lib/service-catalogue.ts @@ -62,6 +62,11 @@ interface ServiceCatalogueProps extends GuStackProps { */ rdsDeletionProtection?: boolean; multiAz?: boolean; + + /** + * The GitHub org to search for repositories in. + */ + gitHubOrg?: string; } export class ServiceCatalogue extends GuStack { @@ -71,7 +76,11 @@ export class ServiceCatalogue extends GuStack { const { stage, stack } = this; const app = props.app ?? 'service-catalogue'; - const { rdsDeletionProtection = true, multiAz = false } = props; + const { + rdsDeletionProtection = true, + multiAz = false, + gitHubOrg = 'guardian', + } = props; const nonProdSchedule = props.schedule; @@ -181,6 +190,7 @@ export class ServiceCatalogue extends GuStack { snykCredentials: snykReadOnlyKey, loggingStreamName, logShippingPolicy, + gitHubOrg, }); const anghammaradTopicParameter = @@ -238,6 +248,7 @@ export class ServiceCatalogue extends GuStack { interactiveMonitor.topic, applicationToPostgresSecurityGroup, githubCredentials, + gitHubOrg, ); addDataAuditLambda(this, { diff --git a/packages/repocop/src/config.ts b/packages/repocop/src/config.ts index 6c8435f82..ec703bb14 100644 --- a/packages/repocop/src/config.ts +++ b/packages/repocop/src/config.ts @@ -72,6 +72,11 @@ export interface Config extends PrismaConfig { * The ARN of the Dependency Graph Integrator input topic. */ dependencyGraphIntegratorTopic: string; + + /** + * The name of the GitHub organisation that owns the repositories. + */ + gitHubOrg: string; } export async function getConfig(): Promise { @@ -106,5 +111,6 @@ export async function getConfig(): Promise { dependencyGraphIntegratorTopic: getEnvOrThrow( 'DEPENDENCY_GRAPH_INPUT_TOPIC_ARN', ), + gitHubOrg: process.env['GITHUB_ORG'] ?? 'guardian', }; } diff --git a/packages/repocop/src/index.ts b/packages/repocop/src/index.ts index 84b3b570d..792232475 100644 --- a/packages/repocop/src/index.ts +++ b/packages/repocop/src/index.ts @@ -90,7 +90,11 @@ export async function main() { const productionRepos = unarchivedRepos.filter((repo) => isProduction(repo)); const productionDependabotVulnerabilities: RepocopVulnerability[] = - await getDependabotVulnerabilities(productionRepos, octokit); + await getDependabotVulnerabilities( + productionRepos, + config.gitHubOrg, + octokit, + ); console.log(productionDependabotVulnerabilities); diff --git a/packages/repocop/src/query.ts b/packages/repocop/src/query.ts index 984af7a55..787fcd739 100644 --- a/packages/repocop/src/query.ts +++ b/packages/repocop/src/query.ts @@ -128,17 +128,19 @@ export async function getRepositoryLanguages( async function getAlertsForRepo( octokit: Octokit, - name: string, + orgName: string, + repoName: string, ): Promise { - if (name.startsWith('guardian/')) { - name = name.replace('guardian/', ''); + const prefix = `${orgName}/`; + if (repoName.startsWith(prefix)) { + repoName = repoName.replace(prefix, ''); } try { const alert: DependabotVulnResponse = await octokit.rest.dependabot.listAlertsForRepo({ - owner: 'guardian', - repo: name, + owner: orgName, + repo: repoName, per_page: 100, severity: 'critical,high', state: 'open', @@ -152,7 +154,7 @@ async function getAlertsForRepo( return openRuntimeDependencies; } catch (error) { console.debug( - `Dependabot - ${name}: Could not get alerts. Dependabot may not be enabled.`, + `Dependabot - ${repoName}: Could not get alerts. Dependabot may not be enabled.`, ); console.debug(error); // Return undefined if dependabot is not enabled, to distinguish from @@ -163,12 +165,13 @@ async function getAlertsForRepo( export async function getDependabotVulnerabilities( repos: Repository[], + orgName: string, octokit: Octokit, ) { const dependabotVulnerabilities: RepocopVulnerability[] = ( await Promise.all( repos.map(async (repo) => { - const alerts = await getAlertsForRepo(octokit, repo.name); + const alerts = await getAlertsForRepo(octokit, orgName, repo.name); if (alerts) { return alerts.map((a) => dependabotAlertToRepocopVulnerability(repo.full_name, a), diff --git a/packages/repocop/src/remediation/topics/topic-monitor-production.ts b/packages/repocop/src/remediation/topics/topic-monitor-production.ts index 4d4d7c607..1315c87ab 100644 --- a/packages/repocop/src/remediation/topics/topic-monitor-production.ts +++ b/packages/repocop/src/remediation/topics/topic-monitor-production.ts @@ -135,12 +135,11 @@ async function applyProductionTopicToOneRepoAndMessageTeams( octokit: Octokit, config: Config, ): Promise { - const owner = 'guardian'; const topic = 'production'; const shortRepoName = removeRepoOwner(fullRepoName); const { stage } = config; if (stage === 'PROD') { - await applyTopics(shortRepoName, owner, octokit, topic); + await applyTopics(shortRepoName, config.gitHubOrg, octokit, topic); } else { console.log( `Would have applied the ${topic} topic to ${shortRepoName} with stack ${stackName} if stage was PROD.`,