From c543bf7215a61bae5b060cefaccfcd2d291fee64 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 09:56:33 +0100 Subject: [PATCH 01/54] Add scanning with tests and github action. Use different auth strat add scanning --- .github/workflows/scan-exposures.yml | 97 +++++++++++++++ package.json | 2 + scripts/exposure-scanning/args/index.js | 28 +++++ scripts/exposure-scanning/args/index.test.js | 35 ++++++ scripts/exposure-scanning/fetch/index.js | 64 ++++++++++ scripts/exposure-scanning/fetch/index.test.js | 111 ++++++++++++++++++ scripts/exposure-scanning/index.js | 51 ++++++++ scripts/exposure-scanning/index.test.js | 99 ++++++++++++++++ scripts/exposure-scanning/patch/index.js | 85 ++++++++++++++ scripts/exposure-scanning/patch/index.test.js | 59 ++++++++++ scripts/exposure-scanning/scan/index.js | 61 ++++++++++ scripts/exposure-scanning/scan/index.test.js | 90 ++++++++++++++ yarn.lock | 44 ++++++- 13 files changed, 825 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/scan-exposures.yml create mode 100644 scripts/exposure-scanning/args/index.js create mode 100644 scripts/exposure-scanning/args/index.test.js create mode 100644 scripts/exposure-scanning/fetch/index.js create mode 100644 scripts/exposure-scanning/fetch/index.test.js create mode 100644 scripts/exposure-scanning/index.js create mode 100644 scripts/exposure-scanning/index.test.js create mode 100644 scripts/exposure-scanning/patch/index.js create mode 100644 scripts/exposure-scanning/patch/index.test.js create mode 100644 scripts/exposure-scanning/scan/index.js create mode 100644 scripts/exposure-scanning/scan/index.test.js diff --git a/.github/workflows/scan-exposures.yml b/.github/workflows/scan-exposures.yml new file mode 100644 index 0000000000..2bd48e7117 --- /dev/null +++ b/.github/workflows/scan-exposures.yml @@ -0,0 +1,97 @@ +name: "Scan Exposures" + +on: + pull_request: + branches: + - '**' + issues: + types: [opened, edited] + issue_comment: + types: [created, edited, deleted] + pull_request_review: + types: [submitted, edited] + pull_request_review_comment: + types: [created, edited, deleted] + + +jobs: + + scan_prs: + name: Scan PR Exposures + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Cache Node Modules + id: cache + uses: actions/cache@v2 + with: + path: node_modules + key: node-modules-${{ hashFiles('yarn.lock') }} + + - name: Install Node Modules + if: steps.cache.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + + - name: Scan Issue + shell: bash + if: ${{ github.event_name == 'issues' }} + env: + ISSUE_NUMBER: ${{ github.issue.number }} + PATTERN: ${{ secrets.PATTERN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" + + - name: Scan PR + shell: bash + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' || github.event_name == 'pull_request_review_comment' }} + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PATTERN: ${{ secrets.PATTERN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" + + scan_comments: + name: Scan Comment Exposures + runs-on: ubuntu-latest + # The issue_comment event is triggered by comments on issues and PRs. + if: ${{ github.event_name == 'issue_comment' }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Cache Node Modules + id: cache + uses: actions/cache@v2 + with: + path: node_modules + key: node-modules-${{ hashFiles('yarn.lock') }} + + - name: Install Node Modules + if: steps.cache.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + + - name: Scan Issue + shell: bash + # This step only runs on issue comments + if: ${{ !github.event.issue.pull_request }} + env: + ISSUE_NUMBER: ${{ github.event.issue.number }} + PATTERN: ${{ secrets.PATTERN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" + + - name: Scan PR + shell: bash + # This step only runs on PR comments + if: ${{ github.event.issue.pull_request }} + env: + PR_NUMBER: ${{ github.event.issue.number }} + PATTERN: ${{ secrets.PATTERN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" diff --git a/package.json b/package.json index 31bcb5f047..9e14a1ae2d 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "preinstall": "node scripts/check-package-manager.js && sh scripts/enforceVersions.sh && [ \"$npm_config_refer\" != \"ci\" ]", "postinstall": "yarn packages:check && yarn build", "publish": "node scripts/publish", + "scan:exposures": "node scripts/scan-exposures", "storybook": "NODE_ENV=development start-storybook -p 8180 -s .storybook/static -c .storybook", "test": "yarn test:lint && yarn test:unit", "test:ci": "yarn test:lint && yarn test:unit:ci", @@ -112,6 +113,7 @@ "@loadable/babel-plugin": "^5.13.0", "@loadable/component": "^5.13.1", "@manypkg/cli": "^0.18.0", + "@octokit/rest": "^18.7.0", "@storybook/addon-a11y": "^6.3.4", "@storybook/addon-actions": "^6.3.4", "@storybook/addon-docs": "^6.3.4", diff --git a/scripts/exposure-scanning/args/index.js b/scripts/exposure-scanning/args/index.js new file mode 100644 index 0000000000..0fea6af27e --- /dev/null +++ b/scripts/exposure-scanning/args/index.js @@ -0,0 +1,28 @@ +const isValidId = id => { + try { + return !Number.isNaN(id) && Number.isInteger(parseFloat(id)); + } catch { + return false; + } +}; + +const parseArgs = argv => { + if (argv.length !== 6) { + throw new Error( + 'Error: Incorrect number of args.\nUsage: node scan.js <-pr/-issue> ', + ); + } + + const repo = argv[2]; + const flag = argv[3]; // -pr or -issue + const id = argv[4]; + const regexString = argv[5]; + + if (isValidId(id)) { + return { repo, flag, id, regexString }; + } + + throw new Error('Error: Invalid issue id'); +}; + +export default parseArgs; diff --git a/scripts/exposure-scanning/args/index.test.js b/scripts/exposure-scanning/args/index.test.js new file mode 100644 index 0000000000..afb8cc7efa --- /dev/null +++ b/scripts/exposure-scanning/args/index.test.js @@ -0,0 +1,35 @@ +import parseArgs from '.'; + +describe('Parsing arguments', () => { + it('should return an object containing the flag and id', () => { + expect( + parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 'regex']), + ).toEqual({ + repo: 'simorgh', + flag: '-pr', + id: '1234', + regexString: 'regex', + }); + }); + + it('should throw an error without logging args if the number given is incorrect', () => { + expect(() => + parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 're[g]ex', '-u']), + ).toThrow( + 'Error: Incorrect number of args.\nUsage: node scan.js <-pr/-issue> ', + ); + }); + + it('should throw an error without logging args if an invalid id is given', () => { + expect(() => + parseArgs([ + 'node', + 'scan.js', + 'psammead', + '-issue', + 'add-topic-tags', + 'regex', + ]), + ).toThrow('Error: Invalid issue id'); + }); +}); diff --git a/scripts/exposure-scanning/fetch/index.js b/scripts/exposure-scanning/fetch/index.js new file mode 100644 index 0000000000..5d9fde7e31 --- /dev/null +++ b/scripts/exposure-scanning/fetch/index.js @@ -0,0 +1,64 @@ +import { Octokit } from '@octokit/rest'; + +const octokit = new Octokit(); + +const fetchPrBody = async reqBody => { + const { + data: { body }, + } = await octokit.request('GET /repos/{owner}/{repo}/pulls/{prId}', reqBody); + return body; +}; + +const fetchPrComments = async reqBody => { + const prComments = await octokit.request( + 'GET /repos/{owner}/{repo}/issues/{prId}/comments', + reqBody, + ); + return prComments.data.map(comment => ({ + id: comment.id, + body: comment.body, + })); +}; + +const fetchPrReviewComments = async reqBody => { + const prReviewComments = await octokit.request( + 'GET /repos/{owner}/{repo}/pulls/{prId}/comments', + reqBody, + ); + return prReviewComments.data.map(comment => ({ + id: comment.id, + body: comment.body, + })); +}; + +const fetchIssueBody = async reqBody => { + const { + data: { body }, + } = await octokit.request( + 'GET /repos/{owner}/{repo}/issues/{issueId}', + reqBody, + ); + return body; +}; + +const fetchIssueComments = async reqBody => { + const issueComments = await octokit.request( + 'GET /repos/{owner}/{repo}/issues/{issueId}/comments', + reqBody, + ); + return issueComments.data.map(comment => ({ + id: comment.id, + body: comment.body, + })); +}; + +export const fetchPr = async reqBody => ({ + body: await fetchPrBody(reqBody), + comments: await fetchPrComments(reqBody), + reviewComments: await fetchPrReviewComments(reqBody), +}); + +export const fetchIssue = async reqBody => ({ + body: await fetchIssueBody(reqBody), + comments: await fetchIssueComments(reqBody), +}); diff --git a/scripts/exposure-scanning/fetch/index.test.js b/scripts/exposure-scanning/fetch/index.test.js new file mode 100644 index 0000000000..ef6a2c426d --- /dev/null +++ b/scripts/exposure-scanning/fetch/index.test.js @@ -0,0 +1,111 @@ +const { Headers, Response } = jest.requireActual('node-fetch'); + +const mockHeaders = new Headers({ 'Content-Type': 'application/json' }); + +const ResponseInit = { + url: 'hello', + headers: mockHeaders, + status: 200, +}; + +const mockPrBodyRes = new Response( + JSON.stringify({ + title: 'PR Title', + body: 'PR Body', + }), + ResponseInit, +); + +const mockPrCommentsRes = new Response( + JSON.stringify([ + { id: 1, body: 'Added translations' }, + { id: 2, body: 'Nice work, LGTM!' }, + ]), + ResponseInit, +); + +const mockPrReviewCommentsRes = new Response( + JSON.stringify([{ id: 3, body: 'Smashed it!' }]), + ResponseInit, +); + +const mockIssueBodyRes = new Response( + JSON.stringify({ + title: 'Issue Title', + body: 'Issue Body', + }), + ResponseInit, +); + +const mockIssueCommentsRes = new Response( + JSON.stringify([ + { id: 4, body: 'Closing this' }, + { id: 5, body: 'Nice issue' }, + ]), + ResponseInit, +); + +const prBodyEndpoint = 'https://api.github.com/repos/bbc/simorgh/pulls/9188'; +const prCommentsEndpoint = + 'https://api.github.com/repos/bbc/simorgh/issues/9188/comments'; +const prReviewCommentsEndpoint = + 'https://api.github.com/repos/bbc/simorgh/pulls/9188/comments'; +const issueBodyEndpoint = + 'https://api.github.com/repos/bbc/psammead/issues/4512'; +const issueCommentsEndpoint = + 'https://api.github.com/repos/bbc/psammead/issues/4512/comments'; + +jest.mock('node-fetch', () => ({ + default: async url => + ({ + [prBodyEndpoint]: mockPrBodyRes, + [prCommentsEndpoint]: mockPrCommentsRes, + [prReviewCommentsEndpoint]: mockPrReviewCommentsRes, + [issueBodyEndpoint]: mockIssueBodyRes, + [issueCommentsEndpoint]: mockIssueCommentsRes, + }[url].clone()), +})); + +const { fetchPr, fetchIssue } = require('.'); + +afterEach(() => { + jest.resetAllMocks(); + jest.restoreAllMocks(); +}); + +describe('Fetching from the GitHub API', () => { + it('should call the correct GitHub API endpoints and construct a pr object', async () => { + const reqBody = { + owner: 'bbc', + repo: 'simorgh', + prId: '9188', + }; + + const pr = await fetchPr(reqBody); + expect(pr).toEqual({ + body: 'PR Body', + comments: [ + { id: 1, body: 'Added translations' }, + { id: 2, body: 'Nice work, LGTM!' }, + ], + reviewComments: [{ id: 3, body: 'Smashed it!' }], + }); + }); + + it('should call the correct GitHub API endpoints and construct an issue object', async () => { + const reqBody = { + owner: 'bbc', + repo: 'psammead', + issueId: '4512', + }; + + const issue = await fetchIssue(reqBody); + expect(issue).toEqual({ + body: 'Issue Body', + comments: [ + { id: 4, body: 'Closing this' }, + { id: 5, body: 'Nice issue' }, + ], + }); + }); +}); diff --git a/scripts/exposure-scanning/index.js b/scripts/exposure-scanning/index.js new file mode 100644 index 0000000000..1ed99b9b11 --- /dev/null +++ b/scripts/exposure-scanning/index.js @@ -0,0 +1,51 @@ +import parseArgs from './args'; +import { fetchPr, fetchIssue } from './fetch'; +import { scanPr, scanIssue } from './scan'; +import { patchPr, patchIssue } from './patch'; + +const scanExposures = async () => { + const { repo, flag, id, regexString } = parseArgs(process.argv); + + const regex = new RegExp(regexString, 'gi'); + + if (flag === '-pr') { + const reqBody = { + owner: 'bbc', + repo, + prId: id, + }; + + const pr = await fetchPr(reqBody); + + const scannedPr = await scanPr(pr, regex); + + await patchPr(reqBody, scannedPr); + + if (scannedPr.foundMatch) { + return Promise.reject(new Error('Match found.')); + } + } else if (flag === '-issue') { + const reqBody = { + owner: 'bbc', + repo, + issueId: id, + }; + + const issue = await fetchIssue(reqBody); + + const scannedIssue = await scanIssue(issue, regex); + + await patchIssue(reqBody, scannedIssue); + + if (scannedIssue.foundMatch) { + return Promise.reject(new Error('Match found.')); + } + } else { + return Promise.reject( + new Error('Incorrect flag. Please use -pr or -issue.'), + ); + } + return Promise.resolve('No matches found.'); +}; + +export default scanExposures; diff --git a/scripts/exposure-scanning/index.test.js b/scripts/exposure-scanning/index.test.js new file mode 100644 index 0000000000..b8103c5379 --- /dev/null +++ b/scripts/exposure-scanning/index.test.js @@ -0,0 +1,99 @@ +import parseArgs from './args'; +import { fetchPr, fetchIssue } from './fetch'; +// import { patchPr } from './patch'; +import scanExposures from '.'; + +jest.mock('./args'); +jest.mock('./fetch'); +jest.mock('./patch'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. + +describe('Expected PR scanning', () => { + beforeEach(() => { + jest.clearAllMocks(); + parseArgs.mockReturnValueOnce({ + repo: 'psammead', + id: '100000', + flag: '-pr', + regexString: 'foo|bar', + }); + }); + + it('should not throw when there are no matches in a PR', async () => { + fetchPr.mockResolvedValueOnce({ + body: 'Pr Body', + comments: [], + reviewComments: [], + }); + + await expect(scanExposures()).resolves.toBe('No matches found.'); + }); + + it('should throw when there is a match in the PR body', async () => { + fetchPr.mockResolvedValueOnce({ + body: 'foo and bar', + comments: [], + reviewComments: [], + }); + + await expect(scanExposures()).rejects.toThrow(); + }); + + it('should throw when there is a match in the PR comments', async () => { + fetchPr.mockResolvedValueOnce({ + body: 'PR Body', + comments: [{ id: 3, body: 'foooooo' }], + reviewComments: [], + }); + + await expect(scanExposures()).rejects.toThrow(); + }); + + it('should throw when there is a match in the PR review comments', async () => { + fetchPr.mockResolvedValueOnce({ + body: 'Pr Body', + comments: [], + reviewComments: [{ id: 1, body: 'foo bar bar foo' }], + }); + + await expect(scanExposures()).rejects.toThrow(); + }); +}); + +describe('Expected issue scanning', () => { + beforeEach(() => { + jest.clearAllMocks(); + parseArgs.mockReturnValueOnce({ + repo: 'psammead', + id: '100000', + flag: '-issue', + regexString: 'foo|bar', + }); + }); + + it('should not throw when there are no matches in an issue', async () => { + fetchIssue.mockResolvedValueOnce({ + body: 'Issue Body', + comments: [], + }); + + await expect(scanExposures()).resolves.toBe('No matches found.'); + }); + + it('should throw when there is a match in the issue body', async () => { + fetchIssue.mockResolvedValueOnce({ + body: 'foobar', + reviewComments: [], + }); + + await expect(scanExposures()).rejects.toThrow(); + }); + + it('should throw when there is a match in the issue comments', async () => { + fetchIssue.mockResolvedValueOnce({ + body: 'Issue Body', + reviewComments: [{ id: 0, body: 'bar.' }], + }); + + await expect(scanExposures()).rejects.toThrow(); + }); +}); diff --git a/scripts/exposure-scanning/patch/index.js b/scripts/exposure-scanning/patch/index.js new file mode 100644 index 0000000000..e53229ac60 --- /dev/null +++ b/scripts/exposure-scanning/patch/index.js @@ -0,0 +1,85 @@ +import { Octokit } from '@octokit/rest'; + +const octokit = new Octokit(); + +const patchPrBody = async (reqBody, body) => { + const res = await octokit.request( + 'PATCH /repos/{owner}/{repo}/pulls/{prId}', + { + ...reqBody, + body, + }, + ); + + return res; +}; + +const patchPrReviewComments = async (reqBody, comments) => { + const res = comments.map(async ({ id, body }) => { + await octokit.request( + 'PATCH /repos/{owner}/{repo}/pulls/comments/{commentId}', + { + owner: reqBody.owner, + repo: reqBody.repo, + commentId: id, + body, + }, + ); + }); + + return res; +}; + +const patchPrComments = async (reqBody, comments) => { + const res = comments.forEach(async ({ id, body }) => { + await octokit.request( + 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', + { + ...reqBody, + commentId: id, + body, + }, + ); + }); + + return res; +}; + +const patchIssueBody = async (reqBody, body) => { + await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{prId}', { + ...reqBody, + body, + }); +}; + +const patchIssueComments = async (reqBody, comments) => { + comments.forEach(async ({ id, body }) => { + await octokit.request( + 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', + { + ...reqBody, + commentId: id, + body, + }, + ); + }); +}; + +export const patchPr = async (reqBody, { body, comments, reviewComments }) => { + const patchedPr = await Promise.all([ + patchPrBody(reqBody, body), + patchPrComments(reqBody, comments), + patchPrReviewComments(reqBody, reviewComments), + ]); + + return patchedPr; +}; + +export const patchIssue = async (reqBody, { body, comments }) => { + const patchedIssue = await Promise.all([ + patchIssueBody(reqBody, body), + patchIssueComments(reqBody, comments), + ]); + + return patchedIssue; +}; diff --git a/scripts/exposure-scanning/patch/index.test.js b/scripts/exposure-scanning/patch/index.test.js new file mode 100644 index 0000000000..15f0d7312d --- /dev/null +++ b/scripts/exposure-scanning/patch/index.test.js @@ -0,0 +1,59 @@ +/* eslint-disable no-console */ +/* eslint-disable no-empty */ +jest.mock('node-fetch'); + +console.log = jest.fn(); +console.error = jest.fn(); + +const nodeFetch = require('node-fetch'); + +const { patchPr, patchIssue } = require('.'); + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe('Patching', () => { + it('should call fetch 5 times for the body, comments and review comments endpoints', async () => { + const reqBody = { + owner: 'bbc', + repo: 'psamemad', + prId: '1234', + }; + + const pr = { + body: 'PR *', + comments: [ + { id: 1, body: 'Added *' }, + { id: 2, body: 'Nice work, LGTM!' }, + ], + reviewComments: [ + { id: 3, body: 'Smashed it!' }, + { id: 4, body: 'Should there be * * for this?' }, + ], + }; + + try { + await patchPr(reqBody, pr); + } catch (error) {} + expect(nodeFetch).toHaveBeenCalledTimes(5); + }); + + it('should call fetch 2 times for the body and comments endpoints', async () => { + const reqBody = { + owner: 'bbc', + repo: 'psammead', + issueId: '4512', + }; + + const issue = { + body: '* Body', + comments: [{ id: 5, body: 'Nice *' }], + }; + + try { + await patchIssue(reqBody, issue); + } catch (error) {} + expect(nodeFetch).toHaveBeenCalledTimes(2); + }); +}); diff --git a/scripts/exposure-scanning/scan/index.js b/scripts/exposure-scanning/scan/index.js new file mode 100644 index 0000000000..96f9501a57 --- /dev/null +++ b/scripts/exposure-scanning/scan/index.js @@ -0,0 +1,61 @@ +const replacementStr = '*'; + +let foundMatch = false; + +const scanText = (text, regex) => { + const scannedText = text.replace(regex, replacementStr); + + if (scannedText !== text) { + foundMatch = true; + } + + return scannedText; +}; + +const scanComments = (comments, regex) => { + const scannedComments = []; + + comments.forEach(({ id, body }) => { + if (regex.test(body)) { + foundMatch = true; + scannedComments.push({ + id, + body: body.replace(regex, replacementStr), + }); + } + }); + + return scannedComments; +}; + +export const scanPr = (pr, regex) => { + let scannedPr = pr; + + try { + scannedPr = { + body: scanText(pr.body, regex), + comments: scanComments(pr.comments, regex), + reviewComments: scanComments(pr.reviewComments, regex), + }; + } catch (error) { + throw new Error(`Encountered an error when scanning: ${error}`); + } + + return { ...scannedPr, foundMatch }; +}; + +export const scanIssue = (issue, regex) => { + foundMatch = false; + let scannedIssue = issue; + + try { + scannedIssue = { + body: scanText(issue.body, regex), + comments: scanComments(issue.comments, regex), + }; + } catch (error) { + throw new Error(`Encountered an error when scanning: ${error}`); + } + + return { ...scannedIssue, foundMatch }; +}; diff --git a/scripts/exposure-scanning/scan/index.test.js b/scripts/exposure-scanning/scan/index.test.js new file mode 100644 index 0000000000..36745b4636 --- /dev/null +++ b/scripts/exposure-scanning/scan/index.test.js @@ -0,0 +1,90 @@ +import { scanPr, scanIssue } from '.'; + +describe('Scanning a PR', () => { + const pr = { + body: 'PR Body', + comments: [ + { id: 1, body: 'Added Translations' }, + { id: 2, body: 'Nice work, LGTM!' }, + ], + reviewComments: [ + { id: 3, body: 'Smashed it!' }, + { id: 4, body: 'Should there be new translations for this?' }, + ], + }; + + it('should return the original PR with empty comment arrays if no matches have been found', () => { + const regex = new RegExp('foo|bar', 'gi'); + const scannedPr = scanPr(pr, regex); + expect(scannedPr).toEqual({ + ...pr, + foundMatch: false, + comments: [], + reviewComments: [], + }); + }); + + it('should return the PR with a redacted body and empty comment arrays if a body match was found', () => { + const regex = new RegExp('Body', 'gi'); + const scannedPr = scanPr(pr, regex); + + expect(scannedPr).toEqual({ + body: 'PR *', + foundMatch: true, + comments: [], + reviewComments: [], + }); + }); + + it('should return the PR with redacted comments and review comments if a match has been found', () => { + const regex = new RegExp('translations', 'gi'); + const scannedPr = scanPr(pr, regex); + + expect(scannedPr).toEqual({ + ...pr, + foundMatch: true, + comments: [{ id: 1, body: 'Added *' }], + reviewComments: [{ id: 4, body: 'Should there be new * for this?' }], + }); + }); +}); + +describe('Scanning an Issue', () => { + const issue = { + body: 'Issue Body', + comments: [ + { id: 4, body: 'Closing this' }, + { id: 5, body: 'Nice issue' }, + ], + }; + + it('should return the original issue with empty comment arrays if there are no matches', () => { + const regex = new RegExp('foo|bar'); + const scannedIssue = scanIssue(issue, regex); + expect(scannedIssue).toEqual({ + ...issue, + foundMatch: false, + comments: [], + }); + }); + + it('should return the issue with a redacted body if there is a match in the issue body', () => { + const regex = new RegExp('body', 'gi'); + const scannedIssue = scanIssue(issue, regex); + expect(scannedIssue).toEqual({ + body: 'Issue *', + foundMatch: true, + comments: [], + }); + }); + + it('should return the issue with redacted body and comments if a match occurs in either', () => { + const regex = new RegExp('issue', 'gi'); + const scannedIssue = scanIssue(issue, regex); + expect(scannedIssue).toEqual({ + body: '* Body', + foundMatch: true, + comments: [{ id: 5, body: 'Nice *' }], + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 9eab423afb..a4f0d744a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1780,7 +1780,7 @@ dependencies: "@octokit/types" "^6.0.3" -"@octokit/core@^3.0.0": +"@octokit/core@^3.0.0", "@octokit/core@^3.5.0": version "3.5.1" resolved "https://registry.yarnpkg.com/@octokit/core/-/core-3.5.1.tgz#8601ceeb1ec0e1b1b8217b960a413ed8e947809b" integrity sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw== @@ -1816,6 +1816,11 @@ resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-8.1.4.tgz#20684667b50176b5d24cdb89799a8a12d38c3775" integrity sha512-NnGr4NNDqO5wjSDJo5nxrGtzZUwoT23YasqK2H4Pav/6vSgeVTxuqCL9Aeh+cWfTxDomj1M4Os5BrXFsvl7qiQ== +"@octokit/openapi-types@^9.1.0": + version "9.1.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.1.0.tgz#7f73adbde8ba2d2512de90ea8a5da68b25d65d0a" + integrity sha512-XBP03pG4XuTU+VgeJM1ozRdmZJerMG4tk6wA+raFKycC4qV9jtD2UQroAg9bAcmI3Q0zWvifeDGtPqsFjMzkLg== + "@octokit/plugin-paginate-rest@^2.2.3": version "2.13.6" resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.13.6.tgz#03633839b0ec57a108d2ca1c4b1f64b749d3506c" @@ -1823,6 +1828,26 @@ dependencies: "@octokit/types" "^6.17.3" +"@octokit/plugin-paginate-rest@^2.6.2": + version "2.14.0" + resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.14.0.tgz#f469cb4a908792fb44679c5973d8bba820c88b0f" + integrity sha512-S2uEu2uHeI7Vf+Lvj8tv3O5/5TCAa8GHS0dUQN7gdM7vKA6ZHAbR6HkAVm5yMb1mbedLEbxOuQ+Fa0SQ7tCDLA== + dependencies: + "@octokit/types" "^6.18.0" + +"@octokit/plugin-request-log@^1.0.2": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz#5e50ed7083a613816b1e4a28aeec5fb7f1462e85" + integrity sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA== + +"@octokit/plugin-rest-endpoint-methods@5.5.0": + version "5.5.0" + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.5.0.tgz#28f94a089029dffb8589447366eca5c441bb8d34" + integrity sha512-v4dNoHF8cXNx7C67yQx7oarHs5Wg2IiafWvp/ULkNcCOuXgQdBOkJtwidpYqPiRPUw4uHDkI6Tgfje+nXB+Deg== + dependencies: + "@octokit/types" "^6.21.0" + deprecation "^2.3.1" + "@octokit/plugin-rest-endpoint-methods@^4.0.0": version "4.15.1" resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.15.1.tgz#91a064bee99d0ffcef74a04357e1cf15c27d1cd0" @@ -1852,6 +1877,16 @@ node-fetch "^2.6.1" universal-user-agent "^6.0.0" +"@octokit/rest@^18.7.0": + version "18.7.0" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.7.0.tgz#3a0f6498a9495f3ecede582ab15efe38b298fac8" + integrity sha512-8K8BJFyPFRSfnwu+aSbdjU5w3EtxC33PkDlEi5tyVTYC+t4n7gaqygRg5ajJLCpb/ZzVaXXFJXC9OxQ9TvFRAw== + dependencies: + "@octokit/core" "^3.5.0" + "@octokit/plugin-paginate-rest" "^2.6.2" + "@octokit/plugin-request-log" "^1.0.2" + "@octokit/plugin-rest-endpoint-methods" "5.5.0" + "@octokit/types@^6.0.3", "@octokit/types@^6.13.0", "@octokit/types@^6.16.1", "@octokit/types@^6.17.3": version "6.17.4" resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.17.4.tgz#5ec011bfe3d08b7605c7d92a15796c470ee541b6" @@ -1859,6 +1894,13 @@ dependencies: "@octokit/openapi-types" "^8.1.4" +"@octokit/types@^6.18.0", "@octokit/types@^6.21.0": + version "6.21.0" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.21.0.tgz#723d0296d35296d6ec91247e770f24e6cd51d4ee" + integrity sha512-VPSxn9uhCoOUMpxCsOAQhf8DgIx+uzFjZRYDiZS5+TvrKaEwBrWkjr/5NmUVvPbW6xdPC2n3yL3XCnoxa4rxvg== + dependencies: + "@octokit/openapi-types" "^9.1.0" + "@pmmmwh/react-refresh-webpack-plugin@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.4.3.tgz#1eec460596d200c0236bf195b078a5d1df89b766" From a0a088c2451490855dceecc811b253b08610191c Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 11:49:49 +0100 Subject: [PATCH 02/54] Fix es module errors --- package.json | 2 +- scripts/exposure-scanning/fetch/index.js | 4 +++- scripts/exposure-scanning/index.js | 19 +++++++++++++++---- scripts/exposure-scanning/index.test.js | 10 +++++++--- scripts/exposure-scanning/package.json | 7 +++++++ scripts/exposure-scanning/patch/index.js | 4 +++- 6 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 scripts/exposure-scanning/package.json diff --git a/package.json b/package.json index 9e14a1ae2d..380e8eebfe 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "preinstall": "node scripts/check-package-manager.js && sh scripts/enforceVersions.sh && [ \"$npm_config_refer\" != \"ci\" ]", "postinstall": "yarn packages:check && yarn build", "publish": "node scripts/publish", - "scan:exposures": "node scripts/scan-exposures", + "scan:exposures": "node scripts/exposure-scanning --experimental-modules", "storybook": "NODE_ENV=development start-storybook -p 8180 -s .storybook/static -c .storybook", "test": "yarn test:lint && yarn test:unit", "test:ci": "yarn test:lint && yarn test:unit:ci", diff --git a/scripts/exposure-scanning/fetch/index.js b/scripts/exposure-scanning/fetch/index.js index 5d9fde7e31..c2ce07403b 100644 --- a/scripts/exposure-scanning/fetch/index.js +++ b/scripts/exposure-scanning/fetch/index.js @@ -1,4 +1,6 @@ -import { Octokit } from '@octokit/rest'; +import pkg from '@octokit/rest'; + +const { Octokit } = pkg; const octokit = new Octokit(); diff --git a/scripts/exposure-scanning/index.js b/scripts/exposure-scanning/index.js index 1ed99b9b11..9f4fe50e0d 100644 --- a/scripts/exposure-scanning/index.js +++ b/scripts/exposure-scanning/index.js @@ -1,7 +1,13 @@ -import parseArgs from './args'; -import { fetchPr, fetchIssue } from './fetch'; -import { scanPr, scanIssue } from './scan'; -import { patchPr, patchIssue } from './patch'; +/* eslint-disable import/extensions */ +import parseArgs from './args/index.js'; +import { fetchPr, fetchIssue } from './fetch/index.js'; +import { scanPr, scanIssue } from './scan/index.js'; +import { patchPr, patchIssue } from './patch/index.js'; + +// const parseArgs = require('./args'); +// const { fetchPr, fetchIssue } = require('./fetch'); +// const { scanPr, scanIssue } = require('./scan'); +// const { patchPr, patchIssue } = require('./patch'); const scanExposures = async () => { const { repo, flag, id, regexString } = parseArgs(process.argv); @@ -48,4 +54,9 @@ const scanExposures = async () => { return Promise.resolve('No matches found.'); }; +(async () => { + await scanExposures(); +})(); +scanExposures(); + export default scanExposures; diff --git a/scripts/exposure-scanning/index.test.js b/scripts/exposure-scanning/index.test.js index b8103c5379..c5464ceff9 100644 --- a/scripts/exposure-scanning/index.test.js +++ b/scripts/exposure-scanning/index.test.js @@ -1,12 +1,16 @@ -import parseArgs from './args'; -import { fetchPr, fetchIssue } from './fetch'; -// import { patchPr } from './patch'; +/* eslint-disable import/extensions */ +import parseArgs from './args/index.js'; +import { fetchPr, fetchIssue } from './fetch/index.js'; import scanExposures from '.'; jest.mock('./args'); jest.mock('./fetch'); jest.mock('./patch'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. +// const parseArgs = require('./args'); +// const { fetchPr, fetchIssue } = require('./fetch'); +// const scanExposures = require('.'); + describe('Expected PR scanning', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/scripts/exposure-scanning/package.json b/scripts/exposure-scanning/package.json new file mode 100644 index 0000000000..86e5311233 --- /dev/null +++ b/scripts/exposure-scanning/package.json @@ -0,0 +1,7 @@ +{ + "name": "exposure-scanning", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "type": "module" +} diff --git a/scripts/exposure-scanning/patch/index.js b/scripts/exposure-scanning/patch/index.js index e53229ac60..07ca580dca 100644 --- a/scripts/exposure-scanning/patch/index.js +++ b/scripts/exposure-scanning/patch/index.js @@ -1,4 +1,6 @@ -import { Octokit } from '@octokit/rest'; +import pkg from '@octokit/rest'; + +const { Octokit } = pkg; const octokit = new Octokit(); From 9e2438e36fbe506c122e9e889adad3f5eb5f170f Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 11:56:46 +0100 Subject: [PATCH 03/54] Add flags to cli invocation --- package.json | 2 +- scripts/exposure-scanning/args/index.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 380e8eebfe..6b0ced6606 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "preinstall": "node scripts/check-package-manager.js && sh scripts/enforceVersions.sh && [ \"$npm_config_refer\" != \"ci\" ]", "postinstall": "yarn packages:check && yarn build", "publish": "node scripts/publish", - "scan:exposures": "node scripts/exposure-scanning --experimental-modules", + "scan:exposures": "node scripts/exposure-scanning --experimental-modules --unhandled-rejections=strict", "storybook": "NODE_ENV=development start-storybook -p 8180 -s .storybook/static -c .storybook", "test": "yarn test:lint && yarn test:unit", "test:ci": "yarn test:lint && yarn test:unit:ci", diff --git a/scripts/exposure-scanning/args/index.js b/scripts/exposure-scanning/args/index.js index 0fea6af27e..d2ca3f9f61 100644 --- a/scripts/exposure-scanning/args/index.js +++ b/scripts/exposure-scanning/args/index.js @@ -7,16 +7,16 @@ const isValidId = id => { }; const parseArgs = argv => { - if (argv.length !== 6) { + if (argv.length !== 8) { throw new Error( - 'Error: Incorrect number of args.\nUsage: node scan.js <-pr/-issue> ', + 'Error: Incorrect number of args.\nUsage: node scan.js --experimental-modules --unhandled-rejections=strict <-pr/-issue> ', ); } - const repo = argv[2]; - const flag = argv[3]; // -pr or -issue - const id = argv[4]; - const regexString = argv[5]; + const repo = argv[4]; + const flag = argv[5]; // -pr or -issue + const id = argv[6]; + const regexString = argv[7]; if (isValidId(id)) { return { repo, flag, id, regexString }; From b35cd288e9a35fc67764d91dd1fb23d14827f65a Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 12:22:19 +0100 Subject: [PATCH 04/54] Fix args tests --- package.json | 2 +- scripts/exposure-scanning/args/index.js | 4 +-- scripts/exposure-scanning/args/index.test.js | 29 +++++++++++++++++--- scripts/exposure-scanning/scan/index.js | 4 +-- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 6b0ced6606..c80348ebed 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "preinstall": "node scripts/check-package-manager.js && sh scripts/enforceVersions.sh && [ \"$npm_config_refer\" != \"ci\" ]", "postinstall": "yarn packages:check && yarn build", "publish": "node scripts/publish", - "scan:exposures": "node scripts/exposure-scanning --experimental-modules --unhandled-rejections=strict", + "scan:exposures": "node --experimental-modules --unhandled-rejections=strict scripts/exposure-scanning ", "storybook": "NODE_ENV=development start-storybook -p 8180 -s .storybook/static -c .storybook", "test": "yarn test:lint && yarn test:unit", "test:ci": "yarn test:lint && yarn test:unit:ci", diff --git a/scripts/exposure-scanning/args/index.js b/scripts/exposure-scanning/args/index.js index d2ca3f9f61..202f4eea38 100644 --- a/scripts/exposure-scanning/args/index.js +++ b/scripts/exposure-scanning/args/index.js @@ -9,7 +9,7 @@ const isValidId = id => { const parseArgs = argv => { if (argv.length !== 8) { throw new Error( - 'Error: Incorrect number of args.\nUsage: node scan.js --experimental-modules --unhandled-rejections=strict <-pr/-issue> ', + 'Incorrect number of args.\nUsage: node --experimental-modules --unhandled-rejections=strict scan.js <-pr/-issue> ', ); } @@ -22,7 +22,7 @@ const parseArgs = argv => { return { repo, flag, id, regexString }; } - throw new Error('Error: Invalid issue id'); + throw new Error('Invalid issue id'); }; export default parseArgs; diff --git a/scripts/exposure-scanning/args/index.test.js b/scripts/exposure-scanning/args/index.test.js index afb8cc7efa..8c6fc60149 100644 --- a/scripts/exposure-scanning/args/index.test.js +++ b/scripts/exposure-scanning/args/index.test.js @@ -3,7 +3,16 @@ import parseArgs from '.'; describe('Parsing arguments', () => { it('should return an object containing the flag and id', () => { expect( - parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 'regex']), + parseArgs([ + 'node', + '--experimental-modules', + '--unhandled-rejections=strict', + 'scan.js', + 'simorgh', + '-pr', + '1234', + 'regex', + ]), ).toEqual({ repo: 'simorgh', flag: '-pr', @@ -14,9 +23,19 @@ describe('Parsing arguments', () => { it('should throw an error without logging args if the number given is incorrect', () => { expect(() => - parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 're[g]ex', '-u']), + parseArgs([ + 'node', + '--experimental-modules', + '--unhandled-rejections=strict', + 'scan.js', + 'simorgh', + '-pr', + '1234', + 're[g]ex', + '-u', + ]), ).toThrow( - 'Error: Incorrect number of args.\nUsage: node scan.js <-pr/-issue> ', + 'Incorrect number of args.\nUsage: node --experimental-modules --unhandled-rejections=strict scan.js <-pr/-issue> ', ); }); @@ -24,12 +43,14 @@ describe('Parsing arguments', () => { expect(() => parseArgs([ 'node', + '--experimental-modules', + '--unhandled-rejections=strict', 'scan.js', 'psammead', '-issue', 'add-topic-tags', 'regex', ]), - ).toThrow('Error: Invalid issue id'); + ).toThrow('Invalid issue id'); }); }); diff --git a/scripts/exposure-scanning/scan/index.js b/scripts/exposure-scanning/scan/index.js index 96f9501a57..bbf44950e6 100644 --- a/scripts/exposure-scanning/scan/index.js +++ b/scripts/exposure-scanning/scan/index.js @@ -38,7 +38,7 @@ export const scanPr = (pr, regex) => { reviewComments: scanComments(pr.reviewComments, regex), }; } catch (error) { - throw new Error(`Encountered an error when scanning: ${error}`); + throw new Error(`Encountered an error when scanning.`); } return { ...scannedPr, foundMatch }; @@ -54,7 +54,7 @@ export const scanIssue = (issue, regex) => { comments: scanComments(issue.comments, regex), }; } catch (error) { - throw new Error(`Encountered an error when scanning: ${error}`); + throw new Error(`Encountered an error when scanning.`); } return { ...scannedIssue, foundMatch }; From f26d0cbffb1e3e1b84ed32606da7ae6eef549428 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 12:38:34 +0100 Subject: [PATCH 05/54] Fix parseArgs issues --- scripts/exposure-scanning/args/index.js | 14 ++++----- scripts/exposure-scanning/args/index.test.js | 31 ++++---------------- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/scripts/exposure-scanning/args/index.js b/scripts/exposure-scanning/args/index.js index 202f4eea38..adf4358dbc 100644 --- a/scripts/exposure-scanning/args/index.js +++ b/scripts/exposure-scanning/args/index.js @@ -7,22 +7,22 @@ const isValidId = id => { }; const parseArgs = argv => { - if (argv.length !== 8) { + if (argv.length !== 6) { throw new Error( - 'Incorrect number of args.\nUsage: node --experimental-modules --unhandled-rejections=strict scan.js <-pr/-issue> ', + 'Incorrect number of args.\nUsage: node scripts/exposure-scanning <-pr/-issue> ', ); } - const repo = argv[4]; - const flag = argv[5]; // -pr or -issue - const id = argv[6]; - const regexString = argv[7]; + const repo = argv[2]; + const flag = argv[3]; // -pr or -issue + const id = argv[4]; + const regexString = argv[5]; if (isValidId(id)) { return { repo, flag, id, regexString }; } - throw new Error('Invalid issue id'); + throw new Error('Invalid issue/pr id'); }; export default parseArgs; diff --git a/scripts/exposure-scanning/args/index.test.js b/scripts/exposure-scanning/args/index.test.js index 8c6fc60149..7cc6379131 100644 --- a/scripts/exposure-scanning/args/index.test.js +++ b/scripts/exposure-scanning/args/index.test.js @@ -1,18 +1,9 @@ import parseArgs from '.'; describe('Parsing arguments', () => { - it('should return an object containing the flag and id', () => { + it('should return an object containing the repo, flag, id and regex string', () => { expect( - parseArgs([ - 'node', - '--experimental-modules', - '--unhandled-rejections=strict', - 'scan.js', - 'simorgh', - '-pr', - '1234', - 'regex', - ]), + parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 'regex']), ).toEqual({ repo: 'simorgh', flag: '-pr', @@ -23,19 +14,9 @@ describe('Parsing arguments', () => { it('should throw an error without logging args if the number given is incorrect', () => { expect(() => - parseArgs([ - 'node', - '--experimental-modules', - '--unhandled-rejections=strict', - 'scan.js', - 'simorgh', - '-pr', - '1234', - 're[g]ex', - '-u', - ]), + parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 're[g]ex', '-u']), ).toThrow( - 'Incorrect number of args.\nUsage: node --experimental-modules --unhandled-rejections=strict scan.js <-pr/-issue> ', + 'Incorrect number of args.\nUsage: node scripts/exposure-scanning <-pr/-issue> ', ); }); @@ -43,14 +24,12 @@ describe('Parsing arguments', () => { expect(() => parseArgs([ 'node', - '--experimental-modules', - '--unhandled-rejections=strict', 'scan.js', 'psammead', '-issue', 'add-topic-tags', 'regex', ]), - ).toThrow('Invalid issue id'); + ).toThrow('Invalid issue/pr id'); }); }); From b1481025082b8a57ec046e80b4c73c560139bc9f Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 12:48:05 +0100 Subject: [PATCH 06/54] Add more comprehensive arg parsing --- scripts/exposure-scanning/args/index.js | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/exposure-scanning/args/index.js b/scripts/exposure-scanning/args/index.js index adf4358dbc..dccb4831e7 100644 --- a/scripts/exposure-scanning/args/index.js +++ b/scripts/exposure-scanning/args/index.js @@ -6,6 +6,15 @@ const isValidId = id => { } }; +const isValidRegex = regex => { + try { + RegExp(regex); + } catch { + return false; + } + return regex !== ''; +}; + const parseArgs = argv => { if (argv.length !== 6) { throw new Error( @@ -18,11 +27,19 @@ const parseArgs = argv => { const id = argv[4]; const regexString = argv[5]; - if (isValidId(id)) { - return { repo, flag, id, regexString }; + if (!isValidRegex(regexString)) { + throw new Error('Invalid regex argument given.'); + } + + if (!isValidId(id)) { + throw new Error('Invalid issue/pr id.'); + } + + if (flag !== '-pr' && flag !== 'issue') { + throw new Error('Invalid flag argument given.'); } - throw new Error('Invalid issue/pr id'); + return { repo, flag, id, regexString }; }; export default parseArgs; From ba16a6a593efe2ce57373d7a24080a9bc427d129 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 12:49:12 +0100 Subject: [PATCH 07/54] Remove useless else --- scripts/exposure-scanning/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/exposure-scanning/index.js b/scripts/exposure-scanning/index.js index 9f4fe50e0d..9830047576 100644 --- a/scripts/exposure-scanning/index.js +++ b/scripts/exposure-scanning/index.js @@ -46,10 +46,6 @@ const scanExposures = async () => { if (scannedIssue.foundMatch) { return Promise.reject(new Error('Match found.')); } - } else { - return Promise.reject( - new Error('Incorrect flag. Please use -pr or -issue.'), - ); } return Promise.resolve('No matches found.'); }; From b728fc795e115f92b4e5258388087a1cfee534bf Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 12:56:05 +0100 Subject: [PATCH 08/54] Added more parseArgs tests --- scripts/exposure-scanning/args/index.test.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/exposure-scanning/args/index.test.js b/scripts/exposure-scanning/args/index.test.js index 7cc6379131..b95924b6d2 100644 --- a/scripts/exposure-scanning/args/index.test.js +++ b/scripts/exposure-scanning/args/index.test.js @@ -32,4 +32,23 @@ describe('Parsing arguments', () => { ]), ).toThrow('Invalid issue/pr id'); }); + + it('should throw an error without logging args if an invalid regex is given', () => { + expect(() => + parseArgs(['node', 'scan.js', 'psammead', '-issue', '12341234', '']), + ).toThrow('Invalid regex argument given.'); + }); + + it('should throw an error without logging args if an invalid flag is given', () => { + expect(() => + parseArgs([ + 'node', + 'scan.js', + 'psammead', + '-not-an-issue', + '12341234', + 'regex', + ]), + ).toThrow('Invalid flag argument given.'); + }); }); From 249984ac0e97e20279bf0c067e0d973f574a20d1 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 12:58:46 +0100 Subject: [PATCH 09/54] Remove commonjs comments --- scripts/exposure-scanning/index.js | 5 ----- scripts/exposure-scanning/index.test.js | 4 ---- 2 files changed, 9 deletions(-) diff --git a/scripts/exposure-scanning/index.js b/scripts/exposure-scanning/index.js index 9830047576..dc993f0aba 100644 --- a/scripts/exposure-scanning/index.js +++ b/scripts/exposure-scanning/index.js @@ -4,11 +4,6 @@ import { fetchPr, fetchIssue } from './fetch/index.js'; import { scanPr, scanIssue } from './scan/index.js'; import { patchPr, patchIssue } from './patch/index.js'; -// const parseArgs = require('./args'); -// const { fetchPr, fetchIssue } = require('./fetch'); -// const { scanPr, scanIssue } = require('./scan'); -// const { patchPr, patchIssue } = require('./patch'); - const scanExposures = async () => { const { repo, flag, id, regexString } = parseArgs(process.argv); diff --git a/scripts/exposure-scanning/index.test.js b/scripts/exposure-scanning/index.test.js index c5464ceff9..16c400701b 100644 --- a/scripts/exposure-scanning/index.test.js +++ b/scripts/exposure-scanning/index.test.js @@ -7,10 +7,6 @@ jest.mock('./args'); jest.mock('./fetch'); jest.mock('./patch'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. -// const parseArgs = require('./args'); -// const { fetchPr, fetchIssue } = require('./fetch'); -// const scanExposures = require('.'); - describe('Expected PR scanning', () => { beforeEach(() => { jest.clearAllMocks(); From 7054561aa76d9dd4249074479ca1b52d7c887eda Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 16:13:41 +0100 Subject: [PATCH 10/54] Fix mock issues --- scripts/exposure-scanning/fetch/index.js | 4 +- scripts/exposure-scanning/index.js | 61 +++-------------- scripts/exposure-scanning/patch/index.js | 37 ++++------- scripts/exposure-scanning/patch/index.test.js | 65 +++++++++++++++---- scripts/exposure-scanning/scanExposures.js | 49 ++++++++++++++ .../{index.test.js => scanExposures.test.js} | 8 +-- 6 files changed, 125 insertions(+), 99 deletions(-) create mode 100644 scripts/exposure-scanning/scanExposures.js rename scripts/exposure-scanning/{index.test.js => scanExposures.test.js} (91%) diff --git a/scripts/exposure-scanning/fetch/index.js b/scripts/exposure-scanning/fetch/index.js index c2ce07403b..5d9fde7e31 100644 --- a/scripts/exposure-scanning/fetch/index.js +++ b/scripts/exposure-scanning/fetch/index.js @@ -1,6 +1,4 @@ -import pkg from '@octokit/rest'; - -const { Octokit } = pkg; +import { Octokit } from '@octokit/rest'; const octokit = new Octokit(); diff --git a/scripts/exposure-scanning/index.js b/scripts/exposure-scanning/index.js index dc993f0aba..f26570dd03 100644 --- a/scripts/exposure-scanning/index.js +++ b/scripts/exposure-scanning/index.js @@ -1,53 +1,10 @@ /* eslint-disable import/extensions */ -import parseArgs from './args/index.js'; -import { fetchPr, fetchIssue } from './fetch/index.js'; -import { scanPr, scanIssue } from './scan/index.js'; -import { patchPr, patchIssue } from './patch/index.js'; - -const scanExposures = async () => { - const { repo, flag, id, regexString } = parseArgs(process.argv); - - const regex = new RegExp(regexString, 'gi'); - - if (flag === '-pr') { - const reqBody = { - owner: 'bbc', - repo, - prId: id, - }; - - const pr = await fetchPr(reqBody); - - const scannedPr = await scanPr(pr, regex); - - await patchPr(reqBody, scannedPr); - - if (scannedPr.foundMatch) { - return Promise.reject(new Error('Match found.')); - } - } else if (flag === '-issue') { - const reqBody = { - owner: 'bbc', - repo, - issueId: id, - }; - - const issue = await fetchIssue(reqBody); - - const scannedIssue = await scanIssue(issue, regex); - - await patchIssue(reqBody, scannedIssue); - - if (scannedIssue.foundMatch) { - return Promise.reject(new Error('Match found.')); - } - } - return Promise.resolve('No matches found.'); -}; - -(async () => { - await scanExposures(); -})(); -scanExposures(); - -export default scanExposures; +import scanExposures from './scanExposures.js'; + +try { + (async () => { + await scanExposures(); + })(); +} catch (error) { + throw new Error(`An error occured while scanning for exposures: ${error}`); +} diff --git a/scripts/exposure-scanning/patch/index.js b/scripts/exposure-scanning/patch/index.js index 07ca580dca..a3754a692b 100644 --- a/scripts/exposure-scanning/patch/index.js +++ b/scripts/exposure-scanning/patch/index.js @@ -1,23 +1,16 @@ -import pkg from '@octokit/rest'; - -const { Octokit } = pkg; +import { Octokit } from '@octokit/rest'; const octokit = new Octokit(); const patchPrBody = async (reqBody, body) => { - const res = await octokit.request( - 'PATCH /repos/{owner}/{repo}/pulls/{prId}', - { - ...reqBody, - body, - }, - ); - - return res; + await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{prId}', { + ...reqBody, + body, + }); }; const patchPrReviewComments = async (reqBody, comments) => { - const res = comments.map(async ({ id, body }) => { + await comments.map(async ({ id, body }) => { await octokit.request( 'PATCH /repos/{owner}/{repo}/pulls/comments/{commentId}', { @@ -28,12 +21,10 @@ const patchPrReviewComments = async (reqBody, comments) => { }, ); }); - - return res; }; const patchPrComments = async (reqBody, comments) => { - const res = comments.forEach(async ({ id, body }) => { + await comments.forEach(async ({ id, body }) => { await octokit.request( 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', { @@ -43,19 +34,17 @@ const patchPrComments = async (reqBody, comments) => { }, ); }); - - return res; }; const patchIssueBody = async (reqBody, body) => { - await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{prId}', { + await octokit.request('PATCH /repos/{owner}/{repo}/issues/{issueId}', { ...reqBody, body, }); }; const patchIssueComments = async (reqBody, comments) => { - comments.forEach(async ({ id, body }) => { + await comments.forEach(async ({ id, body }) => { await octokit.request( 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', { @@ -68,20 +57,16 @@ const patchIssueComments = async (reqBody, comments) => { }; export const patchPr = async (reqBody, { body, comments, reviewComments }) => { - const patchedPr = await Promise.all([ + await Promise.all([ patchPrBody(reqBody, body), patchPrComments(reqBody, comments), patchPrReviewComments(reqBody, reviewComments), ]); - - return patchedPr; }; export const patchIssue = async (reqBody, { body, comments }) => { - const patchedIssue = await Promise.all([ + await Promise.all([ patchIssueBody(reqBody, body), patchIssueComments(reqBody, comments), ]); - - return patchedIssue; }; diff --git a/scripts/exposure-scanning/patch/index.test.js b/scripts/exposure-scanning/patch/index.test.js index 15f0d7312d..c6212ac043 100644 --- a/scripts/exposure-scanning/patch/index.test.js +++ b/scripts/exposure-scanning/patch/index.test.js @@ -1,11 +1,27 @@ -/* eslint-disable no-console */ -/* eslint-disable no-empty */ -jest.mock('node-fetch'); +const { Headers, Response } = jest.requireActual('node-fetch'); -console.log = jest.fn(); -console.error = jest.fn(); +const mockHeaders = new Headers({ 'Content-Type': 'application/json' }); -const nodeFetch = require('node-fetch'); +const ResponseInit = { + url: 'hello', + headers: mockHeaders, + status: 200, +}; + +const mockRes = new Response( + JSON.stringify({ + test: 'test', + }), + ResponseInit, +); + +const mockFetch = jest.fn(); +jest.mock('node-fetch', () => ({ + default: async url => { + mockFetch(url); + return mockRes.clone(); + }, +})); const { patchPr, patchIssue } = require('.'); @@ -33,10 +49,25 @@ describe('Patching', () => { ], }; - try { - await patchPr(reqBody, pr); - } catch (error) {} - expect(nodeFetch).toHaveBeenCalledTimes(5); + await patchPr(reqBody, pr); + + expect(mockFetch).toHaveBeenCalledTimes(5); + + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psamemad/pulls/1234', + ); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psamemad/issues/comments/1', + ); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psamemad/issues/comments/2', + ); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psamemad/pulls/comments/3', + ); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psamemad/pulls/comments/4', + ); }); it('should call fetch 2 times for the body and comments endpoints', async () => { @@ -51,9 +82,15 @@ describe('Patching', () => { comments: [{ id: 5, body: 'Nice *' }], }; - try { - await patchIssue(reqBody, issue); - } catch (error) {} - expect(nodeFetch).toHaveBeenCalledTimes(2); + await patchIssue(reqBody, issue); + + expect(mockFetch).toHaveBeenCalledTimes(2); + + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psammead/issues/4512', + ); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psammead/issues/comments/5', + ); }); }); diff --git a/scripts/exposure-scanning/scanExposures.js b/scripts/exposure-scanning/scanExposures.js new file mode 100644 index 0000000000..cfc7af4a03 --- /dev/null +++ b/scripts/exposure-scanning/scanExposures.js @@ -0,0 +1,49 @@ +/* eslint-disable no-console */ +/* eslint-disable import/extensions */ +import parseArgs from './args/index.js'; +import { fetchPr, fetchIssue } from './fetch/index.js'; +import { scanPr, scanIssue } from './scan/index.js'; +import { patchPr, patchIssue } from './patch/index.js'; + +const scanExposures = async () => { + const { repo, flag, id, regexString } = parseArgs(process.argv); + + const regex = new RegExp(regexString, 'gi'); + + if (flag === '-pr') { + const reqBody = { + owner: 'bbc', + repo, + prId: id, + }; + + const pr = await fetchPr(reqBody); + + const scannedPr = await scanPr(pr, regex); + + await patchPr(reqBody, scannedPr); + + if (scannedPr.foundMatch) { + return Promise.reject(new Error('Match found.')); + } + } else if (flag === '-issue') { + const reqBody = { + owner: 'bbc', + repo, + issueId: id, + }; + + const issue = await fetchIssue(reqBody); + + const scannedIssue = await scanIssue(issue, regex); + + await patchIssue(reqBody, scannedIssue); + + if (scannedIssue.foundMatch) { + return Promise.reject(new Error('Match found.')); + } + } + return Promise.resolve('No matches found.'); +}; + +export default scanExposures; diff --git a/scripts/exposure-scanning/index.test.js b/scripts/exposure-scanning/scanExposures.test.js similarity index 91% rename from scripts/exposure-scanning/index.test.js rename to scripts/exposure-scanning/scanExposures.test.js index 16c400701b..62f6ede0eb 100644 --- a/scripts/exposure-scanning/index.test.js +++ b/scripts/exposure-scanning/scanExposures.test.js @@ -1,11 +1,11 @@ /* eslint-disable import/extensions */ import parseArgs from './args/index.js'; import { fetchPr, fetchIssue } from './fetch/index.js'; -import scanExposures from '.'; +import scanExposures from './scanExposures.js'; -jest.mock('./args'); -jest.mock('./fetch'); -jest.mock('./patch'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. +jest.mock('./args/index.js'); +jest.mock('./fetch/index.js'); +jest.mock('./patch/index.js'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. describe('Expected PR scanning', () => { beforeEach(() => { From 1774667dc9800e17502a26d2a1250579bd5072db Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 16:24:22 +0100 Subject: [PATCH 11/54] Conditionally import octokit --- package.json | 1 + scripts/exposure-scanning/fetch/index.js | 12 +++++-- scripts/exposure-scanning/patch/index.js | 11 ++++-- yarn.lock | 43 ++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index c80348ebed..e4b766822c 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ "@loadable/babel-plugin": "^5.13.0", "@loadable/component": "^5.13.1", "@manypkg/cli": "^0.18.0", + "@octokit/action": "^3.11.1", "@octokit/rest": "^18.7.0", "@storybook/addon-a11y": "^6.3.4", "@storybook/addon-actions": "^6.3.4", diff --git a/scripts/exposure-scanning/fetch/index.js b/scripts/exposure-scanning/fetch/index.js index 5d9fde7e31..c8e7a8ece3 100644 --- a/scripts/exposure-scanning/fetch/index.js +++ b/scripts/exposure-scanning/fetch/index.js @@ -1,6 +1,14 @@ -import { Octokit } from '@octokit/rest'; +/* eslint-disable global-require */ +let octokit; -const octokit = new Octokit(); +if (process.env.GITHUB_ACTION) { + console.log(process.env.GITHUB_ACTION); + const { Octokit } = require('@octokit/action'); + octokit = new Octokit(); +} else { + const { Octokit } = require('@octokit/rest'); + octokit = new Octokit(); +} const fetchPrBody = async reqBody => { const { diff --git a/scripts/exposure-scanning/patch/index.js b/scripts/exposure-scanning/patch/index.js index a3754a692b..fe8d74bc12 100644 --- a/scripts/exposure-scanning/patch/index.js +++ b/scripts/exposure-scanning/patch/index.js @@ -1,6 +1,13 @@ -import { Octokit } from '@octokit/rest'; +/* eslint-disable global-require */ +let octokit; -const octokit = new Octokit(); +if (process.env.GITHUB_ACTION) { + const { Octokit } = require('@octokit/action'); + octokit = new Octokit(); +} else { + const { Octokit } = require('@octokit/rest'); + octokit = new Octokit(); +} const patchPrBody = async (reqBody, body) => { await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{prId}', { diff --git a/yarn.lock b/yarn.lock index a4f0d744a9..e9a6f67242 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1773,7 +1773,26 @@ mkdirp "^1.0.4" rimraf "^3.0.2" -"@octokit/auth-token@^2.4.4": +"@octokit/action@^3.11.1": + version "3.11.1" + resolved "https://registry.yarnpkg.com/@octokit/action/-/action-3.11.1.tgz#7c249dcf60258311de176320f5822ae716b8f7c4" + integrity sha512-f4C6Hk5YYK6sw6xLEQqvdgl3WtKuk89scmv8kSroLrtDP0InJxuzqgJ7jMT8DeT2EZnVGaHZ2lagnH83bKVvww== + dependencies: + "@octokit/auth-action" "^1.2.0" + "@octokit/core" "^3.0.0" + "@octokit/plugin-paginate-rest" "^2.2.4" + "@octokit/plugin-rest-endpoint-methods" "5.5.1" + "@octokit/types" "^6.16.1" + +"@octokit/auth-action@^1.2.0": + version "1.3.3" + resolved "https://registry.yarnpkg.com/@octokit/auth-action/-/auth-action-1.3.3.tgz#20004fbf0b4a7012f4f7fc2c54d263749239cd5f" + integrity sha512-8v4c/pw6HTxsF7pCgJoox/q4KKov4zkgLxEGGqLOZPSZaHf1LqdLlj5m5x5c1bKNn38uQXNvJKEnKX1qJlGeQQ== + dependencies: + "@octokit/auth-token" "^2.4.0" + "@octokit/types" "^6.0.3" + +"@octokit/auth-token@^2.4.0", "@octokit/auth-token@^2.4.4": version "2.4.5" resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-2.4.5.tgz#568ccfb8cb46f36441fac094ce34f7a875b197f3" integrity sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA== @@ -1821,6 +1840,11 @@ resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.1.0.tgz#7f73adbde8ba2d2512de90ea8a5da68b25d65d0a" integrity sha512-XBP03pG4XuTU+VgeJM1ozRdmZJerMG4tk6wA+raFKycC4qV9jtD2UQroAg9bAcmI3Q0zWvifeDGtPqsFjMzkLg== +"@octokit/openapi-types@^9.1.1": + version "9.1.1" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.1.1.tgz#fb87f2e2f44b95a5720d61dee409a9f1fbc59217" + integrity sha512-xmyPP9tVb4T4A6Lk6SL6ScnIqAHpPV4jfMZI8VtY286212ri9J/6IFGuLsZ26daADUmriuLejake4k+azEfnaw== + "@octokit/plugin-paginate-rest@^2.2.3": version "2.13.6" resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.13.6.tgz#03633839b0ec57a108d2ca1c4b1f64b749d3506c" @@ -1828,7 +1852,7 @@ dependencies: "@octokit/types" "^6.17.3" -"@octokit/plugin-paginate-rest@^2.6.2": +"@octokit/plugin-paginate-rest@^2.2.4", "@octokit/plugin-paginate-rest@^2.6.2": version "2.14.0" resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.14.0.tgz#f469cb4a908792fb44679c5973d8bba820c88b0f" integrity sha512-S2uEu2uHeI7Vf+Lvj8tv3O5/5TCAa8GHS0dUQN7gdM7vKA6ZHAbR6HkAVm5yMb1mbedLEbxOuQ+Fa0SQ7tCDLA== @@ -1848,6 +1872,14 @@ "@octokit/types" "^6.21.0" deprecation "^2.3.1" +"@octokit/plugin-rest-endpoint-methods@5.5.1": + version "5.5.1" + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.5.1.tgz#31cce8fc3eda4d186bd90828cb7a2203ad95e3d1" + integrity sha512-Al57+OZmO65JpiPk4JS6u6kQ2y9qjoZtY1IWiSshc4N+F7EcrK8Rgy/cUJBB4WIcSFUQyF66EJQK1oKgXWeRNw== + dependencies: + "@octokit/types" "^6.21.1" + deprecation "^2.3.1" + "@octokit/plugin-rest-endpoint-methods@^4.0.0": version "4.15.1" resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.15.1.tgz#91a064bee99d0ffcef74a04357e1cf15c27d1cd0" @@ -1901,6 +1933,13 @@ dependencies: "@octokit/openapi-types" "^9.1.0" +"@octokit/types@^6.21.1": + version "6.21.1" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.21.1.tgz#d0f2b7598c88e13d0bd87e330d975e3fb2a90180" + integrity sha512-PP+m3T5EWZKawru4zi/FvX8KL2vkO5f1fLthx78/7743p7RtJUevt3z7698k+7oAYRA7YuVqfXthSEHqkDvZ8g== + dependencies: + "@octokit/openapi-types" "^9.1.1" + "@pmmmwh/react-refresh-webpack-plugin@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.4.3.tgz#1eec460596d200c0236bf195b078a5d1df89b766" From 7c38d0ae598df52b528924680f5d79644b366915 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 26 Jul 2021 16:43:23 +0100 Subject: [PATCH 12/54] Ensure a github token env variable exists before initialising octokit/action --- scripts/exposure-scanning/fetch/index.js | 3 +-- scripts/exposure-scanning/patch/index.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/exposure-scanning/fetch/index.js b/scripts/exposure-scanning/fetch/index.js index c8e7a8ece3..fa94af35a0 100644 --- a/scripts/exposure-scanning/fetch/index.js +++ b/scripts/exposure-scanning/fetch/index.js @@ -1,8 +1,7 @@ /* eslint-disable global-require */ let octokit; -if (process.env.GITHUB_ACTION) { - console.log(process.env.GITHUB_ACTION); +if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { const { Octokit } = require('@octokit/action'); octokit = new Octokit(); } else { diff --git a/scripts/exposure-scanning/patch/index.js b/scripts/exposure-scanning/patch/index.js index fe8d74bc12..33134a786a 100644 --- a/scripts/exposure-scanning/patch/index.js +++ b/scripts/exposure-scanning/patch/index.js @@ -1,7 +1,7 @@ /* eslint-disable global-require */ let octokit; -if (process.env.GITHUB_ACTION) { +if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { const { Octokit } = require('@octokit/action'); octokit = new Octokit(); } else { From f64078a81e698715d47e22fb822b77b4098cf1ae Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 27 Jul 2021 09:28:58 +0100 Subject: [PATCH 13/54] Use conditional ES import() --- scripts/exposure-scanning/fetch/index.js | 16 +++++++++------- scripts/exposure-scanning/patch/index.js | 16 +++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/scripts/exposure-scanning/fetch/index.js b/scripts/exposure-scanning/fetch/index.js index fa94af35a0..eeec1b8512 100644 --- a/scripts/exposure-scanning/fetch/index.js +++ b/scripts/exposure-scanning/fetch/index.js @@ -1,13 +1,15 @@ /* eslint-disable global-require */ let octokit; -if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { - const { Octokit } = require('@octokit/action'); - octokit = new Octokit(); -} else { - const { Octokit } = require('@octokit/rest'); - octokit = new Octokit(); -} +(async () => { + if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { + const { Octokit } = await import('@octokit/action'); + octokit = new Octokit(); + } else { + const { Octokit } = await import('@octokit/rest'); + octokit = new Octokit(); + } +})(); const fetchPrBody = async reqBody => { const { diff --git a/scripts/exposure-scanning/patch/index.js b/scripts/exposure-scanning/patch/index.js index 33134a786a..744b000e31 100644 --- a/scripts/exposure-scanning/patch/index.js +++ b/scripts/exposure-scanning/patch/index.js @@ -1,13 +1,15 @@ /* eslint-disable global-require */ let octokit; -if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { - const { Octokit } = require('@octokit/action'); - octokit = new Octokit(); -} else { - const { Octokit } = require('@octokit/rest'); - octokit = new Octokit(); -} +(async () => { + if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { + const { Octokit } = await import('@octokit/action'); + octokit = new Octokit(); + } else { + const { Octokit } = await import('@octokit/rest'); + octokit = new Octokit(); + } +})(); const patchPrBody = async (reqBody, body) => { await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{prId}', { From 6ce4575f683c7e7e8bbb21e22c8ff280e9892ae5 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 11:43:59 +0100 Subject: [PATCH 14/54] No longer patch if no match found --- scripts/exposure-scanning/scan/index.js | 40 ++++++++++++------- scripts/exposure-scanning/scanExposures.js | 6 +-- .../exposure-scanning/scanExposures.test.js | 14 +++++-- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/scripts/exposure-scanning/scan/index.js b/scripts/exposure-scanning/scan/index.js index bbf44950e6..e1d7a7d38b 100644 --- a/scripts/exposure-scanning/scan/index.js +++ b/scripts/exposure-scanning/scan/index.js @@ -1,23 +1,23 @@ const replacementStr = '*'; -let foundMatch = false; - const scanText = (text, regex) => { + let foundTextMatch = false; const scannedText = text.replace(regex, replacementStr); if (scannedText !== text) { - foundMatch = true; + foundTextMatch = true; } - return scannedText; + return { body: scannedText, foundTextMatch }; }; const scanComments = (comments, regex) => { + let foundCommentMatch = false; const scannedComments = []; comments.forEach(({ id, body }) => { if (regex.test(body)) { - foundMatch = true; + foundCommentMatch = true; scannedComments.push({ id, body: body.replace(regex, replacementStr), @@ -25,37 +25,49 @@ const scanComments = (comments, regex) => { } }); - return scannedComments; + return { comments: scannedComments, foundCommentMatch }; }; export const scanPr = (pr, regex) => { let scannedPr = pr; + const { body, foundTextMatch } = scanText(pr.body, regex); + const { comments, foundCommentMatch } = scanComments(pr.comments, regex); + const { + reviewComments, + foundCommentMatch: foundReviewCommentsMatch, + } = scanComments(pr.reviewComments, regex); + try { scannedPr = { - body: scanText(pr.body, regex), - comments: scanComments(pr.comments, regex), - reviewComments: scanComments(pr.reviewComments, regex), + body, + comments, + reviewComments, }; } catch (error) { throw new Error(`Encountered an error when scanning.`); } - return { ...scannedPr, foundMatch }; + return { + ...scannedPr, + foundMatch: foundTextMatch || foundCommentMatch || foundReviewCommentsMatch, + }; }; export const scanIssue = (issue, regex) => { - foundMatch = false; let scannedIssue = issue; + const { body, foundTextMatch } = scanText(issue.body, regex); + const { comments, foundCommentMatch } = scanComments(issue.comments, regex); + try { scannedIssue = { - body: scanText(issue.body, regex), - comments: scanComments(issue.comments, regex), + body, + comments, }; } catch (error) { throw new Error(`Encountered an error when scanning.`); } - return { ...scannedIssue, foundMatch }; + return { ...scannedIssue, foundMatch: foundTextMatch || foundCommentMatch }; }; diff --git a/scripts/exposure-scanning/scanExposures.js b/scripts/exposure-scanning/scanExposures.js index cfc7af4a03..b35f9d6e32 100644 --- a/scripts/exposure-scanning/scanExposures.js +++ b/scripts/exposure-scanning/scanExposures.js @@ -21,9 +21,8 @@ const scanExposures = async () => { const scannedPr = await scanPr(pr, regex); - await patchPr(reqBody, scannedPr); - if (scannedPr.foundMatch) { + await patchPr(reqBody, scannedPr); return Promise.reject(new Error('Match found.')); } } else if (flag === '-issue') { @@ -37,9 +36,8 @@ const scanExposures = async () => { const scannedIssue = await scanIssue(issue, regex); - await patchIssue(reqBody, scannedIssue); - if (scannedIssue.foundMatch) { + await patchIssue(reqBody, scannedIssue); return Promise.reject(new Error('Match found.')); } } diff --git a/scripts/exposure-scanning/scanExposures.test.js b/scripts/exposure-scanning/scanExposures.test.js index 62f6ede0eb..55627e3e27 100644 --- a/scripts/exposure-scanning/scanExposures.test.js +++ b/scripts/exposure-scanning/scanExposures.test.js @@ -1,6 +1,7 @@ /* eslint-disable import/extensions */ import parseArgs from './args/index.js'; import { fetchPr, fetchIssue } from './fetch/index.js'; +import { patchPr, patchIssue } from './patch/index.js'; import scanExposures from './scanExposures.js'; jest.mock('./args/index.js'); @@ -18,7 +19,7 @@ describe('Expected PR scanning', () => { }); }); - it('should not throw when there are no matches in a PR', async () => { + it('should not throw or patch when there are no matches in a PR', async () => { fetchPr.mockResolvedValueOnce({ body: 'Pr Body', comments: [], @@ -26,6 +27,7 @@ describe('Expected PR scanning', () => { }); await expect(scanExposures()).resolves.toBe('No matches found.'); + expect(patchPr).not.toHaveBeenCalled(); }); it('should throw when there is a match in the PR body', async () => { @@ -36,6 +38,7 @@ describe('Expected PR scanning', () => { }); await expect(scanExposures()).rejects.toThrow(); + expect(patchPr).toHaveBeenCalled(); }); it('should throw when there is a match in the PR comments', async () => { @@ -46,6 +49,7 @@ describe('Expected PR scanning', () => { }); await expect(scanExposures()).rejects.toThrow(); + expect(patchPr).toHaveBeenCalled(); }); it('should throw when there is a match in the PR review comments', async () => { @@ -56,6 +60,7 @@ describe('Expected PR scanning', () => { }); await expect(scanExposures()).rejects.toThrow(); + expect(patchPr).toHaveBeenCalled(); }); }); @@ -77,23 +82,26 @@ describe('Expected issue scanning', () => { }); await expect(scanExposures()).resolves.toBe('No matches found.'); + expect(patchIssue).not.toHaveBeenCalled(); }); it('should throw when there is a match in the issue body', async () => { fetchIssue.mockResolvedValueOnce({ body: 'foobar', - reviewComments: [], + comments: [], }); await expect(scanExposures()).rejects.toThrow(); + expect(patchIssue).toHaveBeenCalled(); }); it('should throw when there is a match in the issue comments', async () => { fetchIssue.mockResolvedValueOnce({ body: 'Issue Body', - reviewComments: [{ id: 0, body: 'bar.' }], + comments: [{ id: 0, body: 'bar.' }], }); await expect(scanExposures()).rejects.toThrow(); + expect(patchIssue).toHaveBeenCalled(); }); }); From cdec339e23395500de96db92f15b638c19b3c4ac Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 15:56:44 +0100 Subject: [PATCH 15/54] Turn in to psammead utility --- .../exposure-scanning/args/index.js | 0 .../exposure-scanning/args/index.test.js | 0 .../exposure-scanning/fetch/index.js | 0 .../exposure-scanning/fetch/index.test.js | 0 .../utilities}/exposure-scanning/index.js | 0 .../utilities/exposure-scanning/package.json | 26 +++++++++++++++++++ .../exposure-scanning/patch/index.js | 0 .../exposure-scanning/patch/index.test.js | 0 .../exposure-scanning/scan/index.js | 0 .../exposure-scanning/scan/index.test.js | 0 .../exposure-scanning/scanExposures.js | 0 .../exposure-scanning/scanExposures.test.js | 0 scripts/exposure-scanning/package.json | 7 ----- 13 files changed, 26 insertions(+), 7 deletions(-) rename {scripts => packages/utilities}/exposure-scanning/args/index.js (100%) rename {scripts => packages/utilities}/exposure-scanning/args/index.test.js (100%) rename {scripts => packages/utilities}/exposure-scanning/fetch/index.js (100%) rename {scripts => packages/utilities}/exposure-scanning/fetch/index.test.js (100%) rename {scripts => packages/utilities}/exposure-scanning/index.js (100%) create mode 100644 packages/utilities/exposure-scanning/package.json rename {scripts => packages/utilities}/exposure-scanning/patch/index.js (100%) rename {scripts => packages/utilities}/exposure-scanning/patch/index.test.js (100%) rename {scripts => packages/utilities}/exposure-scanning/scan/index.js (100%) rename {scripts => packages/utilities}/exposure-scanning/scan/index.test.js (100%) rename {scripts => packages/utilities}/exposure-scanning/scanExposures.js (100%) rename {scripts => packages/utilities}/exposure-scanning/scanExposures.test.js (100%) delete mode 100644 scripts/exposure-scanning/package.json diff --git a/scripts/exposure-scanning/args/index.js b/packages/utilities/exposure-scanning/args/index.js similarity index 100% rename from scripts/exposure-scanning/args/index.js rename to packages/utilities/exposure-scanning/args/index.js diff --git a/scripts/exposure-scanning/args/index.test.js b/packages/utilities/exposure-scanning/args/index.test.js similarity index 100% rename from scripts/exposure-scanning/args/index.test.js rename to packages/utilities/exposure-scanning/args/index.test.js diff --git a/scripts/exposure-scanning/fetch/index.js b/packages/utilities/exposure-scanning/fetch/index.js similarity index 100% rename from scripts/exposure-scanning/fetch/index.js rename to packages/utilities/exposure-scanning/fetch/index.js diff --git a/scripts/exposure-scanning/fetch/index.test.js b/packages/utilities/exposure-scanning/fetch/index.test.js similarity index 100% rename from scripts/exposure-scanning/fetch/index.test.js rename to packages/utilities/exposure-scanning/fetch/index.test.js diff --git a/scripts/exposure-scanning/index.js b/packages/utilities/exposure-scanning/index.js similarity index 100% rename from scripts/exposure-scanning/index.js rename to packages/utilities/exposure-scanning/index.js diff --git a/packages/utilities/exposure-scanning/package.json b/packages/utilities/exposure-scanning/package.json new file mode 100644 index 0000000000..747c702fe9 --- /dev/null +++ b/packages/utilities/exposure-scanning/package.json @@ -0,0 +1,26 @@ +{ + "name": "@bbc/exposure-scanning", + "version": "0.1.0-alpha.1", + "description": "A utility that scans PRs or issues for exposures.", + "repository": { + "type": "git", + "url": "https://github.com/bbc/psammead/tree/latest/packages/utilities/exposure-scanning" + }, + "author": { + "name": "Psammead Maintainers", + "email": "PsammeadMaintainers@bbc.co.uk" + }, + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/bbc/psammead/issues" + }, + "homepage": "https://github.com/bbc/psammead/blob/latest/packages/utilities/exposure-scanning", + "keywords": [ + "bbc", + "utilities", + "exposure", + "scanning" + ], + "private": true, + "tag": "alpha" +} diff --git a/scripts/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js similarity index 100% rename from scripts/exposure-scanning/patch/index.js rename to packages/utilities/exposure-scanning/patch/index.js diff --git a/scripts/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js similarity index 100% rename from scripts/exposure-scanning/patch/index.test.js rename to packages/utilities/exposure-scanning/patch/index.test.js diff --git a/scripts/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js similarity index 100% rename from scripts/exposure-scanning/scan/index.js rename to packages/utilities/exposure-scanning/scan/index.js diff --git a/scripts/exposure-scanning/scan/index.test.js b/packages/utilities/exposure-scanning/scan/index.test.js similarity index 100% rename from scripts/exposure-scanning/scan/index.test.js rename to packages/utilities/exposure-scanning/scan/index.test.js diff --git a/scripts/exposure-scanning/scanExposures.js b/packages/utilities/exposure-scanning/scanExposures.js similarity index 100% rename from scripts/exposure-scanning/scanExposures.js rename to packages/utilities/exposure-scanning/scanExposures.js diff --git a/scripts/exposure-scanning/scanExposures.test.js b/packages/utilities/exposure-scanning/scanExposures.test.js similarity index 100% rename from scripts/exposure-scanning/scanExposures.test.js rename to packages/utilities/exposure-scanning/scanExposures.test.js diff --git a/scripts/exposure-scanning/package.json b/scripts/exposure-scanning/package.json deleted file mode 100644 index 86e5311233..0000000000 --- a/scripts/exposure-scanning/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "exposure-scanning", - "version": "1.0.0", - "main": "index.js", - "license": "MIT", - "type": "module" -} From 8f66881c176b544ab0b04b8280a0f12a81409105 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:01:21 +0100 Subject: [PATCH 16/54] Comment out action --- .github/workflows/scan-exposures.yml | 164 +++++++++++++-------------- package.json | 1 - 2 files changed, 82 insertions(+), 83 deletions(-) diff --git a/.github/workflows/scan-exposures.yml b/.github/workflows/scan-exposures.yml index 2bd48e7117..ff03f5e73c 100644 --- a/.github/workflows/scan-exposures.yml +++ b/.github/workflows/scan-exposures.yml @@ -1,97 +1,97 @@ -name: "Scan Exposures" +# name: "Scan Exposures" -on: - pull_request: - branches: - - '**' - issues: - types: [opened, edited] - issue_comment: - types: [created, edited, deleted] - pull_request_review: - types: [submitted, edited] - pull_request_review_comment: - types: [created, edited, deleted] +# on: +# pull_request: +# branches: +# - '**' +# issues: +# types: [opened, edited] +# issue_comment: +# types: [created, edited, deleted] +# pull_request_review: +# types: [submitted, edited] +# pull_request_review_comment: +# types: [created, edited, deleted] -jobs: +# jobs: - scan_prs: - name: Scan PR Exposures - runs-on: ubuntu-latest +# scan_prs: +# name: Scan PR Exposures +# runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 +# steps: +# - name: Checkout +# uses: actions/checkout@v2 - - name: Cache Node Modules - id: cache - uses: actions/cache@v2 - with: - path: node_modules - key: node-modules-${{ hashFiles('yarn.lock') }} +# - name: Cache Node Modules +# id: cache +# uses: actions/cache@v2 +# with: +# path: node_modules +# key: node-modules-${{ hashFiles('yarn.lock') }} - - name: Install Node Modules - if: steps.cache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile +# - name: Install Node Modules +# if: steps.cache.outputs.cache-hit != 'true' +# run: yarn install --frozen-lockfile - - name: Scan Issue - shell: bash - if: ${{ github.event_name == 'issues' }} - env: - ISSUE_NUMBER: ${{ github.issue.number }} - PATTERN: ${{ secrets.PATTERN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" +# - name: Scan Issue +# shell: bash +# if: ${{ github.event_name == 'issues' }} +# env: +# ISSUE_NUMBER: ${{ github.issue.number }} +# PATTERN: ${{ secrets.PATTERN }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: | +# yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" - - name: Scan PR - shell: bash - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' || github.event_name == 'pull_request_review_comment' }} - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - PATTERN: ${{ secrets.PATTERN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" +# - name: Scan PR +# shell: bash +# if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' || github.event_name == 'pull_request_review_comment' }} +# env: +# PR_NUMBER: ${{ github.event.pull_request.number }} +# PATTERN: ${{ secrets.PATTERN }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" - scan_comments: - name: Scan Comment Exposures - runs-on: ubuntu-latest - # The issue_comment event is triggered by comments on issues and PRs. - if: ${{ github.event_name == 'issue_comment' }} +# scan_comments: +# name: Scan Comment Exposures +# runs-on: ubuntu-latest +# # The issue_comment event is triggered by comments on issues and PRs. +# if: ${{ github.event_name == 'issue_comment' }} - steps: - - name: Checkout - uses: actions/checkout@v2 +# steps: +# - name: Checkout +# uses: actions/checkout@v2 - - name: Cache Node Modules - id: cache - uses: actions/cache@v2 - with: - path: node_modules - key: node-modules-${{ hashFiles('yarn.lock') }} +# - name: Cache Node Modules +# id: cache +# uses: actions/cache@v2 +# with: +# path: node_modules +# key: node-modules-${{ hashFiles('yarn.lock') }} - - name: Install Node Modules - if: steps.cache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile +# - name: Install Node Modules +# if: steps.cache.outputs.cache-hit != 'true' +# run: yarn install --frozen-lockfile - - name: Scan Issue - shell: bash - # This step only runs on issue comments - if: ${{ !github.event.issue.pull_request }} - env: - ISSUE_NUMBER: ${{ github.event.issue.number }} - PATTERN: ${{ secrets.PATTERN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" +# - name: Scan Issue +# shell: bash +# # This step only runs on issue comments +# if: ${{ !github.event.issue.pull_request }} +# env: +# ISSUE_NUMBER: ${{ github.event.issue.number }} +# PATTERN: ${{ secrets.PATTERN }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: | +# yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" - - name: Scan PR - shell: bash - # This step only runs on PR comments - if: ${{ github.event.issue.pull_request }} - env: - PR_NUMBER: ${{ github.event.issue.number }} - PATTERN: ${{ secrets.PATTERN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" +# - name: Scan PR +# shell: bash +# # This step only runs on PR comments +# if: ${{ github.event.issue.pull_request }} +# env: +# PR_NUMBER: ${{ github.event.issue.number }} +# PATTERN: ${{ secrets.PATTERN }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" diff --git a/package.json b/package.json index 81626f322e..636d47244e 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "preinstall": "node scripts/check-package-manager.js && sh scripts/enforceVersions.sh && [ \"$npm_config_refer\" != \"ci\" ]", "postinstall": "yarn packages:check && yarn build", "publish": "node scripts/publish", - "scan:exposures": "node --experimental-modules --unhandled-rejections=strict scripts/exposure-scanning ", "storybook": "NODE_ENV=development start-storybook -p 8180 -s .storybook/static -c .storybook", "test": "yarn test:lint && yarn test:unit", "test:ci": "yarn test:lint && yarn test:unit:ci", From 1bbc25a5ae1bb4e938c4ee11847716392ee91e15 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:01:43 +0100 Subject: [PATCH 17/54] Remove private attribute from package.json --- packages/utilities/exposure-scanning/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/package.json b/packages/utilities/exposure-scanning/package.json index 747c702fe9..2439533b97 100644 --- a/packages/utilities/exposure-scanning/package.json +++ b/packages/utilities/exposure-scanning/package.json @@ -21,6 +21,5 @@ "exposure", "scanning" ], - "private": true, "tag": "alpha" } From 531598620298fc750e6486d2bfad3e8d451bfc30 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:04:38 +0100 Subject: [PATCH 18/54] Add changelog --- packages/utilities/exposure-scanning/CHANGELOG.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/utilities/exposure-scanning/CHANGELOG.md diff --git a/packages/utilities/exposure-scanning/CHANGELOG.md b/packages/utilities/exposure-scanning/CHANGELOG.md new file mode 100644 index 0000000000..e69de29bb2 From 2a657dd0d207d2caf1d32deb92305826a8c087bd Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:12:21 +0100 Subject: [PATCH 19/54] Fix scan unit tests --- packages/utilities/exposure-scanning/scan/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js index e1d7a7d38b..fccd085d2c 100644 --- a/packages/utilities/exposure-scanning/scan/index.js +++ b/packages/utilities/exposure-scanning/scan/index.js @@ -34,7 +34,7 @@ export const scanPr = (pr, regex) => { const { body, foundTextMatch } = scanText(pr.body, regex); const { comments, foundCommentMatch } = scanComments(pr.comments, regex); const { - reviewComments, + comments: reviewComments, foundCommentMatch: foundReviewCommentsMatch, } = scanComments(pr.reviewComments, regex); From c959431d6b316fe24c1348796d89682c251ddda8 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:23:50 +0100 Subject: [PATCH 20/54] Add changelog --- packages/utilities/exposure-scanning/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/utilities/exposure-scanning/CHANGELOG.md b/packages/utilities/exposure-scanning/CHANGELOG.md index e69de29bb2..b1d91eb4a8 100644 --- a/packages/utilities/exposure-scanning/CHANGELOG.md +++ b/packages/utilities/exposure-scanning/CHANGELOG.md @@ -0,0 +1,7 @@ +# Exposure Scanning Changelog + + +| Version | Description | +|---------|-------------| +| 0.1.0-alpha.1 | [PR#4526](https://github.com/bbc/psammead/pull/4526) Initialise package | + From 70bcdb1b05f3158f02e27139e52b6ad995f9ddcc Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:27:48 +0100 Subject: [PATCH 21/54] Move root level files into src --- packages/utilities/exposure-scanning/{ => src}/index.js | 0 .../exposure-scanning/{ => src}/scanExposures.js | 8 ++++---- .../exposure-scanning/{ => src}/scanExposures.test.js | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename packages/utilities/exposure-scanning/{ => src}/index.js (100%) rename packages/utilities/exposure-scanning/{ => src}/scanExposures.js (82%) rename packages/utilities/exposure-scanning/{ => src}/scanExposures.test.js (94%) diff --git a/packages/utilities/exposure-scanning/index.js b/packages/utilities/exposure-scanning/src/index.js similarity index 100% rename from packages/utilities/exposure-scanning/index.js rename to packages/utilities/exposure-scanning/src/index.js diff --git a/packages/utilities/exposure-scanning/scanExposures.js b/packages/utilities/exposure-scanning/src/scanExposures.js similarity index 82% rename from packages/utilities/exposure-scanning/scanExposures.js rename to packages/utilities/exposure-scanning/src/scanExposures.js index b35f9d6e32..fbe9ad3421 100644 --- a/packages/utilities/exposure-scanning/scanExposures.js +++ b/packages/utilities/exposure-scanning/src/scanExposures.js @@ -1,9 +1,9 @@ /* eslint-disable no-console */ /* eslint-disable import/extensions */ -import parseArgs from './args/index.js'; -import { fetchPr, fetchIssue } from './fetch/index.js'; -import { scanPr, scanIssue } from './scan/index.js'; -import { patchPr, patchIssue } from './patch/index.js'; +import parseArgs from '../args/index.js'; +import { fetchPr, fetchIssue } from '../fetch/index.js'; +import { scanPr, scanIssue } from '../scan/index.js'; +import { patchPr, patchIssue } from '../patch/index.js'; const scanExposures = async () => { const { repo, flag, id, regexString } = parseArgs(process.argv); diff --git a/packages/utilities/exposure-scanning/scanExposures.test.js b/packages/utilities/exposure-scanning/src/scanExposures.test.js similarity index 94% rename from packages/utilities/exposure-scanning/scanExposures.test.js rename to packages/utilities/exposure-scanning/src/scanExposures.test.js index 55627e3e27..67cc1e5210 100644 --- a/packages/utilities/exposure-scanning/scanExposures.test.js +++ b/packages/utilities/exposure-scanning/src/scanExposures.test.js @@ -1,7 +1,7 @@ /* eslint-disable import/extensions */ -import parseArgs from './args/index.js'; -import { fetchPr, fetchIssue } from './fetch/index.js'; -import { patchPr, patchIssue } from './patch/index.js'; +import parseArgs from '../args/index.js'; +import { fetchPr, fetchIssue } from '../fetch/index.js'; +import { patchPr, patchIssue } from '../patch/index.js'; import scanExposures from './scanExposures.js'; jest.mock('./args/index.js'); From 42a84669b98eccd06a1ee6d13df9f35f87d652c0 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:31:57 +0100 Subject: [PATCH 22/54] Remove action --- .github/workflows/scan-exposures.yml | 97 ---------------------------- 1 file changed, 97 deletions(-) delete mode 100644 .github/workflows/scan-exposures.yml diff --git a/.github/workflows/scan-exposures.yml b/.github/workflows/scan-exposures.yml deleted file mode 100644 index ff03f5e73c..0000000000 --- a/.github/workflows/scan-exposures.yml +++ /dev/null @@ -1,97 +0,0 @@ -# name: "Scan Exposures" - -# on: -# pull_request: -# branches: -# - '**' -# issues: -# types: [opened, edited] -# issue_comment: -# types: [created, edited, deleted] -# pull_request_review: -# types: [submitted, edited] -# pull_request_review_comment: -# types: [created, edited, deleted] - - -# jobs: - -# scan_prs: -# name: Scan PR Exposures -# runs-on: ubuntu-latest - -# steps: -# - name: Checkout -# uses: actions/checkout@v2 - -# - name: Cache Node Modules -# id: cache -# uses: actions/cache@v2 -# with: -# path: node_modules -# key: node-modules-${{ hashFiles('yarn.lock') }} - -# - name: Install Node Modules -# if: steps.cache.outputs.cache-hit != 'true' -# run: yarn install --frozen-lockfile - -# - name: Scan Issue -# shell: bash -# if: ${{ github.event_name == 'issues' }} -# env: -# ISSUE_NUMBER: ${{ github.issue.number }} -# PATTERN: ${{ secrets.PATTERN }} -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# run: | -# yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" - -# - name: Scan PR -# shell: bash -# if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' || github.event_name == 'pull_request_review_comment' }} -# env: -# PR_NUMBER: ${{ github.event.pull_request.number }} -# PATTERN: ${{ secrets.PATTERN }} -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" - -# scan_comments: -# name: Scan Comment Exposures -# runs-on: ubuntu-latest -# # The issue_comment event is triggered by comments on issues and PRs. -# if: ${{ github.event_name == 'issue_comment' }} - -# steps: -# - name: Checkout -# uses: actions/checkout@v2 - -# - name: Cache Node Modules -# id: cache -# uses: actions/cache@v2 -# with: -# path: node_modules -# key: node-modules-${{ hashFiles('yarn.lock') }} - -# - name: Install Node Modules -# if: steps.cache.outputs.cache-hit != 'true' -# run: yarn install --frozen-lockfile - -# - name: Scan Issue -# shell: bash -# # This step only runs on issue comments -# if: ${{ !github.event.issue.pull_request }} -# env: -# ISSUE_NUMBER: ${{ github.event.issue.number }} -# PATTERN: ${{ secrets.PATTERN }} -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# run: | -# yarn scan:exposures psammead -issue "$ISSUE_NUMBER" "$PATTERN" - -# - name: Scan PR -# shell: bash -# # This step only runs on PR comments -# if: ${{ github.event.issue.pull_request }} -# env: -# PR_NUMBER: ${{ github.event.issue.number }} -# PATTERN: ${{ secrets.PATTERN }} -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# run: yarn scan:exposures psammead -pr "$PR_NUMBER" "$PATTERN" From 278a637df475653b716b4a520332422efc3b558b Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:33:53 +0100 Subject: [PATCH 23/54] Export function at index --- .../utilities/exposure-scanning/src/index.js | 55 ++++++++++++++++--- .../{scanExposures.test.js => index.test.js} | 2 +- .../exposure-scanning/src/scanExposures.js | 47 ---------------- 3 files changed, 47 insertions(+), 57 deletions(-) rename packages/utilities/exposure-scanning/src/{scanExposures.test.js => index.test.js} (98%) delete mode 100644 packages/utilities/exposure-scanning/src/scanExposures.js diff --git a/packages/utilities/exposure-scanning/src/index.js b/packages/utilities/exposure-scanning/src/index.js index f26570dd03..fbe9ad3421 100644 --- a/packages/utilities/exposure-scanning/src/index.js +++ b/packages/utilities/exposure-scanning/src/index.js @@ -1,10 +1,47 @@ +/* eslint-disable no-console */ /* eslint-disable import/extensions */ -import scanExposures from './scanExposures.js'; - -try { - (async () => { - await scanExposures(); - })(); -} catch (error) { - throw new Error(`An error occured while scanning for exposures: ${error}`); -} +import parseArgs from '../args/index.js'; +import { fetchPr, fetchIssue } from '../fetch/index.js'; +import { scanPr, scanIssue } from '../scan/index.js'; +import { patchPr, patchIssue } from '../patch/index.js'; + +const scanExposures = async () => { + const { repo, flag, id, regexString } = parseArgs(process.argv); + + const regex = new RegExp(regexString, 'gi'); + + if (flag === '-pr') { + const reqBody = { + owner: 'bbc', + repo, + prId: id, + }; + + const pr = await fetchPr(reqBody); + + const scannedPr = await scanPr(pr, regex); + + if (scannedPr.foundMatch) { + await patchPr(reqBody, scannedPr); + return Promise.reject(new Error('Match found.')); + } + } else if (flag === '-issue') { + const reqBody = { + owner: 'bbc', + repo, + issueId: id, + }; + + const issue = await fetchIssue(reqBody); + + const scannedIssue = await scanIssue(issue, regex); + + if (scannedIssue.foundMatch) { + await patchIssue(reqBody, scannedIssue); + return Promise.reject(new Error('Match found.')); + } + } + return Promise.resolve('No matches found.'); +}; + +export default scanExposures; diff --git a/packages/utilities/exposure-scanning/src/scanExposures.test.js b/packages/utilities/exposure-scanning/src/index.test.js similarity index 98% rename from packages/utilities/exposure-scanning/src/scanExposures.test.js rename to packages/utilities/exposure-scanning/src/index.test.js index 67cc1e5210..d0eb0c4312 100644 --- a/packages/utilities/exposure-scanning/src/scanExposures.test.js +++ b/packages/utilities/exposure-scanning/src/index.test.js @@ -2,7 +2,7 @@ import parseArgs from '../args/index.js'; import { fetchPr, fetchIssue } from '../fetch/index.js'; import { patchPr, patchIssue } from '../patch/index.js'; -import scanExposures from './scanExposures.js'; +import scanExposures from './index.js'; jest.mock('./args/index.js'); jest.mock('./fetch/index.js'); diff --git a/packages/utilities/exposure-scanning/src/scanExposures.js b/packages/utilities/exposure-scanning/src/scanExposures.js deleted file mode 100644 index fbe9ad3421..0000000000 --- a/packages/utilities/exposure-scanning/src/scanExposures.js +++ /dev/null @@ -1,47 +0,0 @@ -/* eslint-disable no-console */ -/* eslint-disable import/extensions */ -import parseArgs from '../args/index.js'; -import { fetchPr, fetchIssue } from '../fetch/index.js'; -import { scanPr, scanIssue } from '../scan/index.js'; -import { patchPr, patchIssue } from '../patch/index.js'; - -const scanExposures = async () => { - const { repo, flag, id, regexString } = parseArgs(process.argv); - - const regex = new RegExp(regexString, 'gi'); - - if (flag === '-pr') { - const reqBody = { - owner: 'bbc', - repo, - prId: id, - }; - - const pr = await fetchPr(reqBody); - - const scannedPr = await scanPr(pr, regex); - - if (scannedPr.foundMatch) { - await patchPr(reqBody, scannedPr); - return Promise.reject(new Error('Match found.')); - } - } else if (flag === '-issue') { - const reqBody = { - owner: 'bbc', - repo, - issueId: id, - }; - - const issue = await fetchIssue(reqBody); - - const scannedIssue = await scanIssue(issue, regex); - - if (scannedIssue.foundMatch) { - await patchIssue(reqBody, scannedIssue); - return Promise.reject(new Error('Match found.')); - } - } - return Promise.resolve('No matches found.'); -}; - -export default scanExposures; From a413cf61f86acdcb63ad39e8095fded18b3b9a2d Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 2 Aug 2021 16:35:30 +0100 Subject: [PATCH 24/54] Update mock paths --- packages/utilities/exposure-scanning/src/index.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/utilities/exposure-scanning/src/index.test.js b/packages/utilities/exposure-scanning/src/index.test.js index d0eb0c4312..669a7444c3 100644 --- a/packages/utilities/exposure-scanning/src/index.test.js +++ b/packages/utilities/exposure-scanning/src/index.test.js @@ -4,9 +4,9 @@ import { fetchPr, fetchIssue } from '../fetch/index.js'; import { patchPr, patchIssue } from '../patch/index.js'; import scanExposures from './index.js'; -jest.mock('./args/index.js'); -jest.mock('./fetch/index.js'); -jest.mock('./patch/index.js'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. +jest.mock('../args/index.js'); +jest.mock('../fetch/index.js'); +jest.mock('../patch/index.js'); // Please keep this mock so that you don't accidentally edit a PR or issue by running these tests. describe('Expected PR scanning', () => { beforeEach(() => { From db927019701a31d8cf258d69eb977d2261c8081d Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 3 Aug 2021 13:04:01 +0100 Subject: [PATCH 25/54] Put package dependencies in local package.json and refactor index invocations --- package.json | 2 - .../utilities/exposure-scanning/package.json | 4 ++ .../utilities/exposure-scanning/src/index.js | 58 +++++++++--------- yarn.lock | 59 ++++++++----------- 4 files changed, 60 insertions(+), 63 deletions(-) diff --git a/package.json b/package.json index 636d47244e..7d494e501e 100644 --- a/package.json +++ b/package.json @@ -112,8 +112,6 @@ "@loadable/babel-plugin": "^5.13.0", "@loadable/component": "^5.13.1", "@manypkg/cli": "^0.18.0", - "@octokit/action": "^3.11.1", - "@octokit/rest": "^18.7.0", "@storybook/addon-a11y": "^6.3.4", "@storybook/addon-actions": "^6.3.4", "@storybook/addon-docs": "^6.3.4", diff --git a/packages/utilities/exposure-scanning/package.json b/packages/utilities/exposure-scanning/package.json index 2439533b97..933f81f9df 100644 --- a/packages/utilities/exposure-scanning/package.json +++ b/packages/utilities/exposure-scanning/package.json @@ -15,6 +15,10 @@ "url": "https://github.com/bbc/psammead/issues" }, "homepage": "https://github.com/bbc/psammead/blob/latest/packages/utilities/exposure-scanning", + "dependencies": { + "@octokit/action": "3.13.0", + "@octokit/rest": "18.8.0" + }, "keywords": [ "bbc", "utilities", diff --git a/packages/utilities/exposure-scanning/src/index.js b/packages/utilities/exposure-scanning/src/index.js index fbe9ad3421..2f25f35b3a 100644 --- a/packages/utilities/exposure-scanning/src/index.js +++ b/packages/utilities/exposure-scanning/src/index.js @@ -5,42 +5,44 @@ import { fetchPr, fetchIssue } from '../fetch/index.js'; import { scanPr, scanIssue } from '../scan/index.js'; import { patchPr, patchIssue } from '../patch/index.js'; -const scanExposures = async () => { - const { repo, flag, id, regexString } = parseArgs(process.argv); +const redactPr = async (reqBody, regex) => { + const pr = await fetchPr(reqBody); - const regex = new RegExp(regexString, 'gi'); + const scannedPr = await scanPr(pr, regex); - if (flag === '-pr') { - const reqBody = { - owner: 'bbc', - repo, - prId: id, - }; + if (scannedPr.foundMatch) { + await patchPr(reqBody, scannedPr); + throw new Error('Match found.'); + } +}; - const pr = await fetchPr(reqBody); +const redactIssue = async (reqBody, regex) => { + const issue = await fetchIssue(reqBody); - const scannedPr = await scanPr(pr, regex); + const scannedIssue = await scanIssue(issue, regex); - if (scannedPr.foundMatch) { - await patchPr(reqBody, scannedPr); - return Promise.reject(new Error('Match found.')); - } - } else if (flag === '-issue') { - const reqBody = { - owner: 'bbc', - repo, - issueId: id, - }; + if (scannedIssue.foundMatch) { + await patchIssue(reqBody, scannedIssue); + throw new Error('Match found.'); + } +}; - const issue = await fetchIssue(reqBody); +const scanExposures = async () => { + const { repo, flag, id, regexString } = parseArgs(process.argv); + + const regex = new RegExp(regexString, 'gi'); - const scannedIssue = await scanIssue(issue, regex); + const reqBody = { + owner: 'bbc', + repo, + prId: id, + }; + + await { + '-pr': redactPr, + '-issue': redactIssue, + }[flag](reqBody, regex); - if (scannedIssue.foundMatch) { - await patchIssue(reqBody, scannedIssue); - return Promise.reject(new Error('Match found.')); - } - } return Promise.resolve('No matches found.'); }; diff --git a/yarn.lock b/yarn.lock index e9a6f67242..01f779c273 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1773,16 +1773,17 @@ mkdirp "^1.0.4" rimraf "^3.0.2" -"@octokit/action@^3.11.1": - version "3.11.1" - resolved "https://registry.yarnpkg.com/@octokit/action/-/action-3.11.1.tgz#7c249dcf60258311de176320f5822ae716b8f7c4" - integrity sha512-f4C6Hk5YYK6sw6xLEQqvdgl3WtKuk89scmv8kSroLrtDP0InJxuzqgJ7jMT8DeT2EZnVGaHZ2lagnH83bKVvww== +"@octokit/action@3.13.0": + version "3.13.0" + resolved "https://registry.yarnpkg.com/@octokit/action/-/action-3.13.0.tgz#db04527900241c11cfef6a43f1a312cfb2d8afaa" + integrity sha512-DXyu16O7opeEKEOu3blgbBGWcQhvVrxDFbfNDGFOya2KpVKQsUSntuZM5/r+nyHPrYTACVmtZmFud7BdCNiB3w== dependencies: "@octokit/auth-action" "^1.2.0" "@octokit/core" "^3.0.0" "@octokit/plugin-paginate-rest" "^2.2.4" - "@octokit/plugin-rest-endpoint-methods" "5.5.1" + "@octokit/plugin-rest-endpoint-methods" "5.6.0" "@octokit/types" "^6.16.1" + https-proxy-agent "^5.0.0" "@octokit/auth-action@^1.2.0": version "1.3.3" @@ -1840,10 +1841,10 @@ resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.1.0.tgz#7f73adbde8ba2d2512de90ea8a5da68b25d65d0a" integrity sha512-XBP03pG4XuTU+VgeJM1ozRdmZJerMG4tk6wA+raFKycC4qV9jtD2UQroAg9bAcmI3Q0zWvifeDGtPqsFjMzkLg== -"@octokit/openapi-types@^9.1.1": - version "9.1.1" - resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.1.1.tgz#fb87f2e2f44b95a5720d61dee409a9f1fbc59217" - integrity sha512-xmyPP9tVb4T4A6Lk6SL6ScnIqAHpPV4jfMZI8VtY286212ri9J/6IFGuLsZ26daADUmriuLejake4k+azEfnaw== +"@octokit/openapi-types@^9.3.0": + version "9.3.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.3.0.tgz#160347858d727527901c6aae7f7d5c2414cc1f2e" + integrity sha512-oz60hhL+mDsiOWhEwrj5aWXTOMVtQgcvP+sRzX4C3cH7WOK9QSAoEtjWh0HdOf6V3qpdgAmUMxnQPluzDWR7Fw== "@octokit/plugin-paginate-rest@^2.2.3": version "2.13.6" @@ -1864,20 +1865,12 @@ resolved "https://registry.yarnpkg.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz#5e50ed7083a613816b1e4a28aeec5fb7f1462e85" integrity sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA== -"@octokit/plugin-rest-endpoint-methods@5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.5.0.tgz#28f94a089029dffb8589447366eca5c441bb8d34" - integrity sha512-v4dNoHF8cXNx7C67yQx7oarHs5Wg2IiafWvp/ULkNcCOuXgQdBOkJtwidpYqPiRPUw4uHDkI6Tgfje+nXB+Deg== - dependencies: - "@octokit/types" "^6.21.0" - deprecation "^2.3.1" - -"@octokit/plugin-rest-endpoint-methods@5.5.1": - version "5.5.1" - resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.5.1.tgz#31cce8fc3eda4d186bd90828cb7a2203ad95e3d1" - integrity sha512-Al57+OZmO65JpiPk4JS6u6kQ2y9qjoZtY1IWiSshc4N+F7EcrK8Rgy/cUJBB4WIcSFUQyF66EJQK1oKgXWeRNw== +"@octokit/plugin-rest-endpoint-methods@5.6.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.6.0.tgz#c28833b88d0f07bf94093405d02d43d73c7de99b" + integrity sha512-2G7lIPwjG9XnTlNhe/TRnpI8yS9K2l68W4RP/ki3wqw2+sVeTK8hItPxkqEI30VeH0UwnzpuksMU/yHxiVVctw== dependencies: - "@octokit/types" "^6.21.1" + "@octokit/types" "^6.23.0" deprecation "^2.3.1" "@octokit/plugin-rest-endpoint-methods@^4.0.0": @@ -1909,15 +1902,15 @@ node-fetch "^2.6.1" universal-user-agent "^6.0.0" -"@octokit/rest@^18.7.0": - version "18.7.0" - resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.7.0.tgz#3a0f6498a9495f3ecede582ab15efe38b298fac8" - integrity sha512-8K8BJFyPFRSfnwu+aSbdjU5w3EtxC33PkDlEi5tyVTYC+t4n7gaqygRg5ajJLCpb/ZzVaXXFJXC9OxQ9TvFRAw== +"@octokit/rest@18.8.0": + version "18.8.0" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.8.0.tgz#ba24f7ba554f015a7ae2b7cc2aecef5386ddfea5" + integrity sha512-lsuNRhgzGnLMn/NmQTNCit/6jplFWiTUlPXhqN0zCMLwf2/9pseHzsnTW+Cjlp4bLMEJJNPa5JOzSLbSCOahKw== dependencies: "@octokit/core" "^3.5.0" "@octokit/plugin-paginate-rest" "^2.6.2" "@octokit/plugin-request-log" "^1.0.2" - "@octokit/plugin-rest-endpoint-methods" "5.5.0" + "@octokit/plugin-rest-endpoint-methods" "5.6.0" "@octokit/types@^6.0.3", "@octokit/types@^6.13.0", "@octokit/types@^6.16.1", "@octokit/types@^6.17.3": version "6.17.4" @@ -1926,19 +1919,19 @@ dependencies: "@octokit/openapi-types" "^8.1.4" -"@octokit/types@^6.18.0", "@octokit/types@^6.21.0": +"@octokit/types@^6.18.0": version "6.21.0" resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.21.0.tgz#723d0296d35296d6ec91247e770f24e6cd51d4ee" integrity sha512-VPSxn9uhCoOUMpxCsOAQhf8DgIx+uzFjZRYDiZS5+TvrKaEwBrWkjr/5NmUVvPbW6xdPC2n3yL3XCnoxa4rxvg== dependencies: "@octokit/openapi-types" "^9.1.0" -"@octokit/types@^6.21.1": - version "6.21.1" - resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.21.1.tgz#d0f2b7598c88e13d0bd87e330d975e3fb2a90180" - integrity sha512-PP+m3T5EWZKawru4zi/FvX8KL2vkO5f1fLthx78/7743p7RtJUevt3z7698k+7oAYRA7YuVqfXthSEHqkDvZ8g== +"@octokit/types@^6.23.0": + version "6.23.0" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.23.0.tgz#b39f242b20036e89fa8f34f7962b4e9b7ff8f65b" + integrity sha512-eG3clC31GSS7K3oBK6C6o7wyXPrkP+mu++eus8CSZdpRytJ5PNszYxudOQ0spWZQ3S9KAtoTG6v1WK5prJcJrA== dependencies: - "@octokit/openapi-types" "^9.1.1" + "@octokit/openapi-types" "^9.3.0" "@pmmmwh/react-refresh-webpack-plugin@^0.4.3": version "0.4.3" From 8fff350264f7399f7bbe6a90d493d266677eab28 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Thu, 5 Aug 2021 11:41:00 +0100 Subject: [PATCH 26/54] Update usage msg --- packages/utilities/exposure-scanning/args/index.js | 2 +- packages/utilities/exposure-scanning/args/index.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/utilities/exposure-scanning/args/index.js b/packages/utilities/exposure-scanning/args/index.js index dccb4831e7..7946312715 100644 --- a/packages/utilities/exposure-scanning/args/index.js +++ b/packages/utilities/exposure-scanning/args/index.js @@ -18,7 +18,7 @@ const isValidRegex = regex => { const parseArgs = argv => { if (argv.length !== 6) { throw new Error( - 'Incorrect number of args.\nUsage: node scripts/exposure-scanning <-pr/-issue> ', + 'Incorrect number of args.\nUsage: node path/to/exposure-scanning <-pr/-issue> ', ); } diff --git a/packages/utilities/exposure-scanning/args/index.test.js b/packages/utilities/exposure-scanning/args/index.test.js index b95924b6d2..d0c8e07d0d 100644 --- a/packages/utilities/exposure-scanning/args/index.test.js +++ b/packages/utilities/exposure-scanning/args/index.test.js @@ -16,7 +16,7 @@ describe('Parsing arguments', () => { expect(() => parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 're[g]ex', '-u']), ).toThrow( - 'Incorrect number of args.\nUsage: node scripts/exposure-scanning <-pr/-issue> ', + 'Incorrect number of args.\nUsage: node path/to/exposure-scanning <-pr/-issue> ', ); }); From a62f936c63802a8617ac7602085afc07bd15eda0 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Thu, 5 Aug 2021 11:42:18 +0100 Subject: [PATCH 27/54] Update test name --- packages/utilities/exposure-scanning/args/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/args/index.test.js b/packages/utilities/exposure-scanning/args/index.test.js index d0c8e07d0d..fec14c7a0c 100644 --- a/packages/utilities/exposure-scanning/args/index.test.js +++ b/packages/utilities/exposure-scanning/args/index.test.js @@ -12,7 +12,7 @@ describe('Parsing arguments', () => { }); }); - it('should throw an error without logging args if the number given is incorrect', () => { + it('should throw an error without logging args if the number of args given is incorrect', () => { expect(() => parseArgs(['node', 'scan.js', 'simorgh', '-pr', '1234', 're[g]ex', '-u']), ).toThrow( From 502ecb3ad51f3a423fddce7bbd36e293b2fa1ef2 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Thu, 5 Aug 2021 11:43:43 +0100 Subject: [PATCH 28/54] Remove clone call --- packages/utilities/exposure-scanning/fetch/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/fetch/index.test.js b/packages/utilities/exposure-scanning/fetch/index.test.js index ef6a2c426d..7231c052dd 100644 --- a/packages/utilities/exposure-scanning/fetch/index.test.js +++ b/packages/utilities/exposure-scanning/fetch/index.test.js @@ -63,7 +63,7 @@ jest.mock('node-fetch', () => ({ [prReviewCommentsEndpoint]: mockPrReviewCommentsRes, [issueBodyEndpoint]: mockIssueBodyRes, [issueCommentsEndpoint]: mockIssueCommentsRes, - }[url].clone()), + }[url]), })); const { fetchPr, fetchIssue } = require('.'); From cb1b4dd75b4be0e57fbeb88c80ed025974c39f58 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Thu, 5 Aug 2021 13:21:34 +0100 Subject: [PATCH 29/54] Reconfigure try/catch in scan methods --- .../utilities/exposure-scanning/scan/index.js | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/utilities/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js index fccd085d2c..d57dc880fd 100644 --- a/packages/utilities/exposure-scanning/scan/index.js +++ b/packages/utilities/exposure-scanning/scan/index.js @@ -30,44 +30,48 @@ const scanComments = (comments, regex) => { export const scanPr = (pr, regex) => { let scannedPr = pr; - - const { body, foundTextMatch } = scanText(pr.body, regex); - const { comments, foundCommentMatch } = scanComments(pr.comments, regex); - const { - comments: reviewComments, - foundCommentMatch: foundReviewCommentsMatch, - } = scanComments(pr.reviewComments, regex); + let foundMatch = false; try { + const { body, foundTextMatch } = scanText(pr.body, regex); + const { comments, foundCommentMatch } = scanComments(pr.comments, regex); + const { + comments: reviewComments, + foundCommentMatch: foundReviewCommentsMatch, + } = scanComments(pr.reviewComments, regex); + scannedPr = { body, comments, reviewComments, }; + + foundMatch = + foundTextMatch || foundCommentMatch || foundReviewCommentsMatch; } catch (error) { throw new Error(`Encountered an error when scanning.`); } - return { - ...scannedPr, - foundMatch: foundTextMatch || foundCommentMatch || foundReviewCommentsMatch, - }; + return { ...scannedPr, foundMatch }; }; export const scanIssue = (issue, regex) => { let scannedIssue = issue; - - const { body, foundTextMatch } = scanText(issue.body, regex); - const { comments, foundCommentMatch } = scanComments(issue.comments, regex); + let foundMatch = false; try { + const { body, foundTextMatch } = scanText(issue.body, regex); + const { comments, foundCommentMatch } = scanComments(issue.comments, regex); + scannedIssue = { body, comments, }; + + foundMatch = foundTextMatch || foundCommentMatch; } catch (error) { throw new Error(`Encountered an error when scanning.`); } - return { ...scannedIssue, foundMatch: foundTextMatch || foundCommentMatch }; + return { ...scannedIssue, foundMatch }; }; From 455a69158bd363d88df686bd0a5d888fe2b17141 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Fri, 6 Aug 2021 09:54:10 +0100 Subject: [PATCH 30/54] Use --- .../exposure-scanning/patch/index.js | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index 744b000e31..10e2aaeb93 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -19,30 +19,34 @@ const patchPrBody = async (reqBody, body) => { }; const patchPrReviewComments = async (reqBody, comments) => { - await comments.map(async ({ id, body }) => { - await octokit.request( - 'PATCH /repos/{owner}/{repo}/pulls/comments/{commentId}', - { - owner: reqBody.owner, - repo: reqBody.repo, - commentId: id, - body, - }, - ); - }); + await Promise.all( + comments.map(async ({ id, body }) => { + await octokit.request( + 'PATCH /repos/{owner}/{repo}/pulls/comments/{commentId}', + { + owner: reqBody.owner, + repo: reqBody.repo, + commentId: id, + body, + }, + ); + }), + ); }; const patchPrComments = async (reqBody, comments) => { - await comments.forEach(async ({ id, body }) => { - await octokit.request( - 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', - { - ...reqBody, - commentId: id, - body, - }, - ); - }); + await Promise.all( + comments.map(async ({ id, body }) => { + await octokit.request( + 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', + { + ...reqBody, + commentId: id, + body, + }, + ); + }), + ); }; const patchIssueBody = async (reqBody, body) => { @@ -53,16 +57,18 @@ const patchIssueBody = async (reqBody, body) => { }; const patchIssueComments = async (reqBody, comments) => { - await comments.forEach(async ({ id, body }) => { - await octokit.request( - 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', - { - ...reqBody, - commentId: id, - body, - }, - ); - }); + await Promise.all( + comments.map(async ({ id, body }) => { + await octokit.request( + 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', + { + ...reqBody, + commentId: id, + body, + }, + ); + }), + ); }; export const patchPr = async (reqBody, { body, comments, reviewComments }) => { From f742c24dd3010a4df6beaf6722a25a88aa4325b3 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Fri, 6 Aug 2021 14:58:20 +0100 Subject: [PATCH 31/54] Add checking the text content to patch tests --- .../utilities/exposure-scanning/patch/index.test.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index c6212ac043..2f1a3242cf 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -17,8 +17,8 @@ const mockRes = new Response( const mockFetch = jest.fn(); jest.mock('node-fetch', () => ({ - default: async url => { - mockFetch(url); + default: async (url, params) => { + mockFetch(url, JSON.parse(params.body).body); return mockRes.clone(); }, })); @@ -53,20 +53,27 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledTimes(5); + console.log(mockFetch.mock.calls); + expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/1234', + pr.body, ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/issues/comments/1', + pr.comments[0].body, ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/issues/comments/2', + pr.comments[1].body, ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/comments/3', + pr.reviewComments[0].body, ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/comments/4', + pr.reviewComments[1].body, ); }); @@ -88,9 +95,11 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/issues/4512', + issue.body, ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/issues/comments/5', + issue.comments[0].body, ); }); }); From 19ca6291dadca9c3d4ee29d45b39a7b4f41d9728 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Fri, 6 Aug 2021 15:47:59 +0100 Subject: [PATCH 32/54] Update packages/utilities/exposure-scanning/src/index.js Co-authored-by: Andrew Bennett --- packages/utilities/exposure-scanning/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/src/index.js b/packages/utilities/exposure-scanning/src/index.js index 2f25f35b3a..2e52e39e6b 100644 --- a/packages/utilities/exposure-scanning/src/index.js +++ b/packages/utilities/exposure-scanning/src/index.js @@ -43,7 +43,7 @@ const scanExposures = async () => { '-issue': redactIssue, }[flag](reqBody, regex); - return Promise.resolve('No matches found.'); + return 'No matches found.'; }; export default scanExposures; From 1b5e4a96e8c8a772b3600ea2c85e889e3c917df8 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Fri, 6 Aug 2021 16:51:10 +0100 Subject: [PATCH 33/54] add tests for regex omission in logs --- .../utilities/exposure-scanning/scan/index.js | 8 +++--- .../exposure-scanning/scan/index.test.js | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/utilities/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js index d57dc880fd..3dfeca65c3 100644 --- a/packages/utilities/exposure-scanning/scan/index.js +++ b/packages/utilities/exposure-scanning/scan/index.js @@ -1,6 +1,6 @@ const replacementStr = '*'; -const scanText = (text, regex) => { +export const scanText = (text, regex) => { let foundTextMatch = false; const scannedText = text.replace(regex, replacementStr); @@ -11,7 +11,7 @@ const scanText = (text, regex) => { return { body: scannedText, foundTextMatch }; }; -const scanComments = (comments, regex) => { +export const scanComments = (comments, regex) => { let foundCommentMatch = false; const scannedComments = []; @@ -49,7 +49,7 @@ export const scanPr = (pr, regex) => { foundMatch = foundTextMatch || foundCommentMatch || foundReviewCommentsMatch; } catch (error) { - throw new Error(`Encountered an error when scanning.`); + throw new Error('Encountered an error when scanning.'); } return { ...scannedPr, foundMatch }; @@ -70,7 +70,7 @@ export const scanIssue = (issue, regex) => { foundMatch = foundTextMatch || foundCommentMatch; } catch (error) { - throw new Error(`Encountered an error when scanning.`); + throw new Error('Encountered an error when scanning.'); } return { ...scannedIssue, foundMatch }; diff --git a/packages/utilities/exposure-scanning/scan/index.test.js b/packages/utilities/exposure-scanning/scan/index.test.js index 36745b4636..46f155a6cf 100644 --- a/packages/utilities/exposure-scanning/scan/index.test.js +++ b/packages/utilities/exposure-scanning/scan/index.test.js @@ -47,6 +47,19 @@ describe('Scanning a PR', () => { reviewComments: [{ id: 4, body: 'Should there be new * for this?' }], }); }); + + it('should not log the regex when the scanPr method throws', () => { + const regex = new RegExp('foobar', 'gi'); + const invalidPr = { + body: ['test'], + comments: [], + reviewcomments: [], + }; + + expect(() => { + scanPr(invalidPr, regex); + }).toThrow('Encountered an error when scanning.'); + }); }); describe('Scanning an Issue', () => { @@ -87,4 +100,16 @@ describe('Scanning an Issue', () => { comments: [{ id: 5, body: 'Nice *' }], }); }); + + it('should not log the regex when scanIssue throws', () => { + const invalidIssue = { + body: 'text', + comments: 'non-iterable', + reviewComments: [], + }; + const regex = new RegExp('foobar', 'gi'); + expect(() => { + scanIssue(invalidIssue, regex); + }).toThrow('Encountered an error when scanning.'); + }); }); From f7f5fa76b0b83758a9bee37103848dfd951042d8 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 09:25:58 +0100 Subject: [PATCH 34/54] Remove console.log call --- packages/utilities/exposure-scanning/patch/index.test.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index 2f1a3242cf..a7ff27b343 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -53,8 +53,6 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledTimes(5); - console.log(mockFetch.mock.calls); - expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/1234', pr.body, From c661fba5973ec798f6e8858d97098504b45ae6d8 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 09:29:31 +0100 Subject: [PATCH 35/54] Refactor fetch comments methods --- .../exposure-scanning/fetch/index.js | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/utilities/exposure-scanning/fetch/index.js b/packages/utilities/exposure-scanning/fetch/index.js index eeec1b8512..786c8c54ec 100644 --- a/packages/utilities/exposure-scanning/fetch/index.js +++ b/packages/utilities/exposure-scanning/fetch/index.js @@ -19,24 +19,24 @@ const fetchPrBody = async reqBody => { }; const fetchPrComments = async reqBody => { - const prComments = await octokit.request( + const { data } = await octokit.request( 'GET /repos/{owner}/{repo}/issues/{prId}/comments', reqBody, ); - return prComments.data.map(comment => ({ - id: comment.id, - body: comment.body, + return data.map(({ id, body }) => ({ + id, + body, })); }; const fetchPrReviewComments = async reqBody => { - const prReviewComments = await octokit.request( + const { data } = await octokit.request( 'GET /repos/{owner}/{repo}/pulls/{prId}/comments', reqBody, ); - return prReviewComments.data.map(comment => ({ - id: comment.id, - body: comment.body, + return data.map(({ id, body }) => ({ + id, + body, })); }; @@ -51,13 +51,13 @@ const fetchIssueBody = async reqBody => { }; const fetchIssueComments = async reqBody => { - const issueComments = await octokit.request( + const { data } = await octokit.request( 'GET /repos/{owner}/{repo}/issues/{issueId}/comments', reqBody, ); - return issueComments.data.map(comment => ({ - id: comment.id, - body: comment.body, + return data.map(({ id, body }) => ({ + id, + body, })); }; From e9c64c77267b808822669b05be687464ce2e7620 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 09:33:39 +0100 Subject: [PATCH 36/54] Use Promise.all correctly --- .../exposure-scanning/patch/index.js | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index 10e2aaeb93..af6f7ad9a4 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -20,8 +20,8 @@ const patchPrBody = async (reqBody, body) => { const patchPrReviewComments = async (reqBody, comments) => { await Promise.all( - comments.map(async ({ id, body }) => { - await octokit.request( + comments.map(({ id, body }) => + octokit.request( 'PATCH /repos/{owner}/{repo}/pulls/comments/{commentId}', { owner: reqBody.owner, @@ -29,23 +29,23 @@ const patchPrReviewComments = async (reqBody, comments) => { commentId: id, body, }, - ); - }), + ), + ), ); }; const patchPrComments = async (reqBody, comments) => { await Promise.all( - comments.map(async ({ id, body }) => { - await octokit.request( + comments.map(({ id, body }) => + octokit.request( 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', { ...reqBody, commentId: id, body, }, - ); - }), + ), + ), ); }; @@ -58,16 +58,16 @@ const patchIssueBody = async (reqBody, body) => { const patchIssueComments = async (reqBody, comments) => { await Promise.all( - comments.map(async ({ id, body }) => { - await octokit.request( + comments.map(({ id, body }) => + octokit.request( 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', { ...reqBody, commentId: id, body, }, - ); - }), + ), + ), ); }; From 8ea0dc3246e37ae66c3d8220de5ca7093a1e1b05 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 09:34:57 +0100 Subject: [PATCH 37/54] Avoid variable mutation in scanText --- packages/utilities/exposure-scanning/scan/index.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/utilities/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js index 3dfeca65c3..6fbdd43a04 100644 --- a/packages/utilities/exposure-scanning/scan/index.js +++ b/packages/utilities/exposure-scanning/scan/index.js @@ -1,12 +1,8 @@ const replacementStr = '*'; export const scanText = (text, regex) => { - let foundTextMatch = false; const scannedText = text.replace(regex, replacementStr); - - if (scannedText !== text) { - foundTextMatch = true; - } + const foundTextMatch = scannedText !== text; return { body: scannedText, foundTextMatch }; }; From 2ae5fb2bc260764eda02d350a64b151cd90157d1 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 09:45:54 +0100 Subject: [PATCH 38/54] Use map and filter for scanComments --- .../utilities/exposure-scanning/scan/index.js | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/utilities/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js index 6fbdd43a04..5f71da02bd 100644 --- a/packages/utilities/exposure-scanning/scan/index.js +++ b/packages/utilities/exposure-scanning/scan/index.js @@ -8,18 +8,18 @@ export const scanText = (text, regex) => { }; export const scanComments = (comments, regex) => { - let foundCommentMatch = false; - const scannedComments = []; - - comments.forEach(({ id, body }) => { - if (regex.test(body)) { - foundCommentMatch = true; - scannedComments.push({ - id, - body: body.replace(regex, replacementStr), - }); - } - }); + const scannedComments = comments + .map(({ id, body }) => { + if (regex.test(body)) { + return { + id, + body: body.replace(regex, replacementStr), + }; + } + return false; + }) + .filter(Boolean); + const foundCommentMatch = scannedComments.length > 0; return { comments: scannedComments, foundCommentMatch }; }; From b6e94cf9036e385766000e1e1f25d418b747d694 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 09:50:07 +0100 Subject: [PATCH 39/54] Clean up scan methods --- .../utilities/exposure-scanning/scan/index.js | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/packages/utilities/exposure-scanning/scan/index.js b/packages/utilities/exposure-scanning/scan/index.js index 5f71da02bd..1b62c4fbee 100644 --- a/packages/utilities/exposure-scanning/scan/index.js +++ b/packages/utilities/exposure-scanning/scan/index.js @@ -25,9 +25,6 @@ export const scanComments = (comments, regex) => { }; export const scanPr = (pr, regex) => { - let scannedPr = pr; - let foundMatch = false; - try { const { body, foundTextMatch } = scanText(pr.body, regex); const { comments, foundCommentMatch } = scanComments(pr.comments, regex); @@ -36,38 +33,25 @@ export const scanPr = (pr, regex) => { foundCommentMatch: foundReviewCommentsMatch, } = scanComments(pr.reviewComments, regex); - scannedPr = { + return { body, comments, reviewComments, + foundMatch: + foundTextMatch || foundCommentMatch || foundReviewCommentsMatch, }; - - foundMatch = - foundTextMatch || foundCommentMatch || foundReviewCommentsMatch; } catch (error) { throw new Error('Encountered an error when scanning.'); } - - return { ...scannedPr, foundMatch }; }; export const scanIssue = (issue, regex) => { - let scannedIssue = issue; - let foundMatch = false; - try { const { body, foundTextMatch } = scanText(issue.body, regex); const { comments, foundCommentMatch } = scanComments(issue.comments, regex); - scannedIssue = { - body, - comments, - }; - - foundMatch = foundTextMatch || foundCommentMatch; + return { body, comments, foundMatch: foundTextMatch || foundCommentMatch }; } catch (error) { throw new Error('Encountered an error when scanning.'); } - - return { ...scannedIssue, foundMatch }; }; From ed955c117466e89efd6da54db2199f9987246305 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 10:38:53 +0100 Subject: [PATCH 40/54] Use id instead of issueId or prId --- .../exposure-scanning/fetch/index.js | 13 +++--- .../exposure-scanning/fetch/index.test.js | 4 +- .../exposure-scanning/patch/index.js | 45 ++++++++----------- .../utilities/exposure-scanning/src/index.js | 2 +- 4 files changed, 26 insertions(+), 38 deletions(-) diff --git a/packages/utilities/exposure-scanning/fetch/index.js b/packages/utilities/exposure-scanning/fetch/index.js index 786c8c54ec..75070e20ed 100644 --- a/packages/utilities/exposure-scanning/fetch/index.js +++ b/packages/utilities/exposure-scanning/fetch/index.js @@ -14,13 +14,13 @@ let octokit; const fetchPrBody = async reqBody => { const { data: { body }, - } = await octokit.request('GET /repos/{owner}/{repo}/pulls/{prId}', reqBody); + } = await octokit.request('GET /repos/{owner}/{repo}/pulls/{id}', reqBody); return body; }; const fetchPrComments = async reqBody => { const { data } = await octokit.request( - 'GET /repos/{owner}/{repo}/issues/{prId}/comments', + 'GET /repos/{owner}/{repo}/issues/{id}/comments', reqBody, ); return data.map(({ id, body }) => ({ @@ -31,7 +31,7 @@ const fetchPrComments = async reqBody => { const fetchPrReviewComments = async reqBody => { const { data } = await octokit.request( - 'GET /repos/{owner}/{repo}/pulls/{prId}/comments', + 'GET /repos/{owner}/{repo}/pulls/{id}/comments', reqBody, ); return data.map(({ id, body }) => ({ @@ -43,16 +43,13 @@ const fetchPrReviewComments = async reqBody => { const fetchIssueBody = async reqBody => { const { data: { body }, - } = await octokit.request( - 'GET /repos/{owner}/{repo}/issues/{issueId}', - reqBody, - ); + } = await octokit.request('GET /repos/{owner}/{repo}/issues/{id}', reqBody); return body; }; const fetchIssueComments = async reqBody => { const { data } = await octokit.request( - 'GET /repos/{owner}/{repo}/issues/{issueId}/comments', + 'GET /repos/{owner}/{repo}/issues/{id}/comments', reqBody, ); return data.map(({ id, body }) => ({ diff --git a/packages/utilities/exposure-scanning/fetch/index.test.js b/packages/utilities/exposure-scanning/fetch/index.test.js index 7231c052dd..349e9544cc 100644 --- a/packages/utilities/exposure-scanning/fetch/index.test.js +++ b/packages/utilities/exposure-scanning/fetch/index.test.js @@ -78,7 +78,7 @@ describe('Fetching from the GitHub API', () => { const reqBody = { owner: 'bbc', repo: 'simorgh', - prId: '9188', + id: '9188', }; const pr = await fetchPr(reqBody); @@ -96,7 +96,7 @@ describe('Fetching from the GitHub API', () => { const reqBody = { owner: 'bbc', repo: 'psammead', - issueId: '4512', + id: '4512', }; const issue = await fetchIssue(reqBody); diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index af6f7ad9a4..e740b871c2 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -12,7 +12,7 @@ let octokit; })(); const patchPrBody = async (reqBody, body) => { - await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{prId}', { + await octokit.request('PATCH /repos/{owner}/{repo}/pulls/{id}', { ...reqBody, body, }); @@ -21,15 +21,12 @@ const patchPrBody = async (reqBody, body) => { const patchPrReviewComments = async (reqBody, comments) => { await Promise.all( comments.map(({ id, body }) => - octokit.request( - 'PATCH /repos/{owner}/{repo}/pulls/comments/{commentId}', - { - owner: reqBody.owner, - repo: reqBody.repo, - commentId: id, - body, - }, - ), + octokit.request('PATCH /repos/{owner}/{repo}/pulls/comments/{id}', { + owner: reqBody.owner, + repo: reqBody.repo, + id, + body, + }), ), ); }; @@ -37,20 +34,17 @@ const patchPrReviewComments = async (reqBody, comments) => { const patchPrComments = async (reqBody, comments) => { await Promise.all( comments.map(({ id, body }) => - octokit.request( - 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', - { - ...reqBody, - commentId: id, - body, - }, - ), + octokit.request('PATCH /repos/{owner}/{repo}/issues/comments/{id}', { + ...reqBody, + id, + body, + }), ), ); }; const patchIssueBody = async (reqBody, body) => { - await octokit.request('PATCH /repos/{owner}/{repo}/issues/{issueId}', { + await octokit.request('PATCH /repos/{owner}/{repo}/issues/{id}', { ...reqBody, body, }); @@ -59,14 +53,11 @@ const patchIssueBody = async (reqBody, body) => { const patchIssueComments = async (reqBody, comments) => { await Promise.all( comments.map(({ id, body }) => - octokit.request( - 'PATCH /repos/{owner}/{repo}/issues/comments/{commentId}', - { - ...reqBody, - commentId: id, - body, - }, - ), + octokit.request('PATCH /repos/{owner}/{repo}/issues/comments/{id}', { + ...reqBody, + id, + body, + }), ), ); }; diff --git a/packages/utilities/exposure-scanning/src/index.js b/packages/utilities/exposure-scanning/src/index.js index 2e52e39e6b..4dc9e67b4e 100644 --- a/packages/utilities/exposure-scanning/src/index.js +++ b/packages/utilities/exposure-scanning/src/index.js @@ -35,7 +35,7 @@ const scanExposures = async () => { const reqBody = { owner: 'bbc', repo, - prId: id, + id, }; await { From 548c2d127e4ae047b68d40c3a1ed4e5599545ef5 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 10:44:35 +0100 Subject: [PATCH 41/54] Remove duplicate code --- .../exposure-scanning/fetch/index.js | 21 +++++------------- .../exposure-scanning/patch/index.js | 22 +++++-------------- .../exposure-scanning/patch/index.test.js | 4 ++-- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/packages/utilities/exposure-scanning/fetch/index.js b/packages/utilities/exposure-scanning/fetch/index.js index 75070e20ed..140bb8570b 100644 --- a/packages/utilities/exposure-scanning/fetch/index.js +++ b/packages/utilities/exposure-scanning/fetch/index.js @@ -18,7 +18,7 @@ const fetchPrBody = async reqBody => { return body; }; -const fetchPrComments = async reqBody => { +const fetchComments = async reqBody => { const { data } = await octokit.request( 'GET /repos/{owner}/{repo}/issues/{id}/comments', reqBody, @@ -29,7 +29,7 @@ const fetchPrComments = async reqBody => { })); }; -const fetchPrReviewComments = async reqBody => { +const fetchReviewComments = async reqBody => { const { data } = await octokit.request( 'GET /repos/{owner}/{repo}/pulls/{id}/comments', reqBody, @@ -47,24 +47,13 @@ const fetchIssueBody = async reqBody => { return body; }; -const fetchIssueComments = async reqBody => { - const { data } = await octokit.request( - 'GET /repos/{owner}/{repo}/issues/{id}/comments', - reqBody, - ); - return data.map(({ id, body }) => ({ - id, - body, - })); -}; - export const fetchPr = async reqBody => ({ body: await fetchPrBody(reqBody), - comments: await fetchPrComments(reqBody), - reviewComments: await fetchPrReviewComments(reqBody), + comments: await fetchComments(reqBody), + reviewComments: await fetchReviewComments(reqBody), }); export const fetchIssue = async reqBody => ({ body: await fetchIssueBody(reqBody), - comments: await fetchIssueComments(reqBody), + comments: await fetchComments(reqBody), }); diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index e740b871c2..069a7bb1db 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -18,7 +18,7 @@ const patchPrBody = async (reqBody, body) => { }); }; -const patchPrReviewComments = async (reqBody, comments) => { +const patchReviewComments = async (reqBody, comments) => { await Promise.all( comments.map(({ id, body }) => octokit.request('PATCH /repos/{owner}/{repo}/pulls/comments/{id}', { @@ -31,7 +31,7 @@ const patchPrReviewComments = async (reqBody, comments) => { ); }; -const patchPrComments = async (reqBody, comments) => { +const patchComments = async (reqBody, comments) => { await Promise.all( comments.map(({ id, body }) => octokit.request('PATCH /repos/{owner}/{repo}/issues/comments/{id}', { @@ -50,29 +50,17 @@ const patchIssueBody = async (reqBody, body) => { }); }; -const patchIssueComments = async (reqBody, comments) => { - await Promise.all( - comments.map(({ id, body }) => - octokit.request('PATCH /repos/{owner}/{repo}/issues/comments/{id}', { - ...reqBody, - id, - body, - }), - ), - ); -}; - export const patchPr = async (reqBody, { body, comments, reviewComments }) => { await Promise.all([ patchPrBody(reqBody, body), - patchPrComments(reqBody, comments), - patchPrReviewComments(reqBody, reviewComments), + patchComments(reqBody, comments), + patchReviewComments(reqBody, reviewComments), ]); }; export const patchIssue = async (reqBody, { body, comments }) => { await Promise.all([ patchIssueBody(reqBody, body), - patchIssueComments(reqBody, comments), + patchComments(reqBody, comments), ]); }; diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index a7ff27b343..3e75ea23eb 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -34,7 +34,7 @@ describe('Patching', () => { const reqBody = { owner: 'bbc', repo: 'psamemad', - prId: '1234', + id: '1234', }; const pr = { @@ -79,7 +79,7 @@ describe('Patching', () => { const reqBody = { owner: 'bbc', repo: 'psammead', - issueId: '4512', + id: '4512', }; const issue = { From 68570a42eeaa7b3a023d0a1494e761984eac9621 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 14:27:30 +0100 Subject: [PATCH 42/54] Add a limit to the number of patch comment invocations in one go --- .../exposure-scanning/patch/index.js | 7 +++-- .../exposure-scanning/patch/index.test.js | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index 069a7bb1db..e6a7f6cd75 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -1,6 +1,9 @@ /* eslint-disable global-require */ let octokit; +// Used to limit the number of calls to the patch comment and review comment endpoint +const maxCommentPatches = 20; + (async () => { if (process.env.GITHUB_ACTION && process.env.GITHUB_TOKEN) { const { Octokit } = await import('@octokit/action'); @@ -20,7 +23,7 @@ const patchPrBody = async (reqBody, body) => { const patchReviewComments = async (reqBody, comments) => { await Promise.all( - comments.map(({ id, body }) => + comments.slice(0, maxCommentPatches).map(({ id, body }) => octokit.request('PATCH /repos/{owner}/{repo}/pulls/comments/{id}', { owner: reqBody.owner, repo: reqBody.repo, @@ -33,7 +36,7 @@ const patchReviewComments = async (reqBody, comments) => { const patchComments = async (reqBody, comments) => { await Promise.all( - comments.map(({ id, body }) => + comments.slice(0, maxCommentPatches).map(({ id, body }) => octokit.request('PATCH /repos/{owner}/{repo}/issues/comments/{id}', { ...reqBody, id, diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index 3e75ea23eb..a82043deb5 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -100,4 +100,34 @@ describe('Patching', () => { issue.comments[0].body, ); }); + + it.only('should not patch any more than 20 comments', async () => { + const reqBody = { + owner: 'bbc', + repo: 'psammead', + id: '100000', + }; + + const comments = new Array(45).fill().map((_, i) => ({ + id: i, + body: `Comment number ${i}`, + })); + + const issue = { + body: 'PR Body', + comments, + }; + + await patchIssue(reqBody, issue); + + expect(mockFetch).toHaveBeenCalledTimes(21); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psammead/issues/comments/19', + 'Comment number 19', + ); + expect(mockFetch).not.toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psammead/issues/comments/20', + 'Comment number 20', + ); + }); }); From 5f7df867671bec27602245b130e2ef570d193c22 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 14:27:49 +0100 Subject: [PATCH 43/54] Remove it.only use --- packages/utilities/exposure-scanning/patch/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index a82043deb5..f78c735f19 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -101,7 +101,7 @@ describe('Patching', () => { ); }); - it.only('should not patch any more than 20 comments', async () => { + it('should not patch any more than 20 comments', async () => { const reqBody = { owner: 'bbc', repo: 'psammead', From 040d48d2357b365e2f7a66baa6ef7d72035673c2 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 9 Aug 2021 14:30:57 +0100 Subject: [PATCH 44/54] Add patch limit test for review comments --- .../exposure-scanning/patch/index.test.js | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index f78c735f19..3e75ea9142 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -130,4 +130,35 @@ describe('Patching', () => { 'Comment number 20', ); }); + + it('should not patch any more than 21 review comments', async () => { + const reqBody = { + owner: 'bbc', + repo: 'psammead', + id: '100000', + }; + + const reviewComments = new Array(45).fill().map((_, i) => ({ + id: i, + body: `Review comment number ${i}`, + })); + + const pr = { + body: 'PR Body', + comments: [], + reviewComments, + }; + + await patchPr(reqBody, pr); + + expect(mockFetch).toHaveBeenCalledTimes(21); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psammead/pulls/comments/19', + 'Review comment number 19', + ); + expect(mockFetch).not.toHaveBeenCalledWith( + 'https://api.github.com/repos/bbc/psammead/pulls/comments/20', + 'Review comment number 20', + ); + }); }); From be93e450927c94e76553c4db60a7962259570dbe Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 10 Aug 2021 09:56:51 +0100 Subject: [PATCH 45/54] Ensure fetch method is PATCH --- .../utilities/exposure-scanning/patch/index.test.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/patch/index.test.js b/packages/utilities/exposure-scanning/patch/index.test.js index 3e75ea9142..55d7ae6947 100644 --- a/packages/utilities/exposure-scanning/patch/index.test.js +++ b/packages/utilities/exposure-scanning/patch/index.test.js @@ -18,7 +18,7 @@ const mockRes = new Response( const mockFetch = jest.fn(); jest.mock('node-fetch', () => ({ default: async (url, params) => { - mockFetch(url, JSON.parse(params.body).body); + mockFetch(url, JSON.parse(params.body).body, params.method); return mockRes.clone(); }, })); @@ -56,22 +56,27 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/1234', pr.body, + 'PATCH', ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/issues/comments/1', pr.comments[0].body, + 'PATCH', ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/issues/comments/2', pr.comments[1].body, + 'PATCH', ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/comments/3', pr.reviewComments[0].body, + 'PATCH', ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psamemad/pulls/comments/4', pr.reviewComments[1].body, + 'PATCH', ); }); @@ -94,10 +99,12 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/issues/4512', issue.body, + 'PATCH', ); expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/issues/comments/5', issue.comments[0].body, + 'PATCH', ); }); @@ -124,10 +131,12 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/issues/comments/19', 'Comment number 19', + 'PATCH', ); expect(mockFetch).not.toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/issues/comments/20', 'Comment number 20', + 'PATCH', ); }); @@ -155,10 +164,12 @@ describe('Patching', () => { expect(mockFetch).toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/pulls/comments/19', 'Review comment number 19', + 'PATCH', ); expect(mockFetch).not.toHaveBeenCalledWith( 'https://api.github.com/repos/bbc/psammead/pulls/comments/20', 'Review comment number 20', + 'PATCH', ); }); }); From 61eb55943c5be3e7b48994d1090c2f1754e2a703 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 10 Aug 2021 10:00:22 +0100 Subject: [PATCH 46/54] Remove unnecessary lint comments --- packages/utilities/exposure-scanning/fetch/index.js | 1 - packages/utilities/exposure-scanning/patch/index.js | 1 - packages/utilities/exposure-scanning/src/index.js | 10 ++++------ packages/utilities/exposure-scanning/src/index.test.js | 9 ++++----- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/utilities/exposure-scanning/fetch/index.js b/packages/utilities/exposure-scanning/fetch/index.js index 140bb8570b..77c49822c7 100644 --- a/packages/utilities/exposure-scanning/fetch/index.js +++ b/packages/utilities/exposure-scanning/fetch/index.js @@ -1,4 +1,3 @@ -/* eslint-disable global-require */ let octokit; (async () => { diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index e6a7f6cd75..aa435db757 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -1,4 +1,3 @@ -/* eslint-disable global-require */ let octokit; // Used to limit the number of calls to the patch comment and review comment endpoint diff --git a/packages/utilities/exposure-scanning/src/index.js b/packages/utilities/exposure-scanning/src/index.js index 4dc9e67b4e..9b5c1dea7a 100644 --- a/packages/utilities/exposure-scanning/src/index.js +++ b/packages/utilities/exposure-scanning/src/index.js @@ -1,9 +1,7 @@ -/* eslint-disable no-console */ -/* eslint-disable import/extensions */ -import parseArgs from '../args/index.js'; -import { fetchPr, fetchIssue } from '../fetch/index.js'; -import { scanPr, scanIssue } from '../scan/index.js'; -import { patchPr, patchIssue } from '../patch/index.js'; +import parseArgs from '../args/index'; +import { fetchPr, fetchIssue } from '../fetch/index'; +import { scanPr, scanIssue } from '../scan/index'; +import { patchPr, patchIssue } from '../patch/index'; const redactPr = async (reqBody, regex) => { const pr = await fetchPr(reqBody); diff --git a/packages/utilities/exposure-scanning/src/index.test.js b/packages/utilities/exposure-scanning/src/index.test.js index 669a7444c3..646fe15430 100644 --- a/packages/utilities/exposure-scanning/src/index.test.js +++ b/packages/utilities/exposure-scanning/src/index.test.js @@ -1,8 +1,7 @@ -/* eslint-disable import/extensions */ -import parseArgs from '../args/index.js'; -import { fetchPr, fetchIssue } from '../fetch/index.js'; -import { patchPr, patchIssue } from '../patch/index.js'; -import scanExposures from './index.js'; +import parseArgs from '../args/index'; +import { fetchPr, fetchIssue } from '../fetch/index'; +import { patchPr, patchIssue } from '../patch/index'; +import scanExposures from './index'; jest.mock('../args/index.js'); jest.mock('../fetch/index.js'); From e4b83654775ace59c70bf1f4478ddf5362f05014 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 10 Aug 2021 13:25:51 +0100 Subject: [PATCH 47/54] Add comment explaining patch limit further --- packages/utilities/exposure-scanning/patch/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/utilities/exposure-scanning/patch/index.js b/packages/utilities/exposure-scanning/patch/index.js index aa435db757..74c1831a62 100644 --- a/packages/utilities/exposure-scanning/patch/index.js +++ b/packages/utilities/exposure-scanning/patch/index.js @@ -1,6 +1,9 @@ let octokit; // Used to limit the number of calls to the patch comment and review comment endpoint +// since each comment is uniquely identified in the context of the repository instead +// of the individual PR/issue, and we only wish to update the comments and review comments +// of that single PR or issue. const maxCommentPatches = 20; (async () => { From 7d7983825c948aae2f1c98c3c7b2bfacb24ec085 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Thu, 12 Aug 2021 17:20:53 +0100 Subject: [PATCH 48/54] add-scanning --- .../utilities/exposure-scanning/README.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 packages/utilities/exposure-scanning/README.md diff --git a/packages/utilities/exposure-scanning/README.md b/packages/utilities/exposure-scanning/README.md new file mode 100644 index 0000000000..829ba1e23f --- /dev/null +++ b/packages/utilities/exposure-scanning/README.md @@ -0,0 +1,47 @@ +# exposure-scanning - [![Known Vulnerabilities](https://snyk.io/test/github/bbc/psammead/badge.svg?targetFile=packages%2Futilities%2Fexposure-scanning%2Fpackage.json)](https://snyk.io/test/github/bbc/psammead?targetFile=packages%2Futilities%2Fexposure-scanning%2Fpackage.json) [![Dependency Status](https://david-dm.org/bbc/psammead.svg?path=packages/utilities/exposure-scanning)](https://david-dm.org/bbc/psammead?path=packages/utilities/exposure-scanning) [![peerDependencies Status](https://david-dm.org/bbc/psammead/peer-status.svg?path=packages/utilities/exposure-scanning)](https://david-dm.org/bbc/psammead?path=packages/utilities/exposure-scanning&type=peer) [![GitHub license](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/bbc/psammead/blob/latest/LICENSE) [![npm version](https://img.shields.io/npm/v/@bbc/exposure-scanning.svg)](https://www.npmjs.com/package/@bbc/exposure-scanning) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/bbc/psammead/blob/latest/CONTRIBUTING.md) + +This package provides a utility to scan and patch issues and pull requests for content that matches a given regular expression. + + +## Installation + +```jsx +npm install @bbc/exposure-scanning --save +``` + +## Usage + +The script can be imported and executed like so: + +```jsx +// /scripts/scan-exposures/index.jsx +import scanExposures from '@bbc/exposure-scanning'; + +(async () => { + await scanExposures(); +})(); +``` + +Then, this can be executed in the command line: + +```sh +./scripts/scan-exposures psammead -pr 1234 "foo|bar" +``` + +The command line arguments are as follows: + - repository (psammead in the example) + - content type (-pr or -issue) + - id (of the issue or pull request) + - regex ("foo|bar" in the example) + +## Contributing + +Psammead is completely open source. We are grateful for any contributions, whether they be new components, bug fixes or general improvements. Please see our primary contributing guide which can be found at [the root of the Psammead respository](https://github.com/bbc/psammead/blob/latest/CONTRIBUTING.md). + +### [Code of Conduct](https://github.com/bbc/psammead/blob/latest/CODE_OF_CONDUCT.md) + +We welcome feedback and help on this work. By participating in this project, you agree to abide by the [code of conduct](https://github.com/bbc/psammead/blob/latest/CODE_OF_CONDUCT.md). Please take a moment to read it. + +### License + +Psammead is [Apache 2.0 licensed](https://github.com/bbc/psammead/blob/latest/LICENSE). From 659cd40226ecbd7d4aa63291d788156edfe464a2 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 16 Aug 2021 09:37:03 +0100 Subject: [PATCH 49/54] Add alpha component notice to README --- packages/utilities/exposure-scanning/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/utilities/exposure-scanning/README.md b/packages/utilities/exposure-scanning/README.md index 829ba1e23f..f527579bf8 100644 --- a/packages/utilities/exposure-scanning/README.md +++ b/packages/utilities/exposure-scanning/README.md @@ -1,3 +1,6 @@ +# ⛔️ This is an alpha component ⛔️ + +This component is currently tagged as alpha and is not suitable for production use. Following the passing of an accessibility review this component will be marked as ready for production and the alpha tag removed. # exposure-scanning - [![Known Vulnerabilities](https://snyk.io/test/github/bbc/psammead/badge.svg?targetFile=packages%2Futilities%2Fexposure-scanning%2Fpackage.json)](https://snyk.io/test/github/bbc/psammead?targetFile=packages%2Futilities%2Fexposure-scanning%2Fpackage.json) [![Dependency Status](https://david-dm.org/bbc/psammead.svg?path=packages/utilities/exposure-scanning)](https://david-dm.org/bbc/psammead?path=packages/utilities/exposure-scanning) [![peerDependencies Status](https://david-dm.org/bbc/psammead/peer-status.svg?path=packages/utilities/exposure-scanning)](https://david-dm.org/bbc/psammead?path=packages/utilities/exposure-scanning&type=peer) [![GitHub license](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/bbc/psammead/blob/latest/LICENSE) [![npm version](https://img.shields.io/npm/v/@bbc/exposure-scanning.svg)](https://www.npmjs.com/package/@bbc/exposure-scanning) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/bbc/psammead/blob/latest/CONTRIBUTING.md) This package provides a utility to scan and patch issues and pull requests for content that matches a given regular expression. From b05b1c9567a062163ce466ca581c3236ec553087 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 16 Aug 2021 09:38:26 +0100 Subject: [PATCH 50/54] Remove a11y section of notice --- packages/utilities/exposure-scanning/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/README.md b/packages/utilities/exposure-scanning/README.md index f527579bf8..e4eb8c8e10 100644 --- a/packages/utilities/exposure-scanning/README.md +++ b/packages/utilities/exposure-scanning/README.md @@ -1,6 +1,6 @@ # ⛔️ This is an alpha component ⛔️ -This component is currently tagged as alpha and is not suitable for production use. Following the passing of an accessibility review this component will be marked as ready for production and the alpha tag removed. +This component is currently tagged as alpha and is not suitable for production use. # exposure-scanning - [![Known Vulnerabilities](https://snyk.io/test/github/bbc/psammead/badge.svg?targetFile=packages%2Futilities%2Fexposure-scanning%2Fpackage.json)](https://snyk.io/test/github/bbc/psammead?targetFile=packages%2Futilities%2Fexposure-scanning%2Fpackage.json) [![Dependency Status](https://david-dm.org/bbc/psammead.svg?path=packages/utilities/exposure-scanning)](https://david-dm.org/bbc/psammead?path=packages/utilities/exposure-scanning) [![peerDependencies Status](https://david-dm.org/bbc/psammead/peer-status.svg?path=packages/utilities/exposure-scanning)](https://david-dm.org/bbc/psammead?path=packages/utilities/exposure-scanning&type=peer) [![GitHub license](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/bbc/psammead/blob/latest/LICENSE) [![npm version](https://img.shields.io/npm/v/@bbc/exposure-scanning.svg)](https://www.npmjs.com/package/@bbc/exposure-scanning) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/bbc/psammead/blob/latest/CONTRIBUTING.md) This package provides a utility to scan and patch issues and pull requests for content that matches a given regular expression. From 5d7972d048d11ca77446e78f3d0865181eb33399 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Mon, 16 Aug 2021 15:57:44 +0100 Subject: [PATCH 51/54] Include that the match has been patched in error msg --- packages/utilities/exposure-scanning/src/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/utilities/exposure-scanning/src/index.js b/packages/utilities/exposure-scanning/src/index.js index 9b5c1dea7a..4d5f162dcd 100644 --- a/packages/utilities/exposure-scanning/src/index.js +++ b/packages/utilities/exposure-scanning/src/index.js @@ -10,7 +10,7 @@ const redactPr = async (reqBody, regex) => { if (scannedPr.foundMatch) { await patchPr(reqBody, scannedPr); - throw new Error('Match found.'); + throw new Error('Match found and patched.'); } }; @@ -21,7 +21,7 @@ const redactIssue = async (reqBody, regex) => { if (scannedIssue.foundMatch) { await patchIssue(reqBody, scannedIssue); - throw new Error('Match found.'); + throw new Error('Match found and patched.'); } }; From edf92d685b7d1df7971d79b5aa0957dc8a99d373 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 17 Aug 2021 11:56:25 +0100 Subject: [PATCH 52/54] Update packages/utilities/exposure-scanning/args/index.js Co-authored-by: Jonathan Roebuck --- packages/utilities/exposure-scanning/args/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/args/index.js b/packages/utilities/exposure-scanning/args/index.js index 7946312715..61a28c3bb0 100644 --- a/packages/utilities/exposure-scanning/args/index.js +++ b/packages/utilities/exposure-scanning/args/index.js @@ -35,7 +35,9 @@ const parseArgs = argv => { throw new Error('Invalid issue/pr id.'); } - if (flag !== '-pr' && flag !== 'issue') { + const isValidFlag = ['-pr', 'issue'].includes(flag) + + if (!isValidFlag) { throw new Error('Invalid flag argument given.'); } From 7ad32fba0f62084a641fdcde827b9b7e1bc62854 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 17 Aug 2021 12:01:18 +0100 Subject: [PATCH 53/54] Add semicolon :) --- packages/utilities/exposure-scanning/args/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utilities/exposure-scanning/args/index.js b/packages/utilities/exposure-scanning/args/index.js index 61a28c3bb0..e64b3a57d9 100644 --- a/packages/utilities/exposure-scanning/args/index.js +++ b/packages/utilities/exposure-scanning/args/index.js @@ -35,7 +35,7 @@ const parseArgs = argv => { throw new Error('Invalid issue/pr id.'); } - const isValidFlag = ['-pr', 'issue'].includes(flag) + const isValidFlag = ['-pr', 'issue'].includes(flag); if (!isValidFlag) { throw new Error('Invalid flag argument given.'); From 4f98e69dabd070281b0dfcd8fe86dc94d00349a9 Mon Sep 17 00:00:00 2001 From: HarryVerhoef Date: Tue, 17 Aug 2021 12:28:03 +0100 Subject: [PATCH 54/54] Use promise.all for fetchPr and fetchIssue --- .../exposure-scanning/fetch/index.js | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/utilities/exposure-scanning/fetch/index.js b/packages/utilities/exposure-scanning/fetch/index.js index 77c49822c7..a1865c65c7 100644 --- a/packages/utilities/exposure-scanning/fetch/index.js +++ b/packages/utilities/exposure-scanning/fetch/index.js @@ -46,13 +46,28 @@ const fetchIssueBody = async reqBody => { return body; }; -export const fetchPr = async reqBody => ({ - body: await fetchPrBody(reqBody), - comments: await fetchComments(reqBody), - reviewComments: await fetchReviewComments(reqBody), -}); - -export const fetchIssue = async reqBody => ({ - body: await fetchIssueBody(reqBody), - comments: await fetchComments(reqBody), -}); +export const fetchPr = async reqBody => { + const [body, comments, reviewComments] = await Promise.all([ + fetchPrBody(reqBody), + fetchComments(reqBody), + fetchReviewComments(reqBody), + ]); + + return { + body, + comments, + reviewComments, + }; +}; + +export const fetchIssue = async reqBody => { + const [body, comments] = await Promise.all([ + fetchIssueBody(reqBody), + fetchComments(reqBody), + ]); + + return { + body, + comments, + }; +};