From a24dd7caba24e5af2003cec6573caa5c43259157 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Thu, 18 Feb 2021 16:22:49 +0800 Subject: [PATCH] feat: use the github client authed by installation id (#62) * feat: use the github client authed by installation id * docs: update config doc * fix: fix typo and revert the package-lock * fix: lint fix * fix: revert package-lock.json * fix: move out octokit from loop --- app.yml | 49 +++++++++++++++++----------------- docs/config.md | 3 +-- docs/design.md | 16 ++++++------ nodemon.json | 7 +++++ src/events/app/index.ts | 58 +++++++++++++++++++++++++---------------- src/events/common.ts | 32 ++++++++++++++++++++--- src/index.ts | 15 +++-------- 7 files changed, 108 insertions(+), 72 deletions(-) create mode 100644 nodemon.json diff --git a/app.yml b/app.yml index 9c84f9d..2bdcffc 100644 --- a/app.yml +++ b/app.yml @@ -22,30 +22,30 @@ default_events: # - deployment_status # - fork # - gollum - # - issue_comment + - issue_comment - issues -# - label -# - milestone -# - member -# - membership -# - org_block -# - organization -# - page_build -# - project -# - project_card -# - project_column -# - public + # - label + # - milestone + # - member + # - membership + # - org_block + # - organization + # - page_build + # - project + # - project_card + # - project_column + # - public - pull_request - pull_request_review - pull_request_review_comment -# - push -# - release - - repository -# - repository_import -# - status -# - team -# - team_add -# - watch + # - push + # - release + # - repository + # - repository_import + # - status + # - team + # - team_add + # - watch # The set of permissions needed by the GitHub App. The format of the object uses # the permission name for the key (for example, issues) and the access type for @@ -70,7 +70,7 @@ default_permissions: # Issues and related comments, assignees, labels, and milestones. # https://developer.github.com/v3/apps/permissions/#permission-on-issues - issues: write + issues: read # Search repositories, list collaborators, and access repository metadata. # https://developer.github.com/v3/apps/permissions/#metadata-permissions @@ -123,14 +123,15 @@ default_permissions: # Get notified of, and update, content references. # https://developer.github.com/v3/apps/permissions/ # organization_administration: read + # The name of the GitHub App. Defaults to the name specified in package.json - name: ti-sync-bot +name: ti-sync-bot # The homepage of your GitHub App. - url: https://github.com/ti-community-infra/ti-sync-bot +url: https://github.com/ti-community-infra/ti-sync-bot # A description of the GitHub App. - description: A Github App that used to sync tidb community info that contains pull request, issue, comment and contributor info. +description: A Github App that used to sync tidb community info that contains pull request, issue, comment and contributor info. # Set to true when your GitHub App is available to the public or false when it is only accessible to the owner of the app. # Default: true diff --git a/docs/config.md b/docs/config.md index c74f19b..6cae4fa 100644 --- a/docs/config.md +++ b/docs/config.md @@ -6,5 +6,4 @@ | 选项 | 选项说明 | | ------------------------- | --------------------------------------------------------------------------- | -| GITHUB_ACCESS_TOKEN | 在全量同步的过程中，需要使用该 ACCESS TOKEN 来访问 Github 的一些 API，你可以在 [开发者设置](https://github.com/settings/tokens/new) 当中生成该 TOKEN。 | -| SYNC_REPOS | 为了测试方便，如果你在 `.env` 文件当中指定 `SYNC_REPOS` 配置，Bot 在程序启动时将会针对该配置中指定的仓库进行全量同步，例如：`SYNC_REPOS=tikv/tikv,pingcap/tipocket`。 | +| SYNC_REPOS | 为了测试方便，如果你在 `.env` 文件当中指定 `SYNC_REPOS` 配置，Bot 在程序启动时将会针对该配置中指定的仓库进行全量同步，例如：`SYNC_REPOS=tikv/tikv,pingcap/tipocket`。需要注意的是，指定的仓库必须事先安装该 GitHub App。 | diff --git a/docs/design.md b/docs/design.md index dc4d7ab..028165e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -2,7 +2,7 @@ ## 整体思路 -ti-sync-bot（以下简称 Bot ）主要负责将 tidb 社区 Github 上的一些数据同步到数据库当中，它的工作模式主要分为全量同步和增量同步两种模式。 +ti-sync-bot（以下简称 Bot ）主要负责将 tidb 社区 GitHub 上的一些数据同步到数据库当中，它的工作模式主要分为全量同步和增量同步两种模式。 ### 全量同步 @@ -10,19 +10,19 @@ ti-sync-bot（以下简称 Bot ）主要负责将 tidb 社区 Github 上的一 | 事件名称 | 事件类型 | 触发说明 | | ------------------------------- | ----------- | ----------------------------------------- | -| app.start_up | 自定义事件 | 在程序启动时触发，Bot 会获取所有安装了该 Github App 的仓库列表逐一进行全量同步。 | +| app.start_up | 自定义事件 | 在程序启动时触发，Bot 会获取所有安装了该 GitHub App 的仓库列表逐一进行全量同步。 | | installation.created | WebHook 事件 | 当用户初次将 Bot 安装到用户账号或组织账号时触发，用户在安装时可以选择安装到所有仓库或指定仓库，Bot 会针对安装的仓库进行逐一全量同步。 | -| installation_repositories.added | WebHook 事件 | 用户可以在 Github App 设置对已安装仓库进行添加或删除，当新添加一个仓库时，会触发该事件，Bot 会针对新添加的仓库进行逐一全量同步。 | +| installation_repositories.added | WebHook 事件 | 用户可以在 GitHub App 设置对已安装仓库进行添加或删除，当新添加一个仓库时，会触发该事件，Bot 会针对新添加的仓库进行逐一全量同步。 | 在全量同步过程当中，同步 Pull Request 和 同步 Issue 两个过程并发进行，同步 Contributor Email 的过程依赖于同步 PR 的数据，因此会在同步 Pull Request 完成之后执行。 ### 增量同步 -增量同步的设计目的是为了能够更加及时的将 Github 上的数据同步到数据库当中，避免全量同步在一个较为集中的时间段内产生大量的数据库操作和 API 接口请求。 +增量同步的设计目的是为了能够更加及时的将 GitHub 上的数据同步到数据库当中，避免全量同步在一个较为集中的时间段内产生大量的数据库操作和 API 接口请求。 -增量同步是基于 Github 的 WebHook 机制实现的，为了能够在 Bot 启动过程中及时处理 WebHook 发送过来的数据，全量同步和增量同步被设计成并发进行。 +增量同步是基于 GitHub 的 WebHook 机制实现的，为了能够在 Bot 启动过程中及时处理 WebHook 发送过来的数据，全量同步和增量同步被设计成并发进行。 -Bot 通过监听以下类型事件来对 Github 数据进行增量同步： +Bot 通过监听以下类型事件来对 GitHub 数据进行增量同步： | 事件类型 | 动作类型 | 触发行为 | | --------------- | ----------- | ----------- | @@ -44,7 +44,7 @@ Bot 在将收到的数据同步数据库之前会对收到的 Pull Request、Iss | 字段名称 | 字段说明 | | --------------- | ----------- | -| status | PR 的状态可以分为 `open`、`closed` 和 `merged` 三种状态，Github 只提供了 `open` 和 `closed` 两种状态，如果 PR 的 `merged_at` 不为空，则可以判定为 `merged` 状态。 | +| status | PR 的状态可以分为 `open`、`closed` 和 `merged` 三种状态，GitHub 只提供了 `open` 和 `closed` 两种状态，如果 PR 的 `merged_at` 不为空，则可以判定为 `merged` 状态。 | | label | 使用逗号分隔多个标签名。 | | relation | 描述的是 PR 作者与公司的关系，其类型包括：`member`、`not member`。 | | association | 即 author_association，描述的是 PR 作者与当前仓库所属 org 的关系，其类型包括：`COLLABORATOR`、`FIRST_TIME_CONTRIBUTOR`、`CONTRIBUTOR`、`MEMBER`、`NONE`。 | @@ -69,7 +69,7 @@ Bot 会将处于 open 状态的 Pull Request 的相关状态信息同步到数 `review comment` 指的是在 review 过程中针对指定代码添加的评论内容，可以通过 [pulls.listComments](https://docs.github.com/en/free-pro-team@latest/rest/reference/pulls#get-a-review-comment-for-a-pull-request) 接口 review comment 列表。 -比较特别的是，在 review 代码的过程中如果使用了 Github 的 "Add single comment" 功能，Github 会自动地添加一条无内容的 review，然后将实际评论（review comment 类型）与之关联。 +比较特别的是，在 review 代码的过程中如果使用了 GitHub 的 "Add single comment" 功能，GitHub 会自动地添加一条无内容的 review，然后将实际评论（review comment 类型）与之关联。 | 字段名称 | 字段说明 | | --------------- | ----------- | diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000..0ca9c8b --- /dev/null +++ b/nodemon.json @@ -0,0 +1,7 @@ +{ + "signal": "SIGINT", + "watch": ["lib", ".env"], + "delay": 2000, + "verbose": true, + "exec": "npm run start" +} diff --git a/src/events/app/index.ts b/src/events/app/index.ts index 0bdd58e..1230bc5 100644 --- a/src/events/app/index.ts +++ b/src/events/app/index.ts @@ -8,6 +8,7 @@ import { IContributorService } from "../../services/ContributorService"; import { RepoKey } from "../../common/types"; import { + fetchAllInstallations, fetchAllTypeComments, fetchIssueComments, fetchPullRequestCommits, @@ -19,7 +20,6 @@ import { /** * Handle the event that triggered when the program start up. * @param app - * @param github * @param pullService * @param issueService * @param commentService @@ -27,7 +27,6 @@ import { */ export async function handleAppStartUpEvent( app: Probot, - github: InstanceType, pullService: IPullService, issueService: IIssueService, commentService: ICommentService, @@ -43,8 +42,7 @@ export async function handleAppStartUpEvent( await handleSyncRepos( repoConfigs, - github, - app.log, + app, pullService, issueService, commentService, @@ -55,6 +53,7 @@ export async function handleAppStartUpEvent( /** * handle the event that triggered when the user first installs the bot to the account. * @param context + * @param app * @param pullService * @param issueService * @param commentService @@ -62,6 +61,7 @@ export async function handleAppStartUpEvent( */ export async function handleAppInstallOnAccountEvent( context: Context, + app: Probot, pullService: IPullService, issueService: IIssueService, commentService: ICommentService, @@ -79,8 +79,7 @@ export async function handleAppInstallOnAccountEvent( await handleSyncRepos( repoKeys, - context.octokit, - context.log, + app, pullService, issueService, commentService, @@ -92,6 +91,7 @@ export async function handleAppInstallOnAccountEvent( * Handle the event that triggered when the user installs the bot to another new repository * of the account, which has already installed the bot. * @param context + * @param app * @param pullService * @param issueService * @param commentService @@ -99,6 +99,7 @@ export async function handleAppInstallOnAccountEvent( */ export async function handleAppInstallOnRepoEvent( context: Context, + app: Probot, pullService: IPullService, issueService: IIssueService, commentService: ICommentService, @@ -116,8 +117,7 @@ export async function handleAppInstallOnRepoEvent( await handleSyncRepos( repoKeys, - context.octokit, - context.log, + app, pullService, issueService, commentService, @@ -126,10 +126,9 @@ export async function handleAppInstallOnRepoEvent( } /** - * General handling for syncing repositories. + * General handling for syncing repositories of all owners. * @param repoKeys - * @param github - * @param log + * @param app * @param pullService * @param issueService * @param commentService @@ -137,18 +136,24 @@ export async function handleAppInstallOnRepoEvent( */ async function handleSyncRepos( repoKeys: RepoKey[], - github: InstanceType, - log: Logger, + app: Probot, pullService: IPullService, issueService: IIssueService, commentService: ICommentService, contributorService: IContributorService ) { + const octokit = await app.auth(); + for (const repoKey of repoKeys) { + const { data: installation } = await octokit.apps.getRepoInstallation( + repoKey + ); + const github = await app.auth(installation.id); + await handleSyncRepo( repoKey, github, - log, + app.log, pullService, issueService, commentService @@ -156,12 +161,12 @@ async function handleSyncRepos( } // Notice: Synchronizing contributor email must be performed after the synchronization PR is completed. - await handleSyncContributorEmail(github, log, pullService, contributorService) + await handleSyncContributorEmail(app, pullService, contributorService) .then(() => { - log.info("Finish syncing contributor email"); + app.log.info("Finish syncing contributor email"); }) .catch((err) => { - log.error(err, "Failed to sync contributor email"); + app.log.error(err, "Failed to sync contributor email"); }); } @@ -362,19 +367,21 @@ async function handleSyncIssues( /** * Synchronize contributor email according to the patch of pull request. - * @param log + * @param app * @param pullService * @param contributorService - * @param github */ async function handleSyncContributorEmail( - github: InstanceType, - log: Logger, + app: Probot, pullService: IPullService, contributorService: IContributorService ) { - log.info("Syncing contributor email"); + app.log.info("Syncing contributor email"); + + // Obtain all installation IDs so that they can be obtained directly according to the owner name. + const installationIdMap = await fetchAllInstallations(app); + // Only contributors who have not recorded their email are traversed. const noEmailContributorLogins = await contributorService.listNoEmailContributorsLogin(); for (const login of noEmailContributorLogins) { @@ -387,7 +394,12 @@ async function handleSyncContributorEmail( pull_number: pull.pullNumber, }; - const patch = await getPullRequestPatch(pullKey, github, log); + // Obtain the github client authorized by the installation id associated with owner name. + const installationId = installationIdMap.get(pull.owner); + const github = await app.auth(installationId); + + // Obtain the email from the PR's patch format file. + const patch = await getPullRequestPatch(pullKey, github, app.log); if (patch === null) continue; diff --git a/src/events/common.ts b/src/events/common.ts index 4e4a5af..248b205 100644 --- a/src/events/common.ts +++ b/src/events/common.ts @@ -10,8 +10,8 @@ export async function getSyncRepositoryListFromInstallation( app: Probot ): Promise { const syncRepos: RepoKey[] = []; - const github = await app.auth(); - const { data: installations } = await github.apps.listInstallations(); + const octokit = await app.auth(); + const { data: installations } = await octokit.apps.listInstallations(); for (let i of installations) { const github = await app.auth(i.id); @@ -97,7 +97,13 @@ export async function getPullRequestPatch( }, }); } catch (err) { - log.error(err, "Failed to get patch file of pull request."); + log.error( + err, + "Failed to get patch file of pull request %s/%s#%s", + pullKey.owner, + pullKey.repo, + pullKey.pull_number + ); } if (patchResponse?.status === 200) { @@ -131,3 +137,23 @@ export async function fetchIssueComments( ) { return await github.paginate(github.issues.listComments, issueKey); } + +/** + * Fetch all installations. + * @param app + */ +export async function fetchAllInstallations( + app: Probot +): Promise> { + const octokit = await app.auth(); + const installations = await octokit.paginate(octokit.apps.listInstallations); + const installationIdMap = new Map(); + + installations.forEach((installation) => { + if (installation.account?.login !== undefined) { + installationIdMap.set(installation.account.login, installation.id); + } + }); + + return installationIdMap; +} diff --git a/src/index.ts b/src/index.ts index 0ef3b81..7bb2c95 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import "reflect-metadata"; -import { Context, Probot, ProbotOctokit } from "probot"; +import { Context, Probot } from "probot"; import Container from "typedi"; import { createConnection, useContainer } from "typeorm"; @@ -26,16 +26,6 @@ export = async (app: Probot) => { useContainer(Container); Container.set(ILoggerToken, app.log); - // TODO: use the github client authed by installation id. - // Init Github client. - // Notice: This github client uses a TOKEN as the bot github account for access, in this case, we do not need to - // authorize for each installation through the Github APP, but this will also bring some restrictions. - const github = new ProbotOctokit({ - auth: { - token: process.env.GITHUB_ACCESS_TOKEN, - }, - }); - // Connect database. createConnection() .then(() => { @@ -44,7 +34,6 @@ export = async (app: Probot) => { // WebHook-based incremental sync, and the two are executed concurrently. handleAppStartUpEvent( app, - github, Container.get(IPullServiceToken), Container.get(IIssueServiceToken), Container.get(ICommentServiceToken), @@ -68,6 +57,7 @@ export = async (app: Probot) => { app.on("installation.created", async (context) => { await handleAppInstallOnAccountEvent( context, + app, Container.get(IPullServiceToken), Container.get(IIssueServiceToken), Container.get(ICommentServiceToken), @@ -78,6 +68,7 @@ export = async (app: Probot) => { app.on("installation_repositories.added", async (context) => { await handleAppInstallOnRepoEvent( context, + app, Container.get(IPullServiceToken), Container.get(IIssueServiceToken), Container.get(ICommentServiceToken),