feat: use the github client authed by installation id (#62)

* feat: use the github client authed by installation id * docs: update config doc * fix: fix typo and revert the package-lock * fix: lint fix * fix: revert package-lock.json * fix: move out octokit from loop
ti-community-infra · Feb 18, 2021 · a24dd7c · a24dd7c
1 parent a658b99
commit a24dd7c
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 72 deletions.
diff --git a/app.yml b/app.yml
@@ -22,30 +22,30 @@ default_events:
   # - deployment_status
   # - fork
   # - gollum
-  # - issue_comment
+  - issue_comment
   - issues
-# - label
-# - milestone
-# - member
-# - membership
-# - org_block
-# - organization
-# - page_build
-# - project
-# - project_card
-# - project_column
-# - public
+  # - label
+  # - milestone
+  # - member
+  # - membership
+  # - org_block
+  # - organization
+  # - page_build
+  # - project
+  # - project_card
+  # - project_column
+  # - public
   - pull_request
   - pull_request_review
   - pull_request_review_comment
-# - push
-# - release
-  - repository
-# - repository_import
-# - status
-# - team
-# - team_add
-# - watch
+  # - push
+  # - release
+  # - repository
+  # - repository_import
+  # - status
+  # - team
+  # - team_add
+  # - watch
 
 # The set of permissions needed by the GitHub App. The format of the object uses
 # the permission name for the key (for example, issues) and the access type for
@@ -70,7 +70,7 @@ default_permissions:
 
   # Issues and related comments, assignees, labels, and milestones.
   # https://developer.github.com/v3/apps/permissions/#permission-on-issues
-  issues: write
+  issues: read
 
   # Search repositories, list collaborators, and access repository metadata.
   # https://developer.github.com/v3/apps/permissions/#metadata-permissions
@@ -123,14 +123,15 @@ default_permissions:
   # Get notified of, and update, content references.
   # https://developer.github.com/v3/apps/permissions/
   # organization_administration: read
+
 # The name of the GitHub App. Defaults to the name specified in package.json
-  name: ti-sync-bot
+name: ti-sync-bot
 
 # The homepage of your GitHub App.
-  url: https://github.com/ti-community-infra/ti-sync-bot
+url: https://github.com/ti-community-infra/ti-sync-bot
 
 # A description of the GitHub App.
-  description: A Github App that used to sync tidb community info that contains pull request, issue, comment and contributor info.
+description: A Github App that used to sync tidb community info that contains pull request, issue, comment and contributor info.
 
 # Set to true when your GitHub App is available to the public or false when it is only accessible to the owner of the app.
 # Default: true

diff --git a/docs/config.md b/docs/config.md
@@ -6,5 +6,4 @@
 
 | 选项                       | 选项说明                                   |
 | ------------------------- | --------------------------------------------------------------------------- |
-| GITHUB_ACCESS_TOKEN       | 在全量同步的过程中，需要使用该 ACCESS TOKEN 来访问 Github 的一些 API，你可以在 [开发者设置](https://github.com/settings/tokens/new) 当中生成该 TOKEN。           |
-| SYNC_REPOS                | 为了测试方便，如果你在 `.env` 文件当中指定 `SYNC_REPOS` 配置，Bot 在程序启动时将会针对该配置中指定的仓库进行全量同步，例如：`SYNC_REPOS=tikv/tikv,pingcap/tipocket`。 |
+| SYNC_REPOS                | 为了测试方便，如果你在 `.env` 文件当中指定 `SYNC_REPOS` 配置，Bot 在程序启动时将会针对该配置中指定的仓库进行全量同步，例如：`SYNC_REPOS=tikv/tikv,pingcap/tipocket`。需要注意的是，指定的仓库必须事先安装该 GitHub App。 |
diff --git a/docs/design.md b/docs/design.md
@@ -2,27 +2,27 @@
 
 ## 整体思路
 
-ti-sync-bot（以下简称 Bot ）主要负责将 tidb 社区 Github 上的一些数据同步到数据库当中，它的工作模式主要分为全量同步和增量同步两种模式。
+ti-sync-bot（以下简称 Bot ）主要负责将 tidb 社区 GitHub 上的一些数据同步到数据库当中，它的工作模式主要分为全量同步和增量同步两种模式。
 
 ### 全量同步
 
 全量同步的设计目的是为了让 Bot 能够将新安装仓库中过去的数据或者在 Bot 因为故障导致的停机过程中产生的数据同步到数据库当中。
 
 | 事件名称                         | 事件类型     | 触发说明                                     |
 | ------------------------------- | ----------- | ----------------------------------------- |
-| app.start_up                    | 自定义事件    | 在程序启动时触发，Bot 会获取所有安装了该 Github App 的仓库列表逐一进行全量同步。 |
+| app.start_up                    | 自定义事件    | 在程序启动时触发，Bot 会获取所有安装了该 GitHub App 的仓库列表逐一进行全量同步。 |
 | installation.created            | WebHook 事件 | 当用户初次将 Bot 安装到用户账号或组织账号时触发，用户在安装时可以选择安装到所有仓库或指定仓库，Bot 会针对安装的仓库进行逐一全量同步。 |
-| installation_repositories.added | WebHook 事件 | 用户可以在 Github App 设置对已安装仓库进行添加或删除，当新添加一个仓库时，会触发该事件，Bot 会针对新添加的仓库进行逐一全量同步。 |
+| installation_repositories.added | WebHook 事件 | 用户可以在 GitHub App 设置对已安装仓库进行添加或删除，当新添加一个仓库时，会触发该事件，Bot 会针对新添加的仓库进行逐一全量同步。 |
 
 在全量同步过程当中，同步 Pull Request 和 同步 Issue 两个过程并发进行，同步 Contributor Email 的过程依赖于同步 PR 的数据，因此会在同步 Pull Request 完成之后执行。
 
 ### 增量同步
 
-增量同步的设计目的是为了能够更加及时的将 Github 上的数据同步到数据库当中，避免全量同步在一个较为集中的时间段内产生大量的数据库操作和 API 接口请求。
+增量同步的设计目的是为了能够更加及时的将 GitHub 上的数据同步到数据库当中，避免全量同步在一个较为集中的时间段内产生大量的数据库操作和 API 接口请求。
 
-增量同步是基于 Github 的 WebHook 机制实现的，为了能够在 Bot 启动过程中及时处理 WebHook 发送过来的数据，全量同步和增量同步被设计成并发进行。
+增量同步是基于 GitHub 的 WebHook 机制实现的，为了能够在 Bot 启动过程中及时处理 WebHook 发送过来的数据，全量同步和增量同步被设计成并发进行。
 
-Bot 通过监听以下类型事件来对 Github 数据进行增量同步：
+Bot 通过监听以下类型事件来对 GitHub 数据进行增量同步：
 
 | 事件类型          | 动作类型      | 触发行为     |
 | --------------- | ----------- | ----------- |
@@ -44,7 +44,7 @@ Bot 在将收到的数据同步数据库之前会对收到的 Pull Request、Iss
 
 | 字段名称         | 字段说明     |
 | --------------- | ----------- |
-| status          | PR 的状态可以分为 `open`、`closed` 和 `merged` 三种状态，Github 只提供了 `open` 和 `closed` 两种状态，如果 PR 的 `merged_at` 不为空，则可以判定为 `merged` 状态。 |
+| status          | PR 的状态可以分为 `open`、`closed` 和 `merged` 三种状态，GitHub 只提供了 `open` 和 `closed` 两种状态，如果 PR 的 `merged_at` 不为空，则可以判定为 `merged` 状态。 |
 | label           | 使用逗号分隔多个标签名。 |
 | relation        | 描述的是 PR 作者与公司的关系，其类型包括：`member`、`not member`。 |
 | association     | 即 author_association，描述的是 PR 作者与当前仓库所属 org 的关系，其类型包括：`COLLABORATOR`、`FIRST_TIME_CONTRIBUTOR`、`CONTRIBUTOR`、`MEMBER`、`NONE`。 |
@@ -69,7 +69,7 @@ Bot 会将处于 open 状态的 Pull Request 的相关状态信息同步到数
 
 `review comment` 指的是在 review 过程中针对指定代码添加的评论内容，可以通过 [pulls.listComments](https://docs.github.com/en/free-pro-team@latest/rest/reference/pulls#get-a-review-comment-for-a-pull-request) 接口 review comment 列表。
 
-比较特别的是，在 review 代码的过程中如果使用了 Github 的 "Add single comment" 功能，Github 会自动地添加一条无内容的 review，然后将实际评论（review comment 类型）与之关联。
+比较特别的是，在 review 代码的过程中如果使用了 GitHub 的 "Add single comment" 功能，GitHub 会自动地添加一条无内容的 review，然后将实际评论（review comment 类型）与之关联。
 
 | 字段名称         | 字段说明     |
 | --------------- | ----------- |

diff --git a/nodemon.json b/nodemon.json
@@ -0,0 +1,7 @@
+{
+  "signal": "SIGINT",
+  "watch": ["lib", ".env"],
+  "delay": 2000,
+  "verbose": true,
+  "exec": "npm run start"
+}
diff --git a/src/events/app/index.ts b/src/events/app/index.ts
@@ -8,6 +8,7 @@ import { IContributorService } from "../../services/ContributorService";
 
 import { RepoKey } from "../../common/types";
 import {
+  fetchAllInstallations,
   fetchAllTypeComments,
   fetchIssueComments,
   fetchPullRequestCommits,
@@ -19,15 +20,13 @@ import {
 /**
  * Handle the event that triggered when the program start up.
  * @param app
- * @param github
  * @param pullService
  * @param issueService
  * @param commentService
  * @param contributorService
  */
 export async function handleAppStartUpEvent(
   app: Probot,
-  github: InstanceType<typeof ProbotOctokit>,
   pullService: IPullService,
   issueService: IIssueService,
   commentService: ICommentService,
@@ -43,8 +42,7 @@ export async function handleAppStartUpEvent(
 
   await handleSyncRepos(
     repoConfigs,
-    github,
-    app.log,
+    app,
     pullService,
     issueService,
     commentService,
@@ -55,13 +53,15 @@ export async function handleAppStartUpEvent(
 /**
  * handle the event that triggered when the user first installs the bot to the account.
  * @param context
+ * @param app
  * @param pullService
  * @param issueService
  * @param commentService
  * @param contributorService
  */
 export async function handleAppInstallOnAccountEvent(
   context: Context<EventPayloads.WebhookPayloadInstallation>,
+  app: Probot,
   pullService: IPullService,
   issueService: IIssueService,
   commentService: ICommentService,
@@ -79,8 +79,7 @@ export async function handleAppInstallOnAccountEvent(
 
   await handleSyncRepos(
     repoKeys,
-    context.octokit,
-    context.log,
+    app,
     pullService,
     issueService,
     commentService,
@@ -92,13 +91,15 @@ export async function handleAppInstallOnAccountEvent(
  * Handle the event that triggered when the user installs the bot to another new repository
  * of the account, which has already installed the bot.
  * @param context
+ * @param app
  * @param pullService
  * @param issueService
  * @param commentService
  * @param contributorService
  */
 export async function handleAppInstallOnRepoEvent(
   context: Context<EventPayloads.WebhookPayloadInstallationRepositories>,
+  app: Probot,
   pullService: IPullService,
   issueService: IIssueService,
   commentService: ICommentService,
@@ -116,8 +117,7 @@ export async function handleAppInstallOnRepoEvent(
 
   await handleSyncRepos(
     repoKeys,
-    context.octokit,
-    context.log,
+    app,
     pullService,
     issueService,
     commentService,
@@ -126,42 +126,47 @@ export async function handleAppInstallOnRepoEvent(
 }
 
 /**
- * General handling for syncing repositories.
+ * General handling for syncing repositories of all owners.
  * @param repoKeys
- * @param github
- * @param log
+ * @param app
  * @param pullService
  * @param issueService
  * @param commentService
  * @param contributorService
  */
 async function handleSyncRepos(
   repoKeys: RepoKey[],
-  github: InstanceType<typeof ProbotOctokit>,
-  log: Logger,
+  app: Probot,
   pullService: IPullService,
   issueService: IIssueService,
   commentService: ICommentService,
   contributorService: IContributorService
 ) {
+  const octokit = await app.auth();
+
   for (const repoKey of repoKeys) {
+    const { data: installation } = await octokit.apps.getRepoInstallation(
+      repoKey
+    );
+    const github = await app.auth(installation.id);
+
     await handleSyncRepo(
       repoKey,
       github,
-      log,
+      app.log,
       pullService,
       issueService,
       commentService
     );
   }
 
   // Notice: Synchronizing contributor email must be performed after the synchronization PR is completed.
-  await handleSyncContributorEmail(github, log, pullService, contributorService)
+  await handleSyncContributorEmail(app, pullService, contributorService)
     .then(() => {
-      log.info("Finish syncing contributor email");
+      app.log.info("Finish syncing contributor email");
     })
     .catch((err) => {
-      log.error(err, "Failed to sync contributor email");
+      app.log.error(err, "Failed to sync contributor email");
     });
 }
 
@@ -362,19 +367,21 @@ async function handleSyncIssues(
 
 /**
  * Synchronize contributor email according to the patch of pull request.
- * @param log
+ * @param app
  * @param pullService
  * @param contributorService
- * @param github
  */
 async function handleSyncContributorEmail(
-  github: InstanceType<typeof ProbotOctokit>,
-  log: Logger,
+  app: Probot,
   pullService: IPullService,
   contributorService: IContributorService
 ) {
-  log.info("Syncing contributor email");
+  app.log.info("Syncing contributor email");
+
+  // Obtain all installation IDs so that they can be obtained directly according to the owner name.
+  const installationIdMap = await fetchAllInstallations(app);
 
+  // Only contributors who have not recorded their email are traversed.
   const noEmailContributorLogins = await contributorService.listNoEmailContributorsLogin();
 
   for (const login of noEmailContributorLogins) {
@@ -387,7 +394,12 @@ async function handleSyncContributorEmail(
         pull_number: pull.pullNumber,
       };
 
-      const patch = await getPullRequestPatch(pullKey, github, log);
+      // Obtain the github client authorized by the installation id associated with owner name.
+      const installationId = installationIdMap.get(pull.owner);
+      const github = await app.auth(installationId);
+
+      // Obtain the email from the PR's patch format file.
+      const patch = await getPullRequestPatch(pullKey, github, app.log);
 
       if (patch === null) continue;
 

diff --git a/src/events/common.ts b/src/events/common.ts
@@ -10,8 +10,8 @@ export async function getSyncRepositoryListFromInstallation(
   app: Probot
 ): Promise<RepoKey[]> {
   const syncRepos: RepoKey[] = [];
-  const github = await app.auth();
-  const { data: installations } = await github.apps.listInstallations();
+  const octokit = await app.auth();
+  const { data: installations } = await octokit.apps.listInstallations();
 
   for (let i of installations) {
     const github = await app.auth(i.id);
@@ -97,7 +97,13 @@ export async function getPullRequestPatch(
       },
     });
   } catch (err) {
-    log.error(err, "Failed to get patch file of pull request.");
+    log.error(
+      err,
+      "Failed to get patch file of pull request %s/%s#%s",
+      pullKey.owner,
+      pullKey.repo,
+      pullKey.pull_number
+    );
   }
 
   if (patchResponse?.status === 200) {
@@ -131,3 +137,23 @@ export async function fetchIssueComments(
 ) {
   return await github.paginate(github.issues.listComments, issueKey);
 }
+
+/**
+ * Fetch all installations.
+ * @param app
+ */
+export async function fetchAllInstallations(
+  app: Probot
+): Promise<Map<string, number>> {
+  const octokit = await app.auth();
+  const installations = await octokit.paginate(octokit.apps.listInstallations);
+  const installationIdMap = new Map();
+
+  installations.forEach((installation) => {
+    if (installation.account?.login !== undefined) {
+      installationIdMap.set(installation.account.login, installation.id);
+    }
+  });
+
+  return installationIdMap;
+}