feat: sync delete mode (#398)

> 为了避免部分 npm 包误封、误删，导致生产环境影响，新增 syncDeleteMode 配置，允许自定义同步策略

* 新增 `syncDeleteMode` : 'ignore' | 'block' | 'delete'
  * delete: 目前默认值，同步删包事件
  * ignore: 忽略 upstream 所有删包事件
  * block: 不做物理删除，只新增 block 记录，不允许访问，除非管理员手动恢复并更新 `syncPackageBlockList`
* `npm-security-holder` 场景也判断为删包事件
* 更新原有删包流程，统一处理，调整部分日志输出

---------------

> New `syncDeleteMode` to allow custom syncing policy to avoid some npm
packages being blocked or deleted by mistake.

* Add `syncDeleteMode` : 'ignore' | 'block' | 'delete'
  * delete: by default, sync delete events
  * ignore: ignore all upstream delete events
* block: only add block records, cant access unless the administrator
manually restores and update `syncPackageBlockList`.
* `npm-security-holder` event is also determined to be a delete event
* Update the original packet deletion process, update log output by the
way

app/common/constants.ts

@@ -1,3 +1,13 @@
 export const BUG_VERSIONS = 'bug-versions';
 export const LATEST_TAG = 'latest';
 export const GLOBAL_WORKER = 'GLOBAL_WORKER';
+export enum SyncMode {
+  none = 'none',
+  exist = 'exist',
+  all = 'all',
+}
+export enum SyncDeleteMode {
+  ignore = 'ignore',
+  block = 'block',
+  delete = 'delete',
+}

app/core/service/PackageSyncerService.ts

@@ -10,7 +10,8 @@ import {
 } from 'egg';
 import { setTimeout } from 'timers/promises';
 import { rm } from 'fs/promises';
-import { NPMRegistry } from '../../common/adapter/NPMRegistry';
+import semver from 'semver';
+import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
 import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil';
 import { downloadToTempfile } from '../../common/FileUtil';
 import { TaskState, TaskType } from '../../common/enum/Task';
@@ -31,6 +32,16 @@ import { Registry } from '../entity/Registry';
 import { BadRequestError } from 'egg-errors';
 import { ScopeManagerService } from './ScopeManagerService';
 import { EventCorkAdvice } from './EventCorkerAdvice';
+import { SyncDeleteMode } from '../../common/constants';
+
+type syncDeletePkgOptions = {
+  task: Task,
+  pkg: Package | null,
+  logUrl: string,
+  url: string,
+  logs: string[],
+  data: any,
+};
 
 function isoNow() {
   return new Date().toISOString();
@@ -209,6 +220,89 @@ export class PackageSyncerService extends AbstractService {
     await this.taskService.appendTaskLog(task, logs.join('\n'));
   }
 
+  private isRemovedInRemote(remoteFetchResult: RegistryResponse) {
+    const { status, data } = remoteFetchResult;
+
+    // deleted or blocked
+    if (status === 404 || status === 451) {
+      return true;
+    }
+
+    const hasMaintainers = data?.maintainers && data?.maintainers.length !== 0;
+    if (hasMaintainers) {
+      return false;
+    }
+
+    // unpublished
+    const timeMap = data.time || {};
+    if (timeMap.unpublished) {
+      return true;
+    }
+
+    // security holder
+    // test/fixtures/registry.npmjs.org/security-holding-package.json
+    let isSecurityHolder = true;
+    for (const versionInfo of Object.entries<{ _npmUser?: { name: string } }>(data.versions || {})) {
+      const [ v, info ] = versionInfo;
+      // >=0.0.1-security <0.0.2-0
+      const isSecurityVersion = semver.satisfies(v, '^0.0.1-security');
+      const isNpmUser = info?._npmUser?.name === 'npm';
+      if (!isSecurityVersion || !isNpmUser) {
+        isSecurityHolder = false;
+        break;
+      }
+    }
+
+    return isSecurityHolder;
+  }
+
+  // sync deleted package, deps on the syncDeleteMode
+  // - ignore: do nothing, just finish the task
+  // - delete: remove the package from local registry
+  // - block: block the package, update the manifest.block, instead of delete versions
+  // 根据 syncDeleteMode 配置，处理删包场景
+  // - ignore: 不做任何处理，直接结束任务
+  // - delete: 删除包数据，包括 manifest 存储
+  // - block: 软删除 将包标记为 block，用户无法直接使用
+  private async syncDeletePkg({ task, pkg, logUrl, url, logs, data }: syncDeletePkgOptions) {
+    const fullname = task.targetName;
+    const failEnd = `❌❌❌❌❌ ${url || fullname} ❌❌❌❌❌`;
+    const syncDeleteMode: SyncDeleteMode = this.config.cnpmcore.syncDeleteMode;
+    logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was removed in remote registry, response data: ${JSON.stringify(data)}, config.syncDeleteMode = ${syncDeleteMode}`);
+
+    // pkg not exists in local registry
+    if (!pkg) {
+      task.error = `Package not exists, response data: ${JSON.stringify(data)}`;
+      logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
+      logs.push(`[${isoNow()}] ${failEnd}`);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s',
+        task.taskId, task.targetName, task.error);
+      return;
+    }
+
+    if (syncDeleteMode === SyncDeleteMode.ignore) {
+      // ignore deleted package
+      logs.push(`[${isoNow()}] 🟢 Skip remove since config.syncDeleteMode = ignore`);
+    } else if (syncDeleteMode === SyncDeleteMode.block) {
+      // block deleted package
+      await this.packageManagerService.blockPackage(pkg, 'Removed in remote registry');
+      logs.push(`[${isoNow()}] 🟢 Block the package since config.syncDeleteMode = block`);
+    } else if (syncDeleteMode === SyncDeleteMode.delete) {
+      // delete package
+      await this.packageManagerService.unpublishPackage(pkg);
+      logs.push(`[${isoNow()}] 🟢 Delete the package since config.syncDeleteMode = delete`);
+    }
+
+    // update log
+    logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
+    logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
+    await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
+    this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s',
+      task.taskId, task.targetName);
+
+  }
+
   // 初始化对应的 Registry
   // 1. 优先从 pkg.registryId 获取 (registryId 一经设置 不应改变)
   // 1. 其次从 task.data.registryId (创建单包同步任务时传入)
@@ -314,11 +408,11 @@ export class PackageSyncerService extends AbstractService {
       return;
     }
 
-    let result: any;
+    let registryFetchResult: RegistryResponse;
     try {
-      result = await this.npmRegistry.getFullManifests(fullname);
+      registryFetchResult = await this.npmRegistry.getFullManifests(fullname);
     } catch (err: any) {
-      const status = err.status || 'unknow';
+      const status = err.status || 'unknown';
       task.error = `request manifests error: ${err}, status: ${status}`;
       logs.push(`[${isoNow()}] ❌ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`);
       logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
@@ -328,7 +422,7 @@ export class PackageSyncerService extends AbstractService {
       return;
     }
 
-    const { url, data, headers, res, status } = result;
+    const { url, data, headers, res, status } = registryFetchResult;
     let readme = data.readme || '';
     if (typeof readme !== 'string') {
       readme = JSON.stringify(readme);
@@ -342,33 +436,15 @@ export class PackageSyncerService extends AbstractService {
     const contentLength = headers['content-length'] || '-';
     logs.push(`[${isoNow()}] HTTP [${status}] content-length: ${contentLength}, timing: ${JSON.stringify(res.timing)}`);
 
-    // 404 unpublished
-    // 451 blocked
-    const shouldRemovePkg = status === 404 || status === 451;
-    if (shouldRemovePkg) {
-      if (pkg) {
-        await this.packageManagerService.unpublishPackage(pkg);
-        logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was unpublished caused by ${status} response: ${JSON.stringify(data)}`);
-        logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-        logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
-        await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
-        this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s',
-          task.taskId, task.targetName);
-      } else {
-        task.error = `Package not exists, response data: ${JSON.stringify(data)}`;
-        logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
-        logs.push(`[${isoNow()}] ${failEnd}`);
-        await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
-        this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s',
-          task.taskId, task.targetName, task.error);
-      }
+    if (this.isRemovedInRemote(registryFetchResult)) {
+      await this.syncDeletePkg({ task, pkg, logs, logUrl, url, data });
       return;
     }
 
     const versionMap = data.versions || {};
     const distTags = data['dist-tags'] || {};
 
-    // show latest infomations
+    // show latest information
     if (distTags.latest) {
       logs.push(`[${isoNow()}] 📖 ${fullname} latest version: ${distTags.latest ?? '-'}, published time: ${JSON.stringify(timeMap[distTags.latest])}`);
     }
@@ -432,20 +508,6 @@ export class PackageSyncerService extends AbstractService {
       //     }
       //   }
       // }
-      if (timeMap.unpublished) {
-        if (pkg) {
-          await this.packageManagerService.unpublishPackage(pkg);
-          logs.push(`[${isoNow()}] 🟢 Sync unpublished package: ${JSON.stringify(timeMap.unpublished)} success`);
-        } else {
-          logs.push(`[${isoNow()}] 📖 Ignore unpublished package: ${JSON.stringify(timeMap.unpublished)}`);
-        }
-        logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-        logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
-        await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
-        this.logger.info('[PackageSyncerService.executeTask:success] taskId: %s, targetName: %s',
-          task.taskId, task.targetName);
-        return;
-      }
 
       // invalid maintainers, sync fail
       task.error = `invalid maintainers: ${JSON.stringify(maintainers)}`;

app/infra/NFSClientAdapter.ts

@@ -1,10 +1,10 @@
 import {
   AccessLevel,
   EggObjectLifecycle,
-  InitTypeQualifier,
   Inject,
-  ObjectInitType,
   SingletonProto,
+  EggQualifier,
+  EggType,
 } from '@eggjs/tegg';
 import { EggAppConfig, EggLogger } from 'egg';
 import FSClient from 'fs-cnpm';
@@ -17,7 +17,7 @@ import { Readable } from 'stream';
 })
 export class NFSClientAdapter implements EggObjectLifecycle, NFSClient {
   @Inject()
-  @InitTypeQualifier(ObjectInitType.SINGLETON)
+  @EggQualifier(EggType.APP)
   private logger: EggLogger;
 
   @Inject()

config/config.default.ts

@@ -3,6 +3,7 @@ import { join } from 'path';
 import { EggAppConfig, PowerPartial } from 'egg';
 import OSSClient from 'oss-cnpm';
 import { patchAjv } from '../app/port/typebox';
+import { SyncDeleteMode, SyncMode } from '../app/common/constants';
 
 export default (appInfo: EggAppConfig) => {
   const config = {} as PowerPartial<EggAppConfig>;
@@ -22,7 +23,8 @@ export default (appInfo: EggAppConfig) => {
     //  - none: don't sync npm package, just redirect it to sourceRegistry
     //  - all: sync all npm packages
     //  - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled
-    syncMode: 'none',
+    syncMode: SyncMode.none,
+    syncDeleteMode: SyncDeleteMode.delete,
     hookEnable: false,
     syncPackageWorkerMaxConcurrentTasks: 10,
     triggerHookWorkerMaxConcurrentTasks: 10,

test/common/adapter/binary/ImageminBinary.test.ts

@@ -80,7 +80,7 @@ describe('test/common/adapter/binary/ImageminBinary.test.ts', () => {
       data: await TestUtil.readFixturesFile('registry.npmjs.com/advpng-bin.json'),
     });
     let result = await binary.fetch('/', 'advpng-bin');
-    console.log(result?.items.map(_ => _.name));
+    // console.log(result?.items.map(_ => _.name));
     assert(result);
     assert(result.items.length > 0);
     let matchDir1 = false;
@@ -308,7 +308,7 @@ describe('test/common/adapter/binary/ImageminBinary.test.ts', () => {
       data: await TestUtil.readFixturesFile('registry.npmjs.com/guetzli.json'),
     });
     const result = await binary.fetch('/', 'guetzli-bin');
-    console.log(result);
+    // console.log(result);
     assert(result);
     // console.log(result.items);
     assert(result.items.length > 0);