test: add schedule & controller test.

cnpm · Aug 13, 2023 · 68ebc7d · 68ebc7d
1 parent e151c3f
commit 68ebc7d
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 12 deletions.
diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts
@@ -1,20 +1,22 @@
 import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors';
 import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg';
 import { EggHttpClient } from 'egg';
+import { readFile, rm } from 'node:fs/promises';
 import { valid as semverValid } from 'semver';
-import { downloadToTempfile } from '../../common/FileUtil';
-import { NPMRegistry } from '../../common/adapter/NPMRegistry';
-import { ProxyCache } from '../entity/ProxyCache';
-import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskService } from './TaskService';
-import { readFile, rm } from 'node:fs/promises';
+import { CacheService } from './CacheService';
+import { NPMRegistry } from '../../common/adapter/NPMRegistry';
 import { NFSAdapter } from '../../common/adapter/NFSAdapter';
-import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants';
+import { ProxyCache } from '../entity/ProxyCache';
+import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task';
+import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository';
+import { TaskType, TaskState } from '../../common/enum/Task';
+import { downloadToTempfile } from '../../common/FileUtil';
+import { calculateIntegrity } from '../../common/PackageUtil';
 import { DIST_NAMES } from '../entity/Package';
+import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants';
 import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository';
-import { TaskType, TaskState } from '../../common/enum/Task';
-import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task';
 
 function isoNow() {
   return new Date().toISOString();
@@ -24,8 +26,10 @@ export function isPkgManifest(fileType: DIST_NAMES) {
   return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS;
 }
 
-type GetSourceManifestAndCacheReturnType<T> = { storeKey: string, proxyBytes: Buffer,
-  manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType| PackageJSONType :
+type GetSourceManifestAndCacheReturnType<T> = {
+  storeKey: string,
+  proxyBytes: Buffer,
+  manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType :
     T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never;
 };
 
@@ -43,6 +47,8 @@ export class ProxyCacheService extends AbstractService {
   private readonly proxyCacheRepository: ProxyCacheRepository;
   @Inject()
   private readonly taskService: TaskService;
+  @Inject()
+  private readonly cacheService: CacheService;
 
   async getPackageVersionTarBuffer(fullname: string, url: string): Promise<Buffer| null> {
     if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) {
@@ -183,18 +189,19 @@ export class ProxyCacheService extends AbstractService {
     const logs: string[] = [];
     const fullname = (task as CreateUpdateProxyCacheTask).data.fullname;
     const { fileType, version } = (task as CreateUpdateProxyCacheTask).data;
+    let cacheBytes;
     logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`);
     try {
       if (isPkgManifest(fileType)) {
         const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType);
         if (!cachedFiles) throw new Error('task params error, can not found record in repo.');
-        await this.getSourceManifestAndCache<typeof fileType>(fullname, fileType);
+        cacheBytes = (await this.getSourceManifestAndCache<typeof fileType>(fullname, fileType)).proxyBytes;
         ProxyCache.update(cachedFiles);
         await this.proxyCacheRepository.saveProxyCache(cachedFiles);
       } else {
         task.error = 'Unacceptable file type.';
         logs.push(`[${isoNow()}] ❌ ${task.error}`);
-        logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version} ❌❌❌❌❌`);
+        logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`);
         await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
         this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s',
           task.taskId, task.targetName, task.error);
@@ -210,6 +217,13 @@ export class ProxyCacheService extends AbstractService {
       return;
     }
     logs.push(`[${isoNow()}] 🟢 Update Success.`);
+    const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS;
+    const cachedKey = await this.cacheService.getPackageEtag(fullname, isFullManifests);
+    if (cachedKey) {
+      const { shasum: etag } = await calculateIntegrity(cacheBytes);
+      await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, cacheBytes);
+      logs.push(`[${isoNow()}] 🟢 Update Cache Success.`);
+    }
     await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
   }
 

diff --git a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json
@@ -4,6 +4,7 @@
     },
     "modified": "2022-01-26T20:31:13.648Z",
     "name": "foobar",
+    "description": "cnpmcore mock json",
     "versions": {
         "1.0.0": {
             "name": "foobar",

diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts
@@ -2,6 +2,7 @@ import assert from 'assert';
 import { app, mock } from 'egg-mock/bootstrap';
 import { TestUtil } from '../../../../test/TestUtil';
 import { NFSClientAdapter } from '../../../../app/infra/NFSClientAdapter';
+import { SyncMode } from '../../../../app/common/constants';
 
 describe('test/port/controller/package/DownloadPackageVersionTarController.test.ts', () => {
   let publisher: any;
@@ -282,6 +283,17 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test.
       app.expectLog('[middleware:ErrorHandler][syncPackage] create sync package');
     });
 
+    it('should create sync specific version task when package version tgz not found in proxy mode ', async () => {
+      mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy);
+      mock(app.config.cnpmcore, 'redirectNotFound', false);
+      const res = await app.httpRequest()
+        .get('/foobar/-/foobar-1.0.0.tgz')
+        .set('user-agent', publisher.ua + ' node/16.0.0')
+        .set('Accept', 'application/vnd.npm.install-v1+json');
+      assert(res.status === 200);
+      app.expectLog('[DownloadPackageVersionTarController.createSyncTask:success]');
+    });
+
   });
 
   describe('[GET /:fullname/download/:fullname-:version.tgz] deprecatedDownload()', () => {

diff --git a/test/port/controller/package/ShowPackageController.test.ts b/test/port/controller/package/ShowPackageController.test.ts
@@ -7,6 +7,7 @@ import { PackageManagerService } from '../../../../app/core/service/PackageManag
 import { CacheService } from '../../../../app/core/service/CacheService';
 import { DistRepository } from '../../../../app/repository/DistRepository';
 import { BugVersionService } from '../../../../app/core/service/BugVersionService';
+import { SyncMode } from '../../../../app/common/constants';
 
 describe('test/port/controller/package/ShowPackageController.test.ts', () => {
   let packageRepository: PackageRepository;
@@ -863,5 +864,23 @@ describe('test/port/controller/package/ShowPackageController.test.ts', () => {
       assert(res.status === 302);
       assert(res.headers.location === 'https://registry.npmjs.org/egg');
     });
+
+    it('should read manifest from source in proxy mode', async () => {
+      mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy);
+      mock(app.config.cnpmcore, 'redirectNotFound', false);
+      const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json'));
+      app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', {
+        data,
+        persist: false,
+      });
+      const res = await app.httpRequest()
+        .get('/foobar')
+        .set('user-agent', publisher.ua + ' node/16.0.0')
+        .set('Accept', 'application/vnd.npm.install-v1+json');
+      assert(res.status === 200);
+      assert(res.headers.location === app.config.cnpmcore.registry);
+      assert(res.body.data.description === 'cnpmcore mock json');
+      assert(res.body.data.versions['1.0.0'].dist.tarball.includes(app.config.cnpmcore.registry));
+    });
   });
 });
diff --git a/test/port/controller/package/ShowPackageVersionController.test.ts b/test/port/controller/package/ShowPackageVersionController.test.ts
@@ -3,6 +3,7 @@ import { app, mock } from 'egg-mock/bootstrap';
 import { TestUtil } from '../../../../test/TestUtil';
 import { BugVersion } from '../../../../app/core/entity/BugVersion';
 import { BugVersionService } from '../../../../app/core/service/BugVersionService';
+import { SyncMode } from '../../../../app/common/constants';
 
 describe('test/port/controller/package/ShowPackageVersionController.test.ts', () => {
   let publisher;
@@ -368,5 +369,22 @@ describe('test/port/controller/package/ShowPackageVersionController.test.ts', ()
         .expect(200);
       assert(res.body._source_registry_name === 'self');
     });
+
+    it('should read package version manifest from source in proxy mode', async () => {
+      mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy);
+      mock(app.config.cnpmcore, 'redirectNotFound', false);
+      const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/abbreviated.json'));
+      app.mockHttpclient('https://registry.npmjs.org/foobar/1.0.0', 'GET', {
+        data,
+        persist: false,
+      });
+      const res = await app.httpRequest()
+        .get('/foobar/1.0.0')
+        .set('user-agent', publisher.ua + ' node/16.0.0')
+        .set('Accept', 'application/vnd.npm.install-v1+json');
+      assert(res.status === 200);
+      assert(res.headers.location === app.config.cnpmcore.registry);
+      assert(res.body.data.dist.tarball.includes(app.config.cnpmcore.registry));
+    });
   });
 });
diff --git a/test/schedule/CheckProxyCacheUpdateWorker.test.ts b/test/schedule/CheckProxyCacheUpdateWorker.test.ts
@@ -0,0 +1,28 @@
+import assert from 'assert';
+import { app, mock } from 'egg-mock/bootstrap';
+import { SyncMode } from '../../app/common/constants';
+import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository';
+import { ProxyCache } from '../../app/core/entity/ProxyCache';
+import { DIST_NAMES } from '../../app/core/entity/Package';
+import { TaskService } from '../../app/core/service/TaskService';
+import { TaskType } from '../../app/common/enum/Task';
+
+const CheckProxyCacheUpdateWorkerPath = require.resolve('../../app/port/schedule/CheckProxyCacheUpdateWorker');
+
+describe('test/schedule/CheckProxyCacheUpdateWorker.test.ts', () => {
+  it('should create update task by repo', async () => {
+    mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy);
+    mock(app.config.cnpmcore, 'redirectNotFound', false);
+    const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository);
+    const taskService = await app.getEggObject(TaskService);
+    await proxyCacheRepository.saveProxyCache(ProxyCache.create({
+      fullname: 'foo-bar',
+      fileType: DIST_NAMES.FULL_MANIFESTS,
+    }));
+    await app.runSchedule(CheckProxyCacheUpdateWorkerPath);
+    const task = await taskService.findExecuteTask(TaskType.UpdateProxyCache);
+    assert(task);
+    assert.equal(task.targetName, `foo-bar/${DIST_NAMES.FULL_MANIFESTS}`);
+  });
+
+});
diff --git a/test/schedule/SyncProxyCacheWorker.test.ts b/test/schedule/SyncProxyCacheWorker.test.ts
@@ -0,0 +1,37 @@
+import { app, mock } from 'egg-mock/bootstrap';
+import { SyncMode } from '../../app/common/constants';
+import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository';
+import { ProxyCache } from '../../app/core/entity/ProxyCache';
+import { DIST_NAMES } from '../../app/core/entity/Package';
+import { ProxyCacheService } from '../../app/core/service/ProxyCacheService';
+
+const SyncProxyCacheWorkerPath = require.resolve('../../app/port/schedule/SyncProxyCacheWorker');
+
+describe('test/schedule/SyncProxyCacheWorker.test.ts', () => {
+
+  beforeEach(async () => {
+    mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy);
+    mock(app.config.cnpmcore, 'redirectNotFound', false);
+  });
+
+  it('should execute task success', async () => {
+
+    const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository);
+    const proxyCacheService = await app.getEggObject(ProxyCacheService);
+    await proxyCacheRepository.saveProxyCache(ProxyCache.create({
+      fullname: 'foobar',
+      fileType: DIST_NAMES.FULL_MANIFESTS,
+    }));
+
+
+    await proxyCacheService.createTask(`foobar/${DIST_NAMES.ABBREVIATED_MANIFESTS}`, {
+      fullname: 'foobar',
+      fileType: DIST_NAMES.ABBREVIATED_MANIFESTS,
+    });
+
+    await app.runSchedule(SyncProxyCacheWorkerPath);
+    app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:start]');
+    app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:success]');
+  });
+
+});