diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index d1c1414f..59a29835 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,20 +1,22 @@ import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; +import { readFile, rm } from 'node:fs/promises'; import { valid as semverValid } from 'semver'; -import { downloadToTempfile } from '../../common/FileUtil'; -import { NPMRegistry } from '../../common/adapter/NPMRegistry'; -import { ProxyCache } from '../entity/ProxyCache'; -import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; -import { readFile, rm } from 'node:fs/promises'; +import { CacheService } from './CacheService'; +import { NPMRegistry } from '../../common/adapter/NPMRegistry'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; +import { ProxyCache } from '../entity/ProxyCache'; +import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { TaskType, TaskState } from '../../common/enum/Task'; +import { downloadToTempfile } from '../../common/FileUtil'; +import { calculateIntegrity } from '../../common/PackageUtil'; import { DIST_NAMES } from '../entity/Package'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; -import { TaskType, TaskState } from '../../common/enum/Task'; -import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; function isoNow() { return new Date().toISOString(); @@ -24,8 +26,10 @@ export function isPkgManifest(fileType: DIST_NAMES) { return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS; } -type GetSourceManifestAndCacheReturnType = { storeKey: string, proxyBytes: Buffer, - manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType| PackageJSONType : +type GetSourceManifestAndCacheReturnType = { + storeKey: string, + proxyBytes: Buffer, + manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType : T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; }; @@ -43,6 +47,8 @@ export class ProxyCacheService extends AbstractService { private readonly proxyCacheRepository: ProxyCacheRepository; @Inject() private readonly taskService: TaskService; + @Inject() + private readonly cacheService: CacheService; async getPackageVersionTarBuffer(fullname: string, url: string): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { @@ -183,18 +189,19 @@ export class ProxyCacheService extends AbstractService { const logs: string[] = []; const fullname = (task as CreateUpdateProxyCacheTask).data.fullname; const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; + let cacheBytes; logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { if (isPkgManifest(fileType)) { const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); - await this.getSourceManifestAndCache(fullname, fileType); + cacheBytes = (await this.getSourceManifestAndCache(fullname, fileType)).proxyBytes; ProxyCache.update(cachedFiles); await this.proxyCacheRepository.saveProxyCache(cachedFiles); } else { task.error = 'Unacceptable file type.'; logs.push(`[${isoNow()}] ❌ ${task.error}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version} ❌❌❌❌❌`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`); await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error); @@ -210,6 +217,13 @@ export class ProxyCacheService extends AbstractService { return; } logs.push(`[${isoNow()}] 🟢 Update Success.`); + const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS; + const cachedKey = await this.cacheService.getPackageEtag(fullname, isFullManifests); + if (cachedKey) { + const { shasum: etag } = await calculateIntegrity(cacheBytes); + await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, cacheBytes); + logs.push(`[${isoNow()}] 🟢 Update Cache Success.`); + } await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } diff --git a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json index ed0ed4b5..db6c2348 100644 --- a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json +++ b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json @@ -4,6 +4,7 @@ }, "modified": "2022-01-26T20:31:13.648Z", "name": "foobar", + "description": "cnpmcore mock json", "versions": { "1.0.0": { "name": "foobar", diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 21dd720c..4cae91cd 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -2,6 +2,7 @@ import assert from 'assert'; import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { NFSClientAdapter } from '../../../../app/infra/NFSClientAdapter'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/DownloadPackageVersionTarController.test.ts', () => { let publisher: any; @@ -282,6 +283,17 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. app.expectLog('[middleware:ErrorHandler][syncPackage] create sync package'); }); + it('should create sync specific version task when package version tgz not found in proxy mode ', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest() + .get('/foobar/-/foobar-1.0.0.tgz') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + app.expectLog('[DownloadPackageVersionTarController.createSyncTask:success]'); + }); + }); describe('[GET /:fullname/download/:fullname-:version.tgz] deprecatedDownload()', () => { diff --git a/test/port/controller/package/ShowPackageController.test.ts b/test/port/controller/package/ShowPackageController.test.ts index 745c62be..94c45f5c 100644 --- a/test/port/controller/package/ShowPackageController.test.ts +++ b/test/port/controller/package/ShowPackageController.test.ts @@ -7,6 +7,7 @@ import { PackageManagerService } from '../../../../app/core/service/PackageManag import { CacheService } from '../../../../app/core/service/CacheService'; import { DistRepository } from '../../../../app/repository/DistRepository'; import { BugVersionService } from '../../../../app/core/service/BugVersionService'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/ShowPackageController.test.ts', () => { let packageRepository: PackageRepository; @@ -863,5 +864,23 @@ describe('test/port/controller/package/ShowPackageController.test.ts', () => { assert(res.status === 302); assert(res.headers.location === 'https://registry.npmjs.org/egg'); }); + + it('should read manifest from source in proxy mode', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json')); + app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', { + data, + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + assert(res.headers.location === app.config.cnpmcore.registry); + assert(res.body.data.description === 'cnpmcore mock json'); + assert(res.body.data.versions['1.0.0'].dist.tarball.includes(app.config.cnpmcore.registry)); + }); }); }); diff --git a/test/port/controller/package/ShowPackageVersionController.test.ts b/test/port/controller/package/ShowPackageVersionController.test.ts index d99b663d..9585f901 100644 --- a/test/port/controller/package/ShowPackageVersionController.test.ts +++ b/test/port/controller/package/ShowPackageVersionController.test.ts @@ -3,6 +3,7 @@ import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { BugVersion } from '../../../../app/core/entity/BugVersion'; import { BugVersionService } from '../../../../app/core/service/BugVersionService'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/ShowPackageVersionController.test.ts', () => { let publisher; @@ -368,5 +369,22 @@ describe('test/port/controller/package/ShowPackageVersionController.test.ts', () .expect(200); assert(res.body._source_registry_name === 'self'); }); + + it('should read package version manifest from source in proxy mode', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/abbreviated.json')); + app.mockHttpclient('https://registry.npmjs.org/foobar/1.0.0', 'GET', { + data, + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar/1.0.0') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + assert(res.headers.location === app.config.cnpmcore.registry); + assert(res.body.data.dist.tarball.includes(app.config.cnpmcore.registry)); + }); }); }); diff --git a/test/schedule/CheckProxyCacheUpdateWorker.test.ts b/test/schedule/CheckProxyCacheUpdateWorker.test.ts new file mode 100644 index 00000000..20b5e4de --- /dev/null +++ b/test/schedule/CheckProxyCacheUpdateWorker.test.ts @@ -0,0 +1,28 @@ +import assert from 'assert'; +import { app, mock } from 'egg-mock/bootstrap'; +import { SyncMode } from '../../app/common/constants'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { TaskService } from '../../app/core/service/TaskService'; +import { TaskType } from '../../app/common/enum/Task'; + +const CheckProxyCacheUpdateWorkerPath = require.resolve('../../app/port/schedule/CheckProxyCacheUpdateWorker'); + +describe('test/schedule/CheckProxyCacheUpdateWorker.test.ts', () => { + it('should create update task by repo', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const taskService = await app.getEggObject(TaskService); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + await app.runSchedule(CheckProxyCacheUpdateWorkerPath); + const task = await taskService.findExecuteTask(TaskType.UpdateProxyCache); + assert(task); + assert.equal(task.targetName, `foo-bar/${DIST_NAMES.FULL_MANIFESTS}`); + }); + +}); diff --git a/test/schedule/SyncProxyCacheWorker.test.ts b/test/schedule/SyncProxyCacheWorker.test.ts new file mode 100644 index 00000000..42098a06 --- /dev/null +++ b/test/schedule/SyncProxyCacheWorker.test.ts @@ -0,0 +1,37 @@ +import { app, mock } from 'egg-mock/bootstrap'; +import { SyncMode } from '../../app/common/constants'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { ProxyCacheService } from '../../app/core/service/ProxyCacheService'; + +const SyncProxyCacheWorkerPath = require.resolve('../../app/port/schedule/SyncProxyCacheWorker'); + +describe('test/schedule/SyncProxyCacheWorker.test.ts', () => { + + beforeEach(async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + }); + + it('should execute task success', async () => { + + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const proxyCacheService = await app.getEggObject(ProxyCacheService); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + + + await proxyCacheService.createTask(`foobar/${DIST_NAMES.ABBREVIATED_MANIFESTS}`, { + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }); + + await app.runSchedule(SyncProxyCacheWorkerPath); + app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:start]'); + app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:success]'); + }); + +});