diff --git a/app/core/service/PackageManagerService.ts b/app/core/service/PackageManagerService.ts index ea7a5d78..767b0d51 100644 --- a/app/core/service/PackageManagerService.ts +++ b/app/core/service/PackageManagerService.ts @@ -201,26 +201,30 @@ export class PackageManagerService extends AbstractService { integrity: tarDistIntegrity.integrity, }; + // https://github.com/npm/registry/blob/main/docs/responses/package-metadata.md#abbreviated-version-object + // Abbreviated version object const abbreviated = JSON.stringify({ name: cmd.packageJson.name, version: cmd.packageJson.version, deprecated: cmd.packageJson.deprecated, dependencies: cmd.packageJson.dependencies, + acceptDependencies: cmd.packageJson.acceptDependencies, optionalDependencies: cmd.packageJson.optionalDependencies, devDependencies: cmd.packageJson.devDependencies, bundleDependencies: cmd.packageJson.bundleDependencies, peerDependencies: cmd.packageJson.peerDependencies, peerDependenciesMeta: cmd.packageJson.peerDependenciesMeta, bin: cmd.packageJson.bin, + directories: cmd.packageJson.directories, os: cmd.packageJson.os, cpu: cmd.packageJson.cpu, libc: cmd.packageJson.libc, workspaces: cmd.packageJson.workspaces, - directories: cmd.packageJson.directories, dist: cmd.packageJson.dist, engines: cmd.packageJson.engines, _hasShrinkwrap: cmd.packageJson._hasShrinkwrap, hasInstallScript, + funding: cmd.packageJson.funding, // https://github.com/cnpm/npminstall/blob/13efc7eec21a61e509226e3772bfb75cd5605612/lib/install_package.js#L176 // npminstall require publish time to show the recently update versions publish_time: cmd.packageJson.publish_time, diff --git a/app/core/service/PackageSyncerService.ts b/app/core/service/PackageSyncerService.ts index 237bd6c5..768af7d2 100644 --- a/app/core/service/PackageSyncerService.ts +++ b/app/core/service/PackageSyncerService.ts @@ -1,4 +1,6 @@ -import os from 'os'; +import os from 'node:os'; +import { setTimeout } from 'node:timers/promises'; +import { rm } from 'node:fs/promises'; import { AccessLevel, SingletonProto, @@ -6,8 +8,6 @@ import { } from '@eggjs/tegg'; import { Pointcut } from '@eggjs/tegg/aop'; import { EggHttpClient } from 'egg'; -import { setTimeout } from 'timers/promises'; -import { rm } from 'fs/promises'; import { isEqual, isEmpty } from 'lodash'; import semver from 'semver'; import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; @@ -626,7 +626,12 @@ export class PackageSyncerService extends AbstractService { // https://github.com/cnpm/cnpmjs.org/issues/1667 // need libc field https://github.com/cnpm/cnpmcore/issues/187 // fix _npmUser field since https://github.com/cnpm/cnpmcore/issues/553 - const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript', 'deprecated', '_npmUser' ]; + const metaDataKeys = [ + 'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript', + 'deprecated', '_npmUser', 'funding', + // https://github.com/cnpm/cnpmcore/issues/689 + 'acceptDependencies', + ]; const ignoreInAbbreviated = [ '_npmUser' ]; const diffMeta: Partial = {}; for (const key of metaDataKeys) { diff --git a/app/repository/PackageRepository.ts b/app/repository/PackageRepository.ts index 68349332..59ca134b 100644 --- a/app/repository/PackageRepository.ts +++ b/app/repository/PackageRepository.ts @@ -68,6 +68,7 @@ export type PackageJSONType = CnpmcorePatchInfo & { 'bug-versions'?: BugVersionPackages; }; dependencies?: DepInfo; + acceptDependencies?: DepInfo; devDependencies?: DepInfo; peerDependencies?: DepInfo; peerDependenciesMeta?: { @@ -105,7 +106,8 @@ export type PackageJSONType = CnpmcorePatchInfo & { [key: string]: unknown; }; -type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' | 'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions' | 'contributors'; +type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' | +'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions' | 'contributors'; export type CnpmcorePatchInfo = { _cnpmcore_publish_time?: Date; @@ -114,7 +116,10 @@ export type CnpmcorePatchInfo = { block?: string; }; -type AbbreviatedKey = 'name' | 'version' | 'deprecated' | 'dependencies' | 'optionalDependencies' | 'devDependencies' | 'bundleDependencies' | 'peerDependencies' | 'peerDependenciesMeta' | 'bin' | 'os' | 'cpu' | 'libc' | 'workspaces' | 'directories' | 'dist' | 'engines' | 'hasInstallScript' | 'publish_time' | 'block' | '_hasShrinkwrap'; +type AbbreviatedKey = 'name' | 'version' | 'deprecated' | 'dependencies' | 'optionalDependencies' | +'devDependencies' | 'bundleDependencies' | 'peerDependencies' | 'peerDependenciesMeta' | 'bin' | +'os' | 'cpu' | 'libc' | 'workspaces' | 'directories' | 'dist' | 'engines' | 'hasInstallScript' | +'publish_time' | 'block' | '_hasShrinkwrap' | 'acceptDependencies' | 'funding'; type DistType = { tarball: string, diff --git a/test/core/service/PackageSyncerService/executeTask.test.ts b/test/core/service/PackageSyncerService/executeTask.test.ts index bc9137c5..fe10a33f 100644 --- a/test/core/service/PackageSyncerService/executeTask.test.ts +++ b/test/core/service/PackageSyncerService/executeTask.test.ts @@ -1996,6 +1996,67 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(!abbreviatedManifests.data!.versions['2.0.0'].libc); }); + it('should sync missing acceptDependencies with different metas', async () => { + app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/accept-dependencies-module-cnpmsync.json'), + persist: false, + }); + app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-1.0.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), + persist: false, + }); + app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-3.0.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), + persist: false, + }); + + const name = 'accept-dependencies-module-cnpmsync'; + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + await TestUtil.createPackage({ name, version: '2.0.0', isPrivate: false }); + await packageSyncerService.createTask(name, { tips: 'sync test tips here' }); + let task = await packageSyncerService.findExecuteTask(); + assert(task); + await packageSyncerService.executeTask(task); + let stream = await packageSyncerService.findTaskLog(task); + assert(stream); + let log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(log.includes('Synced version 2.0.0 success, different meta: {"peerDependenciesMeta":{"bufferutil":{"optional":true},"utf-8-validate":{"optional":true}},"os":["linux"],"cpu":["x64"],"_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"acceptDependencies":{"webpack":"^4.46.x"}}')); + assert(log.includes('Z] 👉👉👉👉👉 Tips: sync test tips here 👈👈👈👈👈')); + assert(log.includes(', skipDependencies: false')); + const manifests = await packageManagerService.listPackageFullManifests('', name); + assert(manifests.data?.versions['2.0.0']); + assert.equal(manifests.data.versions['2.0.0'].peerDependenciesMeta?.bufferutil.optional, true); + assert.equal(manifests.data.versions['2.0.0'].os?.[0], 'linux'); + assert.equal(manifests.data.versions['2.0.0'].cpu?.[0], 'x64'); + // publishTime + assert.equal(manifests.data.time['1.0.0'], '2021-09-27T08:10:48.747Z'); + const abbreviatedManifests = await packageManagerService.listPackageAbbreviatedManifests('', name); + // console.log(JSON.stringify(abbreviatedManifests.data, null, 2)); + assert(abbreviatedManifests.data?.versions['2.0.0']); + assert.equal(abbreviatedManifests.data!.versions['2.0.0'].peerDependenciesMeta?.bufferutil.optional, true); + assert.equal(abbreviatedManifests.data!.versions['2.0.0'].os?.[0], 'linux'); + assert.equal(abbreviatedManifests.data!.versions['2.0.0'].cpu?.[0], 'x64'); + assert.equal(abbreviatedManifests.data!.versions['2.0.0'].acceptDependencies?.webpack, '^4.46.x'); + app.mockAgent().assertNoPendingInterceptors(); + + // again should skip sync different metas + app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/accept-dependencies-module-cnpmsync.json'), + persist: false, + }); + await packageSyncerService.createTask(name); + task = await packageSyncerService.findExecuteTask(); + assert(task); + await packageSyncerService.executeTask(task); + stream = await packageSyncerService.findTaskLog(task); + assert(stream); + log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(!log.includes('🟢 Synced version 2.0.0 success, different meta:')); + app.mockAgent().assertNoPendingInterceptors(); + }); + it('should sync download data work on enableSyncDownloadData = true', async () => { mock(app.config.cnpmcore, 'syncDownloadDataSourceRegistry', 'https://rold.cnpmjs.org'); mock(app.config.cnpmcore, 'enableSyncDownloadData', true); diff --git a/test/fixtures/registry.npmjs.org/accept-dependencies-module-cnpmsync.json b/test/fixtures/registry.npmjs.org/accept-dependencies-module-cnpmsync.json new file mode 100644 index 00000000..98c41d00 --- /dev/null +++ b/test/fixtures/registry.npmjs.org/accept-dependencies-module-cnpmsync.json @@ -0,0 +1,190 @@ +{ + "_id": "accept-dependencies-module-cnpmsync", + "_rev": "4-5ce9412ef04c734e05c990222217e209", + "name": "accept-dependencies-module-cnpmsync", + "dist-tags": { + "latest": "3.0.0" + }, + "versions": { + "1.0.0": { + "name": "accept-dependencies-module-cnpmsync", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "directories": {}, + "_hasShrinkwrap": false, + "_id": "accept-dependencies-module-cnpmsync@1.0.0", + "_nodeVersion": "12.22.1", + "_npmVersion": "6.14.12", + "dist": { + "integrity": "sha512-piK0lpdbWTc78ondfb8Tz5vtuRILbbTzno+/+S5sqt128+d2h1R2jTegpOT1VO1Qr3bZSn0J8eRKu43sOFiMFg==", + "shasum": "a71a469959855399b7abe75b47352e592cd501fe", + "tarball": "https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-1.0.0.tgz", + "fileCount": 2, + "unpackedSize": 330, + "signatures": [ + { + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig": "MEYCIQCK4nA//tBQz2/KuxlYMfrpGDuLVVg6RyPEiHJwG6iKFwIhAPIhV/06UOaz75081PA74IUb8U0LdTFaAAUSU17NWBJL" + } + ] + }, + "_npmUser": { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + }, + "maintainers": [ + { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + } + ], + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/accept-dependencies-module-cnpmsync_1.0.0_1632730248622_0.3332581028631232" + } + }, + "2.0.0": { + "name": "accept-dependencies-module-cnpmsync", + "version": "2.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "acceptDependencies": { + "webpack": "^4.46.x" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "directories": {}, + "_hasShrinkwrap": false, + "_id": "accept-dependencies-module-cnpmsync@2.0.0", + "_nodeVersion": "12.22.1", + "_npmVersion": "6.14.12", + "dist": { + "integrity": "sha512-VkYtIExG6XmHESMWHxQFjpHH5NNmPHQaSQAKWm2H8e5Ror5gBySsqJjPFn+4rXtefmnHMoFKRtwsLk88ice6Qw==", + "shasum": "93d825fb2d53e0c051bc257e3e17e91572509afa", + "tarball": "https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-2.0.0.tgz", + "fileCount": 2, + "unpackedSize": 436, + "signatures": [ + { + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig": "MEUCIQD4WjKR1KJbuO2Jx6/kqMqWZ1jltTM6CUUmPNIk+cgpxQIgeWijbYGrTkzfKUPKuB6EKAKZMCX8VYtZhayowVId15o=" + } + ] + }, + "_npmUser": { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + }, + "maintainers": [ + { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + } + ], + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/accept-dependencies-module-cnpmsync_2.0.0_1632732483099_0.18902750158956594" + } + }, + "3.0.0": { + "name": "accept-dependencies-module-cnpmsync", + "version": "3.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "acceptDependencies": { + "webpack": "^4.46.x" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + }, + "directories": {}, + "_hasShrinkwrap": false, + "_id": "accept-dependencies-module-cnpmsync@3.0.0", + "_nodeVersion": "12.22.1", + "_npmVersion": "6.14.12", + "dist": { + "integrity": "sha512-4JwhZmW/twQk4rXEKpUbG32iYYoabRHr7HnQMQ0LRQ7u6oOu6Q+AQ5EttZKbB+Gms+NyBVyzJZhihbogoB+Jyw==", + "shasum": "02a247bc483c2e65404e5eb3772d8e1ec11f690d", + "tarball": "https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-3.0.0.tgz", + "fileCount": 2, + "unpackedSize": 376, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJh2zXTCRA9TVsSAnZWagAAmzwP/iUE83lfm4diBan4L1LC\nE+M/bzza17I9CVfV84fZWWXwNNMLTOEBjbAWu+9cFv1PM+WAgg38v3Khd2F0\nvr5wp/X6BiINp0mjxYRyqmU21OHvkmhahCMSxqX+SZ2eFMYiV3TmOL1LWGmI\nL9nEJ+o/bVJ8R4Rg/DTrVFIJvKJN5hlsk3ZFEgIFZz6NZGXQAR7slsEuuDit\n0B4ZjoQH3bdMQXkO43UYQhdrdX3VfmoR815H7qdwfSIOMLREfcWiE+kOPI9u\nBUf0Og2Xiss7RasTShOHqVnsYnBm02LzJ1Av45OgzLR4Bpxvv0rS9BGAVnmR\nIQRSaj3T6NaADSN3dhDeUakOkzTUVyAT6Q+WyDSuy4vXZRspPJpsLxR1yeVa\nZquQEPTLwisM5fyl1vL8hsZ94JGRDQUV6Vb24JBmnF1Yfvi/8aJwzZ0euVSo\n074TG7+WV9gmadT0QvEt+fWnn/bM3kPWCq5Gfp3rk5M0rAbera9sUvnv4TaI\nCT0iLOsG38htO8bCNXhjVQbKy8PHslbBWX1mjJsTgLEfUyBsRFMBUiiYafOE\nbWChEa11BGOdPDA/iwwYf2bwo68a/zn8UHxGu51jzs/WGqnBaDlz/ymfcxCJ\nrmmhHdDdfmYhWUgyIlRobE02owmXe6cdgSNMIPTm+YRL0MEGbEhhoEj8qla6\nNKkE\r\n=awKd\r\n-----END PGP SIGNATURE-----\r\n", + "signatures": [ + { + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig": "MEQCICAhOYMv6PCr8KpNPyEisIKBHguA4cMisNHTxMPxdsemAiBlfaCFSmAPj75r18BvAd87EK9y8kHZGk/ezVtugR20Kg==" + } + ] + }, + "_npmUser": { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + }, + "maintainers": [ + { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + } + ], + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/accept-dependencies-module-cnpmsync_3.0.0_1632733912778_0.7339024277257908" + } + } + }, + "time": { + "created": "2021-09-27T08:10:48.622Z", + "1.0.0": "2021-09-27T08:10:48.747Z", + "modified": "2022-05-09T12:14:25.637Z", + "2.0.0": "2021-09-27T08:48:03.214Z", + "3.0.0": "2021-09-27T09:11:52.897Z" + }, + "maintainers": [ + { + "name": "fengmk2", + "email": "fengmk2@gmail.com" + } + ], + "license": "ISC", + "readme": "ERROR: No README data found!", + "readmeFilename": "" +}