Skip to content

Commit

Permalink
feat: sync missing acceptDependencies on abbreviated format (#691)
Browse files Browse the repository at this point in the history
closes #689

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Added support for `acceptDependencies`, `directories`, and `funding`
metadata in package management.

- **Bug Fixes**
  - Improved package metadata synchronization to include new keys.

- **Tests**
- Added new test cases to verify `acceptDependencies` metadata syncing.

- **Chores**
- Updated import statements to use the `node` namespace for better
compatibility.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
  • Loading branch information
fengmk2 authored May 25, 2024
1 parent 6f9f8ab commit 96648fd
Show file tree
Hide file tree
Showing 5 changed files with 272 additions and 7 deletions.
6 changes: 5 additions & 1 deletion app/core/service/PackageManagerService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -201,26 +201,30 @@ export class PackageManagerService extends AbstractService {
integrity: tarDistIntegrity.integrity,
};

// https://github.com/npm/registry/blob/main/docs/responses/package-metadata.md#abbreviated-version-object
// Abbreviated version object
const abbreviated = JSON.stringify({
name: cmd.packageJson.name,
version: cmd.packageJson.version,
deprecated: cmd.packageJson.deprecated,
dependencies: cmd.packageJson.dependencies,
acceptDependencies: cmd.packageJson.acceptDependencies,
optionalDependencies: cmd.packageJson.optionalDependencies,
devDependencies: cmd.packageJson.devDependencies,
bundleDependencies: cmd.packageJson.bundleDependencies,
peerDependencies: cmd.packageJson.peerDependencies,
peerDependenciesMeta: cmd.packageJson.peerDependenciesMeta,
bin: cmd.packageJson.bin,
directories: cmd.packageJson.directories,
os: cmd.packageJson.os,
cpu: cmd.packageJson.cpu,
libc: cmd.packageJson.libc,
workspaces: cmd.packageJson.workspaces,
directories: cmd.packageJson.directories,
dist: cmd.packageJson.dist,
engines: cmd.packageJson.engines,
_hasShrinkwrap: cmd.packageJson._hasShrinkwrap,
hasInstallScript,
funding: cmd.packageJson.funding,
// https://github.com/cnpm/npminstall/blob/13efc7eec21a61e509226e3772bfb75cd5605612/lib/install_package.js#L176
// npminstall require publish time to show the recently update versions
publish_time: cmd.packageJson.publish_time,
Expand Down
13 changes: 9 additions & 4 deletions app/core/service/PackageSyncerService.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import os from 'os';
import os from 'node:os';
import { setTimeout } from 'node:timers/promises';
import { rm } from 'node:fs/promises';
import {
AccessLevel,
SingletonProto,
Inject,
} from '@eggjs/tegg';
import { Pointcut } from '@eggjs/tegg/aop';
import { EggHttpClient } from 'egg';
import { setTimeout } from 'timers/promises';
import { rm } from 'fs/promises';
import { isEqual, isEmpty } from 'lodash';
import semver from 'semver';
import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
Expand Down Expand Up @@ -626,7 +626,12 @@ export class PackageSyncerService extends AbstractService {
// https://github.com/cnpm/cnpmjs.org/issues/1667
// need libc field https://github.com/cnpm/cnpmcore/issues/187
// fix _npmUser field since https://github.com/cnpm/cnpmcore/issues/553
const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript', 'deprecated', '_npmUser' ];
const metaDataKeys = [
'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript',
'deprecated', '_npmUser', 'funding',
// https://github.com/cnpm/cnpmcore/issues/689
'acceptDependencies',
];
const ignoreInAbbreviated = [ '_npmUser' ];
const diffMeta: Partial<PackageJSONType> = {};
for (const key of metaDataKeys) {
Expand Down
9 changes: 7 additions & 2 deletions app/repository/PackageRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ export type PackageJSONType = CnpmcorePatchInfo & {
'bug-versions'?: BugVersionPackages;
};
dependencies?: DepInfo;
acceptDependencies?: DepInfo;
devDependencies?: DepInfo;
peerDependencies?: DepInfo;
peerDependenciesMeta?: {
Expand Down Expand Up @@ -105,7 +106,8 @@ export type PackageJSONType = CnpmcorePatchInfo & {
[key: string]: unknown;
};

type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' | 'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions' | 'contributors';
type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' |
'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions' | 'contributors';

export type CnpmcorePatchInfo = {
_cnpmcore_publish_time?: Date;
Expand All @@ -114,7 +116,10 @@ export type CnpmcorePatchInfo = {
block?: string;
};

type AbbreviatedKey = 'name' | 'version' | 'deprecated' | 'dependencies' | 'optionalDependencies' | 'devDependencies' | 'bundleDependencies' | 'peerDependencies' | 'peerDependenciesMeta' | 'bin' | 'os' | 'cpu' | 'libc' | 'workspaces' | 'directories' | 'dist' | 'engines' | 'hasInstallScript' | 'publish_time' | 'block' | '_hasShrinkwrap';
type AbbreviatedKey = 'name' | 'version' | 'deprecated' | 'dependencies' | 'optionalDependencies' |
'devDependencies' | 'bundleDependencies' | 'peerDependencies' | 'peerDependenciesMeta' | 'bin' |
'os' | 'cpu' | 'libc' | 'workspaces' | 'directories' | 'dist' | 'engines' | 'hasInstallScript' |
'publish_time' | 'block' | '_hasShrinkwrap' | 'acceptDependencies' | 'funding';

type DistType = {
tarball: string,
Expand Down
61 changes: 61 additions & 0 deletions test/core/service/PackageSyncerService/executeTask.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1996,6 +1996,67 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
assert(!abbreviatedManifests.data!.versions['2.0.0'].libc);
});

it('should sync missing acceptDependencies with different metas', async () => {
app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/accept-dependencies-module-cnpmsync.json'),
persist: false,
});
app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-1.0.0.tgz', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
persist: false,
});
app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-3.0.0.tgz', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
persist: false,
});

const name = 'accept-dependencies-module-cnpmsync';
mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true);
await TestUtil.createPackage({ name, version: '2.0.0', isPrivate: false });
await packageSyncerService.createTask(name, { tips: 'sync test tips here' });
let task = await packageSyncerService.findExecuteTask();
assert(task);
await packageSyncerService.executeTask(task);
let stream = await packageSyncerService.findTaskLog(task);
assert(stream);
let log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Synced version 2.0.0 success, different meta: {"peerDependenciesMeta":{"bufferutil":{"optional":true},"utf-8-validate":{"optional":true}},"os":["linux"],"cpu":["x64"],"_npmUser":{"name":"fengmk2","email":"[email protected]"},"acceptDependencies":{"webpack":"^4.46.x"}}'));
assert(log.includes('Z] 👉👉👉👉👉 Tips: sync test tips here 👈👈👈👈👈'));
assert(log.includes(', skipDependencies: false'));
const manifests = await packageManagerService.listPackageFullManifests('', name);
assert(manifests.data?.versions['2.0.0']);
assert.equal(manifests.data.versions['2.0.0'].peerDependenciesMeta?.bufferutil.optional, true);
assert.equal(manifests.data.versions['2.0.0'].os?.[0], 'linux');
assert.equal(manifests.data.versions['2.0.0'].cpu?.[0], 'x64');
// publishTime
assert.equal(manifests.data.time['1.0.0'], '2021-09-27T08:10:48.747Z');
const abbreviatedManifests = await packageManagerService.listPackageAbbreviatedManifests('', name);
// console.log(JSON.stringify(abbreviatedManifests.data, null, 2));
assert(abbreviatedManifests.data?.versions['2.0.0']);
assert.equal(abbreviatedManifests.data!.versions['2.0.0'].peerDependenciesMeta?.bufferutil.optional, true);
assert.equal(abbreviatedManifests.data!.versions['2.0.0'].os?.[0], 'linux');
assert.equal(abbreviatedManifests.data!.versions['2.0.0'].cpu?.[0], 'x64');
assert.equal(abbreviatedManifests.data!.versions['2.0.0'].acceptDependencies?.webpack, '^4.46.x');
app.mockAgent().assertNoPendingInterceptors();

// again should skip sync different metas
app.mockHttpclient('https://registry.npmjs.org/accept-dependencies-module-cnpmsync', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/accept-dependencies-module-cnpmsync.json'),
persist: false,
});
await packageSyncerService.createTask(name);
task = await packageSyncerService.findExecuteTask();
assert(task);
await packageSyncerService.executeTask(task);
stream = await packageSyncerService.findTaskLog(task);
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(!log.includes('🟢 Synced version 2.0.0 success, different meta:'));
app.mockAgent().assertNoPendingInterceptors();
});

it('should sync download data work on enableSyncDownloadData = true', async () => {
mock(app.config.cnpmcore, 'syncDownloadDataSourceRegistry', 'https://rold.cnpmjs.org');
mock(app.config.cnpmcore, 'enableSyncDownloadData', true);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
{
"_id": "accept-dependencies-module-cnpmsync",
"_rev": "4-5ce9412ef04c734e05c990222217e209",
"name": "accept-dependencies-module-cnpmsync",
"dist-tags": {
"latest": "3.0.0"
},
"versions": {
"1.0.0": {
"name": "accept-dependencies-module-cnpmsync",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"os": [
"linux"
],
"cpu": [
"x64"
],
"directories": {},
"_hasShrinkwrap": false,
"_id": "[email protected]",
"_nodeVersion": "12.22.1",
"_npmVersion": "6.14.12",
"dist": {
"integrity": "sha512-piK0lpdbWTc78ondfb8Tz5vtuRILbbTzno+/+S5sqt128+d2h1R2jTegpOT1VO1Qr3bZSn0J8eRKu43sOFiMFg==",
"shasum": "a71a469959855399b7abe75b47352e592cd501fe",
"tarball": "https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-1.0.0.tgz",
"fileCount": 2,
"unpackedSize": 330,
"signatures": [
{
"keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA",
"sig": "MEYCIQCK4nA//tBQz2/KuxlYMfrpGDuLVVg6RyPEiHJwG6iKFwIhAPIhV/06UOaz75081PA74IUb8U0LdTFaAAUSU17NWBJL"
}
]
},
"_npmUser": {
"name": "fengmk2",
"email": "[email protected]"
},
"maintainers": [
{
"name": "fengmk2",
"email": "[email protected]"
}
],
"_npmOperationalInternal": {
"host": "s3://npm-registry-packages",
"tmp": "tmp/accept-dependencies-module-cnpmsync_1.0.0_1632730248622_0.3332581028631232"
}
},
"2.0.0": {
"name": "accept-dependencies-module-cnpmsync",
"version": "2.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"acceptDependencies": {
"webpack": "^4.46.x"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
},
"os": [
"linux"
],
"cpu": [
"x64"
],
"directories": {},
"_hasShrinkwrap": false,
"_id": "[email protected]",
"_nodeVersion": "12.22.1",
"_npmVersion": "6.14.12",
"dist": {
"integrity": "sha512-VkYtIExG6XmHESMWHxQFjpHH5NNmPHQaSQAKWm2H8e5Ror5gBySsqJjPFn+4rXtefmnHMoFKRtwsLk88ice6Qw==",
"shasum": "93d825fb2d53e0c051bc257e3e17e91572509afa",
"tarball": "https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-2.0.0.tgz",
"fileCount": 2,
"unpackedSize": 436,
"signatures": [
{
"keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA",
"sig": "MEUCIQD4WjKR1KJbuO2Jx6/kqMqWZ1jltTM6CUUmPNIk+cgpxQIgeWijbYGrTkzfKUPKuB6EKAKZMCX8VYtZhayowVId15o="
}
]
},
"_npmUser": {
"name": "fengmk2",
"email": "[email protected]"
},
"maintainers": [
{
"name": "fengmk2",
"email": "[email protected]"
}
],
"_npmOperationalInternal": {
"host": "s3://npm-registry-packages",
"tmp": "tmp/accept-dependencies-module-cnpmsync_2.0.0_1632732483099_0.18902750158956594"
}
},
"3.0.0": {
"name": "accept-dependencies-module-cnpmsync",
"version": "3.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"acceptDependencies": {
"webpack": "^4.46.x"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
},
"directories": {},
"_hasShrinkwrap": false,
"_id": "[email protected]",
"_nodeVersion": "12.22.1",
"_npmVersion": "6.14.12",
"dist": {
"integrity": "sha512-4JwhZmW/twQk4rXEKpUbG32iYYoabRHr7HnQMQ0LRQ7u6oOu6Q+AQ5EttZKbB+Gms+NyBVyzJZhihbogoB+Jyw==",
"shasum": "02a247bc483c2e65404e5eb3772d8e1ec11f690d",
"tarball": "https://registry.npmjs.org/accept-dependencies-module-cnpmsync/-/accept-dependencies-module-cnpmsync-3.0.0.tgz",
"fileCount": 2,
"unpackedSize": 376,
"npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJh2zXTCRA9TVsSAnZWagAAmzwP/iUE83lfm4diBan4L1LC\nE+M/bzza17I9CVfV84fZWWXwNNMLTOEBjbAWu+9cFv1PM+WAgg38v3Khd2F0\nvr5wp/X6BiINp0mjxYRyqmU21OHvkmhahCMSxqX+SZ2eFMYiV3TmOL1LWGmI\nL9nEJ+o/bVJ8R4Rg/DTrVFIJvKJN5hlsk3ZFEgIFZz6NZGXQAR7slsEuuDit\n0B4ZjoQH3bdMQXkO43UYQhdrdX3VfmoR815H7qdwfSIOMLREfcWiE+kOPI9u\nBUf0Og2Xiss7RasTShOHqVnsYnBm02LzJ1Av45OgzLR4Bpxvv0rS9BGAVnmR\nIQRSaj3T6NaADSN3dhDeUakOkzTUVyAT6Q+WyDSuy4vXZRspPJpsLxR1yeVa\nZquQEPTLwisM5fyl1vL8hsZ94JGRDQUV6Vb24JBmnF1Yfvi/8aJwzZ0euVSo\n074TG7+WV9gmadT0QvEt+fWnn/bM3kPWCq5Gfp3rk5M0rAbera9sUvnv4TaI\nCT0iLOsG38htO8bCNXhjVQbKy8PHslbBWX1mjJsTgLEfUyBsRFMBUiiYafOE\nbWChEa11BGOdPDA/iwwYf2bwo68a/zn8UHxGu51jzs/WGqnBaDlz/ymfcxCJ\nrmmhHdDdfmYhWUgyIlRobE02owmXe6cdgSNMIPTm+YRL0MEGbEhhoEj8qla6\nNKkE\r\n=awKd\r\n-----END PGP SIGNATURE-----\r\n",
"signatures": [
{
"keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA",
"sig": "MEQCICAhOYMv6PCr8KpNPyEisIKBHguA4cMisNHTxMPxdsemAiBlfaCFSmAPj75r18BvAd87EK9y8kHZGk/ezVtugR20Kg=="
}
]
},
"_npmUser": {
"name": "fengmk2",
"email": "[email protected]"
},
"maintainers": [
{
"name": "fengmk2",
"email": "[email protected]"
}
],
"_npmOperationalInternal": {
"host": "s3://npm-registry-packages",
"tmp": "tmp/accept-dependencies-module-cnpmsync_3.0.0_1632733912778_0.7339024277257908"
}
}
},
"time": {
"created": "2021-09-27T08:10:48.622Z",
"1.0.0": "2021-09-27T08:10:48.747Z",
"modified": "2022-05-09T12:14:25.637Z",
"2.0.0": "2021-09-27T08:48:03.214Z",
"3.0.0": "2021-09-27T09:11:52.897Z"
},
"maintainers": [
{
"name": "fengmk2",
"email": "[email protected]"
}
],
"license": "ISC",
"readme": "ERROR: No README data found!",
"readmeFilename": ""
}

0 comments on commit 96648fd

Please sign in to comment.