From 8f9387eb9444404c339925dced0f09b70e124f42 Mon Sep 17 00:00:00 2001 From: MarkAckert Date: Thu, 26 Sep 2024 14:59:48 -0400 Subject: [PATCH] add pnpm support, respect lockfiles Signed-off-by: MarkAckert --- .dockerfiles/ort.Dockerfile | 2 + licenses/dependency-scan/package-lock.json | 83 +++++++------------ .../src/actions/base/InstallAction.ts | 27 +++--- .../dependency-scan/src/utils/Utilities.ts | 12 +++ licenses/dependency-scan/yarn.lock | 67 ++++++++------- 5 files changed, 98 insertions(+), 93 deletions(-) diff --git a/.dockerfiles/ort.Dockerfile b/.dockerfiles/ort.Dockerfile index 54a2358..7d048b3 100644 --- a/.dockerfiles/ort.Dockerfile +++ b/.dockerfiles/ort.Dockerfile @@ -26,6 +26,8 @@ ENV PATH="$HOME/.cargo/bin:$PATH" RUN npm install -g yarn +RUN wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.bashrc" SHELL="$(which bash)" bash - + ENV owasp_version=5.3.2 ENV owasp_dc_download="https://github.com/jeremylong/DependencyCheck/releases/download/v${owasp_version}/" diff --git a/licenses/dependency-scan/package-lock.json b/licenses/dependency-scan/package-lock.json index 66fabc0..9f88c56 100644 --- a/licenses/dependency-scan/package-lock.json +++ b/licenses/dependency-scan/package-lock.json @@ -26,13 +26,16 @@ "@types/cross-spawn": "^6.0.0", "@types/fs-extra": "8.0.0", "@types/lodash": "^4.14.202", - "@types/node": "^12.0.2", + "@types/node": "^18.0.2", "@types/rimraf": "4.0.5", "@types/xml2js": "^0.4.5", "@types/yaml": "^1.9.7", "lodash": "^4.17.21", "tslint": "^5.13.1", "typescript": "5.2.2" + }, + "engines": { + "node": ">18.0.0" } }, "node_modules/@babel/code-frame": { @@ -266,13 +269,6 @@ "@types/node": "*" } }, - "node_modules/@types/cross-spawn/node_modules/@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/fs-extra": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.0.0.tgz", @@ -283,13 +279,6 @@ "@types/node": "*" } }, - "node_modules/@types/fs-extra/node_modules/@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/lodash": { "version": "4.14.202", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz", @@ -297,11 +286,13 @@ "dev": true }, "node_modules/@types/node": { - "version": "12.12.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.26.tgz", - "integrity": "sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA==", + "version": "18.19.53", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz", + "integrity": "sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==", "dev": true, - "license": "MIT" + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/rimraf": { "version": "4.0.5", @@ -324,13 +315,6 @@ "@types/node": "*" } }, - "node_modules/@types/xml2js/node_modules/@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/yaml": { "version": "1.9.7", "resolved": "https://registry.npmjs.org/@types/yaml/-/yaml-1.9.7.tgz", @@ -1188,6 +1172,12 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "node_modules/universal-user-agent": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", @@ -1535,14 +1525,6 @@ "dev": true, "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true - } } }, "@types/fs-extra": { @@ -1552,14 +1534,6 @@ "dev": true, "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true - } } }, "@types/lodash": { @@ -1569,10 +1543,13 @@ "dev": true }, "@types/node": { - "version": "12.12.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.26.tgz", - "integrity": "sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA==", - "dev": true + "version": "18.19.53", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz", + "integrity": "sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==", + "dev": true, + "requires": { + "undici-types": "~5.26.4" + } }, "@types/rimraf": { "version": "4.0.5", @@ -1590,14 +1567,6 @@ "dev": true, "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true - } } }, "@types/yaml": { @@ -2198,6 +2167,12 @@ "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "dev": true }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "universal-user-agent": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", diff --git a/licenses/dependency-scan/src/actions/base/InstallAction.ts b/licenses/dependency-scan/src/actions/base/InstallAction.ts index f8253a3..295742f 100644 --- a/licenses/dependency-scan/src/actions/base/InstallAction.ts +++ b/licenses/dependency-scan/src/actions/base/InstallAction.ts @@ -74,11 +74,7 @@ export class InstallAction implements IAction { const bootstrapGradle = spawn.sync(`./bootstrap_gradlew.sh`, [], { cwd: absDir, env: process.env, shell: true }); this.log.logOutputSync(bootstrapGradle, projectDir, "install"); - // let gradleArgs = ["build", "-x", "test", "-x", "check"]; let gradleArgs = ["compileJava"] - /* if (this.repoRules.hasExtraGradleArgs(projectDir)) { - gradleArgs = gradleArgs.concat(this.repoRules.getExtraGradleArgs(projectDir)); - }*/ console.log(`Issuing ./gradlew build in ${absDir} with args ${gradleArgs}`); const installProcess = spawn.sync(`./gradlew`, gradleArgs, { cwd: absDir, env: process.env, shell: true }); this.log.logOutputSync(installProcess, projectDir, "install"); @@ -87,9 +83,22 @@ export class InstallAction implements IAction { if (Utilities.dirHasNodeProject(absDir)) { fs.copyFileSync("resources/private_npmrc/.npmrc", path.join(absDir, ".npmrc")); fs.copyFileSync("resources/private_npmrc/.yarnrc", path.join(absDir, ".yarnrc")); - if (fs.existsSync(path.join(absDir, "package-lock.json"))) { - fs.unlinkSync(path.join(absDir, "package-lock.json")); + + const registry =["--registry", "https://zowe.jfrog.io/zowe/api/npm/npm-release"] + //default npm install prod + let installCmd = "npm"; + let installArgs = ["install", "--omit=dev", ...registry]; + + if (Utilities.hasPnpmLockFile(`${absDir}`)) { + installCmd = "pnpm"; + installArgs = ["install", "--frozen-lockfile", "--prod", ...registry] + } else if (Utilities.hasNpmLockfile(`${absDir}`)) { + installArgs = ["ci", "--omit=dev", ...registry] + } else if (Utilities.hasYarnLockfile(`${absDir}`)) { + installCmd = "yarn"; + installArgs = ["install", "--production", "--frozen-lockfile", "--ignore-engines", ...registry] } + if (fs.existsSync(path.join(absDir, "node_modules"))) { try { rimraf.sync(path.join(absDir, "node_modules"), { maxRetries: 10 }); @@ -103,11 +112,7 @@ export class InstallAction implements IAction { // So far, there are no failures downstream due to an integrity mismatch at this step. /// -- Alternatives to skip-integrity-check are dropping network-concurrency to 1 and/or setting a mutex on yarn install. console.log("Issuing yarn install in " + absDir); - const installProcess = spawn("yarn", ["install", - ((projectDir === "zowe-explorer-vscode") ? "" : "--production"), - "--network-timeout", "300000", "--ignore-engines", - "--registry", "https://zowe.jfrog.io/zowe/api/npm/npm-release", - "--skip-integrity-check", "--network-concurrency", "5"], { cwd: absDir, env: process.env, shell: true }); + const installProcess = spawn(installCmd, installArgs, { cwd: absDir, env: process.env, shell: true }); processPromises.push(this.log.logOutputAsync(installProcess, projectDir, "install")); } if (Utilities.dirHasCargoProject(absDir)) { diff --git a/licenses/dependency-scan/src/utils/Utilities.ts b/licenses/dependency-scan/src/utils/Utilities.ts index 50c485b..4605686 100644 --- a/licenses/dependency-scan/src/utils/Utilities.ts +++ b/licenses/dependency-scan/src/utils/Utilities.ts @@ -106,4 +106,16 @@ export class Utilities { public static dirHasCargoProject(dir: string) { return fs.existsSync(path.join(dir, "Cargo.toml")); } + + public static hasPnpmLockFile(dir: string): boolean { + return fs.existsSync(`${dir}/pnpm-lock.yaml`); + } + + public static hasNpmLockfile(dir: string): boolean { + return fs.existsSync(`${dir}/package-lock.json`) || fs.existsSync(`${dir}/npm-shrinkwrap.json`); + } + + public static hasYarnLockfile(dir: string): boolean { + return fs.existsSync(`${dir}/yarn.lock`); + } } \ No newline at end of file diff --git a/licenses/dependency-scan/yarn.lock b/licenses/dependency-scan/yarn.lock index f91c271..a512e04 100644 --- a/licenses/dependency-scan/yarn.lock +++ b/licenses/dependency-scan/yarn.lock @@ -35,7 +35,7 @@ resolved "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz" integrity sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA== -"@octokit/core@^5.0.0": +"@octokit/core@^5.0.0", "@octokit/core@>=5": version "5.1.0" resolved "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz" integrity sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g== @@ -161,15 +161,10 @@ resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz" integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ== -"@types/node@*": - version "13.7.0" - resolved "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz" - integrity sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ== - -"@types/node@^18.0.2": - version "18.19.11" - resolved "https://registry.npmjs.org/@types/node/-/node-18.19.11.tgz#355cf2a28a8c2edf154b275a5715401b18fe0b63" - integrity sha512-hzdHPKpDdp5bEcRq1XTlZ2ntVjLcHCTV73dEcGg02eSY/+9AZ+jlfz6i00+zOrunMWenjHuI49J8J7Y9uz50JQ== +"@types/node@*", "@types/node@^18.0.2": + version "18.19.53" + resolved "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz" + integrity sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg== dependencies: undici-types "~5.26.4" @@ -295,16 +290,16 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" - integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= - color-name@~1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + commander@^2.12.1: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" @@ -315,6 +310,15 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= +cross-spawn@^7.0.0: + version "7.0.3" + resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + cross-spawn@6.0.5: version "6.0.5" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz" @@ -326,15 +330,6 @@ cross-spawn@6.0.5: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^7.0.0: - version "7.0.3" - resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== - dependencies: - path-key "^3.1.0" - shebang-command "^2.0.0" - which "^2.0.1" - deprecation@^2.0.0: version "2.3.1" resolved "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz" @@ -631,7 +626,16 @@ sprintf-js@~1.0.2: resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= -"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.1.0: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -649,7 +653,14 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -706,14 +717,14 @@ tsutils@^2.29.0: dependencies: tslib "^1.8.1" -typescript@5.2.2: +"typescript@>=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev", "typescript@>=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >=3.0.0-dev || >= 3.1.0-dev || >= 3.2.0-dev", typescript@5.2.2: version "5.2.2" resolved "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz" integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== undici-types@~5.26.4: version "5.26.5" - resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== universal-user-agent@^6.0.0: