From ea4fa7eb07a730f587e7eca7fd056b14b0e71971 Mon Sep 17 00:00:00 2001 From: Mark Ackert <35308966+MarkAckert@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:50:20 -0400 Subject: [PATCH] Support PNPM, Respect Lockfiles - rev2 (#148) * Reapply "add pnpm support, respect lockfiles" This reverts commit 2d1046bc8e8eede11e17a4621018e8d7fb6336d5. Signed-off-by: MarkAckert * Fix V3 scans for client-side & generate separate VSCode notices bundle (#149) * Generate a separate VSCode notices bundle Signed-off-by: Timothy Johnson * Add --ignore-scripts flag to Node project installs Signed-off-by: Timothy Johnson * Update path-scurry dep to fix yarn build Signed-off-by: Timothy Johnson * Install pnpm@8 in license scan workflow Signed-off-by: Timothy Johnson * Try to upgrade ORT tool Signed-off-by: Timothy Johnson * Test workflow with new image Signed-off-by: Timothy Johnson * Exclude zedc cargo project from ZE licenses Signed-off-by: Timothy Johnson * Update repoRules.json Signed-off-by: Timothy Johnson * update docker image tag Signed-off-by: MarkAckert --------- Signed-off-by: Timothy Johnson Signed-off-by: MarkAckert Co-authored-by: Timothy Johnson --------- Signed-off-by: MarkAckert Signed-off-by: Timothy Johnson Co-authored-by: Timothy Johnson --- .dockerfiles/ort.Dockerfile | 4 +- .github/workflows/license-generation.yml | 74 ++++++++++++----- licenses/dependency-scan/package-lock.json | 83 +++++++------------ .../dependency-scan/resources/repoRules.json | 7 +- .../src/actions/base/InstallAction.ts | 27 +++--- .../src/actions/ort/OrtReportAction.ts | 22 ++++- .../src/actions/ort/OrtSbomAction.ts | 3 + .../dependency-scan/src/utils/Utilities.ts | 12 +++ licenses/dependency-scan/yarn.lock | 51 +++++++----- 9 files changed, 175 insertions(+), 108 deletions(-) diff --git a/.dockerfiles/ort.Dockerfile b/.dockerfiles/ort.Dockerfile index 54a2358..24c4651 100644 --- a/.dockerfiles/ort.Dockerfile +++ b/.dockerfiles/ort.Dockerfile @@ -26,6 +26,8 @@ ENV PATH="$HOME/.cargo/bin:$PATH" RUN npm install -g yarn +RUN wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.bashrc" SHELL="$(which bash)" bash - + ENV owasp_version=5.3.2 ENV owasp_dc_download="https://github.com/jeremylong/DependencyCheck/releases/download/v${owasp_version}/" @@ -49,7 +51,7 @@ RUN rustup install stable && rustup default stable RUN cargo install cargo-license RUN cargo install get-license-helper -ARG ORT_VERSION=15.1.0 +ARG ORT_VERSION=33.1.0 RUN git clone https://github.com/oss-review-toolkit/ort WORKDIR /home/build/ort diff --git a/.github/workflows/license-generation.yml b/.github/workflows/license-generation.yml index 82a8269..1a8662e 100644 --- a/.github/workflows/license-generation.yml +++ b/.github/workflows/license-generation.yml @@ -28,10 +28,10 @@ on: type: string required: false default: '' - zowe_sources_branch: + zowe_sources_branch: description: The branch of zowe-install-packaging used to determine sources included in the scan required: true - default: 'v2.x/rc' + default: 'v2.x/rc' dummy_build: description: Creates empty zip files, bypassing license scans. For test purposes only. required: false @@ -52,26 +52,30 @@ on: - 'debug' env: - PUBLISH_RELEASE: ${{ github.event.inputs.publish_release }} + PUBLISH_RELEASE: ${{ github.event.inputs.publish_release }} RELEASE_SUFFIX: ${{ github.event.inputs.release_suffix }} - REPLACE_EXISTING_RELEASE: ${{ github.event.inputs.replace_release }} + REPLACE_EXISTING_RELEASE: ${{ github.event.inputs.replace_release }} ZOWE_RELEASE_BRANCH: ${{ github.event.inputs.zowe_sources_branch }} PENDING_APPROVAL_REPORT_NAME: dependency_approval_action_aggregates.json DEPENDENCY_SCAN_HOME: licenses/dependency-scan MARKDOWN_REPORT_NAME: markdown_dependency_report.md MARKDOWN_CLI_REPORT: cli_dependency_report.md + MARKDOWN_VSCODE_REPORT: vscode_dependency_report.md MARKDOWN_ZOS_REPORT: zos_dependency_report.md NOTICES_AGGREGATE_FILE: notices_aggregate.txt NOTICES_CLI_FILE: notices_cli.txt + NOTICES_VSCODE_FILE: notices_vscode.txt NOTICES_ZOS_FILE: notices_zos.txt ARTIFACT_PATH: org/zowe/licenses ARTIFACT_PATH_SBOM: init_in_step_one VERSION: ${{ github.event.inputs.zowe_version }} AGG_ARTIFACT_NAME: zowe_licenses_full.zip CLI_ARTIFACT_NAME: zowe_licenses_cli.zip + VSCODE_ARTIFACT_NAME: zowe_licenses_vscode.zip ZOS_ARTIFACT_NAME: zowe_licenses_zos.zip AGG_SBOM_ARTIFACT_NAME: sbom_aggregate.spdx.yml CLI_SBOM_ARTIFACT_NAME: sbom_cli.spdx.yml + VSCODE_SBOM_ARTIFACT_NAME: sbom_vscode.spdx.yml ZOS_SBOM_ARTIFACT_NAME: sbom_zos.spdx.yml FILENAME_PATTERN: init_in_step_one ARTIFACT_REPO: init_in_step_one @@ -84,13 +88,13 @@ jobs: create-licenses: runs-on: ubuntu-latest - + container: image: zowe-docker-release.jfrog.io/ompzowe/zowecicd-license-base:latest steps: - name: Update variables if releasing - run: | + run: | if [ "$PUBLISH_RELEASE" = true ]; then echo "ARTIFACT_REPO=libs-release-local" >> $GITHUB_ENV echo "ARTIFACT_VERSION=$VERSION" >> $GITHUB_ENV @@ -102,44 +106,48 @@ jobs: echo "ARTIFACT_PATH_SBOM=org/zowe/${{ env.VERSION }}-SNAPSHOT/sbom" >> $GITHUB_ENV echo "FILENAME_PATTERN={filename}-${{ env.VERSION }}-SNAPSHOT{timestamp}{fileext}" >> $GITHUB_ENV fi - + - name: Checkout current repo uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: '20' - + - name: '[Zowe Actions] Prepare workflow' uses: zowe-actions/shared-actions/prepare-workflow@main - + - name: 'Setup jFrog CLI' uses: jfrog/setup-jfrog-cli@v4 env: JF_ENV_1: ${{ secrets.JF_ARTIFACTORY_TOKEN }} - name: '[TEST-ONLY] Dummy scan step' - if: ${{ github.event.inputs.dummy_build == 'true' }} + if: ${{ github.event.inputs.dummy_build == 'true' }} working-directory: ${{ env.DEPENDENCY_SCAN_HOME }} - run: | + run: | mkdir -p zowe_licenses mkdir -p zowe_cli_licenses + mkdir -p zowe_vscode_licenses mkdir -p zowe_zos_licenses echo "HI" >> dummy.txt cp dummy.txt zowe_licenses cp dummy.txt zowe_cli_licenses + cp dummy.txt zowe_vscode_licenses cp dummy.txt zowe_zos_licenses zip -j ${{ env.AGG_ARTIFACT_NAME }} zowe_licenses/* zip -j ${{ env.CLI_ARTIFACT_NAME }} zowe_cli_licenses/* + zip -j ${{ env.VSCODE_ARTIFACT_NAME }} zowe_vscode_licenses/* zip -j ${{ env.ZOS_ARTIFACT_NAME }} zowe_zos_licenses/* echo "" > ${{ env.AGG_SBOM_ARTIFACT_NAME }} - echo "" > ${{ env.ZOS_SBOM_ARTIFACT_NAME }} echo "" > ${{ env.CLI_SBOM_ARTIFACT_NAME }} + echo "" > ${{ env.VSCODE_SBOM_ARTIFACT_NAME }} + echo "" > ${{ env.ZOS_SBOM_ARTIFACT_NAME }} - name: Scan Licenses on Branch ${{ env.ZOWE_RELEASE_BRANCH }} - if: ${{ github.event.inputs.dummy_build == 'false' }} + if: ${{ github.event.inputs.dummy_build == 'false' }} env: APP_NOTICES_SCAN: true APP_LICENSE_SCAN: true @@ -147,23 +155,26 @@ jobs: working-directory: ${{ env.DEPENDENCY_SCAN_HOME }} run: | # Rustup is set to default in the container, but it's not picked up in this run block - rustup default stable + rustup default stable + npm install -g pnpm@8 yarn install && yarn build node lib/index.js cd build - zip -r logs.zip logs/ + zip -r logs.zip logs/ zip -r license_reports.zip license_reports/ zip -r notice_reports.zip notice_reports/ cd .. mkdir -p zowe_licenses mkdir -p zowe_cli_licenses + mkdir -p zowe_vscode_licenses mkdir -p zowe_zos_licenses cp ../resources/* zowe_licenses/ cp ../resources/* zowe_cli_licenses/ + cp ../resources/* zowe_vscode_licenses/ cp ../resources/* zowe_zos_licenses/ zip -r logs.zip build/logs/* - + # Aggregate cp build/notice_reports/${{ env.NOTICES_AGGREGATE_FILE }} zowe_licenses/zowe_full_notices.txt cp build/license_reports/${{ env.MARKDOWN_REPORT_NAME }} zowe_licenses/zowe_full_dependency_list.md @@ -173,6 +184,12 @@ jobs: cp build/notice_reports/${{ env.NOTICES_CLI_FILE }} zowe_cli_licenses/zowe_cli_notices.txt cp build/license_reports/${{ env.MARKDOWN_CLI_REPORT }} zowe_cli_licenses/zowe_cli_dependency_list.md zip -j ${{ env.CLI_ARTIFACT_NAME }} zowe_cli_licenses/* + + # VSCode + cp build/notice_reports/${{ env.NOTICES_VSCODE_FILE }} zowe_vscode_licenses/zowe_vscode_notices.txt + cp build/license_reports/${{ env.MARKDOWN_VSCODE_REPORT }} zowe_vscode_licenses/zowe_vscode_dependency_list.md + zip -j ${{ env.VSCODE_ARTIFACT_NAME }} zowe_vscode_licenses/* + # z/OS cp build/notice_reports/${{ env.NOTICES_ZOS_FILE }} zowe_zos_licenses/zowe_zos_notices.txt cp build/license_reports/${{ env.MARKDOWN_ZOS_REPORT }} zowe_zos_licenses/zowe_zos_dependency_list.md @@ -181,6 +198,7 @@ jobs: # SBOMs cp build/sbom_reports/${{ env.AGG_SBOM_ARTIFACT_NAME }} ${{ env.AGG_SBOM_ARTIFACT_NAME }} cp build/sbom_reports/${{ env.CLI_SBOM_ARTIFACT_NAME }} ${{ env.CLI_SBOM_ARTIFACT_NAME }} + cp build/sbom_reports/${{ env.VSCODE_SBOM_ARTIFACT_NAME }} ${{ env.VSCODE_SBOM_ARTIFACT_NAME }} cp build/sbom_reports/${{ env.ZOS_SBOM_ARTIFACT_NAME }} ${{ env.ZOS_SBOM_ARTIFACT_NAME }} @@ -199,6 +217,11 @@ jobs: --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \ --url https://zowe.jfrog.io/artifactory \ ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.CLI_ARTIFACT_NAME }} + jfrog rt del\ + --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \ + --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \ + --url https://zowe.jfrog.io/artifactory \ + ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.VSCODE_ARTIFACT_NAME }} jfrog rt del\ --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \ --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \ @@ -214,6 +237,11 @@ jobs: --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \ --url https://zowe.jfrog.io/artifactory \ ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.CLI_SBOM_ARTIFACT_NAME }} + jfrog rt del\ + --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \ + --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \ + --url https://zowe.jfrog.io/artifactory \ + ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }} jfrog rt del\ --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \ --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \ @@ -237,7 +265,8 @@ jobs: sigstore-sign-artifacts: true artifacts: | ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }} - ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }} + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }} + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }} - name: Publish to Artifactory @@ -252,9 +281,10 @@ jobs: artifacts: | ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_SBOM_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_SBOM_ARTIFACT_NAME }} + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_SBOM_ARTIFACT_NAME }} - - name: Archive Aggregates + - name: Archive Aggregates uses: actions/upload-artifact@v4 if: ${{ always() }} with: @@ -262,13 +292,17 @@ jobs: ${{ env.DEPENDENCY_SCAN_HOME }}/logs.zip ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }}.bundle - ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }} - ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}.bundle + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }} + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}.bundle + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_ARTIFACT_NAME }} + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_ARTIFACT_NAME }}.bundle ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}.bundle ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_SBOM_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_SBOM_ARTIFACT_NAME }}.bundle ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_SBOM_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_SBOM_ARTIFACT_NAME }}.bundle + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }} + ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.VSCODE_SBOM_ARTIFACT_NAME }}.bundle ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_SBOM_ARTIFACT_NAME }} ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_SBOM_ARTIFACT_NAME }}.bundle diff --git a/licenses/dependency-scan/package-lock.json b/licenses/dependency-scan/package-lock.json index 66fabc0..9f88c56 100644 --- a/licenses/dependency-scan/package-lock.json +++ b/licenses/dependency-scan/package-lock.json @@ -26,13 +26,16 @@ "@types/cross-spawn": "^6.0.0", "@types/fs-extra": "8.0.0", "@types/lodash": "^4.14.202", - "@types/node": "^12.0.2", + "@types/node": "^18.0.2", "@types/rimraf": "4.0.5", "@types/xml2js": "^0.4.5", "@types/yaml": "^1.9.7", "lodash": "^4.17.21", "tslint": "^5.13.1", "typescript": "5.2.2" + }, + "engines": { + "node": ">18.0.0" } }, "node_modules/@babel/code-frame": { @@ -266,13 +269,6 @@ "@types/node": "*" } }, - "node_modules/@types/cross-spawn/node_modules/@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/fs-extra": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.0.0.tgz", @@ -283,13 +279,6 @@ "@types/node": "*" } }, - "node_modules/@types/fs-extra/node_modules/@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/lodash": { "version": "4.14.202", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz", @@ -297,11 +286,13 @@ "dev": true }, "node_modules/@types/node": { - "version": "12.12.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.26.tgz", - "integrity": "sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA==", + "version": "18.19.53", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz", + "integrity": "sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==", "dev": true, - "license": "MIT" + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/rimraf": { "version": "4.0.5", @@ -324,13 +315,6 @@ "@types/node": "*" } }, - "node_modules/@types/xml2js/node_modules/@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/yaml": { "version": "1.9.7", "resolved": "https://registry.npmjs.org/@types/yaml/-/yaml-1.9.7.tgz", @@ -1188,6 +1172,12 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "node_modules/universal-user-agent": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", @@ -1535,14 +1525,6 @@ "dev": true, "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true - } } }, "@types/fs-extra": { @@ -1552,14 +1534,6 @@ "dev": true, "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true - } } }, "@types/lodash": { @@ -1569,10 +1543,13 @@ "dev": true }, "@types/node": { - "version": "12.12.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.26.tgz", - "integrity": "sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA==", - "dev": true + "version": "18.19.53", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz", + "integrity": "sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg==", + "dev": true, + "requires": { + "undici-types": "~5.26.4" + } }, "@types/rimraf": { "version": "4.0.5", @@ -1590,14 +1567,6 @@ "dev": true, "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "13.7.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", - "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", - "dev": true - } } }, "@types/yaml": { @@ -2198,6 +2167,12 @@ "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "dev": true }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "universal-user-agent": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", diff --git a/licenses/dependency-scan/resources/repoRules.json b/licenses/dependency-scan/resources/repoRules.json index ca9b3dc..f4a404a 100644 --- a/licenses/dependency-scan/resources/repoRules.json +++ b/licenses/dependency-scan/resources/repoRules.json @@ -57,8 +57,13 @@ "pattern": "**/samples/**", "reason": "EXAMPLE_OF", "comment": "Sample APIs" + }, + { + "pattern": "**/zedc/**", + "reason": "TEST_TOOL_OF", + "comment": "Development CLI" } ] } } -} \ No newline at end of file +} diff --git a/licenses/dependency-scan/src/actions/base/InstallAction.ts b/licenses/dependency-scan/src/actions/base/InstallAction.ts index f8253a3..6ae6c51 100644 --- a/licenses/dependency-scan/src/actions/base/InstallAction.ts +++ b/licenses/dependency-scan/src/actions/base/InstallAction.ts @@ -74,11 +74,7 @@ export class InstallAction implements IAction { const bootstrapGradle = spawn.sync(`./bootstrap_gradlew.sh`, [], { cwd: absDir, env: process.env, shell: true }); this.log.logOutputSync(bootstrapGradle, projectDir, "install"); - // let gradleArgs = ["build", "-x", "test", "-x", "check"]; let gradleArgs = ["compileJava"] - /* if (this.repoRules.hasExtraGradleArgs(projectDir)) { - gradleArgs = gradleArgs.concat(this.repoRules.getExtraGradleArgs(projectDir)); - }*/ console.log(`Issuing ./gradlew build in ${absDir} with args ${gradleArgs}`); const installProcess = spawn.sync(`./gradlew`, gradleArgs, { cwd: absDir, env: process.env, shell: true }); this.log.logOutputSync(installProcess, projectDir, "install"); @@ -87,9 +83,22 @@ export class InstallAction implements IAction { if (Utilities.dirHasNodeProject(absDir)) { fs.copyFileSync("resources/private_npmrc/.npmrc", path.join(absDir, ".npmrc")); fs.copyFileSync("resources/private_npmrc/.yarnrc", path.join(absDir, ".yarnrc")); - if (fs.existsSync(path.join(absDir, "package-lock.json"))) { - fs.unlinkSync(path.join(absDir, "package-lock.json")); + + const registry =["--registry", "https://zowe.jfrog.io/zowe/api/npm/npm-release"] + //default npm install prod + let installCmd = "npm"; + let installArgs = ["install", "--omit=dev", ...registry]; + + if (Utilities.hasPnpmLockFile(`${absDir}`)) { + installCmd = "pnpm"; + installArgs = ["install", "--frozen-lockfile", "--prod", "--ignore-scripts", ...registry] + } else if (Utilities.hasNpmLockfile(`${absDir}`)) { + installArgs = ["ci", "--omit=dev", "--ignore-scripts", ...registry] + } else if (Utilities.hasYarnLockfile(`${absDir}`)) { + installCmd = "yarn"; + installArgs = ["install", "--production", "--frozen-lockfile", "--ignore-engines", "--ignore-scripts", ...registry] } + if (fs.existsSync(path.join(absDir, "node_modules"))) { try { rimraf.sync(path.join(absDir, "node_modules"), { maxRetries: 10 }); @@ -103,11 +112,7 @@ export class InstallAction implements IAction { // So far, there are no failures downstream due to an integrity mismatch at this step. /// -- Alternatives to skip-integrity-check are dropping network-concurrency to 1 and/or setting a mutex on yarn install. console.log("Issuing yarn install in " + absDir); - const installProcess = spawn("yarn", ["install", - ((projectDir === "zowe-explorer-vscode") ? "" : "--production"), - "--network-timeout", "300000", "--ignore-engines", - "--registry", "https://zowe.jfrog.io/zowe/api/npm/npm-release", - "--skip-integrity-check", "--network-concurrency", "5"], { cwd: absDir, env: process.env, shell: true }); + const installProcess = spawn(installCmd, installArgs, { cwd: absDir, env: process.env, shell: true }); processPromises.push(this.log.logOutputAsync(installProcess, projectDir, "install")); } if (Utilities.dirHasCargoProject(absDir)) { diff --git a/licenses/dependency-scan/src/actions/ort/OrtReportAction.ts b/licenses/dependency-scan/src/actions/ort/OrtReportAction.ts index 2f6318f..1fee149 100644 --- a/licenses/dependency-scan/src/actions/ort/OrtReportAction.ts +++ b/licenses/dependency-scan/src/actions/ort/OrtReportAction.ts @@ -35,6 +35,7 @@ export class OrtReportAction implements IAction { private readonly AGG_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "markdown_dependency_report.md"); private readonly CLI_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "cli_dependency_report.md") + private readonly VSCODE_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "vscode_dependency_report.md") private readonly ZOS_REPORT_MARKDOWN_FILE = path.resolve(Constants.LICENSE_REPORTS_DIR, "zos_dependency_report.md") private reportQueue: async.AsyncQueue = async.queue(this.reportProject.bind(this), Constants.PARALLEL_REPORT_COUNT); @@ -91,6 +92,7 @@ export class OrtReportAction implements IAction { const sourceDependencies: ZoweManifestSourceDependency[] = this.zoweManifest.sourceDependencies; const aggregateNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_aggregate.txt"); const cliNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_cli.txt"); + const vscodeNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_vscode.txt"); const zosNoticesFile = path.join(Constants.NOTICE_REPORTS_DIR, "notices_zos.txt"); (sourceDependencies).forEach((dependency: ZoweManifestSourceDependency) => { @@ -106,6 +108,8 @@ export class OrtReportAction implements IAction { fs.appendFileSync(aggregateNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n"); if (noticeInstance.destinations.join(",").includes("CLI")) { fs.appendFileSync(cliNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n"); + } else if (noticeInstance.destinations.join(",").includes("Visual Studio Code")) { + fs.appendFileSync(vscodeNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n"); } else { fs.appendFileSync(zosNoticesFile, fs.readFileSync(noticesTxtFile).toString() + "\n"); } @@ -125,9 +129,11 @@ export class OrtReportAction implements IAction { const aggregateReportFile = fs.createWriteStream(this.AGG_REPORT_MARKDOWN_FILE, { flags: "a" }); const cliReportFile = fs.createWriteStream(this.CLI_REPORT_MARKDOWN_FILE) + const vscodeReportFile = fs.createWriteStream(this.VSCODE_REPORT_MARKDOWN_FILE) const zosReportFile = fs.createWriteStream(this.ZOS_REPORT_MARKDOWN_FILE) aggregateReportFile.write("# Zowe Third Party Library Usage\n\n"); cliReportFile.write("# Zowe CLI Third Party Library Usage\n\n"); + vscodeReportFile.write("# Zowe Explorer for VS Code Third Party Library Usage\n\n"); zosReportFile.write("# Zowe z/OS Third Party Library Usage\n\n"); (sourceDependencies).forEach((dependency) => { aggregateReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase() @@ -136,6 +142,9 @@ export class OrtReportAction implements IAction { if (dependency.entries[0].destinations.join(",").includes("CLI")) { cliReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase() + "-dependency-attributions)" + "\n"); + } else if (dependency.entries[0].destinations.join(",").includes("Visual Studio Code")) { + vscodeReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase() + + "-dependency-attributions)" + "\n"); } else { zosReportFile.write("* [" + dependency.componentGroup + "](#" + dependency.componentGroup.replace(/\s/g, "-").toLowerCase() + "-dependency-attributions)" + "\n"); @@ -143,8 +152,9 @@ export class OrtReportAction implements IAction { } }); aggregateReportFile.write("\n"); - zosReportFile.write("\n"); cliReportFile.write("\n"); + vscodeReportFile.write("\n"); + zosReportFile.write("\n"); (sourceDependencies).forEach((dependency: ZoweManifestSourceDependency) => { const reports: ReportInfo[] = (dependency.entries.map((depEntry): ReportInfo => { @@ -157,10 +167,12 @@ export class OrtReportAction implements IAction { let totalDepCt = 0; let cliDepCt = 0; + let vscodeDepCt = 0; let zosDepCt = 0; let missingReport: boolean = false; let fullReportString = ""; let cliReportString = fullReportString + let vscodeReportString = fullReportString let zosReportString = fullReportString reports.forEach((reportInstance: ReportInfo) => { try { @@ -176,6 +188,10 @@ export class OrtReportAction implements IAction { cliDepCt += reportDepCt cliReportString += `### ${dependency.componentGroup} Dependency Attributions\n` cliReportString += lines.join("\n"); + } else if (reportInstance.destinations.join(",").includes("Visual Studio Code")) { + vscodeDepCt += reportDepCt + vscodeReportString += `### ${dependency.componentGroup} Dependency Attributions\n` + vscodeReportString += lines.join("\n"); } else { zosDepCt += reportDepCt zosReportString += `### ${dependency.componentGroup} Dependency Attributions\n` @@ -196,6 +212,10 @@ export class OrtReportAction implements IAction { cliReportFile.write(cliReportString); cliReportFile.write("\n\n") } + if (vscodeDepCt > 0) { + vscodeReportFile.write(vscodeReportString); + vscodeReportFile.write("\n\n") + } if (zosDepCt > 0) { zosReportFile.write(zosReportString); zosReportFile.write("\n\n") diff --git a/licenses/dependency-scan/src/actions/ort/OrtSbomAction.ts b/licenses/dependency-scan/src/actions/ort/OrtSbomAction.ts index a724c0b..3af2954 100644 --- a/licenses/dependency-scan/src/actions/ort/OrtSbomAction.ts +++ b/licenses/dependency-scan/src/actions/ort/OrtSbomAction.ts @@ -34,6 +34,7 @@ export class OrtSbomAction implements IAction { private readonly SBOM_ZOS_REPORT = path.resolve(Constants.SBOM_REPORTS_DIR, "sbom_zos.spdx.yml"); private readonly SBOM_CLI_REPORT = path.resolve(Constants.SBOM_REPORTS_DIR, "sbom_cli.spdx.yml"); + private readonly SBOM_VSCODE_REPORT = path.resolve(Constants.SBOM_REPORTS_DIR, "sbom_vscode.spdx.yml"); private readonly SBOM_AGG_REPORT = path.resolve(Constants.SBOM_REPORTS_DIR, "sbom_aggregate.spdx.yml"); private sbomQueue: async.AsyncQueue = async.queue(this.reportSboms.bind(this), Constants.PARALLEL_NOTICE_REPORT_COUNT); @@ -85,6 +86,8 @@ export class OrtSbomAction implements IAction { fs.appendFileSync(this.SBOM_AGG_REPORT, fs.readFileSync(sbomFile).toString()); if (sbomReport.destinations.join(",").includes("CLI")) { fs.appendFileSync(this.SBOM_CLI_REPORT, fs.readFileSync(sbomFile).toString()); + } else if (sbomReport.destinations.join(",").includes("Visual Studio Code")) { + fs.appendFileSync(this.SBOM_VSCODE_REPORT, fs.readFileSync(sbomFile).toString()); } else { fs.appendFileSync(this.SBOM_ZOS_REPORT, fs.readFileSync(sbomFile).toString()); } diff --git a/licenses/dependency-scan/src/utils/Utilities.ts b/licenses/dependency-scan/src/utils/Utilities.ts index 50c485b..4605686 100644 --- a/licenses/dependency-scan/src/utils/Utilities.ts +++ b/licenses/dependency-scan/src/utils/Utilities.ts @@ -106,4 +106,16 @@ export class Utilities { public static dirHasCargoProject(dir: string) { return fs.existsSync(path.join(dir, "Cargo.toml")); } + + public static hasPnpmLockFile(dir: string): boolean { + return fs.existsSync(`${dir}/pnpm-lock.yaml`); + } + + public static hasNpmLockfile(dir: string): boolean { + return fs.existsSync(`${dir}/package-lock.json`) || fs.existsSync(`${dir}/npm-shrinkwrap.json`); + } + + public static hasYarnLockfile(dir: string): boolean { + return fs.existsSync(`${dir}/yarn.lock`); + } } \ No newline at end of file diff --git a/licenses/dependency-scan/yarn.lock b/licenses/dependency-scan/yarn.lock index f91c271..1a26567 100644 --- a/licenses/dependency-scan/yarn.lock +++ b/licenses/dependency-scan/yarn.lock @@ -161,15 +161,10 @@ resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz" integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ== -"@types/node@*": - version "13.7.0" - resolved "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz" - integrity sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ== - -"@types/node@^18.0.2": - version "18.19.11" - resolved "https://registry.npmjs.org/@types/node/-/node-18.19.11.tgz#355cf2a28a8c2edf154b275a5715401b18fe0b63" - integrity sha512-hzdHPKpDdp5bEcRq1XTlZ2ntVjLcHCTV73dEcGg02eSY/+9AZ+jlfz6i00+zOrunMWenjHuI49J8J7Y9uz50JQ== +"@types/node@*", "@types/node@^18.0.2": + version "18.19.53" + resolved "https://registry.npmjs.org/@types/node/-/node-18.19.53.tgz" + integrity sha512-GLxgUgHhDKO1Edw9Q0lvMbiO/IQXJwJlMaqxSGBXMpPy8uhkCs2iiPFaB2Q/gmobnFkckD3rqTBMVjXdwq+nKg== dependencies: undici-types "~5.26.4" @@ -492,10 +487,10 @@ lodash@^4.17.14, lodash@^4.17.21: resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -"lru-cache@^9.1.1 || ^10.0.0": - version "10.0.1" - resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz" - integrity sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g== +lru-cache@^10.2.0: + version "10.4.3" + resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" + integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== minimatch@^3.0.4: version "3.1.2" @@ -561,11 +556,11 @@ path-parse@^1.0.6: integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-scurry@^1.10.1: - version "1.10.1" - resolved "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz" - integrity sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ== + version "1.11.1" + resolved "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== dependencies: - lru-cache "^9.1.1 || ^10.0.0" + lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" reflect-metadata@^0.1.13: @@ -631,7 +626,16 @@ sprintf-js@~1.0.2: resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= -"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.1.0: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -649,7 +653,14 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -713,7 +724,7 @@ typescript@5.2.2: undici-types@~5.26.4: version "5.26.5" - resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== universal-user-agent@^6.0.0: