Complete licenses automation in github actions (#132)

* use container runner Signed-off-by: MarkAckert <[email protected]> * update release artifact output Signed-off-by: MarkAckert <[email protected]> * chore: change container user Signed-off-by: MarkAckert <[email protected]> * chore: update user Signed-off-by: MarkAckert <[email protected]> * chore: update container to remove jenkins Signed-off-by: MarkAckert <[email protected]> * chore: remove prepare step Signed-off-by: MarkAckert <[email protected]> * chore: install glob Signed-off-by: MarkAckert <[email protected]> * chore: add glob to package Signed-off-by: MarkAckert <[email protected]> * chore: remove updateLiecnse from build Signed-off-by: MarkAckert <[email protected]> * chore: fix missing '&' Signed-off-by: MarkAckert <[email protected]> * chore: add ignore Signed-off-by: MarkAckert <[email protected]> * chore: update typescript vers Signed-off-by: MarkAckert <[email protected]> * chore: update packages, tsconfig Signed-off-by: MarkAckert <[email protected]> * chore: update cds Signed-off-by: MarkAckert <[email protected]> * chore: add missing envs to scan Signed-off-by: MarkAckert <[email protected]> * chore: update vars to publish step Signed-off-by: MarkAckert <[email protected]> * build: add prepare-workflow back Signed-off-by: MarkAckert <[email protected]> * chore: dummy license zips Signed-off-by: MarkAckert <[email protected]> * build: add overwrite Signed-off-by: MarkAckert <[email protected]> * build: support for release Signed-off-by: MarkAckert <[email protected]> * chore: fix commands, add suffix Signed-off-by: MarkAckert <[email protected]> * build: fix rt del variables Signed-off-by: MarkAckert <[email protected]> * build: update file name structure Signed-off-by: MarkAckert <[email protected]> * build: defer init of variable Signed-off-by: MarkAckert <[email protected]> * build: use bash Signed-off-by: MarkAckert <[email protected]> * chore: fix shell Signed-off-by: MarkAckert <[email protected]> * chore: just use sh syntax Signed-off-by: MarkAckert <[email protected]> * chore: GITHUB_OUTPUT should be GITHUB_ENV Signed-off-by: MarkAckert <[email protected]> * chore: remove duplicate snapshot Signed-off-by: MarkAckert <[email protected]> * chore: trigger dummy with a workflow variable Signed-off-by: MarkAckert <[email protected]> * chore: gha expression syntax Signed-off-by: MarkAckert <[email protected]> * chore: change condition again Signed-off-by: MarkAckert <[email protected]> * debug: git ls Signed-off-by: MarkAckert <[email protected]> * chore: use newer checkout Signed-off-by: MarkAckert <[email protected]> * chore: checkout is now at v4 Signed-off-by: MarkAckert <[email protected]> * debug: get more info from git Signed-off-by: MarkAckert <[email protected]> * debug: permission issue? Signed-off-by: MarkAckert <[email protected]> * debug: try adding a safe directory? Signed-off-by: MarkAckert <[email protected]> * debug: safe directory fixed it? Signed-off-by: MarkAckert <[email protected]> * build: clean up step Signed-off-by: MarkAckert <[email protected]> * build: add timestamp to snapshots Signed-off-by: MarkAckert <[email protected]> * build: shift all filename processing to FILENAME_PATTERN Signed-off-by: MarkAckert <[email protected]> * build: clarify some vars, add SNAPSHOT back in Signed-off-by: MarkAckert <[email protected]> * build: update snapshot filename_pattern Signed-off-by: MarkAckert <[email protected]> * build: add ability to remove licenses Signed-off-by: MarkAckert <[email protected]> * chore: duplicate hyphen in snapshotname Signed-off-by: MarkAckert <[email protected]> * chore: wrong field name Signed-off-by: MarkAckert <[email protected]> * build: add required url parameter Signed-off-by: MarkAckert <[email protected]> --------- Signed-off-by: MarkAckert <[email protected]>
zowe · Nov 30, 2023 · 7ba622b · 7ba622b
1 parent 2e8addd
commit 7ba622b
Show file tree

Hide file tree

Showing 5 changed files with 344 additions and 71 deletions.
diff --git a/.github/workflows/license-generation.yml b/.github/workflows/license-generation.yml
@@ -15,18 +15,32 @@ on:
         type: boolean
         required: true
         default: false
-      replace_release:
-        description: Should the license bundle overwrite an existing release in libs-release-local
+      overwrite_release:
+        description: Should the license bundle overwrite and replace an existing artifact
         type: boolean
-        required: true
+        required: false
         default: false
+      release_suffix:
+        description: Should the license bundle have a suffix (useful during RC testing)
+        type: string
+        required: false
+        default: ''
       zowe_sources_branch: 
         description: The branch of zowe-install-packaging used to determine sources included in the scan
         required: true
         default: 'v2.x/rc' 
+      dummy_build:
+        description: Creates empty zip files, bypassing license scans. For test purposes only.
+        required: false
+        type: choice
+        default: 'false'
+        options:
+          - 'true'
+          - 'false'
 
 env:
   PUBLISH_RELEASE:  ${{ github.event.inputs.publish_release }}
+  RELEASE_SUFFIX: ${{ github.event.inputs.release_suffix }}
   REPLACE_EXISTING_RELEASE:  ${{ github.event.inputs.replace_release }} 
   ZOWE_RELEASE_BRANCH: ${{ github.event.inputs.zowe_sources_branch }}
   PENDING_APPROVAL_REPORT_NAME: dependency_approval_action_aggregates.json
@@ -37,57 +51,86 @@ env:
   NOTICES_AGGREGATE_FILE: notices_aggregate.txt
   NOTICES_CLI_FILE: notices_cli.txt
   NOTICES_ZOS_FILE: notices_zos.txt
-  ARTIFACT_REPO: libs-snapshot-local
   ARTIFACT_PATH: org/zowe/licenses
   VERSION: ${{ github.event.inputs.zowe_version }}
-  ARTIFACT_VERSION: ${{ github.event.inputs.zowe_version }}-SNAPSHOT
-  AGG_ARTIFACT_NAME: zowe_licenses_full-SNAPSHOT.zip
-  CLI_ARTIFACT_NAME: zowe_licenses_cli-SNAPSHOT.zip
-  ZOS_ARTIFACT_NAME: zowe_licenses_zos-SNAPSHOT.zip
+  AGG_ARTIFACT_NAME: zowe_licenses_full.zip
+  CLI_ARTIFACT_NAME: zowe_licenses_cli.zip
+  ZOS_ARTIFACT_NAME: zowe_licenses_zos.zip
+  FILENAME_PATTERN: init_in_step_one
+  ARTIFACT_REPO: init_in_step_one
+  ARTIFACT_VERSION: init_in_step_one
 
 jobs:
 
   create-licenses:
 
     runs-on: ubuntu-latest
+
+    container:
+      image: zowe-docker-snapshot.jfrog.io/ompzowe/license-scan-build:v2x
+
     steps:
+
       - name: Update variables if releasing
         run: | 
-          if [[ "$PUBLISH_RELEASE" = true ]]; then
-              echo "ARTIFACT_REPO=libs-release-local" >> $GITHUB_OUTPUT
-              echo "ARTIFACT_VERSION=$VERSION" >> $GITHUB_OUTPUT
-              echo "AGG_ARTIFACT_NAME=zowe_licenses_full.zip" >> $GITHUB_OUTPUT
-              echo "CLI_ARTIFACT_NAME=zowe_licenses_cli.zip" >> $GITHUB_OUTPUT
-              echo "ZOS_ARTIFACT_NAME=zowe_licenses_zos.zip" >> $GITHUB_OUTPUT
+          if [ "$PUBLISH_RELEASE" = true ]; then
+              echo "ARTIFACT_REPO=libs-release-local" >> $GITHUB_ENV
+              echo "ARTIFACT_VERSION=$VERSION" >> $GITHUB_ENV
+              echo "FILENAME_PATTERN={filename}${{ env.RELEASE_SUFFIX }}{fileext}" >> $GITHUB_ENV
+          else
+              echo "ARTIFACT_REPO=libs-snapshot-local" >> $GITHUB_ENV
+              echo "ARTIFACT_VERSION=$VERSION-SNAPSHOT" >> $GITHUB_ENV
+              echo "FILENAME_PATTERN={filename}-${{ env.VERSION }}-SNAPSHOT{timestamp}{fileext}" >> $GITHUB_ENV
           fi
-      
-      - name: Create release artifact path
-        run: | 
-          echo "FULL_ARTIFACT_PATH=$ARTIFACT_REPO/$ARTIFACT_PATH/$ARTIFACT_VERSION/" >> $GITHUB_OUTPUT
-
+    
       - name: Checkout current repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
+      - name: 'Install Ansible'
+        uses: zowe-actions/shared-actions/prepare-workflow@main
+      - name: '[Zowe Actions] Prepare workflow'
+        uses: zowe-actions/shared-actions/prepare-workflow@main
 
       - name: 'Setup jFrog CLI'
         uses: jfrog/setup-jfrog-cli@v2
         env:
           JF_ENV_1: ${{ secrets.JF_ARTIFACTORY_TOKEN }}
-
-      - name: 'Prepare workflow'
-        uses: zowe-actions/shared-actions/prepare-workflow@main
 
-      - name: Download and extract ${{ matrix.ARTIFACT_NAME }}
+      - name: '[TEST-ONLY] Dummy scan step'
+        if: ${{ github.event.inputs.dummy_build == 'true' }} 
+        working-directory: ${{ env.DEPENDENCY_SCAN_HOME }}
+        run: | 
+          mkdir -p zowe_licenses
+          mkdir -p zowe_cli_licenses
+          mkdir -p zowe_zos_licenses
+          echo "HI" >> dummy.txt
+          cp dummy.txt zowe_licenses
+          cp dummy.txt zowe_cli_licenses
+          cp dummy.txt zowe_zos_licenses
+
+          zip -j ${{ env.AGG_ARTIFACT_NAME }} zowe_licenses/*
+          zip -j ${{ env.CLI_ARTIFACT_NAME }} zowe_cli_licenses/*
+          zip -j ${{ env.ZOS_ARTIFACT_NAME }} zowe_zos_licenses/*
+
+
+      - name: Scan Licenses on Branch ${{ env.ZOWE_RELEASE_BRANCH }}
+        if: ${{ github.event.inputs.dummy_build == 'false' }} 
+        env:
+          APP_NOTICES_SCAN: true
+          APP_LICENSE_SCAN: true
+          ZOWE_MANIFEST_BRANCH: ${{ env.ZOWE_RELEASE_BRANCH }}
         working-directory: ${{ env.DEPENDENCY_SCAN_HOME }}
         run: |
-          yarn install & yarn build
+          yarn install && yarn build
           node lib/index.js
-          cd build && zip -r logs.zip logs/
-          cd build && zip -r license_reports.zip license_reports/
-          cd build && zip -r notice_reports.zip notice_reports/
+          cd build
+          zip -r logs.zip logs/ 
+          zip -r license_reports.zip license_reports/
+          zip -r notice_reports.zip notice_reports/
+          cd ..
           mkdir -p zowe_licenses
           mkdir -p zowe_cli_licenses
           mkdir -p zowe_zos_licenses
@@ -117,14 +160,41 @@ jobs:
             ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}      
             ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}
 
+      - name: Remove existing artifacts
+        id: cleanup
+        if: ${{ github.event.inputs.publish_release }} && ${{ github.event.inputs.overwrite_release }}
+        run: |
+          jfrog rt del \
+            --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
+            --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
+            --url https://zowe.jfrog.io/artifactory \
+            ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.AGG_ARTIFACT_NAME }}
+          jfrog rt del \
+            --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
+            --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
+            --url https://zowe.jfrog.io/artifactory \
+            ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.CLI_ARTIFACT_NAME }}
+          jfrog rt del\
+            --user ${{ secrets.ZOWE_JFROG_ELEVATED_USER }} \
+            --password ${{secrets.ZOWE_JFROG_ELEVATED_KEY }} \
+            --url https://zowe.jfrog.io/artifactory \
+            ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/${{ env.ZOS_ARTIFACT_NAME }}
+
+      - name: '[PUBLISH] Fix local git configuration (container+runner UID mismatch)'
+        if: ${{ github.event.inputs.publish_release }}
+        id: debug-git
+        run: |
+          git config --global --add safe.directory /__w/zowe-dependency-scan-pipeline/zowe-dependency-scan-pipeline
 
       - name: Publish to Artifactory
         id: publish
         timeout-minutes: 10
         uses: zowe-actions/shared-actions/publish@main
         with:
-          publishTargetPathPattern: ${{ env.FULL_ARTIFACT_PATH }}
+          publish-target-file-pattern: ${{ env.FILENAME_PATTERN }}
+          publish-target-path-pattern: ${{ env.ARTIFACT_REPO }}/${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_VERSION }}/
+          perform-release: ${{ env.PUBLISH_RELEASE }}
           artifacts: |
             ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.AGG_ARTIFACT_NAME }}
             ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.CLI_ARTIFACT_NAME }}      
-            ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}
+            ${{ env.DEPENDENCY_SCAN_HOME }}/${{ env.ZOS_ARTIFACT_NAME }}
diff --git a/licenses/dependency-scan/package.json b/licenses/dependency-scan/package.json
@@ -4,7 +4,7 @@
   "description": "Scripts written in typescript/javascript to generate dependency attribution reports.",
   "main": "lib/index.js",
   "scripts": {
-    "build": "node scripts/updateLicense.js && tsc --pretty"
+    "build": "tsc --pretty"
   },
   "repository": {
     "type": "git",
@@ -22,7 +22,7 @@
     "fs-extra": "8.1.0",
     "inversify": "^5.0.1",
     "reflect-metadata": "^0.1.13",
-    "rimraf": "^2.6.3",
+    "rimraf": "5.0.5",
     "toml": "^3.0.0",
     "xml2js": "0.4.23"
   },
@@ -33,9 +33,9 @@
     "@types/cross-spawn": "^6.0.0",
     "@types/fs-extra": "8.0.0",
     "@types/node": "^12.0.2",
-    "@types/rimraf": "^2.0.2",
+    "@types/rimraf": "4.0.5",
     "@types/xml2js": "^0.4.5",
     "tslint": "^5.13.1",
-    "typescript": "^3.3.3333"
+    "typescript": "5.2.2"
   }
 }
diff --git a/licenses/dependency-scan/src/actions/base/InstallAction.ts b/licenses/dependency-scan/src/actions/base/InstallAction.ts
@@ -88,7 +88,7 @@ export class InstallAction implements IAction {
                 }
                 if (fs.existsSync(path.join(absDir, "node_modules"))) {
                     try {
-                        rimraf.sync(path.join(absDir, "node_modules"), { maxBusyTries: 10 });
+                        rimraf.sync(path.join(absDir, "node_modules"), { maxRetries: 10 });
                     } catch (rmErr) {
                         console.log(`Issue cleaning node_modules prior to install, will try to continue... ${rmErr}`)
                     }

diff --git a/licenses/dependency-scan/tsconfig.json b/licenses/dependency-scan/tsconfig.json
@@ -5,7 +5,8 @@
       "reflect-metadata"
     ],
     "lib": [
-      "esnext"
+      "esnext",
+      "DOM",
     ],
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
@@ -40,6 +41,7 @@
   "exclude": [
     "lib",
     "node_modules",
+    "**/node_modules/*",
     "**/__mocks__/*",
     "**/__tests__/*",
     "**/*.spec.ts",