Merge branch 'v2.x/staging' into user/markackert/2x-to-3x-sync

Signed-off-by: MarkAckert <[email protected]>

.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: weekly
+    groups:
+      all-version-dependencies:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "patch"
+          - "minor"
+      all-security-minorpatch:
+        applies-to: security-updates  
+        patterns:
+          - "*"
+        update-types:
+          - "patch"
+          - "minor"

.github/scripts/cicd_test/make_matrix.sh

@@ -37,6 +37,10 @@ case $install_test_choice in
   test_force_system="zzow08"
   ;;
 
+"z/OS node v20")
+  test_file="$ZOS_NODE_V20_TESTFILE"
+  ;;
+
 "Non-strict Verify External Certificate")
   test_file="$NON_STRICT_VERIFY_EXTERNAL_CERTIFICATE_TESTFILE"
   ;;

.github/workflows/cicd-test-readme.md

@@ -109,7 +109,7 @@ Background: CICD testing relies on a `zowe.pax` or `zowe-smpe.zip` (for SMPE ins
 When running CICD integration tests during RC stage, the following string will be parsed into the Github Actions matrix. As a result, a total of 21 independent jobs will be spawned.
 
 ```
-basic/install.ts(zzow06,zzow07,zzow08);basic/install-ptf.ts(zzow06,zzow07,zzow08);basic/install-ext.ts(zzow07);extended/keyring.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v18.ts(zzow06,zzow07,zzow08);extended/certificates/nonstrict-verify-external-certificate.ts(zzow06)
+basic/install.ts(zzow06,zzow07,zzow08);basic/install-ptf.ts(zzow06,zzow07,zzow08);basic/install-ext.ts(zzow07);extended/keyring.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v18.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v20.ts(zzow06,zzow07,zzow08);extended/certificates/nonstrict-verify-external-certificate.ts(zzow06)
 ```
 
 Total elapsed time when running in parallel is approximately 3.5 hours on paper idealy if all parallel jobs are executing at the same time. In reality, from numerous tests performed, total elapsed time is around 4 hours.  
@@ -122,6 +122,7 @@ Selected test running elapsed time:
 | Convenience Pax | 53m |
 | SMPE PTF | 68m |
 | z/OS node v18 | 45m |
+| z/OS node v20 | 45m |
 | Keyring | 53m |
 | Non-strict Verify External Certificate | 51m |
 | Extensions | 67m

.github/workflows/cicd-test.yml

@@ -25,6 +25,7 @@ on:
           - Extensions
           - Keyring
           - z/OS node v18
+          - z/OS node v20
           - Non-strict Verify External Certificate
           - Install PTF Twice
           - VSAM Caching Storage Method
@@ -74,6 +75,7 @@ env:
   EXTENSIONS_TESTFILE: basic/install-ext.ts
   KEYRING_TESTFILE: extended/keyring.ts
   ZOS_NODE_V18_TESTFILE: extended/node-versions/node-v18.ts
+  ZOS_NODE_V20_TESTFILE: extended/node-versions/node-v20.ts
   NON_STRICT_VERIFY_EXTERNAL_CERTIFICATE_TESTFILE: extended/certificates/nonstrict-verify-external-certificate.ts
   INSTALL_PTF_TWICE_TESTFILE: extended/install-ptf-two-times.ts
   VSAM_CACHING_STORAGE_METHOD_TESTFILE: extended/caching-storages/vsam-storage.ts
@@ -292,6 +294,9 @@ jobs:
               case 'z/OS node v18':
                 eta = 45
                 break;
+              case 'z/OS node v20':
+                eta = 45
+                break;
               case 'Infinispan Caching Storage Method':
                 eta = 51
                 break;

.github/workflows/diff-schema.yml

@@ -0,0 +1,41 @@
+name: Generate schema diff
+on:
+  push:
+    branches:
+      - v1.x/master
+      - v2.x/master
+      - v3.x/master
+
+  workflow_dispatch:
+    inputs:
+      FROM_COMMIT:
+        description: 'Specify tag (v prefix) to diff from'
+        required: false
+      TO_COMMIT:
+        description: 'Specify tag (v prefix) to diff to'
+        required: false
+
+jobs:
+  diff:
+    runs-on: ubuntu-latest
+    steps:
+      - name: '[Prep 1] Checkout'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: '[Prep 2] Setup Node'
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18
+      - name: '[Prep 3] Get to version'
+        run: node -e "const fs = require('fs'); const currentVersion = JSON.parse(fs.readFileSync('./manifest.json.template', 'utf8')).version; if ('${{ github.event.inputs.TO_COMMIT }}'.length > 0) { console.log('TO=${{ github.event.inputs.TO_COMMIT }}')} else { console.log('TO=v'+currentVersion) }" >> $GITHUB_ENV
+      - name: '[Prep 4] Get from version'
+        run: node -e "if ('${{ github.event.inputs.FROM_COMMIT }}'.length > 0) { console.log('FROM=${{ github.event.inputs.FROM_COMMIT }}') } else { let parts = '${{ env.TO }}'.split('.'); parts[1]--; console.log('FROM='+parts.join('.')); }" >> $GITHUB_ENV
+      - name: '[Build] Make diff'
+        run: git diff ${{ env.FROM }} ${{ env.TO }} -- schemas > schemas.diff
+      - name: '[Upload]'
+        uses: actions/upload-artifact@v3
+        with:
+          name: schemas.diff
+          path: schemas.diff
+          if-no-files-found: error

.github/workflows/diff-yaml

@@ -0,0 +1,41 @@
+name: Generate example yaml diff
+on:
+  push:
+    branches:
+      - v1.x/master
+      - v2.x/master
+      - v3.x/master
+
+  workflow_dispatch:
+    inputs:
+      FROM_COMMIT:
+        description: 'Specify tag (v prefix) to diff from'
+        required: false
+      TO_COMMIT:
+        description: 'Specify tag (v prefix) to diff to'
+        required: false
+
+jobs:
+  diff:
+    runs-on: ubuntu-latest
+    steps:
+      - name: '[Prep 1] Checkout'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: '[Prep 2] Setup Node'
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18
+      - name: '[Prep 3] Get to version'
+        run: node -e "const fs = require('fs'); const currentVersion = JSON.parse(fs.readFileSync('./manifest.json.template', 'utf8')).version; if ('${{ github.event.inputs.TO_COMMIT }}'.length > 0) { console.log('TO=${{ github.event.inputs.TO_COMMIT }}')} else { console.log('TO=v'+currentVersion) }" >> $GITHUB_ENV
+      - name: '[Prep 4] Get from version'
+        run: node -e "if ('${{ github.event.inputs.FROM_COMMIT }}'.length > 0) { console.log('FROM=${{ github.event.inputs.FROM_COMMIT }}') } else { let parts = '${{ env.TO }}'.split('.'); parts[1]--; console.log('FROM='+parts.join('.')); }" >> $GITHUB_ENV
+      - name: '[Build] Make diff'
+        run: git diff ${{ env.FROM }} ${{ env.TO }} -- example-zowe.yaml > example-yaml.diff
+      - name: '[Upload]'
+        uses: actions/upload-artifact@v3
+        with:
+          name: example-yaml.diff
+          path: example-yaml.diff
+          if-no-files-found: error

.github/workflows/snyk-analysis.yml

@@ -55,5 +55,5 @@ jobs:
       uses: snyk/actions/docker@master
       # we don't have original Dockerfile, cannot generate and upload sarif 
       with:
-        image: ${{ matrix.image }}
+        image: ${{ matrix.images }}
         command: test

.gitignore

@@ -46,7 +46,7 @@ manifest.json
 artifactory-download-spec.json
 
 # zowe jobs output
-output/
+./output/
 
 # release temporary folder
 .release/

CHANGELOG.md

@@ -10,7 +10,13 @@ All notable changes to the Zowe Installer will be documented in this file.
 
 <!--Add the PR or issue number to the entry if available.-->
 
-## `2.16.0
+## `2.17.0`
+
+## New features and enhancements
+- Enhancement: `zwe components install` and `zwe components upgrade` now checks the schema validity of any application framework plugins present within a component, so that you will know better if a component is valid prior to running Zowe. [#3866](https://github.com/zowe/zowe-install-packaging/pull/3866) 
+- Enhancement: Added zowe.network.server.tls.attls and zowe.network.client.tls.attls as booleans for controlling global or per-component way to tell Zowe servers that they should operate in a mode compatible with an AT-TLS setup. [#3463](https://github.com/zowe/zowe-install-packaging/pull/3463)
+
+## `2.16.0`
 
 ## Minor enhancements/defect fixes
 - Enhancement: Reduced resource consumption by removal of one shell process per server that was used when starting each server.  (#3812)

bin/commands/components/install/extract/index.ts

@@ -150,6 +150,20 @@ export function execute(componentFile: string, autoEncoding?: string, upgrade?:
     common.printErrorAndExit(`Error ZWEL0167E: Cannot find component name from ${componentFile} package manifest`, undefined, 167);
   }
   common.printDebug(`- Component name found as ${componentName}`);
+
+  // If the component has appfw plugins, their validity should be checked against appfw plugin schema.
+  //   If invalid, the installation will exit with an error message.
+  if (manifest.appfwPlugins) {
+    manifest.appfwPlugins.forEach((appfwPlugin: {path: string})=> {
+      let result = component.getPluginDefinition(pathoid.resolve(tmpDir, appfwPlugin.path), true);
+      //Normally, getPluginDefinition would quit upon failure. But we want to cleanup the tmpDir before that.
+      // So, we pass true to allow it to continue, check for null, and then remove the tmpdir and exit if so.
+      if (result === null) {
+        fs.rmrf(tmpDir);
+        std.exit(1);
+      }
+    });
+  }
 
   const destinationDir = pathoid.resolve(targetDir, componentName);
   const bkpDir = pathoid.resolve(targetDir, `${componentName}_zwebkp`);

bin/commands/init/apfauth/.help

@@ -4,7 +4,7 @@ NOTE: You require proper permission to run APF authorize command.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:

bin/commands/init/certificate/.help

@@ -7,7 +7,7 @@ be written back to your Zowe YAML configuration file:
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:
@@ -148,12 +148,12 @@ zOSMF:
     Zowe the label of existing certificate.
 - If `zowe.verifyCertificates` is not `DISABLED`, and z/OSMF host (`zOSMF.host`)
   is provided, Zowe will try to trust z/OSMF certificate.
-  * If you are using `RACF` security manager, Zowe will try to automatically
-    detect the z/OSMF CA based on certificate owner specified by
+  * If you are using `RACF` or `TSS` security manager, Zowe will try to
+    automatically detect the z/OSMF CA based on certificate owner specified by
     `zowe.setup.certificate.keyring.zOSMF.user`. Default value of this field is
     `IZUSVR`. If the automatic detection failed, you will need to define
     `zowe.setup.certificate.keyring.zOSMF.ca` indicates what is the label of
     z/OSMF root certificate authority.
-  * If you are using `ACF2` or `TSS` (Top Secret) security manager,
+  * If you are using `ACF2` security manager,
     `zowe.setup.certificate.keyring.zOSMF.ca` is required to indicates what is
     the label of z/OSMF root certificate authority.

bin/commands/init/mvs/.help

@@ -2,7 +2,7 @@ This command will prepare Zowe custom data sets.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:

bin/commands/init/security/.help

@@ -4,7 +4,7 @@ NOTE: You require proper permission to run security configuration.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:

bin/commands/init/stc/.help

@@ -5,7 +5,7 @@ NOTE: You require proper permission to write to target procedure library.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:

bin/commands/init/vsam/.help

@@ -1,9 +1,9 @@
-This command will run ZWECSVSM jcl to create VSAM data set for Zowe APIML
+This command will run ZWECSVSM JCL to create VSAM data set for Zowe APIML
 Caching Service.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:
@@ -21,9 +21,9 @@ components:
         name: IBMUSER.ZWE.CUST.CACHE2
 ```
 
-- `zowe.setup.dataset.prefix` shows where the `SZWESAMP` data set is installed,
-- `zowe.setup.dataset.jcllib` is the custom JCL library. Zowe will create customized
-  ZWESECUR JCL here before applying it.
+- `zowe.setup.dataset.prefix` shows where the `SZWESAMP` data set is installed.
+- `zowe.setup.dataset.jcllib` is the custom JCL library. Zowe server command may
+  generate sample JCLs and put into this data set.
 - `zowe.setup.vsam.mode` indicates whether the VSAM will utilize Record Level
   Sharing (RLS) services or not. Valid value is `RLS` or `NONRLS`.
 - `zowe.setup.vsam.volume` indicates the name of volume.

bin/commands/install/.help

@@ -6,7 +6,7 @@ already prepared during SMPE install.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     dataset:
@@ -19,3 +19,4 @@ Expected outputs:
   * `SZWEAUTH` contains few Zowe load modules (++PROGRAM).
   * `SZWESAMP` contains several sample configurations.
   * `SZWEEXEC` contains few utilities used by Zowe.
+  * `SZWELOAD` contains config manager for REXX.

bin/commands/internal/config/output/.examples

@@ -0,0 +1 @@
+zwe internal config output -c /path/to/zowe.yaml

bin/commands/internal/config/output/.help

@@ -0,0 +1 @@
+Outputs the merged YAML used at Zowe runtime into zowe.workspaceDirectory/.env/.zowe-merged.yaml

bin/commands/internal/config/output/cli.ts

@@ -0,0 +1,13 @@
+/*
+  This program and the accompanying materials are made available
+  under the terms of the Eclipse Public License v2.0 which
+  accompanies this distribution, and is available at
+  https://www.eclipse.org/legal/epl-v20.html
+ 
+  SPDX-License-Identifier: EPL-2.0
+ 
+  Copyright Contributors to the Zowe Project.
+*/
+import * as std from 'cm_std';
+import * as index from './index';
+index.execute();

bin/commands/internal/config/output/index.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+#######################################################################
+# This program and the accompanying materials are made available
+# under the terms of the Eclipse Public License v2.0 which
+# accompanies this distribution, and is available at
+# https://www.eclipse.org/legal/epl-v20.html
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Copyright Contributors to the Zowe Project.
+#######################################################################
+
+_CEE_RUNOPTS="XPLINK(ON),HEAPPOOLS(OFF),HEAPPOOLS64(OFF)" ${ZWE_zowe_runtimeDirectory}/bin/utils/configmgr -script "${ZWE_zowe_runtimeDirectory}/bin/commands/internal/config/output/cli.js"

bin/commands/internal/config/output/index.ts

@@ -0,0 +1,18 @@
+/*
+  This program and the accompanying materials are made available
+  under the terms of the Eclipse Public License v2.0 which
+  accompanies this distribution, and is available at
+  https://www.eclipse.org/legal/epl-v20.html
+ 
+  SPDX-License-Identifier: EPL-2.0
+ 
+  Copyright Contributors to the Zowe Project.
+*/
+
+import * as common from '../../../../libs/common';
+import * as config from '../../../../libs/config';
+
+export function execute() {
+  common.requireZoweYaml();
+  const ZOWE_CONFIG=config.getZoweConfig();
+}

bin/commands/internal/config/set/index.ts

@@ -17,7 +17,7 @@ import * as fakejq from '../../../../libs/fakejq';
 
 export function execute(configPath:string, newValue: any, haInstance?: string, valueAsString?: boolean) {
   common.requireZoweYaml();
-  const configFiles=std.getenv('ZWE_CLI_PARAMETER_CONFIG');
+  const configFiles=std.getenv('ZWE_PRIVATE_CONFIG_ORIG');
   const ZOWE_CONFIG=config.getZoweConfig();
 
   if (!valueAsString) {

bin/commands/start/.help

@@ -2,13 +2,14 @@ Start Zowe with main started task.
 
 These Zowe YAML configurations showing with sample values are used:
 
-```
+```yaml
 zowe:
   setup:
     security:
       stcs:
         zowe: ZWESLSTC
-  jobname: ZWE1SV
+  job:
+    name: ZWE1SV
 haInstances:
   [ha-instance]:
     sysname: ZLP1
@@ -21,3 +22,7 @@ haInstances:
 - `haInstances.[ha-instance].sysname` is the SYSNAME of the target HA instance.
   If you pass `--ha-instance` parameter, this is the SYSNAME the start command
   will be routed to.
+
+Note: `zwe start` is only identifying an already configured instance of Zowe.
+Any additional changes to the `zowe.yaml` config could possibly require
+either manual changes or running related `zwe` commands before issuing `zwe start`.