Zowe Suite v1.25.0

CHANGELOG.md

@@ -3,6 +3,15 @@
 All notable changes to the Zlux Server Framework package will be documented in this file.
 This repo is part of the app-server Zowe Component, and the change logs here may appear on Zowe.org in that section.
 
+## 1.25.0
+
+- Enhancement: Improved callRootService when targeting agents such as ZSS to issue the request direct to the destination rather than using an additional loopback request to the app-server first. This should improve performance, reduce the need for the app-server being a client of itself, and allow for more request options when calling the agent.
+- Enhancement: Allow timeout parameter to be specified in a callService or callRootService command, such as when needing a long timeout to request an agent response.
+- Enhancement: Removed need for app-server to be a client of itself when using the callService API, by adding the option for requests to this API to be executed internal to app-server. This option is very compatible with pre-existing use of callService but is disabled by default to avoid disruption. You can enable it by setting the server configuration property `node.internalRouting=true`
+- Enhancement: Allow new pluginDefinition webContent destination variable for substitution, ZWE_EXTERNAL_PORT which matches the environment variable value, or if absent, the gateway or app-server ports conditional to configuration.
+- Bugfix: App-server would ignore when `VERIFY_CERTIFICATE=false` was set, and try to verify APIML servers. This would lead to login failures when APIML server was on a different system than App-server. Now, app-server will or will not verify APIML certificates according to `VERIFY_CERTIFICATE` value.
+- Bugfix: app-server could throw an uncaught exception when a proxied server had a socket error
+
 ## 1.24.0
 
 - Bugfix: Fixed issue where server could not bind to a hostname value for node.https.ipAddresses

lib/assets/i18n/log/messages_en.json

@@ -225,7 +225,7 @@
   "ZWED0037W":"Ending server process due to uncaught exception.",
   "ZWED0038W":"[Path=%s stderr]: %s",
   "ZWED0039W":"Exception at server cleanup function:\n%s",
-  "ZWED0040W":"Callservice: Service call failed. ",
+  "ZWED0040W":"Callservice: Service call to %s:%s%s failed. ",
   "ZWED0041W":"%s Exception caught. Message=%s",
   "ZWED0042W":"Stack trace follows\n%s",
   "ZWED0043W":"%s proxyWS error:%s",
@@ -305,6 +305,7 @@
   "ZWED0176W":"Failed to load client cert/key pair for Caching Service",
   "ZWED0177W":"Unable to load %s for '%s' into config",
   "ZWED0178W":"Skipping authentication plugin %s because it's not HA compatible",
+  "ZWED0179W":"Unable to retrieve the list of certificate authorities from the keyring=%s owner=%s Error: %s",
 
   "ZWED0001E":"RESERVED: Error: %s",
   "ZWED0002E":"Could not stop language manager for types=%s",

lib/index.js

@@ -218,6 +218,7 @@ Server.prototype = {
       hostname: this.userConfig.node.hostname ? this.userConfig.node.hostname : os.hostname(),
       httpPort: httpPort,
       httpsPort: httpsPort,
+      internalRouting: this.userConfig.node.internalRouting,
       productCode: this.appConfig.productCode,
       productDir: this.userConfig.productDir,
       proxiedHost: this.startUpConfig.proxiedHost,

lib/proxy.js

@@ -87,6 +87,7 @@ function makeSimpleProxy(host, port, options, pluginID, serviceName) {
     } else {
       proxyLog.debug('ZWED0172I'); //proxyLog.debug('Callservice: no auth helper');
     }
+
     const req2 = httpApi.request(requestOptions, (res2) => {
       proxyLog.debug("ZWED0173I", res2.statusCode); //proxyLog.debug("status code" + res2.statusCode);
       res1.status(res2.statusCode);
@@ -125,9 +126,8 @@ function makeSimpleProxy(host, port, options, pluginID, serviceName) {
       res2.pipe(res1);
     });
     req2.on('error', (e) => {
-      proxyLog.warn('ZWED0040W', e); //proxyLog.warn('Callservice: Service call failed.');
-      res1.status(500).send(`ZWED0040W - Unable to complete network request to ${host}:${port}: `
-          + e.message, null, null);
+      proxyLog.warn('ZWED0040W', requestOptions.host, requestOptions.port, requestOptions.path, e); //proxyLog.warn('Callservice: Service call to ... failed.');
+      res1.end();
     });
     if ((req1.method == 'POST') || (req1.method == 'PUT') || (req1.method == 'PATCH')) {
       proxyLog.debug('ZWED0175I'); //proxyLog.debug('Callservice: Forwarding request body to service');

lib/util.js

@@ -400,22 +400,41 @@ module.exports.getRemoteIframeTemplate = function(remoteUrl) {
   return defaultRemoteAppTemplate.replace('${remoteUrl}', remoteUrl);
 }
 
-module.exports.makeRemoteUrl = function(destination,req) {
-
+module.exports.makeRemoteUrl = function(destination, req, serverConfig) {
+  let referer = req.get('Referer');
+  loggers.utilLogger.debug(`referer: ${referer}`);
+
   let zoweExternalHost;
-  if(destination.includes('ZOWE_EXTERNAL_HOST')) {
-    let referer = req.get('Referer');
-    loggers.utilLogger.info(`referer: ${referer}`);  
+  let zoweExternalPort;
+
+  if(destination.includes('ZOWE_EXTERNAL_HOST') || destination.includes('ZWE_EXTERNAL_HOST')) {
     if( referer > '') {
       zoweExternalHost = referer.split(':')[1].substring(2); 
+    } else if (process.env.ZWE_EXTERNAL_HOST) {
+      zoweExternalHost = process.env.ZWE_EXTERNAL_HOST;
+    } else if (process.env.ZOWE_EXTERNAL_HOST) {
+      zoweExternalHost = process.env.ZOWE_EXTERNAL_HOST;
     } else {
       zoweExternalHost = process.env.ZOWE_EXPLORER_HOST;
     }
   }
+  if (destination.includes('ZWE_EXTERNAL_PORT')) {
+    if (process.env.ZWE_EXTERNAL_PORT) {
+      zoweExternalPort = process.env.ZWE_EXTERNAL_PORT;
+    } else if (serverConfig.node.mediationLayer && serverConfig.node.mediaitonLayer.server && serverConfig.node.mediationLayer.server.gatewayPort) {
+      zoweExternalPort = serverConfig.node.mediationLayer.server.gatewayPort;
+    } else if (serverConfig.node.https.port) {
+      zoweExternalPort = serverConfig.node.https.port;
+    } else {
+      zoweExternalPort = serverConfig.node.http.port;
+    }
+  }
 
   return destination
           .replace('${ZOWE_EXTERNAL_HOST}', zoweExternalHost)
+          .replace('${ZWE_EXTERNAL_HOST}', zoweExternalHost)
           .replace('${GATEWAY_PORT}', process.env.GATEWAY_PORT)
+          .replace('${ZWE_EXTERNAL_PORT}', zoweExternalPort)
           .replace('${ZOWE_EXPLORER_HOST}', process.env.ZOWE_EXPLORER_HOST);
 }
 

lib/webapp.js

@@ -14,6 +14,10 @@
 const express = require('express');
 const expressApp = express();
 
+//for internal routing fill
+const status = require('statuses');
+const iconv = require('iconv-lite');
+
 const fs = require('fs');
 const util = require('util');
 const url = require('url');
@@ -847,12 +851,13 @@ const staticHandlers = {
  *  This is passed to every other service of the plugin, so that 
  *  the service can be called by other services under the plugin
  */
-function WebServiceHandle(urlPrefix, environment) {
+function WebServiceHandle(urlPrefix, environment, isAgentService) {
   this.urlPrefix = urlPrefix;
   if (!environment.loopbackConfig.port) {
     installLog.severe(`ZWED0003E`, loopbackConfig); //installLog.severe(`loopback configuration not valid,`,loopbackConfig, `loopback calls will fail!`);
   }
   this.environment = environment;
+  this.isAgentService = isAgentService;
 }
 WebServiceHandle.prototype = {
   constructor: WebServiceHandle,
@@ -866,7 +871,7 @@ WebServiceHandle.prototype = {
   port: 0,
   urlPrefix: null,
 
-  call(path, options, originalRequest) {
+  call(path, options, originalRequest, originalRes) {
     return new Promise((resolve, reject) => {
       if (typeof path === "object") {
         options = path;
@@ -892,6 +897,7 @@ WebServiceHandle.prototype = {
         protocol: protocol,
         path: url,
         auth: options.auth,
+        timeout: options.timeout,
         rejectUnauthorized: rejectUnauthorized
       };
       const headers = {};
@@ -917,38 +923,101 @@ WebServiceHandle.prototype = {
           options.body = json;
         }
       }
-      if (options.zluxLoopbackSecret) {
-        //TODO use secret in a crypto scheme
-        headers[ZLUX_LOOPBACK_HEADER] = options.zluxLoopbackSecret;
-      }
       //console.log("headers: ", headers)
       if (Object.getOwnPropertyNames(headers).length > 0) {
         requestOptions.headers = headers;
       }
-      let httpOrHttps = this.environment.loopbackConfig.isHttps ? https : http;
-      const request = httpOrHttps.request(requestOptions, (response) => {
-        var chunks = [];
-        response.on('data',(chunk)=> {
-          utilLog.debug('ZWED0194I'); //utilLog.debug('Callservice: Data received');
-          chunks.push(chunk);
-        });
-        response.on('end',() => {
-          utilLog.debug('ZWED0195I'); //utilLog.debug('Callservice: Service call completed.');
-          response.body = Buffer.concat(chunks).toString();
-          resolve(response);
-        });
+
+      let httpOrHttps;
+      if (this.isAgentService) {
+        requestOptions.hostname = this.environment.agentRequestOptions.host;
+        requestOptions.port = this.environment.agentRequestOptions.port;
+        requestOptions.protocol = this.environment.agentRequestOptions.protocol;
+        requestOptions.rejectUnauthorized = this.environment.agentRequestOptions.rejectUnauthorized;
+        requestOptions.ca = this.environment.agentRequestOptions.ca;
+        requestOptions.ciphers = this.environment.agentRequestOptions.ciphers;
+        requestOptions.secureOptions = this.environment.agentRequestOptions.secureOptions;
+        if (this.environment.agentRequestOptions.apimlPrefix) {
+          requestOptions.path = this.environment.agentRequestOptions.apimlPrefix + requestOptions.path;
+        }
+        httpOrHttps = requestOptions.protocol == 'http:' ? http : https;
+        routingLog.debug(`Call agent host=%s path=%s`,requestOptions.hostname, requestOptions.path);
+      } else if (options.internalRouting) {
+        routingLog.debug(`Call internally path=%s`, requestOptions.path);
+        //clone done to prevent original request alteration, but shallow so hope attributes arent altered
+        let req = Object.assign({},originalRequest);
+        let res = Object.assign({},originalRes);
+
+        req.url = req.originalUrl = requestOptions.path;
+        req.method = requestOptions.method;
+        req.headers = requestOptions.headers;
+        req.rawHeaders=[];
+        let keys = Object.keys(req.headers);
+        for (let i = 0; i < keys.length; i++) {
+          let k = keys[i];
+          let v = req.headers[keys[i]];
+          req.rawHeaders.push(k);
+          req.rawHeaders.push(v);
+          delete req.headers[k];
+          req.headers[k.toLowerCase()]=v;
+
+        }
+
+        if (options.body) {
+          req._body=options.body;
+        } else {
+          delete req._body;
+        }
+        if (!req.headers['content-length']) {
+          req.headers['content-length']='0';
+        }
+
+
+        res.end = (body)=> {
+          utilLog.debug('router returned with body=',body.length);
+          res.body = iconv.decode(body,'utf8');
+          if (!res.statusMessage) {
+            res.statusMessage = status[res.statusCode];
+          }
+          resolve(res);
+        }
+        expressApp._router.handle(req, res);
+      } else {
+        routingLog.debug(`Call loopback path=%s`, requestOptions.path);
+        //loopback call to get to router
+        if (options.zluxLoopbackSecret) {
+          //TODO use secret in a crypto scheme
+          headers[ZLUX_LOOPBACK_HEADER] = options.zluxLoopbackSecret;
+        }
+
+        httpOrHttps = this.environment.loopbackConfig.isHttps ? https : http;
       }
-      );
-      request.on('error', (e) => {
-        utilLog.warn('ZWED0061W'); //utilLog.warn('Callservice: Service call failed.');
-        reject(e);
-      });
-      if (options.body) {
-        request.write(options.body);
+      //if not internal routing
+      if (httpOrHttps) {
+        const request = httpOrHttps.request(requestOptions, (response) => {
+          var chunks = [];
+          response.on('data',(chunk)=> {
+            utilLog.debug('ZWED0194I'); //utilLog.debug('Callservice: Data received');
+            chunks.push(chunk);
+          });
+          response.on('end',() => {
+            utilLog.debug('ZWED0195I'); //utilLog.debug('Callservice: Service call completed.');
+            response.body = Buffer.concat(chunks).toString();
+            resolve(response);
+          });
+        }
+                                           );
+        request.on('error', (e) => {
+          utilLog.warn('ZWED0061W'); //utilLog.warn('Callservice: Service call failed.');
+          reject(e);
+        });
+        if (options.body) {
+          request.write(options.body);
+        }
+        utilLog.debug('ZWED0196I', JSON.stringify(requestOptions, null, 2)); //utilLog.debug('Callservice: Issuing request to service: ' 
+        //+ JSON.stringify(requestOptions, null, 2));
+        request.end();
       }
-      utilLog.debug('ZWED0196I', JSON.stringify(requestOptions, null, 2)); //utilLog.debug('Callservice: Issuing request to service: ' 
-          //+ JSON.stringify(requestOptions, null, 2));
-      request.end();
     }
     );
   }
@@ -964,7 +1033,7 @@ const commonMiddleware = {
    * authentication data on: we'll do that
    */
 
-  addAppSpecificDataToRequest(globalAppData, loopbackSecret) {
+  addAppSpecificDataToRequest(globalAppData, loopbackSecret, internalRouting) {
     return function addAppSpecificData(req, res, next) {
       const appData = Object.create(globalAppData);
       if (!req[`${constants.APP_NAME}Data`]) {
@@ -977,7 +1046,7 @@ const commonMiddleware = {
       	appData.webApp = Object.create(appData.webApp);
       }
       appData.webApp.callRootService = function callRootService(name, url, 
-          options) {
+                                                                options) {
         if (!this.rootServices[name]) {
           throw new Error(`ZWED0050E - Root service ${name} not found`);
         }
@@ -1004,7 +1073,8 @@ const commonMiddleware = {
           const service = allHandles[version];
           options = options || {};
           options.zluxLoopbackSecret = loopbackSecret;
-          return service.call(url, options, req);
+          options.internalRouting = internalRouting;
+          return service.call(url, options, req, res);
         } catch (e) {
           return Promise.reject(e);
         }
@@ -1279,7 +1349,8 @@ function WebApp(options){
   this.setValidReferrers(options.serverConfig.node);
 
   this.wsEnvironment = {
-    loopbackConfig: this.loopbackConfig
+    loopbackConfig: this.loopbackConfig,
+    agentRequestOptions: zluxUtil.getAgentRequestOptions(options.serverConfig, options.tlsOptions, false)
   }
   this.options = zluxUtil.makeOptionsObject(defaultOptions, options);
   this.auth = options.auth;
@@ -1433,7 +1504,7 @@ WebApp.prototype = {
 
   installCommonMiddleware() {
     this.expressApp.use(commonMiddleware.addAppSpecificDataToRequest(
-      this.appData, this.loopbackSecret));
+      this.appData, this.loopbackSecret, this.options.internalRouting));
   },
 
   _installRootService(url, method, handler, {needJson, needAuth, authType, isPseudoSso}) {
@@ -1485,7 +1556,7 @@ WebApp.prototype = {
         this.expressApp.use(proxiedRootService.url, rootServicesMiddleware, this.auth.middleware, middlewareArray);
       }
       serviceHandleMap[name] = new WebServiceHandle(proxiedRootService.url, 
-          this.wsEnvironment);
+                                                    this.wsEnvironment, true);
     }
     this.expressApp.use(rootServicesMiddleware);
 
@@ -1872,7 +1943,7 @@ WebApp.prototype = {
       const destination = plugin.webContent.destination;
       const router = express.Router();
       router.use((req, res, callback) => {
-        const remoteUrl = zluxUtil.makeRemoteUrl(destination, req);
+        const remoteUrl = zluxUtil.makeRemoteUrl(destination, req, this.options.serverConfig);
         installLog.info("ZWED0299I", plugin.identifier, remoteUrl);
         const startingPage = zluxUtil.getRemoteIframeTemplate(remoteUrl);
         res.send(startingPage);

lib/webserver.js

@@ -271,6 +271,31 @@ WebServer.prototype = {
         }
       } 
       config.https.ipAddresses = uniqueIps;
+      if(keyring_js && process.env.KEYSTORE_TYPE == 'JCERACFKS') {
+        const keyringOwner = process.env.KEYRING_OWNER;
+        const keyringName = process.env.KEYRING_NAME;
+        let certificateList;
+        if (!config.https.certificateAuthorities) {
+          config.https.certificateAuthorities = [];
+        }
+        if(keyringOwner && keyringName) {
+          try {
+            certificateList = keyring_js.listKeyring(keyringOwner, keyringName);
+          } catch(e) {
+            bootstrapLogger.warn('ZWED0179W', keyringName, keyringOwner, e);
+          }
+        }
+        if(certificateList) {
+          for(let i = 0; i < certificateList.length; i++) {
+            if(certificateList[i].usage === 'CERTAUTH') {
+              let safKeyring = `safkeyring:////${keyringOwner}/${keyringName}&${certificateList[i].label}`;
+              if(config.https.certificateAuthorities.indexOf(safKeyring) === -1) {
+                config.https.certificateAuthorities.push(safKeyring);
+              }
+            }
+          }
+        }
+      }
     }
     return canRun;
   }),
@@ -309,7 +334,10 @@ WebServer.prototype = {
         this.httpsOptions.ciphers = ciphers;
       }
       bootstrapLogger.debug('Using tls ciphers:',this.httpsOptions.ciphers);
-
+      if (options.enableTrace) {
+        this.httpsOptions.enableTrace = true;
+      }
+      bootstrapLogger.debug('TLS trace:', this.httpsOptions.enableTrace ? 'enabled' : 'disabled');
       readTlsOptionsFromConfig(this.config, this.httpsOptions);
     }
   },