Upload newman runs to Postman

bin/newman.js

@@ -61,9 +61,12 @@ program
     .option('--cookie-jar <path>', 'Specify the path to a custom cookie jar (serialized tough-cookie JSON) ')
     .option('--export-cookie-jar <path>', 'Exports the cookie jar to a file after completing the run')
     .option('--verbose', 'Show detailed information of collection run and each request sent')
+    .option('-p, --publish-workspace <workspace-id>', 'Publishes to given workspace')
+    .option('--publish-workspace-skip-response', 'Skip responses (headers, body, etc) while uploading newman run to Postman')
+    .option('--publish-retry', 'Number of times newman can try to publish before safely erroring out.')
+    .option('--publish-upload-timeout', 'Timeout for uploading newman runs to postman', util.cast.integer, 0)
     .action((collection, command) => {
         let options = util.commanderToObject(command),
-
             // parse custom reporter options
             reporterOptions = util.parseNestedOptions(program._originalArgs, '--reporter-', options.reporters);
 
@@ -74,14 +77,15 @@ program
             acc[key] = _.assignIn(value, reporterOptions._generic); // overrides reporter options with _generic
         }, {});
 
-        newman.run(options, function (err, summary) {
+        newman.run(options, function (err, summary, newmanStatus) {
             const runError = err || summary.run.error || summary.run.failures.length;
 
             if (err) {
                 console.error(`error: ${err.message || err}\n`);
                 err.friendly && console.error(`  ${err.friendly}\n`);
             }
-            runError && !_.get(options, 'suppressExitCode') && process.exit(1);
+
+            runError && !newmanStatus.resultUploaded && !_.get(options, 'suppressExitCode') && process.exit(1);
         });
     });
 
@@ -139,7 +143,7 @@ function run (argv, callback) {
 
 // This hack has been added from https://github.com/nodejs/node/issues/6456#issue-151760275
 // @todo: remove when https://github.com/nodejs/node/issues/6456 has been fixed
-(Number(process.version[1]) >= 6) && [process.stdout, process.stderr].forEach((s) => {
+Number(process.version[1]) >= 6 && [process.stdout, process.stderr].forEach((s) => {
     s && s.isTTY && s._handle && s._handle.setBlocking && s._handle.setBlocking(true);
 });
 

lib/run/index.js

@@ -9,6 +9,7 @@ var _ = require('lodash'),
     getOptions = require('./options'),
     exportFile = require('./export-file'),
     util = require('../util'),
+    uploadRunToPostman = require('./upload-run'),
 
     /**
      * This object describes the various events raised by Newman, and what each event argument contains.
@@ -95,7 +96,12 @@ module.exports = function (options, callback) {
     var emitter = new EventEmitter(), // @todo: create a new inherited constructor
         runner = new runtime.Runner(),
         stopOnFailure,
-        entrypoint;
+        entrypoint,
+        uploadConfig = _.pick(options, ['postmanApiKey', 'publishWorkspace', 'publishWorkspaceSkipResponse',
+            'publishUploadTimeout', 'publishRetry']);
+
+    uploadConfig.collectionUID = util.extractResourceUID(options.collection);
+    uploadConfig.environmentUID = util.extractResourceUID(options.environment);
 
     // get the configuration from various sources
     getOptions(options, function (err, options) {
@@ -302,6 +308,9 @@ module.exports = function (options, callback) {
                     // in case runtime faced an error during run, we do not process any other event and emit `done`.
                     // we do it this way since, an error in `done` callback would have anyway skipped any intermediate
                     // events or callbacks
+
+                    // TODO: Add analytics call here
+
                     if (err) {
                         emitter.emit('done', err, emitter.summary);
                         callback(err, emitter.summary);
@@ -340,10 +349,13 @@ module.exports = function (options, callback) {
                         });
                     });
 
-                    asyncEach(emitter.exports, exportFile, function (err) {
+                    asyncEach(emitter.exports, exportFile, async function (err) {
                         // we now trigger actual done event which we had overridden
                         emitter.emit('done', err, emitter.summary);
-                        callback(err, emitter.summary);
+
+                        var resultUploaded = await new uploadRunToPostman(uploadConfig, emitter.summary).start();
+
+                        callback(err, emitter.summary, { resultUploaded });
                     });
                 }
             });

lib/run/upload-run.js

@@ -0,0 +1,133 @@
+const util = require('../util');
+
+const _ = require('lodash'),
+print = require('../print'),
+retry = require('async-retry'),
+request = require('postman-request'),
+ERROR_GO_TO_LINK = '<TBA>',
+UPLOAD_RUN_API_URL = '<TBA>'
+
+class uploadRunToPostman{
+
+    constructor(uploadConfig, runSummary){
+        this.uploadConfig = uploadConfig,
+        this.runSummary = runSummary;
+    }
+
+    /**
+     * TBA after discussing with Harsh
+     * @param {*} runSummary
+     * @param {*} collectionUID
+     * @param {*} environmentUID
+     */
+    buildRunObject = ( runSummary, collectionUID , environmentUID ) => {}
+
+    /**
+     * @private
+     * Internal upload call
+     *
+     * @param {Object} uploadOptions
+     * @returns {function} Returns an async function which can be used by async-retry library to have retries
+     */
+     _upload = (uploadOptions) => {
+        return async(bail) => await new Promise((resolve, reject) => request.post(uploadOptions , (error, response, body) => {
+                    if(error){
+                        if(error.code === 'ECONNREFUSED') return reject(error) // Retry only if the ERROR is ERRCONNECT
+                        return bail(error); // For other errors , dont retry
+                    }
+
+                    if(200 <= response.statusCode && response.statusCode <= 299){
+                        return resolve(body);
+                    }
+
+                    if( 400 <= response.statusCode && response.statusCode<= 499){
+
+                        if(response.statusCode === 404){
+                            return bail(new Error('Couldn\'t find the postman server route'))
+                        }
+
+                        return bail(new Error(body.message)); // Avoid retry if there is client side error ( API key error / Workspace ID / permission error)
+
+                    }
+
+                    if( 500 <= response.statusCode && response.statusCode <= 599){ // Perform Retry if Server side Error
+                        return reject(`Retrying because of server error: ${body.message}`);
+                    }
+
+                    return reject(); // This should not be activated ( Discuss with Harsh , how to handle 3xx )
+                })
+            );
+    }
+
+    /**
+     * @private
+     * Internal upload function which handles the retry
+     *
+     * @param {*} uploadOptions
+     * @param {*} retryOptions
+     * @returns {Promise}
+     */
+    _uploadWithRetry = (upload, retryOptions) => {
+        return retry( upload,{
+            retries: retryOptions.maxRetries,
+            factor: retryOptions.retryDelayMultiplier,
+            randomize: retryOptions.addJitter || false,
+            maxRetryTime: retryOptions.maxRetryDelay * 1000 , // converting to ms
+            maxTimeout: retryOptions.totalTimeout * 1000, // converting to ms
+            onRetry: retryOptions.onRetry || function(){} // e
+        });
+    }
+
+    /**
+     * @public
+     * Starts the run upload process
+     *
+     * @returns {Boolean}  Indicate if run upload was successful
+     */
+    start = async () => {
+        if(!this.uploadConfig.publishWorkspace){
+            return true;
+        }
+
+        if(!this.uploadConfig.postmanApiKey){
+            print.lf('Postman API Key was not provided , cannot upload newman run w/o Postman API Key');
+            return false;
+        }
+
+        print.lf('Uploading newman run to Postman');
+
+        const run = this.buildRunObject(runSummary, uploadConfig.publishWorkspaceSkipResponse);
+
+        const uploadOptions = {
+            url: UPLOAD_RUN_API_URL,
+            body: JSON.stringify(run),
+            headers: {
+                'Content-Type': 'application/json',
+                'X-API-Header': uploadConfig.postmanApiKey
+            }
+        },
+        retryOptions = {
+            maxRetries: uploadConfig.publishRetry,
+            totalTimeout: uploadConfig.publishUploadTimeout,
+            retryDelayMultiplier: 2,
+            maxRetryDelay : 64,
+            addJitter: true
+        };
+
+        try{
+            const response = await this._uploadWithRetry(this._upload(uploadOptions), retryOptions)
+
+            print.lf(`Uploaded the newman run to postman.
+                    Visit ${response.message} to view the results in postman web`);
+            return true;
+
+        } catch(error) {
+            print.lf(`Unable to upload the results to Postman:
+                      Reason: ${error.message}
+                      You can find solutions to common upload errors here: ${ERROR_GO_TO_LINK}`);
+            return false;
+        }
+    }
+}
+
+module.exports = uploadRunToPostman;

lib/util.js

@@ -284,6 +284,37 @@ util = {
      */
     isFloat: function (value) {
         return (value - parseFloat(value) + 1) >= 0;
+    },
+
+    /**
+     * Helper to get resource ID from URL
+     *
+     * If resourceUrl = https://api.getpostman.com/collections/<collection-UID>?apikey=<api_key>
+     * This method extracts <collection-UID>
+     *
+     * @param {String} resource - url for collection\environment
+     * @param resourceURL
+     * @returns {String}
+     */
+    extractResourceUID: function (resourceURL) {
+        if (_.isObject(resourceURL) || !resourceURL || resourceURL === '') {
+            return '';
+        }
+
+        const uidExtractionPattern = /https:\/\/api\.getpostman\.com\/[collections|environments]+\/(\w+)\?apikey=\w+/g;
+
+        let matches,
+            results = [];
+
+        while ((matches = uidExtractionPattern.exec(resourceURL)) !== null) {
+            if (matches.index === uidExtractionPattern.lastIndex) {
+                uidExtractionPattern.lastIndex++;
+            }
+
+            matches.forEach((match) => { return results.push(match); });
+        }
+
+        return results.length === 0 ? '' : results[1];
     }
 };
 

package.json

@@ -40,6 +40,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "async": "3.2.1",
+    "async-retry": "^1.3.3",
     "chardet": "1.3.0",
     "cli-progress": "3.9.0",
     "cli-table3": "0.6.0",

test/library/run-options.test.js

@@ -239,4 +239,14 @@ describe('Newman run options', function () {
             });
         });
     });
+
+    describe('Newman Run Uploads', function () {
+        it('should upload Newman run to Postman if API Key and WorksSpaceID are correct', function() {
+        // @TODO
+        });
+
+        it('should NOT upload Newman run to Postman if API Key and WorkSpaceID are incorrect', function() {
+        // @TODO
+        });
+    });
 });