feat: storing postman collections as directories + newman enhancements

.nycrc.js

@@ -2,7 +2,17 @@ const TEST_TYPE = ((argv) => {
     let match = argv[argv.length - 1].match(/npm\/test-(\w+).js/);
 
     return match && match[1] || '';
-})(process.argv);
+})(process.argv),
+    RUN_TEST_FILES = [
+        'lib/commands/run/*.js',
+        'lib/reporter/*.js',
+        'lib/config/*.js',
+        'lib/print/*.js',
+        'lib/*.js',
+        'bin/**/*.js'
+    ];
+
+
 
 function configOverrides (testType) {
     switch (testType) {
@@ -12,20 +22,22 @@ function configOverrides (testType) {
                 branches: 65,
                 functions: 85,
                 lines: 80
-            };
+           };
         case 'integration':
             return {
                 statements: 40,
                 branches: 20,
                 functions: 40,
-                lines: 40
+                lines: 40,
+                include: RUN_TEST_FILES
             };
         case 'library':
             return {
                 statements: 55,
                 branches: 40,
                 functions: 55,
-                lines: 55
+                lines: 55,
+                include: RUN_TEST_FILES
             };
         case 'unit':
             return {

DIR_COMMANDS.md

@@ -0,0 +1,211 @@
+# Splitting Postman collection into directories - a better storage format
+
+Postman collections are created using the Postman UI application. These
+collections are represented as a [JSON
+file](https://learning.postman.com/collection-format/getting-started/structure-of-a-collection/)
+by the Postman UI application.  It is a single file that contains all the
+requests/tests/variables etc of the collection.  It is designed to be edited
+within the Postman UI. The `newman` tool allows teams to run Postman collections
+as part of their CI pipelines by supplying the Postman collection file as an
+argument. Typically, teams that rely on Postman for their API testing in CI
+pipelines maintain a copy of the collection file in their source repository.
+When teams maintain the Postman collection in their repository, they will have
+the following workflow to make changes to it and push is upstream:
+
+* Import the collection json file from the repo into Postman UI
+* Make changes to the collection in Postman UI
+* Export the collection to a file using Postman UI
+* Commit and push the changes
+
+There are a few challenges with the above workflow:
+
+1. Collection file diffs are very hard to review in pull requests
+    * Old format collection diff: https://github.com/juspay/hyperswitch/commit/7e29adb5c9dee8b03ef58ccbd85b07b106459380
+      ![Screenshot 2023-10-06 at 5 24 35 PM](https://github.com/knutties/newman/assets/77204/d4c67bb2-59fa-4938-be4a-68f8159584a1)
+    * New format collection diff: https://github.com/juspay/hyperswitch/pull/2117/files
+      ![Screenshot 2023-10-06 at 5 25 47 PM](https://github.com/knutties/newman/assets/77204/1c3798c6-84c0-4589-9e5e-ad067d62bb9f)
+1. Developers cannot use their favourite editor to make changes, add/remove new tests
+
+As a consequence of the above, collection maintenance suffers and often become stale.
+
+## Solution to the above challenges
+
+The core reason for challenges presented above is the file format in which the
+Postman requests are maintained.  A single JSON file that encapsulates the entire
+request code and sequencing does not lend itself for easy reviewing and be an
+all-editor friendly format.
+
+This fork of newman attempts to address the challenge by representing the
+Postman collection as a set of directories/files.  It allows developers to
+create Postman collection using the directory representation and also run them.
+It achieves this by re-constructing the collection json from the directory/file
+structure and leveraging the existing newman `run` implementation.
+
+For example, this is the [directory equivalent
+representation](examples/Sample%20Postman%20Collection) of a [sample
+collection](examples/sample-collection.json) in the examples directory as
+generated by the `dir-export` command.
+
+### Directory representation of collection
+
+The following is a tree structure of the directory that represents the Postman
+collection with annotations of what each file corresponds to in a Postman
+collection.
+```
+examples/Sample Postman Collection
+├── .info.json                # contains the info element from the top of collection json
+├── .meta.json                # contains the ordering of the request folders
+├── .auth.json                # contains the auth element from the top of the collection json
+├── .variable.json            # contains the variable element from the top of the collection json
+├── .event.json               # contains the event element from the top of the collection json
+└── A simple GET request
+    ├── .event.meta.json      # contains the ordering of the scripts - prerequest / test
+    ├── event.test.js         # the test script that runs post the request
+    ├── event.prerequest.js   # the test script that runs before the request
+    ├── request.json          # the request json from the Postman collection
+    └── response.json         # the response json
+```
+
+### Next steps
+This concept of representing Postman collections as a directory opens up programmatic
+pre-processing of request data before running the requests.  This could include things like the following:
+
+* Re-using same data across requests
+* Using other javascript libraries in testing code
+
+The instructions for using the tool are given below.  Please give it spin and let me know it you if it is useful.
+
+## Usage
+
+### Installing newman from this fork
+
+One can install the newman executable in this fork using the command:
+
+```
+npm install -g 'git+ssh://[email protected]:knutties/newman.git#feature/newman-dir'
+```
+
+The following no-arg run shows the new commands added to newman to help manage Postman
+requests maintained as directories/files instead of a single json file.
+
+```
+$ newman
+
+Usage: newman [options] [command]
+
+Options:
+  -v, --version                                               output the version number
+  -h, --help                                                  display help for command
+
+Commands:
+  dir-add-folder [options] <folder-path>                      Add a folder to directory based Postman collection in the given path
+  dir-add-request [options] <request-path>                    Add a request to directory based Postman collection in the given path
+  dir-collection-create [options] <collection-path>           Create a directory based Postman collection in the given path
+  dir-export [options] <postman-collection-file>              Convert a Postman collection file into its directory representation
+  dir-export-import-check [options] <postman-collection-file> Check if an export followed by import results in same collection
+  dir-import [options] <collection-dir>                       Convert a Postman directory representation into a postman collection
+  dir-remove-folder <folder-path>                             Remove folder at given path from directory based Postman collection
+  dir-remove-request <request-path>                           Remove request at given path from directory based Postman collection
+  dir-run [options] <collection-dir>                          Runs the requests in collection-dir, with all the provided options
+  run [options] <collection>                                  Initiate a Postman Collection run from a given URL or path
+
+    To get available options for a command:
+      newman <command> -h
+```
+
+The following sections show the invocation of the different commands added in
+this fork of newman.
+
+### Create a new directory based Postman collection
+```
+newman dir-collection-create new-dir-collection
+```
+
+### Generating directory representation for an existing collection
+```
+newman dir-export examples/sample-collection.json
+```
+
+### Convert a directory representation back to a Postman collection json file
+```
+newman dir-import examples/Sample\ Postman\ Collection/ -o examples/sample-collection.json
+```
+
+### Diff the collection generated by export followed by its import
+This command is typically used to test the tool itself to ensure it can handle
+all collection json use-cases.
+```
+newman dir-export-import-check examples/sample-collection.json
+```
+
+### Execute a Postman collection stored in the directory format
+```
+newman dir-run examples/Sample\ Postman\ Collection/
+```
+The `dir-run` command supports all the options that the stock `run` command of
+newman supports.  This is achieved by re-using the same set of command options
+for both the commands.
+
+Sample output of `dir-run` shown below
+```
+newman
+
+Sample Postman Collection
+
+→ A simple GET request
+  GET https://postman-echo.com/get?source=newman-sample-github-collection [200 OK, 847B, 1054ms]
+  ✓  expect response be 200
+  ✓  expect response json contain args
+
+→ A simple POST request
+  POST https://postman-echo.com/post [200 OK, 1.06kB, 317ms]
+
+→ A simple POST request with JSON body
+  POST https://postman-echo.com/post [200 OK, 1.17kB, 233ms]
+
+┌─────────────────────────┬─────────────────────┬─────────────────────┐
+│                         │            executed │              failed │
+├─────────────────────────┼─────────────────────┼─────────────────────┤
+│              iterations │                   1 │                   0 │
+├─────────────────────────┼─────────────────────┼─────────────────────┤
+│                requests │                   3 │                   0 │
+├─────────────────────────┼─────────────────────┼─────────────────────┤
+│            test-scripts │                   1 │                   0 │
+├─────────────────────────┼─────────────────────┼─────────────────────┤
+│      prerequest-scripts │                   0 │                   0 │
+├─────────────────────────┼─────────────────────┼─────────────────────┤
+│              assertions │                   2 │                   0 │
+├─────────────────────────┴─────────────────────┴─────────────────────┤
+│ total run duration: 1647ms                                          │
+├─────────────────────────────────────────────────────────────────────┤
+│ total data received: 2.13kB (approx)                                │
+├─────────────────────────────────────────────────────────────────────┤
+│ average response time: 534ms [min: 233ms, max: 1054ms, s.d.: 368ms] │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+### Add a folder under a existing directory
+```
+newman dir-add-folder examples/Sample\ Postman\ Collection/folder1
+```
+
+
+### Add a request under a directory
+```
+newman dir-add-request examples/Sample\ Postman\ Collection/test4
+```
+
+This command adds the request to the end of the requests already present in the
+folder.  The order of the requests is stored in a separate file called
+[.meta.json](examples/Sample%20Postman%20Collection/.meta.json).  The order of requests
+can be changed by re-ordering the requests in this file.
+
+### Remove a folder
+```
+newman dir-remove-folder examples/Sample\ Postman\ Collection/folder1
+```
+
+### Remove a request
+```
+newman dir-remove-request examples/Sample\ Postman\ Collection/test4
+```