From 95994c78b7b45ff0ea14e1c562fc0da2772a00b0 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sat, 23 Jun 2018 16:23:58 -0400 Subject: [PATCH 01/38] version bump --- index.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 280c81f..9cf046d 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,7 @@ /** * Lightweight web framework for your serverless applications * @author Jeremy Daly - * @version 0.7.0 + * @version 0.8.0 * @license MIT */ diff --git a/package.json b/package.json index d496119..cb21281 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lambda-api", - "version": "0.7.0", + "version": "0.8.0", "description": "Lightweight web framework for your serverless applications", "main": "index.js", "scripts": { From 5086386508d6a37beb5c83b63f4c34b82c41ce4c Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Wed, 11 Jul 2018 16:42:31 -0400 Subject: [PATCH 02/38] close #52 by adding main handler async support --- index.js | 11 ++++--- lib/request.js | 2 +- lib/response.js | 7 ++-- test/run.js | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 test/run.js diff --git a/index.js b/index.js index 9cf046d..77c0c0f 100644 --- a/index.js +++ b/index.js @@ -126,7 +126,7 @@ class API { // Set the event, context and callback this._event = event - this._context = this.context = context + this._context = this.context = typeof context === 'object' ? context : {} this._cb = cb // Initalize request and response objects @@ -136,7 +136,7 @@ class API { try { // Parse the request - request.parseRequest() + await request.parseRequest() // Loop through the middleware and await response for (const mw of this._middleware) { @@ -175,9 +175,12 @@ class API { } } catch(e) { - this.catchErrors(e,response) + await this.catchErrors(e,response) } + // Return the final response + return response._response + } // end run function @@ -236,7 +239,7 @@ class API { await this._finally(response._request,response) // Execute the primary callback - this._cb(err,res) + typeof this._cb === 'function' && this._cb(err,res) } // end _callback diff --git a/lib/request.js b/lib/request.js index 6e87cae..25e5523 100644 --- a/lib/request.js +++ b/lib/request.js @@ -36,7 +36,7 @@ class REQUEST { } // end constructor // Parse the request - parseRequest() { + async parseRequest() { // Set the method this.method = this.app._event.httpMethod.toUpperCase() diff --git a/lib/response.js b/lib/response.js index ce0d521..c6de6f5 100644 --- a/lib/response.js +++ b/lib/response.js @@ -42,6 +42,9 @@ class RESPONSE { // Default Etag support this._etag = false + + // Default response object + this._response = {} } // Sets the statusCode @@ -386,7 +389,7 @@ class RESPONSE { } // Create the response - const response = { + this._response = { headers: this._headers, statusCode: this._statusCode, body: this._request.method === 'HEAD' ? '' : UTILS.encodeBody(body), @@ -394,7 +397,7 @@ class RESPONSE { } // Trigger the callback function - this.app._callback(null, response, this) + this.app._callback(null, this._response, this) } // end send diff --git a/test/run.js b/test/run.js new file mode 100644 index 0000000..10186eb --- /dev/null +++ b/test/run.js @@ -0,0 +1,86 @@ +'use strict'; + +const expect = require('chai').expect // Assertion library + +// Init API instance +const api = require('../index')({ version: 'v1.0' }) +const api_error = require('../index')({ version: 'v1.0' }) +const api_error_path = require('../index')({ version: 'v1.0' }) + +// NOTE: Set test to true +api._test = true; +api_error._test = true; +api_error_path._test = true; + +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'Content-Type': 'application/json' + } +} + +/******************************************************************************/ +/*** DEFINE TEST ROUTE ***/ +/******************************************************************************/ +api.get('/test', function(req,res) { + res.status(200).json({ + method: 'get', + status: 'ok' + }) +}) + +api.get('/testError', function(req,res) { + res.status(404).error('some error') +}) + +api_error.get('/testErrorThrown', function(req,res) { + throw new Error('some thrown error') +}) + + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +describe('Main handler Async/Await:', function() { + + it('With context object', async function() { + let _event = Object.assign({},event,{}) + let result = await api.run(_event,{}) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) + }) // end it + + it('Without context object', async function() { + let _event = Object.assign({},event,{}) + let result = await api.run(_event) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) + }) // end it + + it('With callback', async function() { + let _event = Object.assign({},event,{}) + let result = await api.run(_event,{},(err,res) => {}) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) + }) // end it + + it('Triggered Error', async function() { + let _event = Object.assign({},event,{ path: '/testError' }) + let result = await api.run(_event,{}) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 404, body: '{"error":"some error"}', isBase64Encoded: false }) + }) // end it + + it('Thrown Error', async function() { + let _event = Object.assign({},event,{ path: '/testErrorThrown' }) + let result = await api_error.run(_event,{}) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"some thrown error"}', isBase64Encoded: false }) + }) // end it + + it('Routes Error', async function() { + let _event = Object.assign({},event,{ path: '/testRoute' }) + let result = await api_error_path.run(_event,{}) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 404, body: '{"error":"Route not found"}', isBase64Encoded: false }) + }) // end it + + +}) // end tests From 77a9d62c00d22478d332019c6290bd43a1983749 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 12 Aug 2018 17:39:11 -0400 Subject: [PATCH 03/38] tag updates --- package.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index cb21281..33acffd 100644 --- a/package.json +++ b/package.json @@ -17,10 +17,9 @@ "serverless", "nodejs", "api", - "awslambda", - "apigateway", - "web", - "framework", + "AWS Lambda", + "API Gateway", + "web framework", "json", "schema", "open" From 09abe130f6fa127635e6cc6afbdbf0e2aaa6d707 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 12 Aug 2018 17:46:05 -0400 Subject: [PATCH 04/38] close #51 with added documentation --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e664e2a..a87a95d 100644 --- a/README.md +++ b/README.md @@ -843,5 +843,8 @@ Routes must be configured in API Gateway in order to support routing to the Lamb Simply create a `{proxy+}` route that uses the `ANY` method and all requests will be routed to your Lambda function and processed by the `lambda-api` module. In order for a "root" path mapping to work, you also need to create an `ANY` route for `/`. +## Reusing Persistent Connections +If you are using persistent connections in your function routes (such as AWS RDS or Elasticache), be sure to set `context.callbackWaitsForEmptyEventLoop = false;` in your main handler. This will allow the freezing of connections and will prevent Lambda from hanging on open connections. See [here](https://www.jeremydaly.com/reuse-database-connections-aws-lambda/) for more information. + ## Contributions Contributions, ideas and bug reports are welcome and greatly appreciated. Please add [issues](https://github.com/jeremydaly/lambda-api/issues) for suggestions and bug reports or create a pull request. From a2259a26a3ae03cc91b588ce6047c94a2f8b9d1b Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 12 Aug 2018 18:19:03 -0400 Subject: [PATCH 05/38] close #50 by adding support for multiple middlewares --- index.js | 20 +++++++---- test/errorHandling.js | 31 +++++++++++++++++ test/middleware.js | 78 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 77c0c0f..b2cac28 100644 --- a/index.js +++ b/index.js @@ -248,16 +248,22 @@ class API { // Middleware handler use(path,handler) { - let fn = typeof path === 'function' ? path : handler + // Extract routes let routes = typeof path === 'string' ? Array.of(path) : (Array.isArray(path) ? path : []) - if (fn.length === 3) { - this._middleware.push([routes,fn]) - } else if (fn.length === 4) { - this._errors.push(fn) - } else { - throw new Error('Middleware must have 3 or 4 parameters') + // Add func args as middleware + for (let arg in arguments) { + if (typeof arguments[arg] === 'function') { + if (arguments[arg].length === 3) { + this._middleware.push([routes,arguments[arg]]) + } else if (arguments[arg].length === 4) { + this._errors.push(arguments[arg]) + } else { + throw new Error('Middleware must have 3 or 4 parameters') + } + } } + } // end use diff --git a/test/errorHandling.js b/test/errorHandling.js index 3f2a2d1..849fb55 100644 --- a/test/errorHandling.js +++ b/test/errorHandling.js @@ -5,9 +5,11 @@ const expect = require('chai').expect // Assertion library // Init API instance const api = require('../index')({ version: 'v1.0' }) +const api2 = require('../index')({ version: 'v1.0' }) // NOTE: Set test to true api._test = true; +api2._test = true; let event = { httpMethod: 'get', @@ -58,6 +60,22 @@ api.use(function(err,req,res,next) { } }); +const errorMiddleware1 = (err,req,res,next) => { + req.errorMiddleware1 = true + next() +} + +const errorMiddleware2 = (err,req,res,next) => { + req.errorMiddleware2 = true + next() +} + +const sendError = (err,req,res,next) => { + res.type('text/plain').send('This is a test error message: ' + req.errorMiddleware1 + '/' + req.errorMiddleware2) +} + +api2.use(errorMiddleware1,errorMiddleware2,sendError) + /******************************************************************************/ /*** DEFINE TEST ROUTES ***/ /******************************************************************************/ @@ -86,6 +104,12 @@ api.get('/testErrorPromise', function(req,res) { res.error('This is a test error message') }) + +api2.get('/testError', function(req,res) { + res.status(500) + res.error('This is a test error message') +}) + /******************************************************************************/ /*** BEGIN TESTS ***/ /******************************************************************************/ @@ -124,4 +148,11 @@ describe('Error Handling Tests:', function() { expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: 123/456', isBase64Encoded: false }) }) // end it + it('Multiple error middlewares', async function() { + let _event = Object.assign({},event,{ path: '/testError'}) + let result = await new Promise(r => api2.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: true/true', isBase64Encoded: false }) + }) // end it + + }) // end ERROR HANDLING tests diff --git a/test/middleware.js b/test/middleware.js index 05cf076..4e282aa 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -7,11 +7,15 @@ const expect = require('chai').expect // Assertion library const api = require('../index')({ version: 'v1.0' }) const api2 = require('../index')({ version: 'v1.0' }) const api3 = require('../index')({ version: 'v1.0' }) +const api4 = require('../index')({ version: 'v1.0' }) +const api5= require('../index')({ version: 'v1.0' }) // NOTE: Set test to true api._test = true; api2._test = true; api3._test = true; +api4._test = true; +api5._test = true; let event = { httpMethod: 'get', @@ -84,6 +88,20 @@ api3.use(['/test','/test/:param1','/test2/*'],function(req,res,next) { next() }) +const middleware1 = (req,res,next) => { + req.middleware1 = true + next() +} + +const middleware2 = (req,res,next) => { + req.middleware2 = true + next() +} + +api4.use(middleware1,middleware2); +api5.use('/test/x',middleware1,middleware2); +api5.use('/test/y',middleware1); + /******************************************************************************/ /*** DEFINE TEST ROUTES ***/ /******************************************************************************/ @@ -134,6 +152,38 @@ api3.get('/test3', function(req,res) { res.status(200).json({ method: 'get', middleware: req.testMiddlewareAll ? true : false }) }) +api4.get('/test', (req,res) => { + res.status(200).json({ + method: 'get', + middleware1: req.middleware1 ? true : false, + middleware2: req.middleware2 ? true : false + }) +}) + +api5.get('/test', (req,res) => { + res.status(200).json({ + method: 'get', + middleware1: req.middleware1 ? true : false, + middleware2: req.middleware2 ? true : false + }) +}) + +api5.get('/test/x', (req,res) => { + res.status(200).json({ + method: 'get', + middleware1: req.middleware1 ? true : false, + middleware2: req.middleware2 ? true : false + }) +}) + +api5.get('/test/y', (req,res) => { + res.status(200).json({ + method: 'get', + middleware1: req.middleware1 ? true : false, + middleware2: req.middleware2 ? true : false + }) +}) + /******************************************************************************/ /*** BEGIN TESTS ***/ /******************************************************************************/ @@ -213,4 +263,32 @@ describe('Middleware Tests:', function() { }) // end it + it('Multiple middlewares (no path)', async function() { + let _event = Object.assign({},event,{ path: '/test' }) + let result = await new Promise(r => api4.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","middleware1":true,"middleware2":true}', isBase64Encoded: false }) + }) // end it + + + it('Multiple middlewares (w/o matching path)', async function() { + let _event = Object.assign({},event,{ path: '/test' }) + let result = await new Promise(r => api5.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","middleware1":false,"middleware2":false}', isBase64Encoded: false }) + }) // end it + + + it('Multiple middlewares (w/ matching path)', async function() { + let _event = Object.assign({},event,{ path: '/test/x' }) + let result = await new Promise(r => api5.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","middleware1":true,"middleware2":true}', isBase64Encoded: false }) + }) // end it + + + it('Single middleware (w/ matching path)', async function() { + let _event = Object.assign({},event,{ path: '/test/y' }) + let result = await new Promise(r => api5.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","middleware1":true,"middleware2":false}', isBase64Encoded: false }) + }) // end it + + }) // end MIDDLEWARE tests From 3562047dbadcf670add8ca1548cce73f1e649374 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 13 Aug 2018 08:44:29 -0400 Subject: [PATCH 06/38] update test reporter for local reports --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 33acffd..4672f3c 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "test": "mocha --check-leaks --recursive", - "test-cov": "nyc --reporter=lcov mocha --check-leaks --recursive", + "test-cov": "nyc --reporter=html mocha --check-leaks --recursive", "test-ci": "nyc npm test && nyc report --reporter=text-lcov | ./node_modules/coveralls/bin/coveralls.js", "lint": "eslint ." }, From 4be1cf01b1b133f30882e0760e70258b9dc9a7b7 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 13 Aug 2018 08:44:54 -0400 Subject: [PATCH 07/38] improve header test coverage --- test/headers.js | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/headers.js b/test/headers.js index 2cb62ef..697fa01 100644 --- a/test/headers.js +++ b/test/headers.js @@ -25,6 +25,11 @@ api.get('/test', function(req,res) { res.status(200).json({ method: 'get', status: 'ok' }) }) +api.get('/testEmpty', function(req,res) { + res.header('test') + res.status(200).json({ method: 'get', status: 'ok' }) +}) + api.get('/testOverride', function(req,res) { res.header('Content-Type','text/html') res.status(200).send('
testHTML
') @@ -89,6 +94,12 @@ api.get('/corsOverride', function(req,res) { }).json({}) }) +api.get('/corsOverride2', function(req,res) { + res.cors().cors({ + methods: 'GET, PUT, POST' + }).json({}) +}) + api.get('/auth', function(req,res) { res.json({ auth: req.auth @@ -109,6 +120,12 @@ describe('Header Tests:', function() { expect(result).to.deep.equal({ headers: { 'content-type': 'application/json', 'test': 'testVal' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) }) // end it + it('Empty Header - Default', async function() { + let _event = Object.assign({},event,{ path: '/testEmpty' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json', 'test': '' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) + }) // end it + it('Override Header: /testOveride -- Content-Type: text/html', async function() { let _event = Object.assign({},event,{ path: '/testOverride'}) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) @@ -198,7 +215,7 @@ describe('Header Tests:', function() { }) }) // end it - it('Override CORS Headers', async function() { + it('Override CORS Headers #1', async function() { let _event = Object.assign({},event,{ path: '/corsOverride'}) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) expect(result).to.deep.equal({ @@ -213,6 +230,21 @@ describe('Header Tests:', function() { isBase64Encoded: false }) }) // end it + + it('Override CORS Headers #2', async function() { + let _event = Object.assign({},event,{ path: '/corsOverride2'}) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'access-control-allow-headers': 'Content-Type, Authorization, Content-Length, X-Requested-With', + 'access-control-allow-methods': 'GET, PUT, POST', + 'access-control-allow-origin': '*' + }, statusCode: 200, + body: '{}', + isBase64Encoded: false + }) + }) // end it }) // end CORS tests From 15ee539bc97383e84f931917462cd6cc29fff45a Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 13 Aug 2018 08:59:25 -0400 Subject: [PATCH 08/38] improve test coverage --- lib/response.js | 1 - test/download.js | 20 ++++++++++++++++++++ test/sendFile.js | 37 +++++++++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/lib/response.js b/lib/response.js index c6de6f5..c2c2d0b 100644 --- a/lib/response.js +++ b/lib/response.js @@ -264,7 +264,6 @@ class RESPONSE { this.header('ETag',data.ETag) } else { - //throw new Error('Invalid S3 path') throw new Error('Invalid S3 path') } diff --git a/test/download.js b/test/download.js index 2354303..70e1309 100644 --- a/test/download.js +++ b/test/download.js @@ -87,6 +87,10 @@ api.get('/download/all', function(req,res) { res.download('test/test.txt', 'test-file.txt', { private: true, maxAge: 3600000 }, err => { res.header('x-callback','true') }) }) +api.get('/download/no-options', function(req,res) { + res.download('test/test.txt', 'test-file.txt', err => { res.header('x-callback','true') }) +}) + // S3 file api.get('/download/s3', function(req,res) { @@ -251,6 +255,22 @@ describe('Download Tests:', function() { }) // end it + it('Text file w/ filename and callback (no options)', async function() { + let _event = Object.assign({},event,{ path: '/download/no-options' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'text/plain', + 'x-callback': 'true', + 'cache-control': 'max-age=0', + 'expires': result.headers.expires, + 'last-modified': result.headers['last-modified'], + 'content-disposition': 'attachment; filename="test-file.txt"' + }, statusCode: 200, body: 'VGVzdCBmaWxlIGZvciBzZW5kRmlsZQo=', isBase64Encoded: true + }) + }) // end it + + it('S3 file', async function() { let _event = Object.assign({},event,{ path: '/download/s3' }) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) diff --git a/test/sendFile.js b/test/sendFile.js index 3d04001..c72bb85 100644 --- a/test/sendFile.js +++ b/test/sendFile.js @@ -41,6 +41,10 @@ api.get('/sendfile', function(req,res) { res.sendFile('./test-missing.txt') }) +api.get('/sendfile/root', function(req,res) { + res.sendFile('test.txt', { root: './test/' }) +}) + api.get('/sendfile/err', function(req,res) { res.sendFile('./test-missing.txt', err => { if (err) { @@ -169,6 +173,10 @@ api.get('/sendfile/s3missing', function(req,res) { res.sendFile('s3://my-test-bucket/file-does-not-exist.txt') }) +api.get('/sendfile/s3-bad-path', function(req,res) { + res.sendFile('s3://my-test-bucket') +}) + // Error Middleware api.use(function(err,req,res,next) { // Set x-error header to test middleware execution @@ -187,10 +195,6 @@ describe('SendFile Tests:', function() { before(function() { // Stub getObjectAsync stub = sinon.stub(S3,'getObject') - //stub.prototype.promise = function() { console.log('test') } - //console.log('proto:',); - //S3.getObject.promise = () => { } - //stub.promise = () => {} }) it('Bad path', async function() { @@ -265,6 +269,19 @@ describe('SendFile Tests:', function() { }) }) // end it + it('Text file w/ root path', async function() { + let _event = Object.assign({},event,{ path: '/sendfile/root' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'text/plain', + 'cache-control': 'max-age=0', + 'expires': result.headers.expires, + 'last-modified': result.headers['last-modified'] + }, statusCode: 200, body: 'VGVzdCBmaWxlIGZvciBzZW5kRmlsZQo=', isBase64Encoded: true + }) + }) // end it + it('Text file w/ headers (private cache)', async function() { let _event = Object.assign({},event,{ path: '/sendfile/headers-private' }) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) @@ -371,6 +388,18 @@ describe('SendFile Tests:', function() { }) }) // end it + + it('S3 bad path error',async function() { + let _event = Object.assign({},event,{ path: '/sendfile/s3-bad-path' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'x-error': 'true' + }, statusCode: 500, body: '{"error":"Invalid S3 path"}', isBase64Encoded: false + }) + }) // end it + after(function() { stub.restore() }) From 8cf127e3607d120bf0e2d60bd5559c3e18ecab65 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 13 Aug 2018 09:43:30 -0400 Subject: [PATCH 09/38] enhance test coverage --- index.js | 6 +++++- lib/request.js | 14 ++++++-------- test/routes.js | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index b2cac28..56f3867 100644 --- a/index.js +++ b/index.js @@ -71,6 +71,10 @@ class API { // METHOD: Adds method and handler to routes METHOD(method, path, handler) { + if (typeof handler !== 'function') { + throw new Error(`No route handler specified for ${method} method on ${path} route.`) + } + // Ensure method is an array let methods = Array.isArray(method) ? method : method.split(',') @@ -263,7 +267,7 @@ class API { } } } - + } // end use diff --git a/lib/request.js b/lib/request.js index 25e5523..519a75f 100644 --- a/lib/request.js +++ b/lib/request.js @@ -17,10 +17,8 @@ class REQUEST { // Create a reference to the app this.app = app - // Init and default the handler - this._handler = function() { - console.log('No handler specified') // eslint-disable-line no-console - } + // Init the handler + this._handler // Expose Namespaces this.namespace = this.ns = app._app @@ -70,11 +68,11 @@ class REQUEST { // Set the requestContext this.requestContext = this.app._event.requestContext - // Parse id from context - this.id = this.app.context.awsRequestId ? this.app.context.awsRequestId : null - // Add context - this.context = typeof this.app.context === 'object' ? this.app.context : {} + this.context = this.app.context && typeof this.app.context === 'object' ? this.app.context : {} + + // Parse id from context + this.id = this.context.awsRequestId ? this.context.awsRequestId : null // Capture the raw body this.rawBody = this.app._event.body diff --git a/test/routes.js b/test/routes.js index 155f35e..9a7cf0e 100644 --- a/test/routes.js +++ b/test/routes.js @@ -53,6 +53,10 @@ api.post('/test', function(req,res) { res.status(200).json({ method: 'post', status: 'ok' }) }) +api.post('/test/base64', function(req,res) { + res.status(200).json({ method: 'post', status: 'ok', body: req.body }) +}) + api.put('/test', function(req,res) { res.status(200).json({ method: 'put', status: 'ok' }) }) @@ -248,6 +252,12 @@ describe('Route Tests:', function() { expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) }) // end it + it('Simple path, no `context`', async function() { + let _event = Object.assign({},event,{}) + let result = await new Promise(r => api.run(_event,null,(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) + }) // end it + it('Simple path w/ trailing slash: /test/', async function() { let _event = Object.assign({},event,{ path: '/test/' }) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) @@ -452,6 +462,12 @@ describe('Route Tests:', function() { expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"post","status":"ok","body":{"test":"123","test2":"456"}}', isBase64Encoded: false }) }) // end it + it('With base64 encoded body', async function() { + let _event = Object.assign({},event,{ path: '/test/base64', httpMethod: 'post', body: 'VGVzdCBmaWxlIGZvciBzZW5kRmlsZQo=', isBase64Encoded: true }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"post","status":"ok","body":"Test file for sendFile\\n"}', isBase64Encoded: false }) + }) // end it + it('Missing path: /not_found', async function() { let _event = Object.assign({},event,{ path: '/not_found', httpMethod: 'post' }) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) @@ -824,6 +840,22 @@ describe('Route Tests:', function() { }) // end method tests + + describe('Configuration errors', function() { + + it('Missing handler', async function() { + let error_message + try { + const api_error1 = require('../index')({ version: 'v1.0' }) + api_error1.get('/test-missing-handler') + } catch(e) { + error_message = e.message + } + expect(error_message).to.equal('No route handler specified for GET method on /test-missing-handler route.') + }) // end it + + }) // end Configuration errors + describe('routes() (debug method)', function() { it('Sample routes', function() { From 404d7315adf5fa5709fcc4ff0783c44a272b206a Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 12:45:59 -0400 Subject: [PATCH 10/38] enhance test coverage for route generation --- index.js | 10 ++- test/errorHandling.js | 152 +++++++++++++++++++++++++++++++----------- test/middleware.js | 61 ++++++++++++++++- test/routes.js | 41 ++++++++++++ 4 files changed, 219 insertions(+), 45 deletions(-) diff --git a/index.js b/index.js index 56f3867..bf67387 100644 --- a/index.js +++ b/index.js @@ -30,10 +30,8 @@ class API { // Stores route mappings this._routes = {} - // Default callback - this._cb = function() { - console.log('No callback specified') // eslint-disable-line no-console - } + // Init callback + this._cb // Middleware stack this._middleware = [] @@ -131,7 +129,7 @@ class API { // Set the event, context and callback this._event = event this._context = this.context = typeof context === 'object' ? context : {} - this._cb = cb + this._cb = cb ? cb : undefined // Initalize request and response objects let request = new REQUEST(this) @@ -289,7 +287,7 @@ class API { // Recursive function to create routes object setRoute(obj, value, path) { if (typeof path === 'string') { - let path = path.split('.') + path = path.split('.') } if (path.length > 1){ diff --git a/test/errorHandling.js b/test/errorHandling.js index 849fb55..57a13cf 100644 --- a/test/errorHandling.js +++ b/test/errorHandling.js @@ -6,10 +6,14 @@ const expect = require('chai').expect // Assertion library // Init API instance const api = require('../index')({ version: 'v1.0' }) const api2 = require('../index')({ version: 'v1.0' }) +const api3 = require('../index')({ version: 'v1.0' }) +const api4 = require('../index')({ version: 'v1.0' }) // NOTE: Set test to true api._test = true; api2._test = true; +api3._test = true; +api4._test = true; let event = { httpMethod: 'get', @@ -76,6 +80,20 @@ const sendError = (err,req,res,next) => { api2.use(errorMiddleware1,errorMiddleware2,sendError) +const returnError = (err,req,res,next) => { + return 'this is an error: ' + (req.errorMiddleware1 ? true : false) +} + +api3.use(returnError,errorMiddleware1) + +const callError = (err,req,res,next) => { + res.send('this is an error: ' + (req.errorMiddleware1 ? true : false)) + + next() +} + +api4.use(callError,errorMiddleware1) + /******************************************************************************/ /*** DEFINE TEST ROUTES ***/ /******************************************************************************/ @@ -104,55 +122,113 @@ api.get('/testErrorPromise', function(req,res) { res.error('This is a test error message') }) - api2.get('/testError', function(req,res) { res.status(500) res.error('This is a test error message') }) +api3.get('/testError', function(req,res) { + res.status(500) + res.error('This is a test error message') +}) + +api4.get('/testError', function(req,res) { + res.status(500) + res.error('This is a test error message') +}) + /******************************************************************************/ /*** BEGIN TESTS ***/ /******************************************************************************/ describe('Error Handling Tests:', function() { - this.slow(300); - it('Called Error', async function() { - let _event = Object.assign({},event,{ path: '/testError'}) - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) - expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test error message"}', isBase64Encoded: false }) - }) // end it - - it('Thrown Error', async function() { - let _event = Object.assign({},event,{ path: '/testErrorThrow'}) - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) - expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test thrown error"}', isBase64Encoded: false }) - }) // end it - - it('Simulated Error', async function() { - let _event = Object.assign({},event,{ path: '/testErrorSimulated'}) - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) - expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 405, body: '{"error":"This is a simulated error"}', isBase64Encoded: false }) - }) // end it - - it('Error Middleware', async function() { - let _event = Object.assign({},event,{ path: '/testErrorMiddleware'}) - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) - expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: 123/456', isBase64Encoded: false }) - }) // end it - - it('Error Middleware w/ Promise', async function() { - let _event = Object.assign({},event,{ path: '/testErrorPromise'}) - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) - expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: 123/456', isBase64Encoded: false }) - }) // end it - - it('Multiple error middlewares', async function() { - let _event = Object.assign({},event,{ path: '/testError'}) - let result = await new Promise(r => api2.run(_event,{},(e,res) => { r(res) })) - expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: true/true', isBase64Encoded: false }) - }) // end it - + describe('Standard', function() { + + it('Called Error', async function() { + let _event = Object.assign({},event,{ path: '/testError'}) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test error message"}', isBase64Encoded: false }) + }) // end it + + it('Thrown Error', async function() { + let _event = Object.assign({},event,{ path: '/testErrorThrow'}) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test thrown error"}', isBase64Encoded: false }) + }) // end it + + it('Simulated Error', async function() { + let _event = Object.assign({},event,{ path: '/testErrorSimulated'}) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 405, body: '{"error":"This is a simulated error"}', isBase64Encoded: false }) + }) // end it + + }) + + describe('Middleware', function() { + + it('Error Middleware', async function() { + let _event = Object.assign({},event,{ path: '/testErrorMiddleware'}) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: 123/456', isBase64Encoded: false }) + }) // end it + + it('Error Middleware w/ Promise', async function() { + let _event = Object.assign({},event,{ path: '/testErrorPromise'}) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: 123/456', isBase64Encoded: false }) + }) // end it + + it('Multiple error middlewares', async function() { + let _event = Object.assign({},event,{ path: '/testError'}) + let result = await new Promise(r => api2.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { 'content-type': 'text/plain' }, statusCode: 500, body: 'This is a test error message: true/true', isBase64Encoded: false }) + }) // end it + + it('Returned error from middleware (async)', async function() { + let _event = Object.assign({},event,{ path: '/testError'}) + let result = await new Promise(r => api3.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { }, statusCode: 500, body: 'this is an error: false', isBase64Encoded: false }) + }) // end it + + it('Returned error from middleware (callback)', async function() { + let _event = Object.assign({},event,{ path: '/testError'}) + let result = await new Promise(r => api4.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ headers: { }, statusCode: 500, body: 'this is an error: false', isBase64Encoded: false }) + }) // end it + }) + + + describe('Logging', function() { + + it('Thrown Error', async function() { + let _log + let _event = Object.assign({},event,{ path: '/testErrorThrow'}) + let logger = console.log + api._test = false + console.log = log => _log = log + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + api._test = true + console.log = logger + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test thrown error"}', isBase64Encoded: false }) + expect(_log.message).to.equal('This is a test thrown error') + }) // end it + + + it('API Error', async function() { + let _log + let _event = Object.assign({},event,{ path: '/testError'}) + let logger = console.log + api._test = false + console.log = (...args) => _log = args.join(' ') + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + api._test = true + console.log = logger + expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test error message"}', isBase64Encoded: false }) + expect(_log).to.equal('API Error: This is a test error message') + }) // end it + + }) }) // end ERROR HANDLING tests diff --git a/test/middleware.js b/test/middleware.js index 4e282aa..9902290 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -8,7 +8,9 @@ const api = require('../index')({ version: 'v1.0' }) const api2 = require('../index')({ version: 'v1.0' }) const api3 = require('../index')({ version: 'v1.0' }) const api4 = require('../index')({ version: 'v1.0' }) -const api5= require('../index')({ version: 'v1.0' }) +const api5 = require('../index')({ version: 'v1.0' }) +const api6 = require('../index')({ version: 'v1.0' }) +const api7 = require('../index')({ version: 'v1.0' }) // NOTE: Set test to true api._test = true; @@ -16,6 +18,8 @@ api2._test = true; api3._test = true; api4._test = true; api5._test = true; +api6._test = true; +api7._test = true; let event = { httpMethod: 'get', @@ -102,6 +106,33 @@ api4.use(middleware1,middleware2); api5.use('/test/x',middleware1,middleware2); api5.use('/test/y',middleware1); + + +api6.use((req,res,next) => { + res.header('middleware1',true) + return 'return from middleware' +}) + +// This shouldn't run +api6.use((req,res,next) => { + res.header('middleware2',true) + next() +}) + + + +api7.use((req,res,next) => { + res.header('middleware1',true) + res.send('return from middleware') + next() +}) + +// This shouldn't run +api7.use((req,res,next) => { + res.header('middleware2',true) + next() +}) + /******************************************************************************/ /*** DEFINE TEST ROUTES ***/ /******************************************************************************/ @@ -184,6 +215,16 @@ api5.get('/test/y', (req,res) => { }) }) +api6.get('/test', (req,res) => { + // This should not run because of the middleware return + res.status(200).send('route response') +}) + +api7.get('/test', (req,res) => { + // This should not run because of the middleware return + res.status(200).send('route response') +}) + /******************************************************************************/ /*** BEGIN TESTS ***/ /******************************************************************************/ @@ -291,4 +332,22 @@ describe('Middleware Tests:', function() { }) // end it + it('Short-circuit route with middleware (async return)', async function() { + let _event = Object.assign({},event,{ path: '/test' }) + let result = await new Promise(r => api6.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json', middleware1: true }, + statusCode: 200, body: 'return from middleware', isBase64Encoded: false }) + }) // end it + + + it('Short-circuit route with middleware (callback)', async function() { + let _event = Object.assign({},event,{ path: '/test' }) + let result = await new Promise(r => api7.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json', middleware1: true }, + statusCode: 200, body: 'return from middleware', isBase64Encoded: false }) + }) // end it + + }) // end MIDDLEWARE tests diff --git a/test/routes.js b/test/routes.js index 9a7cf0e..041317e 100644 --- a/test/routes.js +++ b/test/routes.js @@ -29,6 +29,10 @@ api.get('/', function(req,res) { res.status(200).json({ method: 'get', status: 'ok' }) }) +api.get('/return', async function(req,res) { + return { method: 'get', status: 'ok' } +}) + api2.get('/', function(req,res) { res.status(200).json({ method: 'get', status: 'ok' }) }) @@ -252,6 +256,19 @@ describe('Route Tests:', function() { expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 200, body: '{"method":"get","status":"ok"}', isBase64Encoded: false }) }) // end it + + it('Simple path w/ async return', async function() { + let _event = Object.assign({},event,{ path: '/return' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"method":"get","status":"ok"}', + isBase64Encoded: false + }) + }) // end it + + it('Simple path, no `context`', async function() { let _event = Object.assign({},event,{}) let result = await new Promise(r => api.run(_event,null,(e,res) => { r(res) })) @@ -854,6 +871,30 @@ describe('Route Tests:', function() { expect(error_message).to.equal('No route handler specified for GET method on /test-missing-handler route.') }) // end it + it('Missing callback', async function() { + let _event = Object.assign({},event,{ path: '/test', httpMethod: 'get' }) + let result = await api.run(_event,{}).then(res => { return res }) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"method":"get","status":"ok"}', + isBase64Encoded: false + }) + + }) // end it + + it('Invalid middleware', async function() { + let error_message + try { + const api_error2 = require('../index')({ version: 'v1.0' }) + api_error2.use((err,req) => {}) + } catch(e) { + error_message = e.message + } + expect(error_message).to.equal('Middleware must have 3 or 4 parameters') + }) // end it + }) // end Configuration errors describe('routes() (debug method)', function() { From 9bba041483ee3ed32d13a1818bd99d22a9862198 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 12:49:08 -0400 Subject: [PATCH 11/38] unit tests for route creation --- test/unit.js | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/unit.js diff --git a/test/unit.js b/test/unit.js new file mode 100644 index 0000000..52173a9 --- /dev/null +++ b/test/unit.js @@ -0,0 +1,49 @@ +'use strict'; + +const Promise = require('bluebird') // Promise library +const expect = require('chai').expect // Assertion library + +// Init API instance +const api = require('../index')({ version: 'v1.0' }) + + +// NOTE: Set test to true +api._test = true; + +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'Content-Type': 'application/json' + } +} + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +describe('Unit Tests:', function() { + + it('setRoute - string path', async function() { + let routes = {} + api.setRoute(routes,{['_GET']: { route: '/testPath' } },['testPath']) + api.setRoute(routes,{['_GET']: { route: '/testPath/testx' } },'testPath.testx') + expect(routes).to.deep.equal({ testPath: { _GET: { route: '/testPath' }, testx: { _GET: { route: '/testPath/testx' } } } }) + }) // end it + + + it('setRoute - null path', async function() { + let routes = { testPath: null } + api.setRoute(routes,{['_GET']: { route: '/testPath/testx' } },'testPath.testx') + expect(routes).to.deep.equal({ testPath: { testx: { _GET: { route: '/testPath/testx' } } } }) + }) // end it + + it('setRoute - null single path', async function() { + let routes = { testPath: null } + api.setRoute(routes,{['_GET']: { route: '/testPath' } },['testPath']) + expect(routes).to.deep.equal({ testPath: { _GET: { route: '/testPath' } } }) + }) // end it + + +}) // end UNIT tests From ca795bc09b5060610d76b8cf125b05f3fa90fb18 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 12:56:57 -0400 Subject: [PATCH 12/38] add cacheControl convenience method described in #49 --- lib/response.js | 23 ++++- test/cacheControl.js | 221 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 test/cacheControl.js diff --git a/lib/response.js b/lib/response.js index c2c2d0b..3394925 100644 --- a/lib/response.js +++ b/lib/response.js @@ -289,13 +289,16 @@ class RESPONSE { // Add cache-control headers if (opts.cacheControl !== false) { + // console.log('cache control'); if (opts.cacheControl !== true && opts.cacheControl !== undefined) { - this.header('Cache-Control', opts.cacheControl) + this.cache(opts.cacheControl) } else { - let maxAge = opts.maxAge && !isNaN(opts.maxAge) ? (opts.maxAge/1000|0) : 0 - this.header('Cache-Control', (opts.private === true ? 'private, ' : '') + 'max-age=' + maxAge) - this.header('Expires',new Date(Date.now() + maxAge).toUTCString()) + this.cache( + !isNaN(opts.maxAge) ? opts.maxAge : 0, + opts.private + ) } + } // Add last-modified headers @@ -365,6 +368,18 @@ class RESPONSE { return this } + // Add cacheControl headers + cache(maxAge,isPrivate=false) { + // if custom string value + if (maxAge !== true && maxAge !== undefined && typeof maxAge === 'string') { + this.header('Cache-Control', maxAge) + } else if (maxAge !== false) { + maxAge = maxAge && !isNaN(maxAge) ? (maxAge/1000|0) : 0 + this.header('Cache-Control', (isPrivate === true ? 'private, ' : '') + 'max-age=' + maxAge) + this.header('Expires',new Date(Date.now() + maxAge).toUTCString()) + } + return this + } // Sends the request to the main callback send(body) { diff --git a/test/cacheControl.js b/test/cacheControl.js new file mode 100644 index 0000000..2e45ad7 --- /dev/null +++ b/test/cacheControl.js @@ -0,0 +1,221 @@ +'use strict'; + +const Promise = require('bluebird') // Promise library +const expect = require('chai').expect // Assertion library + +// Init API instance +const api = require('../index')({ version: 'v1.0' }) + + +// NOTE: Set test to true +api._test = true; + +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'Content-Type': 'application/json' + } +} + + +/******************************************************************************/ +/*** DEFINE TEST ROUTES ***/ +/******************************************************************************/ + +api.get('/cache', function(req,res) { + res.cache().send('cache') +}) + +api.get('/cacheTrue', function(req,res) { + res.cache(true).send('cache') +}) + +api.get('/cacheFalse', function(req,res) { + res.cache(false).send('cache') +}) + +api.get('/cacheMaxAge', function(req,res) { + res.cache(1000).send('cache') +}) + +api.get('/cachePrivate', function(req,res) { + res.cache(1000,true).send('cache') +}) + +api.get('/cachePrivateFalse', function(req,res) { + res.cache(1000,false).send('cache') +}) + +api.get('/cachePrivateInvalid', function(req,res) { + res.cache(1000,'test').send('cache') +}) + +api.get('/cacheCustom', function(req,res) { + res.cache('custom value').send('cache') +}) + +api.get('/cacheCustomUndefined', function(req,res) { + res.cache(undefined).send('cache') +}) + +api.get('/cacheCustomNull', function(req,res) { + res.cache(null).send('cache') +}) + + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +describe('cacheControl Tests:', function() { + + it('Basic cacheControl (no options)', async function() { + let _event = Object.assign({},event,{ path: '/cache' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=0', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (true)', async function() { + let _event = Object.assign({},event,{ path: '/cacheTrue' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=0', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (false)', async function() { + let _event = Object.assign({},event,{ path: '/cacheFalse' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json' + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (maxAge)', async function() { + let _event = Object.assign({},event,{ path: '/cacheMaxAge' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=1', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (private)', async function() { + let _event = Object.assign({},event,{ path: '/cachePrivate' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'private, max-age=1', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (disable private)', async function() { + let _event = Object.assign({},event,{ path: '/cachePrivateFalse' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=1', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (invalid private value)', async function() { + let _event = Object.assign({},event,{ path: '/cachePrivateInvalid' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=1', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (undefined)', async function() { + let _event = Object.assign({},event,{ path: '/cacheCustomUndefined' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=0', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Basic cacheControl (null)', async function() { + let _event = Object.assign({},event,{ path: '/cacheCustomNull' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'max-age=0', + 'expires': result.headers.expires + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('Custom cacheControl (string)', async function() { + let _event = Object.assign({},event,{ path: '/cacheCustom' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'cache-control': 'custom value' + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + +}) // end UNIT tests From d0590a7160f60b2e26940c6ff901333d05473bfb Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 13:57:08 -0400 Subject: [PATCH 13/38] add no-cache option to #49 --- lib/response.js | 4 +++- test/cacheControl.js | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/response.js b/lib/response.js index 3394925..84631e0 100644 --- a/lib/response.js +++ b/lib/response.js @@ -373,7 +373,9 @@ class RESPONSE { // if custom string value if (maxAge !== true && maxAge !== undefined && typeof maxAge === 'string') { this.header('Cache-Control', maxAge) - } else if (maxAge !== false) { + } else if (maxAge === false) { + this.header('Cache-Control', 'no-cache, no-store, must-revalidate') + } else { maxAge = maxAge && !isNaN(maxAge) ? (maxAge/1000|0) : 0 this.header('Cache-Control', (isPrivate === true ? 'private, ' : '') + 'max-age=' + maxAge) this.header('Expires',new Date(Date.now() + maxAge).toUTCString()) diff --git a/test/cacheControl.js b/test/cacheControl.js index 2e45ad7..6327186 100644 --- a/test/cacheControl.js +++ b/test/cacheControl.js @@ -106,7 +106,8 @@ describe('cacheControl Tests:', function() { let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) expect(result).to.deep.equal({ headers: { - 'content-type': 'application/json' + 'content-type': 'application/json', + 'cache-control': 'no-cache, no-store, must-revalidate' }, statusCode: 200, body: 'cache', From 66230ad708c8866e51a76860dcea4946684ef51a Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 14:58:01 -0400 Subject: [PATCH 14/38] add modified convenience method for adding last-modified headers #49 --- lib/response.js | 16 +++-- test/lastModified.js | 147 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 test/lastModified.js diff --git a/lib/response.js b/lib/response.js index 84631e0..eba806f 100644 --- a/lib/response.js +++ b/lib/response.js @@ -303,11 +303,9 @@ class RESPONSE { // Add last-modified headers if (opts.lastModified !== false) { - let lastModified = opts.lastModified && typeof opts.lastModified.toUTCString === 'function' ? opts.lastModified : (modified ? modified : new Date()) - this.header('Last-Modified', lastModified.toUTCString()) + this.modified(opts.lastModified ? opts.lastModified : modified) } - // Execute callback await fn() @@ -368,7 +366,7 @@ class RESPONSE { return this } - // Add cacheControl headers + // Add cache-control headers cache(maxAge,isPrivate=false) { // if custom string value if (maxAge !== true && maxAge !== undefined && typeof maxAge === 'string') { @@ -383,6 +381,16 @@ class RESPONSE { return this } + // Add last-modified headers + modified(date) { + if (date !== false) { + let lastModified = date && typeof date.toUTCString === 'function' ? date : + date && Date.parse(date) ? new Date(date) : new Date() + this.header('Last-Modified', lastModified.toUTCString()) + } + return this + } + // Sends the request to the main callback send(body) { diff --git a/test/lastModified.js b/test/lastModified.js new file mode 100644 index 0000000..4e6231b --- /dev/null +++ b/test/lastModified.js @@ -0,0 +1,147 @@ +'use strict'; + +const Promise = require('bluebird') // Promise library +const expect = require('chai').expect // Assertion library + +// Init API instance +const api = require('../index')({ version: 'v1.0' }) + + +// NOTE: Set test to true +api._test = true; + +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'Content-Type': 'application/json' + } +} + + +/******************************************************************************/ +/*** DEFINE TEST ROUTES ***/ +/******************************************************************************/ + +api.get('/modified', function(req,res) { + res.modified().send('cache') +}) + +api.get('/modifiedTrue', function(req,res) { + res.modified(true).send('cache') +}) + +api.get('/modifiedFalse', function(req,res) { + res.modified(false).send('cache') +}) + +api.get('/modifiedDate', function(req,res) { + res.modified(new Date('2018-08-01')).send('cache') +}) + +api.get('/modifiedString', function(req,res) { + res.modified('2018-08-01').send('cache') +}) + +api.get('/modifiedBadString', function(req,res) { + res.modified('test').send('cache') +}) + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +describe('modified Tests:', function() { + + it('modified (no options)', async function() { + let _event = Object.assign({},event,{ path: '/modified' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'last-modified': result.headers['last-modified'] + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + expect(typeof result.headers['last-modified']).to.not.be.null + expect(typeof result.headers['last-modified']).to.not.be.empty + }) // end it + + it('modified (true)', async function() { + let _event = Object.assign({},event,{ path: '/modifiedTrue' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'last-modified': result.headers['last-modified'] + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + expect(typeof result.headers['last-modified']).to.not.be.null + expect(typeof result.headers['last-modified']).to.not.be.empty + }) // end it + + it('modified (false)', async function() { + let _event = Object.assign({},event,{ path: '/modifiedFalse' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json' + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('modified (date)', async function() { + let _event = Object.assign({},event,{ path: '/modifiedDate' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'last-modified': 'Wed, 01 Aug 2018 00:00:00 GMT' + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + }) // end it + + it('modified (string)', async function() { + let _event = Object.assign({},event,{ path: '/modifiedString' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'last-modified': 'Wed, 01 Aug 2018 00:00:00 GMT' + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + + }) // end it + + it('modified (invalid date)', async function() { + let _event = Object.assign({},event,{ path: '/modifiedBadString' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'application/json', + 'last-modified': result.headers['last-modified'] + }, + statusCode: 200, + body: 'cache', + isBase64Encoded: false + }) + expect(new Date(result.headers['last-modified'])).to.be.above(new Date('2018-08-02')) + + }) // end it + +}) // end lastModified tests From 6f87aaa35ac7089fc030adbc44db2fefbbf7486d Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 16:58:52 -0400 Subject: [PATCH 15/38] add documentation for cache and modified methods --- README.md | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a87a95d..a8a1ca4 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Lambda API was written to be extremely lightweight and built specifically for se - [REQUEST](#request) - [RESPONSE](#response) - [attachment()](#attachmentfilename) + - [cache()](#cacheage-private) - [clearCookie()](#clearcookiename-options) - [cookie()](#cookiename-value-options) - [cors()](#corsoptions) @@ -65,6 +66,7 @@ Lambda API was written to be extremely lightweight and built specifically for se - [json()](#jsonbody) - [jsonp()](#jsonpbody) - [location](#locationpath) + - [modified()](#modifieddate) - [redirect()](#redirectstatus-path) - [removeHeader()](#removeheaderkey) - [send()](#sendbody) @@ -332,8 +334,8 @@ The `REQUEST` object contains a parsed and normalized request from API Gateway. - `headers`: An object containing the request headers (properties converted to lowercase for HTTP/2, see [rfc7540 8.1.2. HTTP Header Fields](https://tools.ietf.org/html/rfc7540)) - `rawHeaders`: An object containing the original request headers (property case preserved) - `body`: The body of the request. If the `isBase64Encoded` flag is `true`, it will be decoded automatically. - - If the `Content-Type` header is `application/json`, it will attempt to parse the request using `JSON.parse()` - - If the `Content-Type` header is `application/x-www-form-urlencoded`, it will attempt to parse a URL encoded string using `querystring` + - If the `content-type` header is `application/json`, it will attempt to parse the request using `JSON.parse()` + - If the `content-type` header is `application/x-www-form-urlencoded`, it will attempt to parse a URL encoded string using `querystring` - Otherwise it will be plain text. - `rawBody`: If the `isBase64Encoded` flag is `true`, this is a copy of the original, base64 encoded body - `route`: The matched route of the request @@ -359,11 +361,11 @@ api.get('/users', (req,res) => { ``` ### header(key, value) -The `header` method allows for you to set additional headers to return to the client. By default, just the `Content-Type` header is sent with `application/json` as the value. Headers can be added or overwritten by calling the `header()` method with two string arguments. The first is the name of the header and then second is the value. +The `header` method allows for you to set additional headers to return to the client. By default, just the `content-type` header is sent with `application/json` as the value. Headers can be added or overwritten by calling the `header()` method with two string arguments. The first is the name of the header and then second is the value. ```javascript api.get('/users', (req,res) => { - res.header('Content-Type','text/html').send('
This is HTML
') + res.header('content-type','text/html').send('
This is HTML
') }) ``` @@ -429,7 +431,7 @@ api.get('/users', (req,res) => { ``` ### type(type) -Sets the `Content-Type` header for you based on a single `String` input. There are thousands of MIME types, many of which are likely never to be used by your application. Lambda API stores a list of the most popular file types and will automatically set the correct `Content-Type` based on the input. If the `type` contains the "/" character, then it sets the `Content-Type` to the value of `type`. +Sets the `content-type` header for you based on a single `String` input. There are thousands of MIME types, many of which are likely never to be used by your application. Lambda API stores a list of the most popular file types and will automatically set the correct `content-type` based on the input. If the `type` contains the "/" character, then it sets the `content-type` to the value of `type`. ```javascript res.type('.html'); // => 'text/html' @@ -494,7 +496,7 @@ Defaults can be set by calling `res.cors()` with no properties, or with any comb res.cors({ origin: 'example.com', methods: 'GET, POST, OPTIONS', - headers: 'Content-Type, Authorization', + headers: 'content-type, authorization', maxAge: 84000000 }) ``` @@ -553,8 +555,20 @@ res.clearCookie('fooArray', { path: '/', httpOnly: true }).send() ### etag([boolean]) Enables Etag generation for the response if at value of `true` is passed in. Lambda API will generate an Etag based on the body of the response and return the appropriate header. If the request contains an `If-No-Match` header that matches the generated Etag, a `304 Not Modified` response will be returned with a blank body. +### cache([age][,private]) +Adds `cache-control` header to responses. If the first parameter is an `integer`, it will add a `max-age` to the header. The number should be in milliseconds. If the first parameter is `true`, it will add the cache headers with `max-age` set to `0` and use the current time for the `expires` header. If set to false, it will add a cache header with `no-cache, no-store, must-revalidate` as the value. You can also provide a custom string that will manually set the value of the `cache-control` header. And optional second argument takes a `boolean` and will set the `cache-control` to `private` This method is chainable. + +```javascript +res.cache(false).send() // 'cache-control': 'no-cache, no-store, must-revalidate' +res.cache(1000).send() // 'cache-control': 'max-age=1' +res.cache(30000,true).send() // 'cache-control': 'private, max-age=30' +``` + +### modified(date) +Adds a `last-modified` header to responses. A value of `true` will set the value to the current date and time. A JavaScript `Date` object can also be passed in. Note that it will be converted to UTC if not already. A `string` can also be passed in and will be converted to a date if JavaScript's `Date()` function is able to parse it. A value of `false` will prevent the header from being generated, but will not remove any existing `last-modified` headers. + ### attachment([filename]) -Sets the HTTP response `Content-Disposition` header field to "attachment". If a `filename` is provided, then the `Content-Type` is set based on the file extension using the `type()` method and the "filename=" parameter is added to the `Content-Disposition` header. +Sets the HTTP response `content-disposition` header field to "attachment". If a `filename` is provided, then the `content-type` is set based on the file extension using the `type()` method and the "filename=" parameter is added to the `content-disposition` header. ```javascript res.attachment() @@ -566,7 +580,7 @@ res.attachment('path/to/logo.png') ``` ### download(file [, filename] [, options] [, callback]) -This transfers the `file` (either a local path, S3 file reference, or Javascript `Buffer`) as an "attachment". This is a convenience method that combines `attachment()` and `sendFile()` to prompt the user to download the file. This method optionally takes a `filename` as a second parameter that will overwrite the "filename=" parameter of the `Content-Disposition` header, otherwise it will use the filename from the `file`. An optional `options` object passes through to the [sendFile()](#sendfilefile--options--callback) method and takes the same parameters. Finally, a optional `callback` method can be defined which is passed through to [sendFile()](#sendfilefile--options--callback) as well. +This transfers the `file` (either a local path, S3 file reference, or Javascript `Buffer`) as an "attachment". This is a convenience method that combines `attachment()` and `sendFile()` to prompt the user to download the file. This method optionally takes a `filename` as a second parameter that will overwrite the "filename=" parameter of the `content-disposition` header, otherwise it will use the filename from the `file`. An optional `options` object passes through to the [sendFile()](#sendfilefile--options--callback) method and takes the same parameters. Finally, a optional `callback` method can be defined which is passed through to [sendFile()](#sendfilefile--options--callback) as well. ```javascript res.download('./files/sales-report.pdf') @@ -589,10 +603,10 @@ The `sendFile()` method takes up to three arguments. The first is the `file`. Th | -------- | ---- | ----------- | ------- | | maxAge | `Number` | Set the expiration time relative to the current time in milliseconds. Automatically sets the `Expires` header | 0 | | root | `String` | Root directory for relative filenames. | | -| lastModified | `Boolean` or `String` | Sets the `Last-Modified` header to the last modified date of the file. This can be disabled by setting it to `false`, or overridden by setting it to a valid `Date` object | | +| lastModified | `Boolean` or `String` | Sets the `last-modified` header to the last modified date of the file. This can be disabled by setting it to `false`, or overridden by setting it to a valid `Date` object | | | headers | `Object` | Key value pairs of additional headers to be sent with the file | | -| cacheControl | `Boolean` or `String` | Enable or disable setting `Cache-Control` response header. Override value with custom string. | true | -| private | `Boolean` | Sets the `Cache-Control` to `private`. | false | +| cacheControl | `Boolean` or `String` | Enable or disable setting `cache-control` response header. Override value with custom string. | true | +| private | `Boolean` | Sets the `cache-control` to `private`. | false | ```javascript res.sendFile('./img/logo.png') @@ -662,7 +676,7 @@ Middleware can be used to authenticate requests, log API calls, etc. The `REQUES ```javascript // Auth User api.use((req,res,next) => { - if (req.headers.Authorization === 'some value') { + if (req.headers.authorization === 'some value') { req.authorized = true next() // continue execution } else { From bbb86ee8a57afc52720dad14776cc6a966826c3e Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 17:03:39 -0400 Subject: [PATCH 16/38] documentation updates close #49 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a8a1ca4..a01f146 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Lambda API was written to be extremely lightweight and built specifically for se - [REQUEST](#request) - [RESPONSE](#response) - [attachment()](#attachmentfilename) - - [cache()](#cacheage-private) + - [cache()](#cacheageprivate) - [clearCookie()](#clearcookiename-options) - [cookie()](#cookiename-value-options) - [cors()](#corsoptions) @@ -572,11 +572,11 @@ Sets the HTTP response `content-disposition` header field to "attachment". If a ```javascript res.attachment() -// Content-Disposition: attachment +// content-disposition: attachment res.attachment('path/to/logo.png') -// Content-Disposition: attachment; filename="logo.png" -// Content-Type: image/png +// content-disposition: attachment; filename="logo.png" +// content-type: image/png ``` ### download(file [, filename] [, options] [, callback]) From f7f841e3d0f6043a830fe4d5d8da5c765283eb6c Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 20:47:30 -0400 Subject: [PATCH 17/38] close #56 with built in getSignedUrl --- README.md | 21 +++++ lib/response.js | 56 +++++++------ lib/utils.js | 11 +++ test/getLink.js | 212 ++++++++++++++++++++++++++++++++++++++++++++++++ test/utils.js | 57 +++++++++++++ 5 files changed, 333 insertions(+), 24 deletions(-) create mode 100644 test/getLink.js diff --git a/README.md b/README.md index a01f146..3b7b085 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Lambda API was written to be extremely lightweight and built specifically for se - [error()](#errormessage) - [etag()](#etagboolean) - [getHeader()](#getheaderkey) + - [getLink()](#getlinks3path--expires--callback]) - [hasHeader()](#hasheaderkey) - [header()](#headerkey-value) - [html()](#htmlbody) @@ -380,6 +381,26 @@ Returns a boolean indicating the existence of `key` in the response headers. `ke ### removeHeader(key) Removes header matching `key` from the response headers. `key` is case insensitive. This method is chainable. +### getLink(s3Path[,expires][,callback]) +This returns a signed URL to the referenced file in S3 (using the `s3://{my-bucket}/{path-to-file}` format). You can optionally pass in an integer as the second parameter that will changed the default expiration time of the link. The expiration time is in seconds and defaults to `900`. In order to ensure proper URL signing, the `getLink()` must be asynchronous, and therefore returns a promise. You must either `await` the result or use a `.then` to retrieve the value. + +There is an optional third parameter that takes an error handler callback. If the underlying `getSignedUrl()` call fails, the error will be returned using the standard `res.error()` method. You can override this by providing your own callback. + +```javascript +// async/await +api.get('/getLink', async (req,res) => { + let url = await res.getLink('s3://my-bucket/my-file.pdf') + return { link: url } +}) + +// promises +api.get('/getLink', (req,res) => { + res.getLink('s3://my-bucket/my-file.pdf').then(url => { + res.json({ link: url }) + }) +}) +``` + ### send(body) The `send` methods triggers the API to return data to the API Gateway. The `send` method accepts one parameter and sends the contents through as is, e.g. as an object, string, integer, etc. AWS Gateway expects a string, so the data should be converted accordingly. diff --git a/lib/response.js b/lib/response.js index eba806f..5f26959 100644 --- a/lib/response.js +++ b/lib/response.js @@ -11,6 +11,9 @@ const UTILS = require('./utils.js') const fs = require('fs') // Require Node.js file system const path = require('path') // Require Node.js path +// Require AWS S3 service +const S3 = require('./s3-service') + class RESPONSE { // Create the constructor function. @@ -125,6 +128,27 @@ class RESPONSE { .html(`

${statusCode} Redirecting to ${url}

`) } // end redirect + // Convenience method for retrieving a signed link to an S3 bucket object + async getLink(path,expires,callback) { + let params = UTILS.parseS3(path) + + // Default Expires + params.Expires = !isNaN(expires) ? parseInt(expires) : 900 + + // Default callback + let fn = typeof expires === 'function' ? expires : + typeof callback === 'function' ? callback : e => { if (e) this.error(e) } + + // getSignedUrl doesn't support .promise() + return await new Promise(r => S3.getSignedUrl('getObject',params, async (e,url) => { + if (e) { + // Execute callback with caught error + await fn(e) + this.error(e) // Throw error if not done in callback + } + r(url) // return the url + })) + } // end getLink // Convenience method for setting cookies // see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie @@ -240,32 +264,16 @@ class RESPONSE { // If an S3 file identifier if (/^s3:\/\//i.test(filepath)) { - let s3object = filepath.replace(/^s3:\/\//i,'').split('/') - let s3bucket = s3object.shift() - let s3key = s3object.join('/') - - if (s3bucket.length > 0 && s3key.length > 0) { - - // Require AWS S3 service - const S3 = require('./s3-service') - - let params = { - Bucket: s3bucket, - Key: s3key - } - - // Attempt to get the object from S3 - let data = await S3.getObject(params).promise() + let params = UTILS.parseS3(filepath) - // Set results, type and header - buffer = data.Body - modified = data.LastModified - this.type(data.ContentType) - this.header('ETag',data.ETag) + // Attempt to get the object from S3 + let data = await S3.getObject(params).promise() - } else { - throw new Error('Invalid S3 path') - } + // Set results, type and header + buffer = data.Body + modified = data.LastModified + this.type(data.ContentType) + this.header('ETag',data.ETag) // else try and load the file locally } else { diff --git a/lib/utils.js b/lib/utils.js index 0660519..2acdd68 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -109,3 +109,14 @@ module.exports.extractRoutes = extractRoutes // Generate an Etag for the supplied value module.exports.generateEtag = data => crypto.createHash('sha256').update(encodeBody(data)).digest('hex').substr(0,32) + + +// Check if valid S3 path +module.exports.isS3 = path => /^s3:\/\/.+\/.+/i.test(path) + +// Parse S3 path +module.exports.parseS3 = path => { + if (!this.isS3(path)) throw new Error('Invalid S3 path') + let s3object = path.replace(/^s3:\/\//i,'').split('/') + return { Bucket: s3object.shift(), Key: s3object.join('/') } +} diff --git a/test/getLink.js b/test/getLink.js new file mode 100644 index 0000000..7f81139 --- /dev/null +++ b/test/getLink.js @@ -0,0 +1,212 @@ +'use strict'; + +const Promise = require('bluebird') // Promise library +const expect = require('chai').expect // Assertion library + +// Require Sinon.js library +const sinon = require('sinon') + +const AWS = require('aws-sdk') // AWS SDK (automatically available in Lambda) +// AWS.config.credentials = new AWS.SharedIniFileCredentials({profile: 'madlucas'}) + +const S3 = require('../lib/s3-service') // Init S3 Service + +// Init API instance +const api = require('../index')({ version: 'v1.0' }) + + +// NOTE: Set test to true +api._test = true; + +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'Content-Type': 'application/json' + } +} + + +/******************************************************************************/ +/*** DEFINE TEST ROUTES ***/ +/******************************************************************************/ + +api.get('/s3Link', async function(req,res) { + stub.callsArgWith(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') + let url = await res.getLink('s3://my-test-bucket/test/test.txt') + res.send(url) +}) + +api.get('/s3LinkExpire', async function(req,res) { + stub.callsArgWith(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') + let url = await res.getLink('s3://my-test-bucket/test/test.txt',60) + res.send(url) +}) + +api.get('/s3LinkInvalidExpire', async function(req,res) { + stub.callsArgWith(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') + let url = await res.getLink('s3://my-test-bucket/test/test.txt','test') + res.send(url) +}) + + +api.get('/s3LinkExpireFloat', async function(req,res) { + stub.callsArgWith(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') + let url = await res.getLink('s3://my-test-bucket/test/test.txt',3.145) + res.send(url) +}) + +api.get('/s3LinkError', async function(req,res) { + stub.callsArgWith(2, 'getSignedUrl error', null) + let url = await res.getLink('s3://my-test-bucket/test/test.txt', async (e) => { + return await Promise.delay(100).then(() => {}) + }) + res.send(url) +}) + +api.get('/s3LinkErrorCustom', async function(req,res) { + stub.callsArgWith(2, 'getSignedUrl error', null) + let url = await res.getLink('s3://my-test-bucket/test/test.txt', 60 ,async (e) => { + return await Promise.delay(100).then(() => { + res.error('Custom error') + }) + }) + res.send(url) +}) + +api.get('/s3LinkErrorStandard', async function(req,res) { + stub.callsArgWith(2, 'getSignedUrl error', null) + let url = await res.getLink('s3://my-test-bucket/test/test.txt', 900) + res.send(url) +}) + +api.get('/s3LinkInvalid', async function(req,res) { + //stub.callsArgWith(2, 'getSignedUrl error', null) + let url = await res.getLink('s3://my-test-bucket', 900) + res.send(url) +}) + + + + + + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +let stub + +describe('getLink Tests:', function() { + + this.slow(300) + + before(function() { + // Stub getSignedUrl + stub = sinon.stub(S3,'getSignedUrl') + }) + + it('Simple path', async function() { + let _event = Object.assign({},event,{ path: '/s3Link' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ', + isBase64Encoded: false + }) + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 900 }) + }) // end it + + it('Simple path (with custom expiration)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkExpire' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ', + isBase64Encoded: false + }) + // console.log(stub); + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 60 }) + }) // end it + + it('Simple path (with invalid expiration)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkInvalidExpire' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ', + isBase64Encoded: false + }) + // console.log(stub); + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 900 }) + }) // end it + + it('Simple path (with float expiration)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkExpireFloat' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ', + isBase64Encoded: false + }) + // console.log(stub); + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 3 }) + }) // end it + + it('Error (with delayed callback)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkError' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"error":"getSignedUrl error"}', + isBase64Encoded: false + }) + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 900 }) + }) // end it + + it('Custom Error (with delayed callback)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkErrorCustom' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"error":"Custom error"}', + isBase64Encoded: false + }) + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 60 }) + }) // end it + + it('Error (with default callback)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkErrorStandard' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"error":"getSignedUrl error"}', + isBase64Encoded: false + }) + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 900 }) + }) // end it + + it('Error (invalid S3 path)', async function() { + let _event = Object.assign({},event,{ path: '/s3LinkInvalid' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 500, + body: '{"error":"Invalid S3 path"}', + isBase64Encoded: false + }) + }) // end it + + after(function() { + stub.restore() + }) + +}) // end getLink tests diff --git a/test/utils.js b/test/utils.js index 47c5225..e0c7a55 100644 --- a/test/utils.js +++ b/test/utils.js @@ -290,4 +290,61 @@ describe('Utility Function Tests:', function() { }) // end generateEtag tests + describe('isS3:', function() { + it('Empty path', function() { + expect(utils.isS3('')).to.be.false + }) + + it('Valid S3 path', function() { + expect(utils.isS3('s3://test-bucket/key')).to.be.true + }) + + it('Valid S3 path (uppercase)', function() { + expect(utils.isS3('S3://test-bucket/key')).to.be.true + }) + + it('Invalid S3 path', function() { + expect(utils.isS3('s3://test-bucket')).to.be.false + }) + + it('Empty S3 path', function() { + expect(utils.isS3('s3:///')).to.be.false + }) + + it('URL', function() { + expect(utils.isS3('https://somedomain.com/test')).to.be.false + }) + + it('Relative path', function() { + expect(utils.isS3('../test/file.txt')).to.be.false + }) + }) // end isS3 tests + + + describe('parseS3:', function() { + it('Valid S3 path', function() { + expect(utils.parseS3('s3://test-bucket/key')).to.deep.equal({ Bucket: 'test-bucket', Key: 'key' }) + }) + + it('Valid S3 path (nested key)', function() { + expect(utils.parseS3('s3://test-bucket/key/path/file.txt')).to.deep.equal({ Bucket: 'test-bucket', Key: 'key/path/file.txt' }) + }) + + it('Invalid S3 path (no key)', function() { + let func = () => utils.parseS3('s3://test-bucket') + expect(func).to.throw('Invalid S3 path') + }) + + it('Invalid S3 path (no bucket or key)', function() { + let func = () => utils.parseS3('s3://') + expect(func).to.throw('Invalid S3 path') + }) + + it('Invalid S3 path (empty)', function() { + let func = () => utils.parseS3('') + expect(func).to.throw('Invalid S3 path') + }) + + }) // end parseS3 tests + }) // end UTILITY tests From cea8866746e0d686c17d18d3f26e547ff2133cf1 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 20:50:42 -0400 Subject: [PATCH 18/38] documentation update --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3b7b085..46bc33b 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Lambda API was written to be extremely lightweight and built specifically for se - [REQUEST](#request) - [RESPONSE](#response) - [attachment()](#attachmentfilename) - - [cache()](#cacheageprivate) + - [cache()](#cacheage--private) - [clearCookie()](#clearcookiename-options) - [cookie()](#cookiename-value-options) - [cors()](#corsoptions) @@ -381,7 +381,7 @@ Returns a boolean indicating the existence of `key` in the response headers. `ke ### removeHeader(key) Removes header matching `key` from the response headers. `key` is case insensitive. This method is chainable. -### getLink(s3Path[,expires][,callback]) +### getLink(s3Path [,expires] [,callback]) This returns a signed URL to the referenced file in S3 (using the `s3://{my-bucket}/{path-to-file}` format). You can optionally pass in an integer as the second parameter that will changed the default expiration time of the link. The expiration time is in seconds and defaults to `900`. In order to ensure proper URL signing, the `getLink()` must be asynchronous, and therefore returns a promise. You must either `await` the result or use a `.then` to retrieve the value. There is an optional third parameter that takes an error handler callback. If the underlying `getSignedUrl()` call fails, the error will be returned using the standard `res.error()` method. You can override this by providing your own callback. @@ -576,7 +576,7 @@ res.clearCookie('fooArray', { path: '/', httpOnly: true }).send() ### etag([boolean]) Enables Etag generation for the response if at value of `true` is passed in. Lambda API will generate an Etag based on the body of the response and return the appropriate header. If the request contains an `If-No-Match` header that matches the generated Etag, a `304 Not Modified` response will be returned with a blank body. -### cache([age][,private]) +### cache([age] [,private]) Adds `cache-control` header to responses. If the first parameter is an `integer`, it will add a `max-age` to the header. The number should be in milliseconds. If the first parameter is `true`, it will add the cache headers with `max-age` set to `0` and use the current time for the `expires` header. If set to false, it will add a cache header with `no-cache, no-store, must-revalidate` as the value. You can also provide a custom string that will manually set the value of the `cache-control` header. And optional second argument takes a `boolean` and will set the `cache-control` to `private` This method is chainable. ```javascript From abc03b60a27209f9702e2a3fe86bffd5cba7f7f3 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 20:56:15 -0400 Subject: [PATCH 19/38] documentation updates --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 46bc33b..e895e21 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,9 @@ api.get('/status', (req,res) => { }) // Declare your Lambda handler -module.exports.handler = (event, context, callback) => { +exports.handler = async (event, context) => { // Run the request - api.run(event, context, callback) + return await api.run(event, context) } ``` @@ -60,7 +60,7 @@ Lambda API was written to be extremely lightweight and built specifically for se - [error()](#errormessage) - [etag()](#etagboolean) - [getHeader()](#getheaderkey) - - [getLink()](#getlinks3path--expires--callback]) + - [getLink()](#getlinks3path-expires-callback) - [hasHeader()](#hasheaderkey) - [header()](#headerkey-value) - [html()](#htmlbody) @@ -381,7 +381,7 @@ Returns a boolean indicating the existence of `key` in the response headers. `ke ### removeHeader(key) Removes header matching `key` from the response headers. `key` is case insensitive. This method is chainable. -### getLink(s3Path [,expires] [,callback]) +### getLink(s3Path [, expires] [, callback]) This returns a signed URL to the referenced file in S3 (using the `s3://{my-bucket}/{path-to-file}` format). You can optionally pass in an integer as the second parameter that will changed the default expiration time of the link. The expiration time is in seconds and defaults to `900`. In order to ensure proper URL signing, the `getLink()` must be asynchronous, and therefore returns a promise. You must either `await` the result or use a `.then` to retrieve the value. There is an optional third parameter that takes an error handler callback. If the underlying `getSignedUrl()` call fails, the error will be returned using the standard `res.error()` method. You can override this by providing your own callback. @@ -576,7 +576,7 @@ res.clearCookie('fooArray', { path: '/', httpOnly: true }).send() ### etag([boolean]) Enables Etag generation for the response if at value of `true` is passed in. Lambda API will generate an Etag based on the body of the response and return the appropriate header. If the request contains an `If-No-Match` header that matches the generated Etag, a `304 Not Modified` response will be returned with a blank body. -### cache([age] [,private]) +### cache([age] [, private]) Adds `cache-control` header to responses. If the first parameter is an `integer`, it will add a `max-age` to the header. The number should be in milliseconds. If the first parameter is `true`, it will add the cache headers with `max-age` set to `0` and use the current time for the `expires` header. If set to false, it will add a cache header with `no-cache, no-store, must-revalidate` as the value. You can also provide a custom string that will manually set the value of the `cache-control` header. And optional second argument takes a `boolean` and will set the `cache-control` to `private` This method is chainable. ```javascript From 7988df7093481df26d8d17af64148e7e8df84630 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 20:58:03 -0400 Subject: [PATCH 20/38] documentation update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e895e21..2fc3985 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ Lambda API is a lightweight web framework for use with AWS API Gateway and AWS L const api = require('lambda-api')() // Define a route -api.get('/status', (req,res) => { - res.json({ status: 'ok' }) +api.get('/status', async (req,res) => { + return { status: 'ok' } }) // Declare your Lambda handler From 38b3477d0eec32d6bac13f944180ae48a297f14b Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 21:28:15 -0400 Subject: [PATCH 21/38] add s3 url signing to redirects --- README.md | 7 ++++++- lib/response.js | 34 +++++++++++++++++++++------------- test/responses.js | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 2fc3985..f515c55 100644 --- a/README.md +++ b/README.md @@ -480,7 +480,7 @@ api.get('/redirectToGithub', (req,res) => { ``` ### redirect([status,] path) -The `redirect` convenience method triggers a redirection and ends the current API execution. This method is similar to the `location()` method, but it automatically sets the status code and calls `send()`. The redirection URL (relative/absolute path OR a FQDN) can be specified as the only parameter or as a second parameter when a valid `3xx` status code is supplied as the first parameter. The status code is set to `302` by default, but can be changed to `300`, `301`, `302`, `303`, `307`, or `308` by adding it as the first parameter. +The `redirect` convenience method triggers a redirection and ends the current API execution. This method is similar to the `location()` method, but it automatically sets the status code and calls `send()`. The redirection URL (relative/absolute path, a FQDN, or an S3 path reference) can be specified as the only parameter or as a second parameter when a valid `3xx` status code is supplied as the first parameter. The status code is set to `302` by default, but can be changed to `300`, `301`, `302`, `303`, `307`, or `308` by adding it as the first parameter. ```javascript api.get('/redirectToHome', (req,res) => { @@ -490,6 +490,11 @@ api.get('/redirectToHome', (req,res) => { api.get('/redirectToGithub', (req,res) => { res.redirect(301,'https://github.com') }) + +// This will redirect a signed URL using the getLink method +api.get('/redirectToS3File', (req,res) => { + res.redirect('s3://my-bucket/someFile.pdf') +}) ``` ### cors([options]) diff --git a/lib/response.js b/lib/response.js index 5f26959..023b74a 100644 --- a/lib/response.js +++ b/lib/response.js @@ -108,24 +108,32 @@ class RESPONSE { } // Convenience method for Redirect - redirect(path) { + async redirect(path) { let statusCode = 302 // default - // If status code is provided - if (arguments.length === 2) { - if ([300,301,302,303,307,308].includes(arguments[0])) { - statusCode = arguments[0] - path = arguments[1] - } else { - throw new Error(arguments[0] + ' is an invalid redirect status code') + try { + // If status code is provided + if (arguments.length === 2) { + if ([300,301,302,303,307,308].includes(arguments[0])) { + statusCode = arguments[0] + path = arguments[1] + } else { + throw new Error(arguments[0] + ' is an invalid redirect status code') + } } - } - let url = UTILS.escapeHtml(path) + // Auto convert S3 paths to signed URLs + if (UTILS.isS3(path)) path = await this.getLink(path) + + let url = UTILS.escapeHtml(path) - this.location(path) - .status(statusCode) - .html(`

${statusCode} Redirecting to ${url}

`) + this.location(path) + .status(statusCode) + .html(`

${statusCode} Redirecting to ${url}

`) + + } catch(e) { + this.error(e) + } } // end redirect // Convenience method for retrieving a signed link to an S3 bucket object diff --git a/test/responses.js b/test/responses.js index 92e8d10..30deb80 100644 --- a/test/responses.js +++ b/test/responses.js @@ -1,6 +1,9 @@ 'use strict'; const expect = require('chai').expect // Assertion library +const sinon = require('sinon') // Require Sinon.js library +const AWS = require('aws-sdk') // AWS SDK (automatically available in Lambda) +const S3 = require('../lib/s3-service') // Init S3 Service // Init API instance const api = require('../index')({ version: 'v1.0' }) @@ -77,12 +80,25 @@ api.get('/redirectHTML', function(req,res) { res.redirect('http://www.github.com?foo=bar&bat=baz') }) +api.get('/s3Path', function(req,res) { + stub.callsArgWith(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') + res.redirect('s3://my-test-bucket/test/test.txt') +}) + + /******************************************************************************/ /*** BEGIN TESTS ***/ /******************************************************************************/ +let stub + describe('Response Tests:', function() { + before(function() { + // Stub getSignedUrl + stub = sinon.stub(S3,'getSignedUrl') + }) + it('Object response: convert to string', async function() { let _event = Object.assign({},event,{ path: '/testObjectResponse'}) let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) @@ -173,4 +189,23 @@ describe('Response Tests:', function() { expect(result).to.deep.equal({ headers: { 'content-type': 'text/html', 'location': 'http://www.github.com?foo=bar&bat=baz%3Cscript%3Ealert(\'not%20good\')%3C/script%3E' }, statusCode: 302, body: '

302 Redirecting to http://www.github.com?foo=bar&bat=baz<script>alert('not good')</script>

', isBase64Encoded: false }) }) // end it + it('S3 Path', async function() { + let _event = Object.assign({},event,{ path: '/s3Path' }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + expect(result).to.deep.equal({ + headers: { + 'content-type': 'text/html', + 'location': 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ' + }, + statusCode: 302, + body: '

302 Redirecting to https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ

', + isBase64Encoded: false + }) + expect(stub.lastCall.args[1]).to.deep.equal({ Bucket: 'my-test-bucket', Key: 'test/test.txt', Expires: 900 }) + }) // end it + + after(function() { + stub.restore() + }) + }) // end ERROR HANDLING tests From 117e065a42f010395efea32f388ba243cdbd39ea Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Tue, 14 Aug 2018 21:35:33 -0400 Subject: [PATCH 22/38] update responses test --- test/responses.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/responses.js b/test/responses.js index 30deb80..008fe99 100644 --- a/test/responses.js +++ b/test/responses.js @@ -81,7 +81,7 @@ api.get('/redirectHTML', function(req,res) { }) api.get('/s3Path', function(req,res) { - stub.callsArgWith(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') + stub.callsArgWithAsync(2, null, 'https://s3.amazonaws.com/my-test-bucket/test/test.txt?AWSAccessKeyId=AKXYZ&Expires=1534290845&Signature=XYZ') res.redirect('s3://my-test-bucket/test/test.txt') }) From a65170c91e0f31d5bc4a4a863492913b773e637f Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sat, 18 Aug 2018 02:19:40 -0400 Subject: [PATCH 23/38] add logging and sampling support #45 --- README.md | 16 ++- index.js | 66 ++++++++++-- lib/logger.js | 234 ++++++++++++++++++++++++++++++++++++++++++ lib/request.js | 40 +++++++- lib/response.js | 11 +- test/download.js | 9 +- test/errorHandling.js | 44 ++++---- test/getLink.js | 6 +- test/routes.js | 11 +- test/run.js | 2 +- test/sendFile.js | 4 +- 11 files changed, 383 insertions(+), 60 deletions(-) create mode 100644 lib/logger.js diff --git a/README.md b/README.md index f515c55..2f4c8c3 100644 --- a/README.md +++ b/README.md @@ -535,13 +535,25 @@ res.cors({ }) ``` -### error(message) -An error can be triggered by calling the `error` method. This will cause the API to stop execution and return the message to the client. Custom error handling can be accomplished using the [Error Handling](#error-handling) feature. +### error([code], message [,detail]) +An error can be triggered by calling the `error` method. This will cause the API to stop execution and return the message to the client. The status code can be set by optionally passing in an integer as the first parameter. Additional detail can be added as an optional third parameter (or second parameter if no status code is passed). This will add an additional `detail` property to error logs. Details accepts any value that can be serialized by `JSON.stringify` including objects, strings and arrays. Custom error handling can be accomplished using the [Error Handling](#error-handling) feature. ```javascript api.get('/users', (req,res) => { res.error('This is an error') }) + +api.get('/users', (req,res) => { + res.error(403,'Not authorized') +}) + +api.get('/users', (req,res) => { + res.error('Error', { foo: 'bar' }) +}) + +api.get('/users', (req,res) => { + res.error(404, 'Page not found', 'foo bar') +}) ``` ### cookie(name, value [,options]) diff --git a/index.js b/index.js index bf67387..765284f 100644 --- a/index.js +++ b/index.js @@ -10,6 +10,7 @@ const REQUEST = require('./lib/request.js') // Resquest object const RESPONSE = require('./lib/response.js') // Response object const UTILS = require('./lib/utils.js') // Require utils library +const LOGGER = require('./lib/logger.js') // Require logger library const prettyPrint = require('./lib/prettyPrint') // Pretty print for debugging // Create the API class @@ -24,6 +25,28 @@ class API { this._callbackName = props && props.callback ? props.callback.trim() : 'callback' this._mimeTypes = props && props.mimeTypes && typeof props.mimeTypes === 'object' ? props.mimeTypes : {} + // Set sampling info + this._sampleCounts = {} + + // Init request counter + this._requestCount = 0 + + // Track init date/time + this._initTime = Date.now() + + // Logging levels + this._logLevels = { + trace: 10, + debug: 20, + info: 30, + warn: 40, + error: 50, + fatal: 60 + } + + // Configure logger + this._logger = LOGGER.config(props && props.logger,this._logLevels) + // Prefix stack w/ base this._prefix = this.parseRoute(this._base) @@ -45,12 +68,9 @@ class API { // Executed after the callback this._finally = () => {} - // Global error status + // Global error status (used for response parsing errors) this._errorStatus = 500 - // Testing flag (disables logging) - this._test = false - } // end constructor @@ -188,7 +208,7 @@ class API { // Catch all async/sync errors - async catchErrors(e,response) { + async catchErrors(e,response,code,detail) { // Error messages should never be base64 encoded response._isBase64 = false @@ -198,13 +218,21 @@ class API { let message + let info = { + detail, + statusCode: response._statusCode, + coldStart: response._request.coldStart, + stack: this._logger.stack && e.stack || undefined + } + if (e instanceof Error) { - response.status(this._errorStatus) + response.status(code ? code : this._errorStatus) message = e.message - !this._test && console.log(e) // eslint-disable-line no-console + this.log.fatal(message, info) } else { + response.status(code) message = e - !this._test && console.log('API Error:',e) // eslint-disable-line no-console + this.log.error(message, info) } // If first time through, process error middleware @@ -232,7 +260,7 @@ class API { // Custom callback - async _callback(err, res, response) { + async _callback(err,res,response) { // Set done status response._state = 'done' @@ -240,6 +268,24 @@ class API { // Execute finally await this._finally(response._request,response) + // Output logs + response._request._logs.forEach(log => { + console.log(JSON.stringify(this._logger.detail ? // eslint-disable-line no-console + this._logger.format(log,response._request,response) : log)) + }) + + // Generate access log + if ((this._logger.access || response._request._logs.length > 0) && this._logger.access !== 'never') { + let access = this._logger.log( + 'access', + undefined, + response._request, + response._request.context, + { statusCode: res.statusCode, coldStart: response._request.coldStart } + ) + console.log(JSON.stringify(this._logger.format(access,response._request,response))) // eslint-disable-line no-console + } + // Execute the primary callback typeof this._cb === 'function' && this._cb(err,res) @@ -248,7 +294,7 @@ class API { // Middleware handler - use(path,handler) { + use(path) { // Extract routes let routes = typeof path === 'string' ? Array.of(path) : (Array.isArray(path) ? path : []) diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 0000000..d763062 --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,234 @@ +'use strict' + +/** + * Lightweight web framework for your serverless applications + * @author Jeremy Daly + * @license MIT + */ + + +// Config logger +exports.config = (config,levels) => { + + let cfg = config ? config : {} + + // Add custom logging levels + if (cfg.levels && typeof cfg.levels === 'object') { + for (let lvl in cfg.levels) { + if (!/^[A-Za-z_]\w*$/.test(lvl) || isNaN(cfg.levels[lvl])) { + throw new Error('Invalid level configuration') + } + } + levels = Object.assign(levels,cfg.levels) + } + + // Configure sampling rules + let sampling = cfg.sampling ? parseSamplerConfig(cfg.sampling,levels) : false + + // Parse/default the logging level + let level = cfg === true ? 'info' : + cfg.level && levels[cfg.level.toLowerCase()] ? + cfg.level.toLowerCase() : cfg.level === 'none' ? + 'none' : Object.keys(cfg).length > 0 ? 'info' : 'none' + + let messageKey = cfg.messageKey && typeof cfg.messageKey === 'string' ? + cfg.messageKey.trim() : 'msg' + + let customKey = cfg.customKey && typeof cfg.customKey === 'string' ? + cfg.customKey.trim() : 'custom' + + let timestamp = cfg.timestamp === false ? () => undefined : + typeof cfg.timestamp === 'function' ? cfg.timestamp : () => Date.now() + + let timer = cfg.timer === false ? () => undefined : (start) => (Date.now()-start) + + let nested = cfg.nested === true ? true : false // nest serializers + let stack = cfg.stack === true ? true : false // show stack traces in errors + let access = cfg.access === true ? true : cfg.access === 'never' ? 'never' : false // create access logs + let detail = cfg.detail === true ? true : false // add req/res detail to all logs + + let defaults = { + req: req => { + return { + path: req.path, + ip: req.ip, + ua: req.userAgent, + method: req.method, + version: req.version, + qs: Object.keys(req.query).length > 0 ? req.query : undefined + } + }, + res: () => { + return { } + }, + context: context => { + return { + remaining: context.getRemainingTimeInMillis && context.getRemainingTimeInMillis(), + function: context.functionName && context.functionName, + memory: context.memoryLimitInMB && context.memoryLimitInMB + } + }, + custom: custom => typeof custom === 'object' || nested ? custom : { [customKey]: custom } + } + + let serializers = { + main: cfg.serializers && typeof cfg.serializers.main === 'function' ? cfg.serializers.main : () => {}, + req: cfg.serializers && typeof cfg.serializers.req === 'function' ? cfg.serializers.req : () => {}, + res: cfg.serializers && typeof cfg.serializers.res === 'function' ? cfg.serializers.res : () => {}, + context: cfg.serializers && typeof cfg.serializers.context === 'function' ? cfg.serializers.context : () => {}, + custom: cfg.serializers && typeof cfg.serializers.context === 'function' ? cfg.serializers.context : () => {} + } + + // Main logging function + let log = (level,msg,req,context,custom) => { + + let _context = Object.assign(defaults.context(context),serializers.custom(context)) + let _custom = Object.assign(defaults.custom(custom),serializers.custom(custom)) + + return Object.assign({}, + { + level, + time: timestamp(), + id: req.id, + path: req.path, + [messageKey]: msg, + timer: timer(req._start), + sample: req._sample ? true : undefined + }, + serializers.main(req), + nested ? { [customKey]: _custom } : _custom, + nested ? { context: _context } : _context + ) + } + + // Formatting function for additional log data enrichment + let format = function(info,req,res) { + + let _req = Object.assign(defaults.req(req),serializers.req(req)) + let _res = Object.assign(defaults.res(res),serializers.res(res)) + + return Object.assign({}, + info, + nested ? { req: _req } : _req, + nested ? { res: _res } : _res + ) + } // end format + + // Return logger object + return { + level, + stack, + log, + format, + access, + detail, + sampling + } +} + +// Determine if we should sample this request +exports.sampler = (app,req) => { + + if (app._logger.sampling) { + + // default level to false + let level = false + + // Grab the rule based on route + let rule = app._logger.sampling.rules && app._logger.sampling.rules[req.route] + || app._logger.sampling.defaults + + // Get last sample time (default start, last, fixed count, period count and total count) + let counts = app._sampleCounts[rule.default ? 'default' : req.route] + || Object.assign(app._sampleCounts, { + [rule.default ? 'default' : req.route]: { start: 0, fCount: 0, pCount: 0, tCount: 0 } + })[rule.default ? 'default' : req.route] + + let now = Date.now() + + // If this is a new period, reset values + if ((now-counts.start) > rule.period*1000) { + counts.start = now + counts.pCount = 0 + + // If a rule target is set, sample the start + if (rule.target > 0) { + counts.fCount = 1 + level = rule.level // set the sample level + console.log('\n*********** NEW PERIOD ***********'); + } + // Enable sampling if last sample is passed target split + } else if (rule.target > 0 && + counts.start+Math.floor(rule.period*1000/rule.target*counts.fCount) < now) { + level = rule.level + counts.fCount++ + console.log('\n*********** FIXED ***********'); + } else if (rule.rate > 0 && + counts.start+Math.floor(rule.period*1000/(counts.tCount/(now-app._initTime)*rule.period*1000*rule.rate)*counts.pCount) < now) { + level = rule.level + counts.pCount++ + console.log('\n*********** RATE ***********'); + } + + + // Increment total count + counts.tCount++ + + + // console.log( + // '-----------------------------------', + // '\nlastSample:',app._lastSample, + // '\nsampleCounts:',app._sampleCounts, + // '\nrequestCount:',app._requestCount, + // '\ninitTime:',app._initTime, + // '\nlogger:',JSON.stringify(app._logger,null,2), + // '\nroute:', req.route, + // '\nmethod:', req.method, + // '\n-----------------------------------' + // ) + + + return level + + } // end if sampling + + return false +} + +// Parse sampler configuration +const parseSamplerConfig = (config,levels) => { + + // Default config + let cfg = typeof config === 'object' ? config : config === true ? {} : false + + // Error on invalid config + if (cfg === false) throw new Error('Invalid sampler configuration') + + // Create rule default + let defaults = (inputs) => { + return { // target, rate, period, method, level + target: Number.isInteger(inputs.target) ? inputs.target : 1, + rate: !isNaN(inputs.rate) && inputs.rate <= 1 ? inputs.rate : 0.1, + period: Number.isInteger(inputs.period) ? inputs.period : 60, // in seconds + method: (Array.isArray(inputs.method) ? inputs.method : + typeof inputs.method === 'string' ? + inputs.method.split(',') : ['*']).map(x => x.toString().trim()), + level: Object.keys(levels).includes(inputs.level) ? inputs.level : 'trace' + } + } + + // Parse and default rules + let rules = Array.isArray(cfg.rules) ? cfg.rules.reduce((acc,rule) => { + // Error if missing route or not a string + if (!rule.route || typeof rule.route !== 'string') + throw new Error('Invalid route specified in rule') + + return Object.assign(acc, { [rule.route.trim()]: defaults(rule) }) + },{}) : {} + + return { + defaults: Object.assign(defaults(cfg),{ default:true }), + rules + } + +} // end parseSamplerConfig diff --git a/lib/request.js b/lib/request.js index 519a75f..2032fc0 100644 --- a/lib/request.js +++ b/lib/request.js @@ -8,15 +8,25 @@ const QS = require('querystring') // Require the querystring library const UTILS = require('./utils.js') // Require utils library +const LOGGER = require('./logger.js') // Require logger library class REQUEST { // Create the constructor function. constructor(app) { + // Record start time + this._start = Date.now() + // Create a reference to the app this.app = app + // Flag cold starts + this.coldStart = app._requestCount === 0 ? true : false + + // Increment the requests counter + app._requestCount++ + // Init the handler this._handler @@ -31,6 +41,13 @@ class REQUEST { this.headers = {} + // Init log helpers (message,custom) and create app reference + app.log = this.log = Object.keys(app._logLevels).reduce((acc,lvl) => + Object.assign(acc,{ [lvl]: (m,c) => this.logger(lvl, m, this, this.context, c) }),{}) + + // Init _logs array for storage + this._logs = [] + } // end constructor // Parse the request @@ -45,12 +62,19 @@ class REQUEST { // Set the query parameters this.query = this.app._event.queryStringParameters ? this.app._event.queryStringParameters : {} - // Set the headers + // Set the raw headers this.rawHeaders = this.app._event.headers + // Set the headers to lowercase this.headers = Object.keys(this.rawHeaders).reduce((acc,header) => Object.assign(acc,{[header.toLowerCase()]:this.rawHeaders[header]}), {}) + // Extract IP + this.ip = this.headers['x-forwarded-for'] && this.headers['x-forwarded-for'].split(',')[0].trim() + + // Extract user agent + this.userAgent = this.headers['user-agent'] + // Set and parse cookies this.cookies = this.headers.cookie ? this.headers.cookie.split(';') @@ -66,7 +90,7 @@ class REQUEST { this.auth = UTILS.parseAuth(this.headers.authorization) // Set the requestContext - this.requestContext = this.app._event.requestContext + this.requestContext = this.app._event.requestContext || {} // Add context this.context = this.app.context && typeof this.app.context === 'object' ? this.app.context : {} @@ -144,8 +168,20 @@ class REQUEST { throw new Error('Method not allowed') } + // Enable sampling + this._sample = LOGGER.sampler(this.app,this) + } // end parseRequest + // Main logger + logger(...args) { + this.app._logger.level !== 'none' && + this.app._logLevels[args[0]] >= + this.app._logLevels[this._sample ? this._sample : this.app._logger.level] && + this._logs.push(this.app._logger.log(...args)) + } + + } // end REQUEST class // Export the response object diff --git a/lib/response.js b/lib/response.js index 023b74a..aa1ee72 100644 --- a/lib/response.js +++ b/lib/response.js @@ -19,6 +19,9 @@ class RESPONSE { // Create the constructor function. constructor(app,request) { + // Add a reference to the main app + app._response = this + // Create a reference to the app this.app = app @@ -305,7 +308,6 @@ class RESPONSE { // Add cache-control headers if (opts.cacheControl !== false) { - // console.log('cache control'); if (opts.cacheControl !== true && opts.cacheControl !== undefined) { this.cache(opts.cacheControl) } else { @@ -443,8 +445,11 @@ class RESPONSE { // Trigger API error - error(e) { - this.app.catchErrors(e,this) + error(code,e,detail) { + detail = typeof code !== 'number' && e !== undefined ? e : detail + e = typeof code !== 'number' ? code : e + code = typeof code === 'number' ? code : 500 + this.app.catchErrors(e,this,code,detail) } // end error } // end Response class diff --git a/test/download.js b/test/download.js index 70e1309..444f66b 100644 --- a/test/download.js +++ b/test/download.js @@ -41,7 +41,7 @@ api.get('/download', function(req,res) { api.get('/download/err', function(req,res) { res.download('./test-missing.txt', err => { if (err) { - res.status(404).error('There was an error accessing the requested file') + res.error(404,'There was an error accessing the requested file') } }) }) @@ -50,13 +50,10 @@ api.get('/download/test', function(req,res) { res.download('test/test.txt' + (req.query.test ? req.query.test : ''), err => { // Return a promise - return Promise.try(() => { - for(let i = 0; i<40000000; i++) {} - return true - }).then((x) => { + return Promise.delay(100).then((x) => { if (err) { // set custom error code and message on error - res.status(501).error('Custom File Error') + res.error(501,'Custom File Error') } else { // else set custom response code res.status(201) diff --git a/test/errorHandling.js b/test/errorHandling.js index 57a13cf..2433c4c 100644 --- a/test/errorHandling.js +++ b/test/errorHandling.js @@ -8,12 +8,7 @@ const api = require('../index')({ version: 'v1.0' }) const api2 = require('../index')({ version: 'v1.0' }) const api3 = require('../index')({ version: 'v1.0' }) const api4 = require('../index')({ version: 'v1.0' }) - -// NOTE: Set test to true -api._test = true; -api2._test = true; -api3._test = true; -api4._test = true; +const api5 = require('../index')({ version: 'v1.0', logger: true}) let event = { httpMethod: 'get', @@ -52,10 +47,7 @@ api.use(function(err,req,res,next) { api.use(function(err,req,res,next) { if (req.route === '/testErrorPromise') { let start = Date.now() - Promise.try(() => { - for(let i = 0; i<40000000; i++) {} - return true - }).then((x) => { + Promise.delay(100).then((x) => { res.header('Content-Type','text/plain') res.send('This is a test error message: ' + req.testError1 + '/' + req.testError2) }) @@ -87,8 +79,7 @@ const returnError = (err,req,res,next) => { api3.use(returnError,errorMiddleware1) const callError = (err,req,res,next) => { - res.send('this is an error: ' + (req.errorMiddleware1 ? true : false)) - + res.status(500).send('this is an error: ' + (req.errorMiddleware1 ? true : false)) next() } @@ -99,7 +90,6 @@ api4.use(callError,errorMiddleware1) /******************************************************************************/ api.get('/testError', function(req,res) { - res.status(500) res.error('This is a test error message') }) @@ -113,12 +103,10 @@ api.get('/testErrorSimulated', function(req,res) { }) api.get('/testErrorMiddleware', function(req,res) { - res.status(500) res.error('This test error message should be overridden') }) api.get('/testErrorPromise', function(req,res) { - res.status(500) res.error('This is a test error message') }) @@ -128,15 +116,23 @@ api2.get('/testError', function(req,res) { }) api3.get('/testError', function(req,res) { - res.status(500) res.error('This is a test error message') }) api4.get('/testError', function(req,res) { - res.status(500) + res.error(403,'This is a test error message') +}) + +api5.get('/testError', function(req,res) { res.error('This is a test error message') }) +api5.get('/testErrorThrow', function(req,res) { + throw new Error('This is a test thrown error') +}) + + + /******************************************************************************/ /*** BEGIN TESTS ***/ /******************************************************************************/ @@ -207,12 +203,13 @@ describe('Error Handling Tests:', function() { let _event = Object.assign({},event,{ path: '/testErrorThrow'}) let logger = console.log api._test = false - console.log = log => _log = log - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + console.log = log => { try { _log = JSON.parse(log) } catch(e) { _log = log } } + let result = await new Promise(r => api5.run(_event,{},(e,res) => { r(res) })) api._test = true console.log = logger expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test thrown error"}', isBase64Encoded: false }) - expect(_log.message).to.equal('This is a test thrown error') + expect(_log.level).to.equal('fatal') + expect(_log.msg).to.equal('This is a test thrown error') }) // end it @@ -221,12 +218,13 @@ describe('Error Handling Tests:', function() { let _event = Object.assign({},event,{ path: '/testError'}) let logger = console.log api._test = false - console.log = (...args) => _log = args.join(' ') - let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + console.log = log => { try { _log = JSON.parse(log) } catch(e) { _log = log } } + let result = await new Promise(r => api5.run(_event,{},(e,res) => { r(res) })) api._test = true console.log = logger expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, statusCode: 500, body: '{"error":"This is a test error message"}', isBase64Encoded: false }) - expect(_log).to.equal('API Error: This is a test error message') + expect(_log.level).to.equal('error') + expect(_log.msg).to.equal('This is a test error message') }) // end it }) diff --git a/test/getLink.js b/test/getLink.js index 7f81139..c01be12 100644 --- a/test/getLink.js +++ b/test/getLink.js @@ -163,7 +163,7 @@ describe('getLink Tests:', function() { let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, - statusCode: 200, + statusCode: 500, body: '{"error":"getSignedUrl error"}', isBase64Encoded: false }) @@ -175,7 +175,7 @@ describe('getLink Tests:', function() { let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, - statusCode: 200, + statusCode: 500, body: '{"error":"Custom error"}', isBase64Encoded: false }) @@ -187,7 +187,7 @@ describe('getLink Tests:', function() { let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) expect(result).to.deep.equal({ headers: { 'content-type': 'application/json' }, - statusCode: 200, + statusCode: 500, body: '{"error":"getSignedUrl error"}', isBase64Encoded: false }) diff --git a/test/routes.js b/test/routes.js index 041317e..75f11ec 100644 --- a/test/routes.js +++ b/test/routes.js @@ -3,14 +3,9 @@ const expect = require('chai').expect // Assertion library // Init API instance -const api = require('../index')({ version: 'v1.0' }) -const api2 = require('../index')({ version: 'v1.0' }) -const api3 = require('../index')({ version: 'v1.0' }) - -// NOTE: Set test to true -api._test = true; -api2._test = true; -api3._test = true; +const api = require('../index')({ version: 'v1.0', logger: false }) +const api2 = require('../index')({ version: 'v1.0', logger: false }) +const api3 = require('../index')({ version: 'v1.0', logger: false }) let event = { httpMethod: 'get', diff --git a/test/run.js b/test/run.js index 10186eb..83d06a0 100644 --- a/test/run.js +++ b/test/run.js @@ -32,7 +32,7 @@ api.get('/test', function(req,res) { }) api.get('/testError', function(req,res) { - res.status(404).error('some error') + res.error(404,'some error') }) api_error.get('/testErrorThrown', function(req,res) { diff --git a/test/sendFile.js b/test/sendFile.js index c72bb85..0fdf60b 100644 --- a/test/sendFile.js +++ b/test/sendFile.js @@ -48,7 +48,7 @@ api.get('/sendfile/root', function(req,res) { api.get('/sendfile/err', function(req,res) { res.sendFile('./test-missing.txt', err => { if (err) { - res.status(404).error('There was an error accessing the requested file') + res.error(404,'There was an error accessing the requested file') } }) }) @@ -62,7 +62,7 @@ api.get('/sendfile/test', function(req,res) { }).then((x) => { if (err) { // set custom error code and message on error - res.status(501).error('Custom File Error') + res.error(501,'Custom File Error') } else { // else set custom response code res.status(201) From 31df7c57cedb0187312ef0e2c36e11cde153765d Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sat, 18 Aug 2018 02:23:36 -0400 Subject: [PATCH 24/38] fix test config --- test/errorHandling.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/errorHandling.js b/test/errorHandling.js index 2433c4c..43ac4f2 100644 --- a/test/errorHandling.js +++ b/test/errorHandling.js @@ -8,7 +8,7 @@ const api = require('../index')({ version: 'v1.0' }) const api2 = require('../index')({ version: 'v1.0' }) const api3 = require('../index')({ version: 'v1.0' }) const api4 = require('../index')({ version: 'v1.0' }) -const api5 = require('../index')({ version: 'v1.0', logger: true}) +const api5 = require('../index')({ version: 'v1.0', logger: { access: 'never' }}) let event = { httpMethod: 'get', From fe54a088b003bdc55866b0299c82773054d932a1 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sat, 18 Aug 2018 15:50:37 -0400 Subject: [PATCH 25/38] add additional sampler support for #45 --- lib/logger.js | 89 +++++++++++++++++++++++++++++++++++--------------- lib/request.js | 3 +- lib/utils.js | 33 ++++++++++++------- 3 files changed, 86 insertions(+), 39 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index d763062..5c24739 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -6,6 +6,7 @@ * @license MIT */ +const UTILS = require('./utils') // Require utils library // Config logger exports.config = (config,levels) => { @@ -90,7 +91,8 @@ exports.config = (config,levels) => { level, time: timestamp(), id: req.id, - path: req.path, + route: req.route, + method: req.method, [messageKey]: msg, timer: timer(req._start), sample: req._sample ? true : undefined @@ -131,12 +133,33 @@ exports.sampler = (app,req) => { if (app._logger.sampling) { - // default level to false + // Default level to false let level = false - // Grab the rule based on route - let rule = app._logger.sampling.rules && app._logger.sampling.rules[req.route] - || app._logger.sampling.defaults + // Create local reference to the rulesMap + let map = app._logger.sampling.rulesMap + + // Parse the current route + let route = UTILS.parsePath(req.route) + + // Default wildcard mapping + let wildcard = {} + + // Loop the map and see if this route matches + route.forEach(part => { + // Capture wildcard mappings + if (map['*']) wildcard = map['*'] + // Traverse map + map = map[part] ? map[part] : {} + }) // end for loop + + // Set rule reference based on route + let ref = map['__'+req.method] ? map['__'+req.method] : + map['__ANY'] ? map['__ANY'] : wildcard['__'+req.method] ? + wildcard['__'+req.method] : wildcard['__ANY'] ? + wildcard['__ANY'] : -1 + + let rule = ref >= 0 ? app._logger.sampling.rules[ref] : app._logger.sampling.defaults // Get last sample time (default start, last, fixed count, period count and total count) let counts = app._sampleCounts[rule.default ? 'default' : req.route] @@ -146,6 +169,9 @@ exports.sampler = (app,req) => { let now = Date.now() + // Calculate the current velocity + let velocity = rule.rate > 0 ? rule.period*1000/(counts.tCount/(now-app._initTime)*rule.period*1000*rule.rate) : 0 + // If this is a new period, reset values if ((now-counts.start) > rule.period*1000) { counts.start = now @@ -164,30 +190,15 @@ exports.sampler = (app,req) => { counts.fCount++ console.log('\n*********** FIXED ***********'); } else if (rule.rate > 0 && - counts.start+Math.floor(rule.period*1000/(counts.tCount/(now-app._initTime)*rule.period*1000*rule.rate)*counts.pCount) < now) { + counts.start+Math.floor(velocity*counts.pCount+velocity/2) < now) { level = rule.level counts.pCount++ console.log('\n*********** RATE ***********'); } - // Increment total count counts.tCount++ - - // console.log( - // '-----------------------------------', - // '\nlastSample:',app._lastSample, - // '\nsampleCounts:',app._sampleCounts, - // '\nrequestCount:',app._requestCount, - // '\ninitTime:',app._initTime, - // '\nlogger:',JSON.stringify(app._logger,null,2), - // '\nroute:', req.route, - // '\nmethod:', req.method, - // '\n-----------------------------------' - // ) - - return level } // end if sampling @@ -210,25 +221,49 @@ const parseSamplerConfig = (config,levels) => { target: Number.isInteger(inputs.target) ? inputs.target : 1, rate: !isNaN(inputs.rate) && inputs.rate <= 1 ? inputs.rate : 0.1, period: Number.isInteger(inputs.period) ? inputs.period : 60, // in seconds - method: (Array.isArray(inputs.method) ? inputs.method : - typeof inputs.method === 'string' ? - inputs.method.split(',') : ['*']).map(x => x.toString().trim()), level: Object.keys(levels).includes(inputs.level) ? inputs.level : 'trace' } } + // Init ruleMap + let rulesMap = {} + // Parse and default rules - let rules = Array.isArray(cfg.rules) ? cfg.rules.reduce((acc,rule) => { + let rules = Array.isArray(cfg.rules) ? cfg.rules.map((rule,i) => { // Error if missing route or not a string if (!rule.route || typeof rule.route !== 'string') throw new Error('Invalid route specified in rule') - return Object.assign(acc, { [rule.route.trim()]: defaults(rule) }) + // Parse methods into array (if not already) + let methods = (Array.isArray(rule.method) ? rule.method : + typeof rule.method === 'string' ? + rule.method.split(',') : ['ANY']).map(x => x.toString().trim().toUpperCase()) + + let map = {} + let recursive = map // create recursive reference + + //rule.route.replace(/^\//,'').split('/').forEach(part => { + UTILS.parsePath(rule.route).forEach(part => { + Object.assign(recursive,{ [part === '' ? '/' : part]: {} }) + recursive = recursive[part === '' ? '/' : part] + }) + + Object.assign(recursive, methods.reduce((acc,method) => { + return Object.assign(acc, { ['__'+method]: i }) + },{})) + + // Deep merge the maps + UTILS.deepMerge(rulesMap,map) + + return defaults(rule) },{}) : {} + // console.log(JSON.stringify(rulesMap,null,2)); + return { defaults: Object.assign(defaults(cfg),{ default:true }), - rules + rules, + rulesMap } } // end parseSamplerConfig diff --git a/lib/request.js b/lib/request.js index 2032fc0..eb1f171 100644 --- a/lib/request.js +++ b/lib/request.js @@ -114,7 +114,8 @@ class REQUEST { } // Extract path from event (strip querystring just in case) - let path = this.app._event.path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/') + // let path2 = this.app._event.path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/') + let path = UTILS.parsePath(this.app._event.path) // Init the route this.route = null diff --git a/lib/utils.js b/lib/utils.js index 2acdd68..7bfa241 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -17,7 +17,7 @@ const entityMap = { '\'': ''' } -module.exports.escapeHtml = html => html.replace(/[&<>"']/g, s => entityMap[s]) +exports.escapeHtml = html => html.replace(/[&<>"']/g, s => entityMap[s]) // From encodeurl by Douglas Christopher Wilson @@ -25,7 +25,7 @@ let ENCODE_CHARS_REGEXP = /(?:[^\x21\x25\x26-\x3B\x3D\x3F-\x5B\x5D\x5F\x61-\x7A\ let UNMATCHED_SURROGATE_PAIR_REGEXP = /(^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF]([^\uDC00-\uDFFF]|$)/g let UNMATCHED_SURROGATE_PAIR_REPLACE = '$1\uFFFD$2' -module.exports.encodeUrl = url => String(url) +exports.encodeUrl = url => String(url) .replace(UNMATCHED_SURROGATE_PAIR_REGEXP, UNMATCHED_SURROGATE_PAIR_REPLACE) .replace(ENCODE_CHARS_REGEXP, encodeURI) @@ -34,11 +34,14 @@ module.exports.encodeUrl = url => String(url) const encodeBody = body => typeof body === 'object' ? JSON.stringify(body) : (body && typeof body !== 'string' ? body.toString() : (body ? body : '')) -module.exports.encodeBody = encodeBody +exports.encodeBody = encodeBody +exports.parsePath = path => { + return path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/') +} -module.exports.parseBody = body => { +exports.parseBody = body => { try { return JSON.parse(body) } catch(e) { @@ -65,7 +68,7 @@ const parseAuthValue = (type,value) => { } } -module.exports.parseAuth = authStr => { +exports.parseAuth = authStr => { let auth = authStr && typeof authStr === 'string' ? authStr.split(' ') : [] return auth.length > 1 && ['Bearer','Basic','Digest','OAuth'].includes(auth[0]) ? parseAuthValue(auth[0], auth.slice(1).join(' ').trim()) : @@ -74,10 +77,9 @@ module.exports.parseAuth = authStr => { - const mimeMap = require('./mimemap.js') // MIME Map -module.exports.mimeLookup = (input,custom={}) => { +exports.mimeLookup = (input,custom={}) => { let type = input.trim().replace(/^\./,'') // If it contains a slash, return unmodified @@ -103,20 +105,29 @@ const extractRoutes = (routes,table=[]) => { return table } -module.exports.extractRoutes = extractRoutes +exports.extractRoutes = extractRoutes // Generate an Etag for the supplied value -module.exports.generateEtag = data => +exports.generateEtag = data => crypto.createHash('sha256').update(encodeBody(data)).digest('hex').substr(0,32) // Check if valid S3 path -module.exports.isS3 = path => /^s3:\/\/.+\/.+/i.test(path) +exports.isS3 = path => /^s3:\/\/.+\/.+/i.test(path) + // Parse S3 path -module.exports.parseS3 = path => { +exports.parseS3 = path => { if (!this.isS3(path)) throw new Error('Invalid S3 path') let s3object = path.replace(/^s3:\/\//i,'').split('/') return { Bucket: s3object.shift(), Key: s3object.join('/') } } + + +// Deep Merge +exports.deepMerge = (a,b) => { + Object.keys(b).forEach(key => (key in a) ? + this.deepMerge(a[key],b[key]) : Object.assign(a,b) ) + return a +} From dedd89eb55612e304f447ffe413990f70cff183f Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sat, 18 Aug 2018 20:38:36 -0400 Subject: [PATCH 26/38] add logging tests and fix misc bugs --- index.js | 7 +- lib/logger.js | 18 +- test/log.js | 864 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 877 insertions(+), 12 deletions(-) create mode 100644 test/log.js diff --git a/index.js b/index.js index 765284f..4ed441c 100644 --- a/index.js +++ b/index.js @@ -276,11 +276,8 @@ class API { // Generate access log if ((this._logger.access || response._request._logs.length > 0) && this._logger.access !== 'never') { - let access = this._logger.log( - 'access', - undefined, - response._request, - response._request.context, + let access = Object.assign( + this._logger.log('access',undefined,response._request,response._request.context), { statusCode: res.statusCode, coldStart: response._request.coldStart } ) console.log(JSON.stringify(this._logger.format(access,response._request,response))) // eslint-disable-line no-console diff --git a/lib/logger.js b/lib/logger.js index 5c24739..b05c1b0 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -6,6 +6,9 @@ * @license MIT */ + // IDEA: request counts + // IDEA: unique function identifier + const UTILS = require('./utils') // Require utils library // Config logger @@ -54,7 +57,6 @@ exports.config = (config,levels) => { path: req.path, ip: req.ip, ua: req.userAgent, - method: req.method, version: req.version, qs: Object.keys(req.query).length > 0 ? req.query : undefined } @@ -77,14 +79,15 @@ exports.config = (config,levels) => { req: cfg.serializers && typeof cfg.serializers.req === 'function' ? cfg.serializers.req : () => {}, res: cfg.serializers && typeof cfg.serializers.res === 'function' ? cfg.serializers.res : () => {}, context: cfg.serializers && typeof cfg.serializers.context === 'function' ? cfg.serializers.context : () => {}, - custom: cfg.serializers && typeof cfg.serializers.context === 'function' ? cfg.serializers.context : () => {} + custom: cfg.serializers && typeof cfg.serializers.custom === 'function' ? cfg.serializers.custom : () => {} } // Main logging function let log = (level,msg,req,context,custom) => { - let _context = Object.assign(defaults.context(context),serializers.custom(context)) - let _custom = Object.assign(defaults.custom(custom),serializers.custom(custom)) + let _context = Object.assign({},defaults.context(context),serializers.context(context)) + let _custom = typeof custom === 'string' ? defaults.custom(custom) : + Object.assign({},defaults.custom(custom),serializers.custom(custom)) return Object.assign({}, { @@ -101,13 +104,14 @@ exports.config = (config,levels) => { nested ? { [customKey]: _custom } : _custom, nested ? { context: _context } : _context ) - } + + } // end log // Formatting function for additional log data enrichment let format = function(info,req,res) { - let _req = Object.assign(defaults.req(req),serializers.req(req)) - let _res = Object.assign(defaults.res(res),serializers.res(res)) + let _req = Object.assign({},defaults.req(req),serializers.req(req)) + let _res = Object.assign({},defaults.res(res),serializers.res(res)) return Object.assign({}, info, diff --git a/test/log.js b/test/log.js new file mode 100644 index 0000000..ec90c2c --- /dev/null +++ b/test/log.js @@ -0,0 +1,864 @@ +'use strict' + +const expect = require('chai').expect // Assertion library + +// Init API instance +const api_default = require('../index')({ logger: true }) // default logger +const api_customLevel = require('../index')({ version: 'v1.0', logger: { level: 'debug' } }) +const api_disableLevel = require('../index')({ version: 'v1.0', logger: { level: 'none' } }) +const api_customMsgKey = require('../index')({ version: 'v1.0', logger: { messageKey: 'customKey' } }) +const api_customCustomKey = require('../index')({ version: 'v1.0', logger: { customKey: 'customKey' } }) +const api_noTimer = require('../index')({ version: 'v1.0', logger: { timer: false } }) +const api_noTimestamp = require('../index')({ version: 'v1.0', logger: { timestamp: false } }) +const api_customTimestamp = require('../index')({ version: 'v1.0', logger: { timestamp: () => new Date().toUTCString() } }) +const api_accessLogs = require('../index')({ version: 'v1.0', logger: { access: true } }) +const api_noAccessLogs = require('../index')({ version: 'v1.0', logger: { access: 'never' } }) +const api_nested = require('../index')({ version: 'v1.0', logger: { nested: true, access: true } }) +const api_withDetail = require('../index')({ version: 'v1.0', logger: { detail: true } }) +const api_showStackTrace = require('../index')({ version: 'v1.0', logger: { stack: true } }) +const api_customLevels = require('../index')({ version: 'v1.0', logger: { + levels: { + testDebug: 35, + testInfo: 30, + trace: 90 + } +} }) +const api_customSerializers = require('../index')({ version: 'v1.0', logger: { + serializers: { + main: (req) => { + return { + account: req.requestContext.accountId || 'no account', + id: undefined + } + }, + req: (req) => { + return { + 'TESTPATH': req.path, + method: undefined + } + }, + res: (res) => { + return { + 'STATUS': res._statusCode + } + }, + context: (context) => { + return { + 'TEST_CONTEXT': true + } + }, + custom: (custom) => { + return { + 'TEST_CUSTOM': true + } + } + } +} }) + + + + + +// const api3 = require('../index')({ +// logger: { +// sampling: { +// target: 2, +// rate: 0.3, +// period: 300, // seconds +// level: 'debug' +// } +// } +// }) + + + + +// const api6 = require('../index')({ +// logger: { +// sampling: { +// rules: [ +// { route: '/testError', target: 0, rate:0, method: 'POST, GET, DeLete' }, +// { route: '/testlog', target: 10, rate:0, period:10 }, +// { route: '/testlog/:param', target: 2, rate:0, period:10 }, +// { route: '/testlog/:param2', level: 'xyz', method: ['HEAD','PATCH'] }, +// { route: '/testlog/:param2/test', level: 'xyz', method: ['GET','OPTIONS'] }, +// { route: '/testlog/:param2/XYZ', level: 'xyz', method: ['PUT','DELETE'] }, +// { route: '/', level: 'xyz' } +// ], +// period: 30 +// } +// } +// }) + + + + +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'content-type': 'application/json', + 'x-forwarded-for': '12.34.56.78, 23.45.67.89', + 'User-Agent': 'user-agent-string' + } +} + +let context = { + awsRequestId: 'AWSREQID', + functionName: 'testFunc', + memoryLimitInMB: '2048', + getRemainingTimeInMillis: () => 5000 +} + +api_default.get('/', (req,res) => { + req.log.trace('trace message') + req.log.debug('debug message') + req.log.info('info message') + req.log.warn('warn message') + req.log.error('error message') + req.log.fatal('fatal message') + res.send('done') +}) + +api_default.get('/test', (req,res) => { + res.send('done') +}) + +api_customLevel.get('/', (req,res) => { + req.log.trace('trace message') + req.log.debug('debug message') + req.log.info('info message') + req.log.warn('warn message') + req.log.error('error message') + req.log.fatal('fatal message') + res.send('done') +}) + +api_disableLevel.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_customMsgKey.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_customCustomKey.get('/', (req,res) => { + req.log.info('info message','custom messsage') + res.send('done') +}) + +api_noTimer.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_noTimestamp.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_customTimestamp.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_accessLogs.get('/', (req,res) => { + res.send('done') +}) + +api_accessLogs.get('/test', (req,res) => { + res.send('done') +}) + +api_noAccessLogs.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_nested.get('/', (req,res) => { + req.log.info('info message',{ customMsg: 'custom message' }) + res.send('done') +}) + +api_withDetail.get('/', (req,res) => { + req.log.info('info message') + res.send('done') +}) + +api_customLevels.get('/', (req,res) => { + req.log.testDebug('testDebug message') + req.log.testInfo('testDebug message') + req.log.trace('trace message - higher priority') + res.send('done') +}) + +api_customSerializers.get('/', (req,res) => { + req.log.info('info message',{test:true}) + res.send('done') +}) + +api_showStackTrace.get('/', (req,res) => { + undefinedVar // this should throw an error + res.send('done') +}) + +// +// +// api.get('/testlog', (req,res) => { +// res.send('test') +// }) +// +// api.get('/testlog/:param', (req,res) => { +// res.send('test param') +// }) +// +// api6.get('/testlog/:param', (req,res) => { +// res.send('test param') +// }) +// +// +// api2.get('/test', (req,res) => { +// req.log.trace('log message') +// req.log.debug('log message') +// req.log.info('log message') +// req.log.warn('log message') +// req.log.error('log message',{ foo: 'bar', bar: 'foo' }) +// req.log.fatal('log message','custom string') +// res.send('done') +// }) +// +// api3.get('/test', (req,res) => { +// // req.log('message') +// req.log.trace('log message') +// req.log.debug('log message') +// req.log.info('log message') +// req.log.warn('log message') +// req.log.error('log message',{ foo: 'bar', bar: 'foo' }) +// req.log.fatal('log message','custom string') +// res.send('done') +// }) +// +// api4.get('/test', (req,res) => { +// // req.log('message') +// req.log.x('some message') +// req.log.test('some message') +// req.log.trace('log message') +// req.log.debug('log message') +// req.log.info('log messageX') +// req.log.warn('log message') +// req.log.error('log message',{ foo: 'bar', bar: 'foo' }) +// req.log.fatal('log message','custom string') +// // res.error('error message',{ foo: 'barerror' }) +// //res.error('error message','string') +// res.error('error message') +// }) + + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +describe('Logging Tests:', function() { + + // Create a temporary logger to capture the console.log + let _log = [] + let consoleLog = console.log + const logger = log => { try { _log.push(JSON.parse(log)) } catch(e) { _log.push(log) } } + + // Clear the log before each test + beforeEach(function() { _log = [] }) + + + it('Default options (logging: true)', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_default.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(5) + expect(_log[0].level).to.equal('info') + expect(_log[4].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[4]).to.have.property('coldStart') + expect(_log[4]).to.have.property('statusCode') + expect(_log[4]).to.have.property('path') + expect(_log[4]).to.have.property('ip') + expect(_log[4]).to.have.property('ua') + expect(_log[4]).to.have.property('version') + expect(_log[4]).to.have.property('qs') + }) // end it + + + it('Default options (no logs in routes)', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/test' }) + let result = await new Promise(r => api_default.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(0) + }) // end it + + + it('Custom level (debug)', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_customLevel.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(6) + expect(_log[0].level).to.equal('debug') + expect(_log[5].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[5]).to.have.property('coldStart') + expect(_log[5]).to.have.property('statusCode') + expect(_log[5]).to.have.property('path') + expect(_log[5]).to.have.property('ip') + expect(_log[5]).to.have.property('ua') + expect(_log[5]).to.have.property('version') + }) // end it + + + it('Disable (via level setting)', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_disableLevel.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(0) + }) // end it + + + it('Custom Message Label', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_customMsgKey.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.not.have.property('msg') + expect(_log[0]).to.have.property('customKey') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('statusCode') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + it('Custom "custom" Label', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_customCustomKey.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.not.have.property('custom') + expect(_log[0]).to.have.property('customKey') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('statusCode') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + it('Disable Timer', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_noTimer.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.not.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('statusCode') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + + it('Disable Timestamp', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_noTimestamp.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.not.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('statusCode') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + + it('Custom Timestamp', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_customTimestamp.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0].time).to.be.a('string') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // access log + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('statusCode') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + it('Enable access logs', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_accessLogs.run(_event,context,(e,res) => { r(res) })) + let result2 = await new Promise(r => api_accessLogs.run(Object.assign(_event,{ path: '/test' }),context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + + expect(result2).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('access') + expect(_log[1].level).to.equal('access') + // access log 1 + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.not.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + expect(_log[0]).to.have.property('coldStart') + expect(_log[0]).to.have.property('path') + expect(_log[0]).to.have.property('ip') + expect(_log[0]).to.have.property('ua') + expect(_log[0]).to.have.property('version') + // access log 2 + expect(_log[1]).to.have.property('time') + expect(_log[1]).to.have.property('id') + expect(_log[1]).to.have.property('route') + expect(_log[1]).to.not.have.property('msg') + expect(_log[1]).to.have.property('timer') + expect(_log[1]).to.have.property('remaining') + expect(_log[1]).to.have.property('function') + expect(_log[1]).to.have.property('memory') + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + + it('No access logs (never)', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_noAccessLogs.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(1) + expect(_log[0].level).to.equal('info') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + // these should NOT exist + expect(_log[0]).to.not.have.property('coldStart') + expect(_log[0]).to.not.have.property('statusCode') + expect(_log[0]).to.not.have.property('path') + expect(_log[0]).to.not.have.property('ip') + expect(_log[0]).to.not.have.property('ua') + expect(_log[0]).to.not.have.property('version') + }) // end it + + + it('Nested objects', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_nested.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('context') + expect(_log[0].context).to.have.property('remaining') + expect(_log[0].context).to.have.property('function') + expect(_log[0].context).to.have.property('memory') + expect(_log[0]).to.have.property('custom') + expect(_log[0].custom).to.have.property('customMsg') + // access log + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('statusCode') + expect(_log[1]).to.have.property('req') + expect(_log[1].req).to.have.property('path') + expect(_log[1].req).to.have.property('ip') + expect(_log[1].req).to.have.property('ua') + expect(_log[1].req).to.have.property('version') + expect(_log[1]).to.have.property('res') + }) // end it + + + it('With detail per log', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_withDetail.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + expect(_log[0]).to.have.property('path') + expect(_log[0]).to.have.property('ip') + expect(_log[0]).to.have.property('ua') + expect(_log[0]).to.have.property('version') + + expect(_log[1]).to.have.property('time') + expect(_log[1]).to.have.property('id') + expect(_log[1]).to.have.property('route') + expect(_log[1]).to.have.property('timer') + expect(_log[1]).to.have.property('remaining') + expect(_log[1]).to.have.property('function') + expect(_log[1]).to.have.property('memory') + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + + it('Custom levels', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_customLevels.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + + expect(_log).to.have.length(4) + expect(_log[0].level).to.equal('testDebug') + expect(_log[1].level).to.equal('testInfo') + expect(_log[2].level).to.equal('trace') + expect(_log[3].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + // access log + expect(_log[3]).to.have.property('time') + expect(_log[3]).to.have.property('id') + expect(_log[3]).to.have.property('route') + expect(_log[3]).to.have.property('timer') + expect(_log[3]).to.have.property('remaining') + expect(_log[3]).to.have.property('function') + expect(_log[3]).to.have.property('memory') + expect(_log[3]).to.have.property('coldStart') + expect(_log[3]).to.have.property('path') + expect(_log[3]).to.have.property('ip') + expect(_log[3]).to.have.property('ua') + expect(_log[3]).to.have.property('version') + }) // end it + + + it('Custom serializers', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_customSerializers.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.not.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('TEST_CUSTOM') + expect(_log[0]).to.have.property('TEST_CONTEXT') + expect(_log[0]).to.have.property('test') + // access log + expect(_log[1]).to.have.property('time') + expect(_log[1]).to.not.have.property('id') + expect(_log[1]).to.have.property('route') + expect(_log[1]).to.have.property('timer') + expect(_log[1]).to.have.property('remaining') + expect(_log[1]).to.have.property('function') + expect(_log[1]).to.have.property('memory') + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('TESTPATH') + expect(_log[1]).to.have.property('STATUS') + }) // end it + + + it('Invalid custom levels configuration', async function() { + let error_message + try { + const api_error = require('../index')({ version: 'v1.0', logger: { + levels: { '123': 'test '} + } }) + } catch(e) { + error_message = e.message + } + expect(error_message).to.equal('Invalid level configuration') + }) // end it + + + it('Enable stack traces', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/' }) + let result = await new Promise(r => api_showStackTrace.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 500, + body: '{"error":"undefinedVar is not defined"}', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('fatal') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('memory') + expect(_log[0]).to.have.property('stack') + + expect(_log[1]).to.have.property('time') + expect(_log[1]).to.have.property('id') + expect(_log[1]).to.have.property('route') + expect(_log[1]).to.have.property('timer') + expect(_log[1]).to.have.property('remaining') + expect(_log[1]).to.have.property('function') + expect(_log[1]).to.have.property('memory') + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + }) // end it + + +}) From 7d5d3d7b3cba1b252a3322977073d4bb1c76be0d Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sat, 18 Aug 2018 23:32:35 -0400 Subject: [PATCH 27/38] complete sampler and logger tests for #45 --- lib/logger.js | 12 +- lib/request.js | 3 + test/log.js | 95 +---------- test/sampling.js | 413 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 429 insertions(+), 94 deletions(-) create mode 100644 test/sampling.js diff --git a/lib/logger.js b/lib/logger.js index b05c1b0..4e83432 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -165,6 +165,9 @@ exports.sampler = (app,req) => { let rule = ref >= 0 ? app._logger.sampling.rules[ref] : app._logger.sampling.defaults + // Assign rule reference to the REQUEST + req._sampleRule = rule + // Get last sample time (default start, last, fixed count, period count and total count) let counts = app._sampleCounts[rule.default ? 'default' : req.route] || Object.assign(app._sampleCounts, { @@ -185,19 +188,19 @@ exports.sampler = (app,req) => { if (rule.target > 0) { counts.fCount = 1 level = rule.level // set the sample level - console.log('\n*********** NEW PERIOD ***********'); + // console.log('\n*********** NEW PERIOD ***********'); } // Enable sampling if last sample is passed target split } else if (rule.target > 0 && counts.start+Math.floor(rule.period*1000/rule.target*counts.fCount) < now) { level = rule.level counts.fCount++ - console.log('\n*********** FIXED ***********'); + // console.log('\n*********** FIXED ***********'); } else if (rule.rate > 0 && counts.start+Math.floor(velocity*counts.pCount+velocity/2) < now) { level = rule.level counts.pCount++ - console.log('\n*********** RATE ***********'); + // console.log('\n*********** RATE ***********'); } // Increment total count @@ -246,7 +249,6 @@ const parseSamplerConfig = (config,levels) => { let map = {} let recursive = map // create recursive reference - //rule.route.replace(/^\//,'').split('/').forEach(part => { UTILS.parsePath(rule.route).forEach(part => { Object.assign(recursive,{ [part === '' ? '/' : part]: {} }) recursive = recursive[part === '' ? '/' : part] @@ -262,8 +264,6 @@ const parseSamplerConfig = (config,levels) => { return defaults(rule) },{}) : {} - // console.log(JSON.stringify(rulesMap,null,2)); - return { defaults: Object.assign(defaults(cfg),{ default:true }), rules, diff --git a/lib/request.js b/lib/request.js index eb1f171..efec2e7 100644 --- a/lib/request.js +++ b/lib/request.js @@ -169,6 +169,9 @@ class REQUEST { throw new Error('Method not allowed') } + // Reference to sample rule + this._sampleRule = {} + // Enable sampling this._sample = LOGGER.sampler(this.app,this) diff --git a/test/log.js b/test/log.js index ec90c2c..78f7abf 100644 --- a/test/log.js +++ b/test/log.js @@ -56,43 +56,7 @@ const api_customSerializers = require('../index')({ version: 'v1.0', logger: { } }) - - - -// const api3 = require('../index')({ -// logger: { -// sampling: { -// target: 2, -// rate: 0.3, -// period: 300, // seconds -// level: 'debug' -// } -// } -// }) - - - - -// const api6 = require('../index')({ -// logger: { -// sampling: { -// rules: [ -// { route: '/testError', target: 0, rate:0, method: 'POST, GET, DeLete' }, -// { route: '/testlog', target: 10, rate:0, period:10 }, -// { route: '/testlog/:param', target: 2, rate:0, period:10 }, -// { route: '/testlog/:param2', level: 'xyz', method: ['HEAD','PATCH'] }, -// { route: '/testlog/:param2/test', level: 'xyz', method: ['GET','OPTIONS'] }, -// { route: '/testlog/:param2/XYZ', level: 'xyz', method: ['PUT','DELETE'] }, -// { route: '/', level: 'xyz' } -// ], -// period: 30 -// } -// } -// }) - - - - +// Define default event let event = { httpMethod: 'get', path: '/test', @@ -104,6 +68,7 @@ let event = { } } +// Default context let context = { awsRequestId: 'AWSREQID', functionName: 'testFunc', @@ -111,6 +76,11 @@ let context = { getRemainingTimeInMillis: () => 5000 } + +/******************************************************************************/ +/*** DEFINE TEST ROUTES ***/ +/******************************************************************************/ + api_default.get('/', (req,res) => { req.log.trace('trace message') req.log.debug('debug message') @@ -205,57 +175,6 @@ api_showStackTrace.get('/', (req,res) => { res.send('done') }) -// -// -// api.get('/testlog', (req,res) => { -// res.send('test') -// }) -// -// api.get('/testlog/:param', (req,res) => { -// res.send('test param') -// }) -// -// api6.get('/testlog/:param', (req,res) => { -// res.send('test param') -// }) -// -// -// api2.get('/test', (req,res) => { -// req.log.trace('log message') -// req.log.debug('log message') -// req.log.info('log message') -// req.log.warn('log message') -// req.log.error('log message',{ foo: 'bar', bar: 'foo' }) -// req.log.fatal('log message','custom string') -// res.send('done') -// }) -// -// api3.get('/test', (req,res) => { -// // req.log('message') -// req.log.trace('log message') -// req.log.debug('log message') -// req.log.info('log message') -// req.log.warn('log message') -// req.log.error('log message',{ foo: 'bar', bar: 'foo' }) -// req.log.fatal('log message','custom string') -// res.send('done') -// }) -// -// api4.get('/test', (req,res) => { -// // req.log('message') -// req.log.x('some message') -// req.log.test('some message') -// req.log.trace('log message') -// req.log.debug('log message') -// req.log.info('log messageX') -// req.log.warn('log message') -// req.log.error('log message',{ foo: 'bar', bar: 'foo' }) -// req.log.fatal('log message','custom string') -// // res.error('error message',{ foo: 'barerror' }) -// //res.error('error message','string') -// res.error('error message') -// }) - /******************************************************************************/ /*** BEGIN TESTS ***/ diff --git a/test/sampling.js b/test/sampling.js new file mode 100644 index 0000000..647f7fe --- /dev/null +++ b/test/sampling.js @@ -0,0 +1,413 @@ +'use strict' + +const expect = require('chai').expect // Assertion library +const Promise = require('bluebird'); + +let request = 1 + +// Init API instance +const api_default = require('../index')({ logger: { sampling: true } }) +const api_rules = require('../index')({ + logger: { + access: 'never', + sampling: { + rules: [ + { route: '/testNone', target: 0, rate: 0, period: 1 }, + { route: '/testTarget', target: 3, rate: 0, period: 1 }, + { route: '/testTargetRate', target: 0, rate: 0.2, period: 1 }, + { route: '/testTargetMethod', target: 1, rate: 0.2, period: 1, method:'get,put' }, + { route: '/testTargetMethod', target: 2, rate: 0.1, period: 2, method:['Post','DELETE'], level:'info' }, + { route: '/testParam/:param', target: 10, level: 'debug' }, + { route: '/testParam/:param/deep', target: 20, level: 'info' }, + { route: '/testWildCard/*', target: 30, level: 'debug' }, + { route: '/testWildCard/test', target: 40, level: 'debug', method: 'Any' }, + { route: '/testWildCardMethod/*', target: 50, level: 'debug', method:'get' } + ], + target: 1, + rate: 0.1, + period: 1 + } + } +}) +const api_basePathRules = require('../index')({ + logger: { + access: false, + sampling: { + rules: [ + { route: '/', target: 0, rate: 0, period: 5, method: 'get' }, + ], + target: 1, + rate: 0.1, + period: 5 + } + } +}) + + + +// Define default event +let event = { + httpMethod: 'get', + path: '/test', + body: {}, + headers: { + 'content-type': 'application/json', + 'x-forwarded-for': '12.34.56.78, 23.45.67.89', + 'User-Agent': 'user-agent-string' + } +} + +// Default context +let context = { + awsRequestId: 'AWSREQID', + functionName: 'testFunc', + memoryLimitInMB: '2048', + getRemainingTimeInMillis: () => 5000 +} + + +/******************************************************************************/ +/*** DEFINE TEST ROUTES ***/ +/******************************************************************************/ + +api_rules.get('/', (req,res) => { + req.log.trace('request #'+request++) + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testNone', (req,res) => { + req.log.trace('request #'+request++) + res.send('done') +}) + +api_rules.get('/testTarget', (req,res) => { + req.log.trace('request #'+request++) + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testTargetRate', (req,res) => { + req.log.trace('request #'+request++) + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testTargetMethod', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.post('/testTargetMethod', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testParam/:param', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testParam/:param/deep', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testWildCard/test', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testWildCard/other', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testWildCard/other/deep', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + +api_rules.get('/testWildCardMethod/other', (req,res) => { + res.send({ method: req.method, rule: req._sampleRule }) +}) + + + + +/******************************************************************************/ +/*** BEGIN TESTS ***/ +/******************************************************************************/ + +describe('Sampling Tests:', function() { + + describe('Configurations:', function() { + it('Invalid sampling config', async function() { + let error_message + try { + const api_error = require('../index')({ version: 'v1.0', logger: { + sampling: 'invalid' + } }) + } catch(e) { + error_message = e.message + } + expect(error_message).to.equal('Invalid sampler configuration') + }) // end it + + + it('Missing route in rules', async function() { + let error_message + try { + const api_error = require('../index')({ version: 'v1.0', logger: { + sampling: { + rules: [ + { route: '/testNone', target: 0, rate: 0, period: 5 }, + { target: 0, rate: 0, period: 5 } + ] + } + } }) + } catch(e) { + error_message = e.message + } + expect(error_message).to.equal('Invalid route specified in rule') + }) // end it + + + it('Invalid route in rules', async function() { + let error_message + try { + const api_error = require('../index')({ version: 'v1.0', logger: { + sampling: { + rules: [ + { route: '/testNone', target: 0, rate: 0, period: 5 }, + { route: 1, target: 0, rate: 0, period: 5 } + ] + } + } }) + } catch(e) { + error_message = e.message + } + expect(error_message).to.equal('Invalid route specified in rule') + }) // end it + + }) + + describe('Rule Matching', function() { + + it('Match based on method (GET)', async function() { + let _event = Object.assign({},event,{ path: '/testTargetMethod', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(1) + expect(data.rule.rate).to.equal(0.2) + expect(data.rule.period).to.equal(1) + expect(data.rule.level).to.equal('trace') + }) + + it('Match based on method (POST)', async function() { + let _event = Object.assign({},event,{ httpMethod: 'post', path: '/testTargetMethod', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('POST') + expect(data.rule.target).to.equal(2) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(2) + expect(data.rule.level).to.equal('info') + }) + + it('Match parameterized path', async function() { + let _event = Object.assign({},event,{ path: '/testParam/foo', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(10) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(60) + expect(data.rule.level).to.equal('debug') + }) + + it('Match deep parameterized path', async function() { + let _event = Object.assign({},event,{ path: '/testParam/foo/deep', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(20) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(60) + expect(data.rule.level).to.equal('info') + }) + + it('Match wildcard route', async function() { + let _event = Object.assign({},event,{ path: '/testWildCard/other', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(30) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(60) + expect(data.rule.level).to.equal('debug') + }) + + it('Match static route (w/ wildcard at the same level)', async function() { + let _event = Object.assign({},event,{ path: '/testWildCard/test', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(40) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(60) + expect(data.rule.level).to.equal('debug') + }) + + it('Match deep wildcard route', async function() { + let _event = Object.assign({},event,{ path: '/testWildCard/other/deep', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(30) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(60) + expect(data.rule.level).to.equal('debug') + }) + + it('Match wildcard route (by method)', async function() { + let _event = Object.assign({},event,{ path: '/testWildCardMethod/other', queryStringParameters: { test: true } }) + let result = await new Promise(r => api_rules.run(_event,context,(e,res) => { r(res) })) + let data = JSON.parse(result.body) + + expect(data.method).to.equal('GET') + expect(data.rule.target).to.equal(50) + expect(data.rule.rate).to.equal(0.1) + expect(data.rule.period).to.equal(60) + expect(data.rule.level).to.equal('debug') + }) + }) + + + describe('Simulations', function() { + + // Create a temporary logger to capture the console.log + let consoleLog = console.log + let _log = [] + const logger = log => { try { _log.push(JSON.parse(log)) } catch(e) { _log.push(log) } } + + + it('Default route', async function() { + this.timeout(10000); + this.slow(10000); + _log = [] // clear log + request = 1 // reset requests + api_rules._initTime = Date.now() // reset _initTime for the API + + // Set the number of simulated requests + let requests = 100 + + // Set the default event, init result, override logger, and start the counter + let _event = Object.assign({},event,{ path: '/' }) + let result + console.log = logger + let start = Date.now() + + for(let x = 0; x api_rules.run(_event,context,(e,res) => { r(res) })) + await Promise.delay(Math.random()*25) + } // end for loop + + // End the timer and restore console.log + let end = Date.now() + console.log = consoleLog + + let data = JSON.parse(result.body) + let rules = data.rule + + let totalTime = end - start + let totalFixed = Math.ceil(totalTime/(rules.period*1000)*rules.target) + let totalRate = Math.ceil(requests*rules.rate) + let deviation = Math.abs(((totalFixed+totalRate)/_log.length-1).toFixed(2)) + + expect(deviation).to.be.below(0.12) + }) // end it + + + + it('Fixed target only route', async function() { + this.timeout(10000); + this.slow(10000); + _log = [] // clear log + request = 1 // reset requests + api_rules._initTime = Date.now() // reset _initTime for the API + + // Set the number of simulated requests + let requests = 100 + + // Set the default event, init result, override logger, and start the counter + let _event = Object.assign({},event,{ path: '/testTarget' }) + let result + console.log = logger + let start = Date.now() + + for(let x = 0; x api_rules.run(_event,context,(e,res) => { r(res) })) + await Promise.delay(Math.random()*25) + } // end for loop + + // End the timer and restore console.log + let end = Date.now() + console.log = consoleLog + + let data = JSON.parse(result.body) + let rules = data.rule + + let totalTime = end - start + let totalFixed = Math.ceil(totalTime/(rules.period*1000)*rules.target) + let totalRate = Math.ceil(requests*rules.rate) + let deviation = Math.abs(((totalFixed+totalRate)/_log.length-1).toFixed(2)) + + // console.log(_log.length,totalFixed,totalRate,deviation) + expect(deviation).to.be.below(0.12) + }) // end it + + + + it('Fixed rate only route', async function() { + this.timeout(10000); + this.slow(10000); + _log = [] // clear log + request = 1 // reset requests + api_rules._initTime = Date.now() // reset _initTime for the API + + // Set the number of simulated requests + let requests = 100 + + // Set the default event, init result, override logger, and start the counter + let _event = Object.assign({},event,{ path: '/testTargetRate' }) + let result + console.log = logger + let start = Date.now() + + for(let x = 0; x api_rules.run(_event,context,(e,res) => { r(res) })) + await Promise.delay(Math.random()*25) + // await Promise.delay(20) + } // end for loop + + // End the timer and restore console.log + let end = Date.now() + console.log = consoleLog + + let data = JSON.parse(result.body) + let rules = data.rule + + let totalTime = end - start + let totalFixed = Math.ceil(totalTime/(rules.period*1000)*rules.target) + let totalRate = Math.ceil(requests*rules.rate) + let deviation = Math.abs(((totalFixed+totalRate)/_log.length-1).toFixed(2)) + + // console.log(_log); + // console.log(totalTime,_log.length,totalFixed,totalRate,deviation) + expect(deviation).to.be.below(0.12) + }) // end it + + + + }) // end simulations + +}) From d5e4c366f4aed4a426be5fb7548f9ef5dc8cad3e Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 18:00:51 -0400 Subject: [PATCH 28/38] add clientType, clientCountry and requestCount to request object --- index.js | 2 +- lib/request.js | 16 ++++++- test/headers.js | 121 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 4ed441c..61c45d3 100644 --- a/index.js +++ b/index.js @@ -278,7 +278,7 @@ class API { if ((this._logger.access || response._request._logs.length > 0) && this._logger.access !== 'never') { let access = Object.assign( this._logger.log('access',undefined,response._request,response._request.context), - { statusCode: res.statusCode, coldStart: response._request.coldStart } + { statusCode: res.statusCode, coldStart: response._request.coldStart, count: response._request.requestCount } ) console.log(JSON.stringify(this._logger.format(access,response._request,response))) // eslint-disable-line no-console } diff --git a/lib/request.js b/lib/request.js index efec2e7..967bb9a 100644 --- a/lib/request.js +++ b/lib/request.js @@ -25,7 +25,7 @@ class REQUEST { this.coldStart = app._requestCount === 0 ? true : false // Increment the requests counter - app._requestCount++ + this.requestCount = ++app._requestCount // Init the handler this._handler @@ -98,6 +98,18 @@ class REQUEST { // Parse id from context this.id = this.context.awsRequestId ? this.context.awsRequestId : null + // Determine client type + this.clientType = + this.headers['cloudfront-is-desktop-viewer'] === 'true' ? 'desktop' : + this.headers['cloudfront-is-mobile-viewer'] === 'true' ? 'mobile' : + this.headers['cloudfront-is-smarttv-viewer'] === 'true' ? 'tv' : + this.headers['cloudfront-is-tablet-viewer'] === 'true' ? 'tablet' : + 'unknown' + + // Parse country + this.clientCountry = this.headers['cloudfront-viewer-country'] ? + this.headers['cloudfront-viewer-country'].toUpperCase() : 'unknown' + // Capture the raw body this.rawBody = this.app._event.body @@ -114,7 +126,6 @@ class REQUEST { } // Extract path from event (strip querystring just in case) - // let path2 = this.app._event.path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/') let path = UTILS.parsePath(this.app._event.path) // Init the route @@ -123,6 +134,7 @@ class REQUEST { // Create a local routes reference let routes = this.app._routes + // Init wildcard let wildcard = {} // Loop the routes and see if this matches diff --git a/test/headers.js b/test/headers.js index 697fa01..bb65b10 100644 --- a/test/headers.js +++ b/test/headers.js @@ -106,6 +106,10 @@ api.get('/auth', function(req,res) { }) }) +api.get('/cloudfront', (req,res) => { + res.send({ clientType: req.clientType, clientCountry: req.clientCountry }) +}) + /******************************************************************************/ /*** BEGIN TESTS ***/ @@ -295,4 +299,121 @@ describe('Header Tests:', function() { }) // end Auth tests + describe('CloudFront:', function() { + + it('clientType (desktop)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: { + 'CloudFront-Is-Desktop-Viewer': 'true', + 'CloudFront-Is-Mobile-Viewer': 'false', + 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'false', + 'CloudFront-Viewer-Country': 'US' + } }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"desktop","clientCountry":"US"}', + isBase64Encoded: false }) + }) // end it + + it('clientType (mobile)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: { + 'CloudFront-Is-Desktop-Viewer': 'false', + 'CloudFront-Is-Mobile-Viewer': 'true', + 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'false', + 'CloudFront-Viewer-Country': 'US' + } }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"mobile","clientCountry":"US"}', + isBase64Encoded: false }) + }) // end it + + it('clientType (tv)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: { + 'CloudFront-Is-Desktop-Viewer': 'false', + 'CloudFront-Is-Mobile-Viewer': 'false', + 'CloudFront-Is-SmartTV-Viewer': 'true', + 'CloudFront-Is-Tablet-Viewer': 'false', + 'CloudFront-Viewer-Country': 'US' + } }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"tv","clientCountry":"US"}', + isBase64Encoded: false }) + }) // end it + + it('clientType (tablet)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: { + 'CloudFront-Is-Desktop-Viewer': 'false', + 'CloudFront-Is-Mobile-Viewer': 'false', + 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'true', + 'CloudFront-Viewer-Country': 'US' + } }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"tablet","clientCountry":"US"}', + isBase64Encoded: false }) + }) // end it + + it('clientType (unknown)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: { + 'CloudFront-Is-Desktop-Viewer': 'false', + 'CloudFront-Is-Mobile-Viewer': 'false', + 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'false', + 'CloudFront-Viewer-Country': 'US' + } }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"unknown","clientCountry":"US"}', + isBase64Encoded: false }) + }) // end it + + it('clientType (unknown - missing headers)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: {} }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"unknown","clientCountry":"unknown"}', + isBase64Encoded: false }) + }) // end it + + it('clientCountry (UK)', async function() { + let _event = Object.assign({},event,{ path: '/cloudfront', headers: { + 'CloudFront-Is-Desktop-Viewer': 'false', + 'CloudFront-Is-Mobile-Viewer': 'false', + 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'false', + 'CloudFront-Viewer-Country': 'uk' + } }) + let result = await new Promise(r => api.run(_event,{},(e,res) => { r(res) })) + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: '{"clientType":"unknown","clientCountry":"UK"}', + isBase64Encoded: false }) + }) // end it + + }) + }) // end HEADER tests From 3c00eb1af6d0edaa9ec4b0e3a79d7ce2c46b8ef9 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 18:01:48 -0400 Subject: [PATCH 29/38] update test logger helper function to avoid console.log conflicts --- test/log.js | 6 +++++- test/sampling.js | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/log.js b/test/log.js index 78f7abf..2bb079c 100644 --- a/test/log.js +++ b/test/log.js @@ -185,7 +185,11 @@ describe('Logging Tests:', function() { // Create a temporary logger to capture the console.log let _log = [] let consoleLog = console.log - const logger = log => { try { _log.push(JSON.parse(log)) } catch(e) { _log.push(log) } } + const logger = (...logs) => { + let log + try { log = JSON.parse(logs[0]) } catch(e) { } + if (log && log.level) { _log.push(log) } else { console.info(...logs) } + } // Clear the log before each test beforeEach(function() { _log = [] }) diff --git a/test/sampling.js b/test/sampling.js index 647f7fe..aaf801d 100644 --- a/test/sampling.js +++ b/test/sampling.js @@ -287,8 +287,12 @@ describe('Sampling Tests:', function() { // Create a temporary logger to capture the console.log let consoleLog = console.log let _log = [] - const logger = log => { try { _log.push(JSON.parse(log)) } catch(e) { _log.push(log) } } - + // const logger = log => { try { _log.push(JSON.parse(log)) } catch(e) { _log.push(log) } } + const logger = (...logs) => { + let log + try { log = JSON.parse(logs[0]) } catch(e) { } + if (log && log.level) { _log.push(log) } else { console.info(...logs) } + } it('Default route', async function() { this.timeout(10000); From 2e8725eaa05be0c0b9c180b42915f502e5f50dc0 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 23:01:51 -0400 Subject: [PATCH 30/38] misc tweaks --- lib/logger.js | 8 +++++--- lib/response.js | 1 - test/log.js | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index 4e83432..2dcb2d7 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -6,8 +6,9 @@ * @license MIT */ - // IDEA: request counts - // IDEA: unique function identifier + // IDEA: add unique function identifier + // IDEA: response length + // https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-mapping-template-reference.html#context-variable-reference const UTILS = require('./utils') // Require utils library @@ -61,7 +62,8 @@ exports.config = (config,levels) => { qs: Object.keys(req.query).length > 0 ? req.query : undefined } }, - res: () => { + res: res => { + console.info(res._response); return { } }, context: context => { diff --git a/lib/response.js b/lib/response.js index aa1ee72..e6eee82 100644 --- a/lib/response.js +++ b/lib/response.js @@ -316,7 +316,6 @@ class RESPONSE { opts.private ) } - } // Add last-modified headers diff --git a/test/log.js b/test/log.js index 2bb079c..ee139ac 100644 --- a/test/log.js +++ b/test/log.js @@ -72,7 +72,7 @@ let event = { let context = { awsRequestId: 'AWSREQID', functionName: 'testFunc', - memoryLimitInMB: '2048', + memoryLimitInMB: 2048, getRemainingTimeInMillis: () => 5000 } @@ -186,6 +186,7 @@ describe('Logging Tests:', function() { let _log = [] let consoleLog = console.log const logger = (...logs) => { + // console.info(...logs); let log try { log = JSON.parse(logs[0]) } catch(e) { } if (log && log.level) { _log.push(log) } else { console.info(...logs) } From 71322af66066d1c98bb13a9cbbb83c617d3ed9ce Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 23:02:13 -0400 Subject: [PATCH 31/38] add logging documentation --- README.md | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 301 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2f4c8c3..6388674 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ npm i lambda-api --save ## Requirements - AWS Lambda running **Node 8.10+** -- AWS Gateway using [Proxy Integration](#lambda-proxy-integration) +- AWS API Gateway using [Proxy Integration](#lambda-proxy-integration) ## Configuration Require the `lambda-api` module into your Lambda handler script and instantiate it. You can initialize the API with the following options: @@ -345,6 +345,12 @@ The `REQUEST` object contains a parsed and normalized request from API Gateway. - `namespace` or `ns`: A reference to modules added to the app's namespace (see [namespaces](#namespaces)) - `cookies`: An object containing cookies sent from the browser (see the [cookie](#cookiename-value-options) `RESPONSE` method) - `context`: Reference to the `context` passed into the Lambda handler function +- `coldStart`: Boolean that indicates whether or not the current invocation was a cold start +- `requestCount`: Integer representing the total number of invocations of the current function container (how many times it has been reused) +- `ip`: The IP address of the client making the request +- `userAgent`: The `User-Agent` header sent by the client making the request +- `clientType`: Either `desktop`, `mobile`, `tv`, `tablet` or `unknown` based on CloudFront's analysis of the `User-Agent` header +- `clientCountry`: Two letter country code representing the origin of the requests as determined by CloudFront The request object can be used to pass additional information through the processing chain. For example, if you are using a piece of authentication middleware, you can add additional keys to the `REQUEST` object with information about the user. See [middleware](#middleware) for more information. @@ -699,6 +705,257 @@ api.options('/users/*', (req,res) => { }) ``` +## Logging +Lambda API includes a robust logging engine specifically designed for integration with AWS CloudWatch Logs' native JSON support. Not only is it ridiculously fast, but it's also highly configurable. Logging is disabled by default, but can be enabled by passing `{ logger: true }` when you create the Lambda API instance (or by passing a [Logging Configuration](#logging-configuration) definition). + +The logger is attached to the `REQUEST` object and can be used anywhere the object is available (e.g. routes, middleware, and error handlers). + +```javascript +const api = require('lambda-api')({ logger: true }) + +api.get('/status', (req,res) => { + req.log.info('Some info about this route') + res.send({ status: 'ok' }) +}) +``` + +In addition to manual logging, Lambda API can also generate "access" logs for your API requests. API Gateway can also provide access logs, but they are limited to contextual information about your request (see [here](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-mapping-template-reference.html)). Lambda API allows you to capture the same data **PLUS** additional information directly from within your application. + +### Logging Configuration +Logging can be enabled setting the `logger` option to `true` when creating the Lambda API instance. Logging can be configured by setting `logger` to an object that contains configuration information. The following table contains available logging configuration properties. + +| Property | Type | Description | Default | +| -------- | ---- | ----------- | ------- | +| access | `boolean` or `string` | Enables/disables automatic access log generation for each request. See [Access Logs](#access-logs)) | `false` | +| customKey | `string` | Sets the JSON property name for custom data passed to logs. | `custom` | +| detail | `boolean` | Enables/disables adding `REQUEST` and `RESPONSE` data to all log entries. | `false` | +| level | `string` | Minimum logging level to send logs for. See [Logging Levels](#logging-levels). | `info` | +| levels | `object` | Key/value pairs of custom log levels and their priority. See [Custom Logging Levels](#custom-logging-levels). | | +| messageKey | `string` | Sets the JSON property name of the log "message". | `msg` | +| nested | `boolean` | Enables/disables nesting of JSON logs for serializer data. See [Serializers](#serializers). | `false` | +| timestamp | `boolean` or `function` | By default, timestamps will return the epoch time in milliseconds. A value of `false` disables log timestamps. A function that returns a value can be used to override the default format. | `true` | +| sampling | `object` | Enables log sampling for periodic request tracing. See [Sampling](#sampling). | | +| serializers | `object` | Serializers to manipulate the log format. See [Serializers](#serializers). | | +| stack | `boolean` | Enables/disables the inclusion of stack traces in caught errors. | `false` | + +```javascript +const api = require('lambda-api')({ + logger: { + level: 'debug', + access: true, + customKey: 'detail', + messageKey: 'message', + timestamp: () => new Date().toUTCString(), // custom timestamp + stack: true + } +}) +``` + +### Log Format +Logs are generated using Lambda API's standard JSON format. The log format can be customized using [Serializers](#serializers). + +**Standard log format (manual logging):** +```javascript + { + "level": "info", // log level + "time": 1534724904910, // request timestamp + "id": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9", // awsRequestId + "route": "/user/:userId", // route accessed + "method": "GET", // request method + "msg": "Some info about this route", // log message + "timer": 2, // execution time up until log was generated + "custom": "additional data", // addditional custom log detail + "remaining": 2000, // remaining milliseconds until function timeout + "function": "my-function-v1", // function name + "memory": 2048, // allocated function memory + "sample": true // generated during sampling request + } +``` + +### Access Logs +Access logs generate detailed information about the API request. Access logs are disabled by default, but can be enabled by setting the `access` property to `true` in the logging configuration object. If set to `false`, access logs will *only* be generated when other log entries (`info`, `error`, etc.) are created. If set to the string `'never'`, access logs will never be generated. + +Access logs use the same format as the standard logs above, but include additional information about the request. The access log format can be customized using [Serializers](#serializers). + +**Access log format (automatic logging):** +```javascript + { + ... Standard Log Data ..., + "path": "/user/123", + "ip": "12.34.56.78", + "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6)...", + "version": "v1", + "qs": { + "foo": "bar" + } + } +``` + +### Logging Levels +Logging "levels" allow you to add detailed logging to your functions based on severity. There are six standard log levels as specified in the table below along with their default priority. + +| Level | Priority | +| -------- | ------- | +| `trace` | 10 | +| `debug` | 20 | +| `info` | 30 | +| `warn` | 40 | +| `error` | 50 | +| `fatal` | 60 | + +Logs are written to CloudWatch Logs *ONLY* if they are the same or higher severity than specified in the `level` log configuration. + +```javascript +// Logging level set to "warn" +const api = require('lambda-api')({ logger: { level: 'warn' } }) + +api.get('/', (req,res) => { + req.log.trace('trace log message') // ignored + req.log.debug('debug log message') // ignored + req.log.info('info log message') // ignored + req.log.warn('warn log message') // write to CloudWatch + req.log.error('error log message') // write to CloudWatch + req.log.fatal('fatal log message') // write to CloudWatch + res.send({ hello: 'world' }) +}) +``` + +### Custom Logging Levels +Custom logging "levels" can be added by specifying an object containing "level names" as keys and their priorities as values. You can also adjust the priority of standard levels by adding it to the object. + +```javascript +const api = require('lambda-api')({ + logger: { + levels: { + 'test': 5, // low priority 'test' level + 'customLevel': 35, // between info and warn + 'trace': 70 // set trace to the highest priority + } + } +}) +``` + +In the example above, the `test` level would only generate logs if the priority was set to `test`. `customLevel` would generate logs if `level` was set to anything with the same or lower priority (e.g. `info`). `trace` now has the highest priority and would generate a log entry no matter what the level was set to. + +### Adding Additional Detail +Manual logging also allows you to specify additional detail with each log entry. Details can be added by suppling *any variable type* as a second parameter to the logger function. + +```javascript +req.log.info('This is the main log message','some other detail') // string +req.log.info('This is the main log message',{ foo: 'bar', isAuthorized: someVar }) // object +req.log.info('This is the main log message',25) // number +req.log.info('This is the main log message',['val1','val2','val3']) // array +req.log.info('This is the main log message',true) // boolean +``` + +If an object is provided, the keys will be merged into the main log entry's JSON. If an other value is provided, the value will be assigned to a key using the the `customKey` setting as its name. If `nested` is set to `true`, objects will be nested under the value of `customKey` as well. + +### Serializers +Serializers allow you to customize log formats as well as add additional data from your application. Serializers can be defined by adding a `serializers` property to the `logger` configuration object. A property named for an available serializer (`main`, `req`, `res`, `context` or `custom`) needs to return an anonymous function that takes one argument and returns an object. The returned object will be merged into the main JSON log entry. Existing properties can be removed by returning `undefined` as their values. + +```javascript +const api = require('lambda-api')({ + logger: { + serializers: { + req: (req) => { + return { + apiId: req.requestContext.apiId, // add the apiId + stage: req.requestContext.stage, // add the stage + qs: undefined // remove the query string + } + } + } + } +}) +``` + +Serializers are passed one argument that contains their corresponding object. `req` *and* `main` receive the `REQUEST` object, `res` receives the `RESPONSE` object, `context` receives the `context` object passed into the main `run` function, and `custom` receives custom data passed in to the logging methods. Note that only custom `objects` will trigger the `custom` serializer. + +If the `nested` option is set to true in the `logger` configuration, then JSON log entries will be generated with properties for `req`, `res`, `context` and `custom` with their serialized data as nested objects. + +### Sampling +Sampling allows you to periodically generate log entries for all possible severities within a single request execution. All of the log entries will be written to CloudWatch Logs and can be used to trace an entire request. This can be used for debugging, metric samples, resource response time sampling, etc. + +Sampling can be enabled by adding a `sampling` property to the `logger` configuration object. A value of `true` will enable the default sampling rule. The default can be changed by passing in a configuration object with the following available *optional* properties: + +| Property | Type | Description | Default | +| -------- | ---- | ----------- | ------- | +| target | `number` | The minimum number of samples per `period`. | 1 | +| rate | `number` | The percentage of samples to be taken during the `period`. | 0.1 | +| period | `number` | Number of **seconds** representing the duration of each sampling period. | 60 | + +The example below would sample at least `2` requests every `30` seconds as well as an additional `0.1` (10%) of all other requests during that period. Lambda API tracks the velocity of requests and attempts to distribute the samples as evenly as possible across the specified `period`. + +```javascript +const api = require('lambda-api')({ + logger: { + sampling: { + target: 2, + rate: 0.1, + period: 30 + } + } +}) +``` + +Additional rules can be added by specify a `rules` parameter in the `sampling` configuration object. The `rules` should contain an `array` of "rule" objects with the following properties: + +| Property | Type | Description | Default | Required | +| -------- | ---- | ----------- | ------- | -------- | +| route | `string` | The route (as defined in a route handler) to apply this rule to. | | **Yes** | +| target | `number` | The minimum number of samples per `period`. | 1 | No | +| rate | `number` | The percentage of samples to be taken during the `period`. | 0.1 | No | +| period | `number` | Number of **seconds** representing the duration of each sampling period. | 60 | No | +| method | `string` or `array` | A comma separated list or `array` of HTTP methods to apply this rule to. | | No | + +The `route` property is the only value required and must match a route's path definition (e.g. `/user/:userId`, not `/user/123`) to be activated. Routes can also use wildcards at the end of the route to match multiple routes (e.g. `/user/*` would match `/user/:userId` *AND* `/user/:userId/tags`). A list of `method`s can also be supplied that would limit the rule to just those HTTP methods. A comma separated `string` or an `array` will be properly parsed. + +Sampling rules can be used to disable sampling on certain routes by setting the `target` and `rate` to `0`. For example, if you had a `/status` route that you didn't want to be sampled, you would use the following configuration: + +```javascript +const api = require('lambda-api')({ + logger: { + sampling: { + rules: [ + { route: '/status', target: 0, rate: 0 } + ] + } + } +}) +``` + +You could also use sampling rules to enable sampling on certain routes: + +```javascript +const api = require('lambda-api')({ + logger: { + sampling: { + rules: [ + { route: '/user', target: 1, rate: 0.1 }, // enable for /user route + { route: '/posts/*', target: 1, rate: 0.1 } // enable for all routes that start with /posts + ], + target: 0, // disable sampling default target + rate: 0 // disable sampling default rate + } + } +}) +``` + +If you'd like to disable sampling for `GET` and `POST` requests to user: +```javascript +const api = require('lambda-api')({ + logger: { + sampling: { + rules: [ + // disable GET and POST on /user route + { route: '/user', target: 0, rate: 0, method: ['GET','POST'] } + ] + } + } +}) +``` + + ## Middleware The API supports middleware to preprocess requests before they execute their matching routes. Middleware is defined using the `use` method and requires a function with three parameters for the `REQUEST`, `RESPONSE`, and `next` callback. For example: @@ -709,7 +966,7 @@ api.use((req,res,next) => { }) ``` -Middleware can be used to authenticate requests, log API calls, etc. The `REQUEST` and `RESPONSE` objects behave as they do within routes, allowing you to manipulate either object. In the case of authentication, for example, you could verify a request and update the `REQUEST` with an `authorized` flag and continue execution. Or if the request couldn't be authorized, you could respond with an error directly from the middleware. For example: +Middleware can be used to authenticate requests, create database connections, etc. The `REQUEST` and `RESPONSE` objects behave as they do within routes, allowing you to manipulate either object. In the case of authentication, for example, you could verify a request and update the `REQUEST` with an `authorized` flag and continue execution. Or if the request couldn't be authorized, you could respond with an error directly from the middleware. For example: ```javascript // Auth User @@ -723,13 +980,13 @@ api.use((req,res,next) => { }) ``` -The `next()` callback tells the system to continue executing. If this is not called then the system will hang and eventually timeout unless another request ending call such as `error` is called. You can define as many middleware functions as you want. They will execute serially and synchronously in the order in which they are defined. +The `next()` callback tells the system to continue executing. If this is not called then the system will hang (and eventually timeout) unless another request ending call such as `error` is called. You can define as many middleware functions as you want. They will execute serially and synchronously in the order in which they are defined. -**NOTE:** Middleware can use either callbacks like `res.send()` or `return` to trigger a response to the user. Please note that calling either one of these from within a middleware function will return the response immediately. +**NOTE:** Middleware can use either callbacks like `res.send()` or `return` to trigger a response to the user. Please note that calling either one of these from within a middleware function will return the response immediately and terminate API execution. ### Restricting middleware execution to certain path(s) -By default, middleware will execute on every path. If you only need it to execute for specific paths, pass the path (or array of paths) as the first parameter to the `use` function. +By default, middleware will execute on every path. If you only need it to execute for specific paths, pass the path (or array of paths) as the first parameter to the `use` method. ```javascript // Single path @@ -750,6 +1007,26 @@ api.use(['/comments','/users/:userId','/posts/*'],(req,res,next) => { next() }) Path matching checks both the supplied `path` and the defined `route`. This means that parameterized paths can be matched by either the parameter (e.g. `/users/:param1`) or by an exact matching path (e.g. `/users/123`). +### Specifying multiple middleware + +In addition to restricting middleware to certain paths, you can also add multiple middleware using a single `use` method. This is a convenient way to assign several pieces of middleware to the same path or minimize your code. + +```javascript +const middleware1 = (req,res,next) => { + // middleware code +} + +const middleware2 = (req,res,next) => { + // some other middleware code +} + +// Restrict middleware1 and middleware2 to /users route +api.use('/users', middleware1, middleware2) + +// Add middleware1 and middleware2 to all routes +api.use(middleware1, middleware2) +``` + ## Clean Up The API has a built-in clean up method called 'finally()' that will execute after all middleware and routes have been completed, but before execution is complete. This can be used to close database connections or to perform other clean up functions. A clean up function can be defined using the `finally` method and requires a function with two parameters for the REQUEST and the RESPONSE as its only argument. For example: @@ -763,7 +1040,7 @@ api.finally((req,res) => { The `RESPONSE` **CANNOT** be manipulated since it has already been generated. Only one `finally()` method can be defined and will execute after properly handled errors as well. ## Error Handling -The API has simple built-in error handling that will log the error using `console.log`. These will be available via CloudWatch Logs. By default, errors will trigger a JSON response with the error message. If you would like to define additional error handling, you can define them using the `use` method similar to middleware. Error handling middleware must be defined as a function with **four** arguments instead of three like normal middleware. An additional `error` parameter must be added as the first parameter. This will contain the error object generated. +The API has sophisticated error handling that will automatically catch and log errors using the [Logging](#logging) system. By default, errors will trigger a JSON response with the error message. If you would like to define additional error handling, you can define them using the `use` method similar to middleware. Error handling middleware must be defined as a function with **four** arguments instead of three like normal middleware. An additional `error` parameter must be added as the first parameter. This will contain the error object generated. ```javascript api.use((err,req,res,next) => { @@ -774,6 +1051,24 @@ api.use((err,req,res,next) => { The `next()` callback will cause the script to continue executing and eventually call the standard error handling function. You can short-circuit the default handler by calling a request ending method such as `send`, `html`, or `json` OR by `return`ing data from your handler. +Error handling middleware, like regular middleware, also supports specifying multiple handlers in a single `use` method call. + +```javascript +const errorHandler1 = (err,req,res,next) => { + // do something with the error + next() +}) + +const errorHandler2 = (err,req,res,next) => { + // do something else with the error + next() +}) + +api.use(errorHandler1,errorHandler2) +``` + +**NOTE:** Error handling middleware runs on *ALL* paths. If paths are passed in as the first parameter, they will be ignored by the error handling middleware. + ## Namespaces Lambda API allows you to map specific modules to namespaces that can be accessed from the `REQUEST` object. This is helpful when using the pattern in which you create a module that exports middleware, error, or route functions. In the example below, the `data` namespace is added to the API and then accessed by reference within an included module. From d9b6c5c38276443bacdc6506f7418153de065d1f Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 23:24:07 -0400 Subject: [PATCH 32/38] add device and country to access log --- lib/logger.js | 3 ++- test/log.js | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index 2dcb2d7..08a88db 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -58,12 +58,13 @@ exports.config = (config,levels) => { path: req.path, ip: req.ip, ua: req.userAgent, + device: req.clientType, + country: req.clientCountry, version: req.version, qs: Object.keys(req.query).length > 0 ? req.query : undefined } }, res: res => { - console.info(res._response); return { } }, context: context => { diff --git a/test/log.js b/test/log.js index ee139ac..c98a842 100644 --- a/test/log.js +++ b/test/log.js @@ -64,7 +64,12 @@ let event = { headers: { 'content-type': 'application/json', 'x-forwarded-for': '12.34.56.78, 23.45.67.89', - 'User-Agent': 'user-agent-string' + 'User-Agent': 'user-agent-string', + 'CloudFront-Is-Desktop-Viewer': 'false', + 'CloudFront-Is-Mobile-Viewer': 'true', + 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'false', + 'CloudFront-Viewer-Country': 'US' } } @@ -228,6 +233,8 @@ describe('Logging Tests:', function() { expect(_log[4]).to.have.property('ua') expect(_log[4]).to.have.property('version') expect(_log[4]).to.have.property('qs') + expect(_log[4]).to.have.property('device') + expect(_log[4]).to.have.property('country') }) // end it @@ -249,7 +256,7 @@ describe('Logging Tests:', function() { it('Custom level (debug)', async function() { console.log = logger - let _event = Object.assign({},event,{ path: '/' }) + let _event = Object.assign({},event,{ path: '/', }) let result = await new Promise(r => api_customLevel.run(_event,context,(e,res) => { r(res) })) console.log = consoleLog @@ -278,6 +285,8 @@ describe('Logging Tests:', function() { expect(_log[5]).to.have.property('ip') expect(_log[5]).to.have.property('ua') expect(_log[5]).to.have.property('version') + expect(_log[5]).to.have.property('device') + expect(_log[5]).to.have.property('country') }) // end it @@ -329,6 +338,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -365,6 +376,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -399,6 +412,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -434,6 +449,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -470,6 +487,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -510,6 +529,8 @@ describe('Logging Tests:', function() { expect(_log[0]).to.have.property('ip') expect(_log[0]).to.have.property('ua') expect(_log[0]).to.have.property('version') + expect(_log[0]).to.have.property('device') + expect(_log[0]).to.have.property('country') // access log 2 expect(_log[1]).to.have.property('time') expect(_log[1]).to.have.property('id') @@ -524,6 +545,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -558,6 +581,8 @@ describe('Logging Tests:', function() { expect(_log[0]).to.not.have.property('ip') expect(_log[0]).to.not.have.property('ua') expect(_log[0]).to.not.have.property('version') + expect(_log[0]).to.not.have.property('device') + expect(_log[0]).to.not.have.property('country') }) // end it @@ -628,6 +653,8 @@ describe('Logging Tests:', function() { expect(_log[0]).to.have.property('ip') expect(_log[0]).to.have.property('ua') expect(_log[0]).to.have.property('version') + expect(_log[0]).to.have.property('device') + expect(_log[0]).to.have.property('country') expect(_log[1]).to.have.property('time') expect(_log[1]).to.have.property('id') @@ -641,6 +668,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -683,6 +712,8 @@ describe('Logging Tests:', function() { expect(_log[3]).to.have.property('ip') expect(_log[3]).to.have.property('ua') expect(_log[3]).to.have.property('version') + expect(_log[3]).to.have.property('device') + expect(_log[3]).to.have.property('country') }) // end it @@ -728,6 +759,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('version') expect(_log[1]).to.have.property('TESTPATH') expect(_log[1]).to.have.property('STATUS') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it @@ -782,6 +815,8 @@ describe('Logging Tests:', function() { expect(_log[1]).to.have.property('ip') expect(_log[1]).to.have.property('ua') expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') }) // end it From 7fb74f98b73f8cd219628d34d84b332ff5db3602 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 23:34:58 -0400 Subject: [PATCH 33/38] fix bug with custom array --- lib/logger.js | 8 +++++--- test/log.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index 08a88db..8e52190 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -74,7 +74,8 @@ exports.config = (config,levels) => { memory: context.memoryLimitInMB && context.memoryLimitInMB } }, - custom: custom => typeof custom === 'object' || nested ? custom : { [customKey]: custom } + custom: custom => typeof custom === 'object' && !Array.isArray(custom) + || nested ? custom : { [customKey]: custom } } let serializers = { @@ -89,8 +90,9 @@ exports.config = (config,levels) => { let log = (level,msg,req,context,custom) => { let _context = Object.assign({},defaults.context(context),serializers.context(context)) - let _custom = typeof custom === 'string' ? defaults.custom(custom) : - Object.assign({},defaults.custom(custom),serializers.custom(custom)) + let _custom = typeof custom === 'object' && !Array.isArray(custom) ? + Object.assign({},defaults.custom(custom),serializers.custom(custom)) : + defaults.custom(custom) return Object.assign({}, { diff --git a/test/log.js b/test/log.js index c98a842..34181a0 100644 --- a/test/log.js +++ b/test/log.js @@ -175,6 +175,11 @@ api_customSerializers.get('/', (req,res) => { res.send('done') }) +api_default.get('/array', (req,res) => { + req.log.info('info message',['val1','val2','val3']) + res.send('done') +}) + api_showStackTrace.get('/', (req,res) => { undefinedVar // this should throw an error res.send('done') @@ -764,6 +769,48 @@ describe('Logging Tests:', function() { }) // end it + it('Custom data (array)', async function() { + console.log = logger + let _event = Object.assign({},event,{ path: '/array' }) + let result = await new Promise(r => api_default.run(_event,context,(e,res) => { r(res) })) + console.log = consoleLog + + expect(result).to.deep.equal({ + headers: { 'content-type': 'application/json' }, + statusCode: 200, + body: 'done', + isBase64Encoded: false + }) + expect(_log).to.have.length(2) + expect(_log[0].level).to.equal('info') + expect(_log[1].level).to.equal('access') + // standard log + expect(_log[0]).to.have.property('time') + expect(_log[0]).to.have.property('id') + expect(_log[0]).to.have.property('route') + expect(_log[0]).to.have.property('msg') + expect(_log[0]).to.have.property('timer') + expect(_log[0]).to.have.property('remaining') + expect(_log[0]).to.have.property('function') + expect(_log[0]).to.have.property('custom') + // access log + expect(_log[1]).to.have.property('time') + expect(_log[1]).to.have.property('id') + expect(_log[1]).to.have.property('route') + expect(_log[1]).to.have.property('timer') + expect(_log[1]).to.have.property('remaining') + expect(_log[1]).to.have.property('function') + expect(_log[1]).to.have.property('memory') + expect(_log[1]).to.have.property('coldStart') + expect(_log[1]).to.have.property('path') + expect(_log[1]).to.have.property('ip') + expect(_log[1]).to.have.property('ua') + expect(_log[1]).to.have.property('version') + expect(_log[1]).to.have.property('device') + expect(_log[1]).to.have.property('country') + }) // end it + + it('Invalid custom levels configuration', async function() { let error_message try { From 837476ec61df7ec18fed16294a4a3792ac76d68d Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 19 Aug 2018 23:57:15 -0400 Subject: [PATCH 34/38] add toc for logging and other doc updates --- README.md | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 6388674..7175722 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,15 @@ Lambda API was written to be extremely lightweight and built specifically for se - [Enabling Binary Support](#enabling-binary-support) - [Path Parameters](#path-parameters) - [Wildcard Routes](#wildcard-routes) +- [Logging](#logging) + - [Logging Configuration](#logging-configuration) + - [Log Format](#log-format) + - [Access Logs](#access-logs) + - [Logging Levels](#logging-levels) + - [Custom Logging Levels](#custom-logging-levels) + - [Adding Additional Detail](#adding-additional-detail) + - [Serializers](#serializers) + - [Sampling](#sampling) - [Middleware](#middleware) - [Clean Up](#clean-up) - [Error Handling](#error-handling) @@ -113,6 +122,9 @@ const api = require('lambda-api')({ version: 'v1.0', base: 'v1' }); ## Recent Updates For detailed release notes see [Releases](https://github.com/jeremydaly/lambda-api/releases). +### v0.8: Logging Support and Sampling +Lambda API has added a powerful (and customizable) logging engine that utilizes native JSON support for CloudWatch Logs. Log entries can be manually added using standard severities like `info` and `warn`. In addition, "access logs" can be automatically generated with detailed information about each requests. See [Logging](#logging) for more information about logging and auto sampling for request tracing. + ### v0.7: Restrict middleware execution to certain paths Middleware now supports an optional path parameter that supports multiple paths, wildcards, and parameter matching to better control middleware execution. See [middleware](#middleware) for more information. @@ -706,7 +718,7 @@ api.options('/users/*', (req,res) => { ``` ## Logging -Lambda API includes a robust logging engine specifically designed for integration with AWS CloudWatch Logs' native JSON support. Not only is it ridiculously fast, but it's also highly configurable. Logging is disabled by default, but can be enabled by passing `{ logger: true }` when you create the Lambda API instance (or by passing a [Logging Configuration](#logging-configuration) definition). +Lambda API includes a robust logging engine specifically designed to utilize native JSON support for CloudWatch Logs. Not only is it ridiculously fast, but it's also highly configurable. Logging is disabled by default, but can be enabled by passing `{ logger: true }` when you create the Lambda API instance (or by passing a [Logging Configuration](#logging-configuration) definition). The logger is attached to the `REQUEST` object and can be used anywhere the object is available (e.g. routes, middleware, and error handlers). @@ -722,11 +734,11 @@ api.get('/status', (req,res) => { In addition to manual logging, Lambda API can also generate "access" logs for your API requests. API Gateway can also provide access logs, but they are limited to contextual information about your request (see [here](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-mapping-template-reference.html)). Lambda API allows you to capture the same data **PLUS** additional information directly from within your application. ### Logging Configuration -Logging can be enabled setting the `logger` option to `true` when creating the Lambda API instance. Logging can be configured by setting `logger` to an object that contains configuration information. The following table contains available logging configuration properties. +Logging can be enabled by setting the `logger` option to `true` when creating the Lambda API instance. Logging can be configured by setting `logger` to an object that contains configuration information. The following table contains available logging configuration properties. | Property | Type | Description | Default | | -------- | ---- | ----------- | ------- | -| access | `boolean` or `string` | Enables/disables automatic access log generation for each request. See [Access Logs](#access-logs)) | `false` | +| access | `boolean` or `string` | Enables/disables automatic access log generation for each request. See [Access Logs](#access-logs). | `false` | | customKey | `string` | Sets the JSON property name for custom data passed to logs. | `custom` | | detail | `boolean` | Enables/disables adding `REQUEST` and `RESPONSE` data to all log entries. | `false` | | level | `string` | Minimum logging level to send logs for. See [Logging Levels](#logging-levels). | `info` | @@ -735,9 +747,10 @@ Logging can be enabled setting the `logger` option to `true` when creating the L | nested | `boolean` | Enables/disables nesting of JSON logs for serializer data. See [Serializers](#serializers). | `false` | | timestamp | `boolean` or `function` | By default, timestamps will return the epoch time in milliseconds. A value of `false` disables log timestamps. A function that returns a value can be used to override the default format. | `true` | | sampling | `object` | Enables log sampling for periodic request tracing. See [Sampling](#sampling). | | -| serializers | `object` | Serializers to manipulate the log format. See [Serializers](#serializers). | | +| serializers | `object` | Adds serializers that manipulate the log format. See [Serializers](#serializers). | | | stack | `boolean` | Enables/disables the inclusion of stack traces in caught errors. | `false` | +Example: ```javascript const api = require('lambda-api')({ logger: { @@ -768,7 +781,7 @@ Logs are generated using Lambda API's standard JSON format. The log format can b "remaining": 2000, // remaining milliseconds until function timeout "function": "my-function-v1", // function name "memory": 2048, // allocated function memory - "sample": true // generated during sampling request + "sample": true // is generated during sampling request? } ``` @@ -781,11 +794,13 @@ Access logs use the same format as the standard logs above, but include addition ```javascript { ... Standard Log Data ..., - "path": "/user/123", - "ip": "12.34.56.78", - "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6)...", - "version": "v1", - "qs": { + "path": "/user/123", // path accessed + "ip": "12.34.56.78", // client ip address + "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6)...", // User-Agent + "version": "v1", // specified API version + "device": "mobile", // client device (as determined by CloudFront) + "country": "US", // client country (as determined by CloudFront) + "qs": { // query string parameters "foo": "bar" } } @@ -848,7 +863,7 @@ req.log.info('This is the main log message',['val1','val2','val3']) // array req.log.info('This is the main log message',true) // boolean ``` -If an object is provided, the keys will be merged into the main log entry's JSON. If an other value is provided, the value will be assigned to a key using the the `customKey` setting as its name. If `nested` is set to `true`, objects will be nested under the value of `customKey` as well. +If an `object` is provided, the keys will be merged into the main log entry's JSON. If any other `type` is provided, the value will be assigned to a key using the the `customKey` setting as its property name. If `nested` is set to `true`, objects will be nested under the value of `customKey` as well. ### Serializers Serializers allow you to customize log formats as well as add additional data from your application. Serializers can be defined by adding a `serializers` property to the `logger` configuration object. A property named for an available serializer (`main`, `req`, `res`, `context` or `custom`) needs to return an anonymous function that takes one argument and returns an object. The returned object will be merged into the main JSON log entry. Existing properties can be removed by returning `undefined` as their values. @@ -955,6 +970,7 @@ const api = require('lambda-api')({ }) ``` +Any combination of rules can be provided to customize sampling behavior. Note that each rule tracks requests and velocity separately, which could limit the number of samples for infrequently accessed routes. ## Middleware The API supports middleware to preprocess requests before they execute their matching routes. Middleware is defined using the `use` method and requires a function with three parameters for the `REQUEST`, `RESPONSE`, and `next` callback. For example: From 0d01f455faa441c65a31bdc6ff98edc485ff272a Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 20 Aug 2018 00:51:58 -0400 Subject: [PATCH 35/38] doc updates --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7175722..c08b5a7 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,14 @@ Express.js, Fastify, Koa, Restify, and Hapi are just a few of the many amazing w These other frameworks are extremely powerful, but that benefit comes with the steep price of requiring several additional Node.js modules. Not only is this a bit of a security issue (see Beware of Third-Party Packages in [Securing Serverless](https://www.jeremydaly.com/securing-serverless-a-newbies-guide/)), but it also adds bloat to your codebase, filling your `node_modules` directory with a ton of extra files. For serverless applications that need to load quickly, all of these extra dependencies slow down execution and use more memory than necessary. Express.js has **30 dependencies**, Fastify has **12**, and Hapi has **17**! These numbers don't even include their dependencies' dependencies. -Lambda API has **ZERO** dependencies. +Lambda API has **ZERO** dependencies. *None*. *Zip*. *Zilch*. -Lambda API was written to be extremely lightweight and built specifically for serverless applications using AWS Lambda. It provides support for API routing, serving up HTML pages, issuing redirects, serving binary files and much more. It has a powerful middleware and error handling system, allowing you to implement everything from custom authentication to complex logging systems. Best of all, it was designed to work with Lambda's Proxy Integration, automatically handling all the interaction with API Gateway for you. It parses **REQUESTS** and formats **RESPONSES** for you, allowing you to focus on your application's core functionality, instead of fiddling with inputs and outputs. +Lambda API was written to be *extremely lightweight* and built specifically for **SERVERLESS** applications using AWS Lambda and API Gateway. It provides support for API routing, serving up HTML pages, issuing redirects, serving binary files and much more. Worried about observability? Lambda API has a built-in logging engine that can even periodically sample requests for things like tracing and benchmarking. It has a powerful middleware and error handling system, allowing you to implement just about anything you can dream of. Best of all, it was designed to work with Lambda's Proxy Integration, automatically handling all the interaction with API Gateway for you. It parses **REQUESTS** and formats **RESPONSES**, allowing you to focus on your application's core functionality, instead of fiddling with inputs and outputs. + +### Single Purpose Functions +You may have heard that a serverless "best practice" is to keep your functions small and limit them to a single purpose. I generally agree since building monolith applications is not what serverless was designed for. However, what exactly is a "single purpose" when it comes to building serverless APIs and web services? Should we create a separate function for our "create user" `POST` endpoint and then another one for our "update user" `PUT` endpoint? Should we create yet another function for our "delete user" `DELETE` endpoint? You certainly could, but that seems like a lot of repeated boilerplate code. On the other hand, you could create just one function that handled all your user management features. It may even make sense (in certain circumstances) to create one big serverless function handling several related components that can share your VPC database connections. + +Whatever you decide is best for your use case, **Lambda API** is there to support you. Whether your function has over a hundred routes, or just one, Lambda API's small size and lightning fast load time has virtually no impact on your function's performance. Yet despite its small footprint, it gives you the power of a full-featured web framework. ## Table of Contents - [Installation](#installation) @@ -122,7 +127,7 @@ const api = require('lambda-api')({ version: 'v1.0', base: 'v1' }); ## Recent Updates For detailed release notes see [Releases](https://github.com/jeremydaly/lambda-api/releases). -### v0.8: Logging Support and Sampling +### v0.8: Logging Support with Sampling Lambda API has added a powerful (and customizable) logging engine that utilizes native JSON support for CloudWatch Logs. Log entries can be manually added using standard severities like `info` and `warn`. In addition, "access logs" can be automatically generated with detailed information about each requests. See [Logging](#logging) for more information about logging and auto sampling for request tracing. ### v0.7: Restrict middleware execution to certain paths @@ -1211,3 +1216,6 @@ If you are using persistent connections in your function routes (such as AWS RDS ## Contributions Contributions, ideas and bug reports are welcome and greatly appreciated. Please add [issues](https://github.com/jeremydaly/lambda-api/issues) for suggestions and bug reports or create a pull request. + +## Are you using Lambda API? +If you're using Lambda API and finding it useful, hit me up on [Twitter](https://twitter.com/jeremy_daly) or email me at contact[at]jeremydaly.com. I'd love to hear your stories, ideas, and even your complaints! From c96c6c86ef66d177a6cd96ee5df00daf153af2dd Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 20 Aug 2018 08:07:39 -0400 Subject: [PATCH 36/38] add info about error logging types --- README.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c08b5a7..0a41d24 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Lambda API was written to be *extremely lightweight* and built specifically for ### Single Purpose Functions You may have heard that a serverless "best practice" is to keep your functions small and limit them to a single purpose. I generally agree since building monolith applications is not what serverless was designed for. However, what exactly is a "single purpose" when it comes to building serverless APIs and web services? Should we create a separate function for our "create user" `POST` endpoint and then another one for our "update user" `PUT` endpoint? Should we create yet another function for our "delete user" `DELETE` endpoint? You certainly could, but that seems like a lot of repeated boilerplate code. On the other hand, you could create just one function that handled all your user management features. It may even make sense (in certain circumstances) to create one big serverless function handling several related components that can share your VPC database connections. -Whatever you decide is best for your use case, **Lambda API** is there to support you. Whether your function has over a hundred routes, or just one, Lambda API's small size and lightning fast load time has virtually no impact on your function's performance. Yet despite its small footprint, it gives you the power of a full-featured web framework. +Whatever you decide is best for your use case, **Lambda API** is there to support you. Whether your function has over a hundred routes, or just one, Lambda API's small size and lightning fast load time has virtually no impact on your function's performance. Yet despite its small footprint, it gives you the power of a full-featured web framework. ## Table of Contents - [Installation](#installation) @@ -1061,7 +1061,7 @@ api.finally((req,res) => { The `RESPONSE` **CANNOT** be manipulated since it has already been generated. Only one `finally()` method can be defined and will execute after properly handled errors as well. ## Error Handling -The API has sophisticated error handling that will automatically catch and log errors using the [Logging](#logging) system. By default, errors will trigger a JSON response with the error message. If you would like to define additional error handling, you can define them using the `use` method similar to middleware. Error handling middleware must be defined as a function with **four** arguments instead of three like normal middleware. An additional `error` parameter must be added as the first parameter. This will contain the error object generated. +Lambda API has sophisticated error handling that will automatically catch and log errors using the [Logging](#logging) system. By default, errors will trigger a JSON response with the error message. If you would like to define additional error handling, you can define them using the `use` method similar to middleware. Error handling middleware must be defined as a function with **four** arguments instead of three like normal middleware. An additional `error` parameter must be added as the first parameter. This will contain the error object generated. ```javascript api.use((err,req,res,next) => { @@ -1090,6 +1090,31 @@ api.use(errorHandler1,errorHandler2) **NOTE:** Error handling middleware runs on *ALL* paths. If paths are passed in as the first parameter, they will be ignored by the error handling middleware. +### Error Types +Error logs are generate using either the `error` or `fatal` logging level. Errors can be triggered from within routes and middleware by calling the `error()` method on the `RESPONSE` object. If provided a `string` as an error message, this will generate an `error` level log entry. If you supply a JavaScript `Error` object, or you `throw` an error, a `fatal` log entry will be generated. + +```javascript +api.get('/somePath', (res,req) => { + res.error('This is an error message') // creates 'error' log +}) + +api.get('/someOtherPath', (res,req) => { + res.error(new Error('This is a fatal error')) // creates 'fatal' log +}) + +api.get('/anotherPath', (res,req) => { + throw new Error('Another fatal error') // creates 'fatal' log +}) + +api.get('/finalPath', (res,req) => { + try { + // do something + } catch(e) { + res.error(e) // creates 'fatal' log + } +}) +``` + ## Namespaces Lambda API allows you to map specific modules to namespaces that can be accessed from the `REQUEST` object. This is helpful when using the pattern in which you create a module that exports middleware, error, or route functions. In the example below, the `data` namespace is added to the API and then accessed by reference within an included module. From ee130d1581d0a4999c903c058376d3be86fe9f21 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 20 Aug 2018 09:15:13 -0400 Subject: [PATCH 37/38] fix linting errors --- lib/logger.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index 8e52190..10400fd 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -4,11 +4,11 @@ * Lightweight web framework for your serverless applications * @author Jeremy Daly * @license MIT - */ +*/ - // IDEA: add unique function identifier - // IDEA: response length - // https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-mapping-template-reference.html#context-variable-reference +// IDEA: add unique function identifier +// IDEA: response length +// https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-mapping-template-reference.html#context-variable-reference const UTILS = require('./utils') // Require utils library @@ -64,7 +64,7 @@ exports.config = (config,levels) => { qs: Object.keys(req.query).length > 0 ? req.query : undefined } }, - res: res => { + res: () => { return { } }, context: context => { @@ -92,7 +92,7 @@ exports.config = (config,levels) => { let _context = Object.assign({},defaults.context(context),serializers.context(context)) let _custom = typeof custom === 'object' && !Array.isArray(custom) ? Object.assign({},defaults.custom(custom),serializers.custom(custom)) : - defaults.custom(custom) + defaults.custom(custom) return Object.assign({}, { @@ -198,14 +198,14 @@ exports.sampler = (app,req) => { // Enable sampling if last sample is passed target split } else if (rule.target > 0 && counts.start+Math.floor(rule.period*1000/rule.target*counts.fCount) < now) { - level = rule.level - counts.fCount++ - // console.log('\n*********** FIXED ***********'); + level = rule.level + counts.fCount++ + // console.log('\n*********** FIXED ***********'); } else if (rule.rate > 0 && counts.start+Math.floor(velocity*counts.pCount+velocity/2) < now) { - level = rule.level - counts.pCount++ - // console.log('\n*********** RATE ***********'); + level = rule.level + counts.pCount++ + // console.log('\n*********** RATE ***********'); } // Increment total count From e80422fe918e28769ae5044a57f10eb74a0cea07 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Mon, 20 Aug 2018 12:50:12 -0400 Subject: [PATCH 38/38] update sampling test for consistent results --- test/sampling.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sampling.js b/test/sampling.js index aaf801d..06a3883 100644 --- a/test/sampling.js +++ b/test/sampling.js @@ -312,7 +312,7 @@ describe('Sampling Tests:', function() { for(let x = 0; x api_rules.run(_event,context,(e,res) => { r(res) })) - await Promise.delay(Math.random()*25) + await Promise.delay(20) } // end for loop // End the timer and restore console.log @@ -350,7 +350,7 @@ describe('Sampling Tests:', function() { for(let x = 0; x api_rules.run(_event,context,(e,res) => { r(res) })) - await Promise.delay(Math.random()*25) + await Promise.delay(20) } // end for loop // End the timer and restore console.log @@ -389,7 +389,7 @@ describe('Sampling Tests:', function() { for(let x = 0; x api_rules.run(_event,context,(e,res) => { r(res) })) - await Promise.delay(Math.random()*25) + await Promise.delay(20) // await Promise.delay(20) } // end for loop