From db04ae314b51abdf565c0ccdc81c5aaf2cdca082 Mon Sep 17 00:00:00 2001 From: mapedraza Date: Wed, 27 Sep 2023 20:02:25 +0200 Subject: [PATCH 1/4] add test --- .../jexlBasedTransformations-test.js | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js b/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js index 1611546e0..d7d3fd8e7 100644 --- a/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js +++ b/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js @@ -455,6 +455,48 @@ const iotAgentConfig = { skipValue: null } ] + }, + testNull: { + commands: [], + type: 'testNull', + lazy: [], + active: [ + { + name: 'a', + type: 'Number', + expression: 'v' + }, + { + name: 'b', + type: 'Number', + expression: 'v*3' + }, + { + name: 'c', + type: 'Number', + expression: 'v==null' + }, + { + name: 'd', + type: 'Text', + expression: "v?'soy null':'no soy null'" + }, + { + name: 'e', + type: 'Text', + expression: "v==null?'soy null':'no soy null'" + }, + { + name: 'f', + type: 'Text', + expression: "(v*3)?'soy null':'no soy null'" + }, + { + name: 'g', + type: 'Boolean', + expression: 'v == undefined' + } + ] } }, service: 'smartgondor', @@ -598,6 +640,66 @@ describe('Java expression language (JEXL) based transformations plugin', functio }); }); + describe('When an measure arrives whit null values', function () { + // Case: Update for an attribute with bad expression + const values = [ + { + name: 'v', + type: 'Number', + value: null + } + ]; + + beforeEach(function () { + nock.cleanAll(); + + contextBrokerMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/entities?options=upsert', { + id: 'testNull1', + type: 'testNull', + v: { + value: null, + type: 'Number' + }, + b: { + value: 0, + type: 'Number' + }, + c: { + value: true, + type: 'Number' + }, + d: { + value: 'no soy null', + type: 'Text' + }, + e: { + value: 'soy null', + type: 'Text' + }, + f: { + value: 'no soy null', + type: 'Text' + }, + g: { + value: true, + type: 'Boolean' + } + }) + .reply(204); + }); + + it('it should be handled properly', function (done) { + iotAgentLib.update('testNull1', 'testNull', '', values, function (error) { + should.not.exist(error); + contextBrokerMock.done(); + done(); + }); + }); + }); + describe('When there are expression attributes that are just calculated (not sent by the device)', function () { // Case: Expression which results is sent as a new attribute const values = [ From e41f54233d5db80669e73abf24d6ae83fc804a1b Mon Sep 17 00:00:00 2001 From: mapedraza Date: Thu, 28 Sep 2023 12:44:33 +0200 Subject: [PATCH 2/4] Add more cases --- .../jexlBasedTransformations-test.js | 342 +++++++++++++++++- 1 file changed, 338 insertions(+), 4 deletions(-) diff --git a/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js b/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js index d7d3fd8e7..ad3811c25 100644 --- a/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js +++ b/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js @@ -473,13 +473,105 @@ const iotAgentConfig = { }, { name: 'c', + type: 'Boolean', + expression: 'v==null' + }, + { + name: 'd', + type: 'Text', + expression: "v?'no soy null':'soy null'" + }, + { + name: 'e', + type: 'Text', + expression: "v==null?'soy null':'no soy null'" + }, + { + name: 'f', + type: 'Text', + expression: "(v*3)==null?'soy null':'no soy null'" + }, + { + name: 'g', + type: 'Boolean', + expression: 'v == undefined' + } + ] + }, + testNullSkip: { + commands: [], + type: 'testNullSkip', + lazy: [], + active: [ + { + name: 'a', type: 'Number', + expression: 'v', + skipValue: 'avoidNull' + }, + { + name: 'b', + type: 'Number', + expression: 'v*3', + skipValue: 'avoidNull' + }, + { + name: 'c', + type: 'Boolean', + expression: 'v==null', + skipValue: 'avoidNull' + }, + { + name: 'd', + type: 'Text', + expression: "v?'no soy null':'soy null'", + skipValue: 'avoidNull' + }, + { + name: 'e', + type: 'Text', + expression: "v==null?'soy null':'no soy null'", + skipValue: 'avoidNull' + }, + { + name: 'f', + type: 'Text', + expression: "(v*3)==null?'soy null':'no soy null'", + skipValue: 'avoidNull' + }, + { + name: 'g', + type: 'Boolean', + expression: 'v == undefined', + skipValue: 'avoidNull' + } + ] + }, + testNullExplicit: { + type: 'testNullExplicit', + explicitAttrs: true, + commands: [], + lazy: [], + active: [ + { + name: 'a', + type: 'Number', + expression: 'v' + }, + { + name: 'b', + type: 'Number', + expression: 'v*3' + }, + { + name: 'c', + type: 'Boolean', expression: 'v==null' }, { name: 'd', type: 'Text', - expression: "v?'soy null':'no soy null'" + expression: "v?'no soy null':'soy null'" }, { name: 'e', @@ -489,7 +581,7 @@ const iotAgentConfig = { { name: 'f', type: 'Text', - expression: "(v*3)?'soy null':'no soy null'" + expression: "(v*3)==null?'soy null':'no soy null'" }, { name: 'g', @@ -640,7 +732,7 @@ describe('Java expression language (JEXL) based transformations plugin', functio }); }); - describe('When an measure arrives whit null values', function () { + describe('When applying expressions with null values', function () { // Case: Update for an attribute with bad expression const values = [ { @@ -652,6 +744,7 @@ describe('Java expression language (JEXL) based transformations plugin', functio beforeEach(function () { nock.cleanAll(); + // logger.setLevel('DEBUG'); contextBrokerMock = nock('http://192.168.1.1:1026') .matchHeader('fiware-service', 'smartgondor') @@ -669,12 +762,253 @@ describe('Java expression language (JEXL) based transformations plugin', functio }, c: { value: true, + type: 'Boolean' + }, + d: { + value: 'soy null', + type: 'Text' + }, + e: { + value: 'soy null', + type: 'Text' + }, + f: { + value: 'no soy null', + type: 'Text' + }, + g: { + value: true, + type: 'Boolean' + } + }) + .reply(204); + }); + + it('it should be handled properly', function (done) { + iotAgentLib.update('testNull1', 'testNull', '', values, function (error) { + should.not.exist(error); + contextBrokerMock.done(); + done(); + }); + }); + }); + + describe('When applying expressions without values (NaN)', function () { + // Case: Update for an attribute with bad expression + const values = [ + { + name: 'z', + type: 'Number', + value: null + } + ]; + + beforeEach(function () { + nock.cleanAll(); + + contextBrokerMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/entities?options=upsert', { + id: 'testNull2', + type: 'testNull', + z: { + value: null, + type: 'Number' + }, + c: { + value: true, + type: 'Boolean' + }, + d: { + value: 'soy null', + type: 'Text' + }, + e: { + value: 'soy null', + type: 'Text' + }, + f: { + value: 'no soy null', + type: 'Text' + }, + g: { + value: true, + type: 'Boolean' + } + }) + .reply(204); + }); + + it('it should be handled properly', function (done) { + iotAgentLib.update('testNull2', 'testNull', '', values, function (error) { + should.not.exist(error); + contextBrokerMock.done(); + done(); + }); + }); + }); + + describe('When applying expressions with null values - Skip values disabled', function () { + // Case: Update for an attribute with bad expression + const values = [ + { + name: 'v', + type: 'Number', + value: null + } + ]; + + beforeEach(function () { + nock.cleanAll(); + + contextBrokerMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/entities?options=upsert', { + id: 'testNullSkip1', + type: 'testNullSkip', + v: { + value: null, + type: 'Number' + }, + a: { + value: null, + type: 'Number' + }, + b: { + value: 0, + type: 'Number' + }, + c: { + value: true, + type: 'Boolean' + }, + d: { + value: 'soy null', + type: 'Text' + }, + e: { + value: 'soy null', + type: 'Text' + }, + f: { + value: 'no soy null', + type: 'Text' + }, + g: { + value: true, + type: 'Boolean' + } + }) + .reply(204); + }); + + it('it should be handled properly', function (done) { + iotAgentLib.update('testNullSkip1', 'testNullSkip', '', values, function (error) { + should.not.exist(error); + contextBrokerMock.done(); + done(); + }); + }); + }); + + describe('When applying expressions without values (NaN) - Skip values disabled', function () { + // Case: Update for an attribute with bad expression + const values = [ + { + name: 'z', + type: 'Number', + value: null + } + ]; + + beforeEach(function () { + nock.cleanAll(); + logger.setLevel('DEBUG'); + + contextBrokerMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/entities?options=upsert', { + id: 'testNullSkip2', + type: 'testNullSkip', + z: { + value: null, type: 'Number' }, + a: { + value: null, + type: 'Number' + }, + b: { + value: null, + type: 'Number' + }, + c: { + value: true, + type: 'Boolean' + }, d: { + value: 'soy null', + type: 'Text' + }, + e: { + value: 'soy null', + type: 'Text' + }, + f: { value: 'no soy null', type: 'Text' }, + g: { + value: true, + type: 'Boolean' + } + }) + .reply(204); + }); + + it('it should be handled properly', function (done) { + iotAgentLib.update('testNullSkip2', 'testNullSkip', '', values, function (error) { + should.not.exist(error); + contextBrokerMock.done(); + done(); + }); + }); + }); + + describe('When applying expressions with not explicit measures - explicitAttrs = true', function () { + // Case: Update for an attribute with bad expression + const values = [ + { + name: 'v', + type: 'Number', + value: null + } + ]; + + beforeEach(function () { + nock.cleanAll(); + + contextBrokerMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/entities?options=upsert', { + id: 'testNullExplicit1', + type: 'testNullExplicit', + b: { + value: 0, + type: 'Number' + }, + c: { + value: true, + type: 'Boolean' + }, + d: { + value: 'soy null', + type: 'Text' + }, e: { value: 'soy null', type: 'Text' @@ -692,7 +1026,7 @@ describe('Java expression language (JEXL) based transformations plugin', functio }); it('it should be handled properly', function (done) { - iotAgentLib.update('testNull1', 'testNull', '', values, function (error) { + iotAgentLib.update('testNullExplicit1', 'testNullExplicit', '', values, function (error) { should.not.exist(error); contextBrokerMock.done(); done(); From 150da9063f073110d5fc4a0ea8275a5dc8c18547 Mon Sep 17 00:00:00 2001 From: mapedraza Date: Thu, 28 Sep 2023 16:56:40 +0200 Subject: [PATCH 3/4] Add delete nulls --- lib/plugins/jexlParser.js | 12 ++++++++++++ .../expressions/jexlBasedTransformations-test.js | 13 ++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/plugins/jexlParser.js b/lib/plugins/jexlParser.js index 56a4a329d..59c423da9 100644 --- a/lib/plugins/jexlParser.js +++ b/lib/plugins/jexlParser.js @@ -125,12 +125,24 @@ function extractContext(attributeList) { function applyExpression(expression, context, typeInformation) { logContext = fillService(logContext, typeInformation); + // Delete null values from context. Related: + // https://github.com/telefonicaid/iotagent-node-lib/issues/1440 + // https://github.com/TomFrost/Jexl/issues/133 + deleteNulls(context); const result = parse(expression, context); logger.debug(logContext, 'applyExpression "[%j]" over "[%j]" result "[%j]" ', expression, context, result); const expressionResult = result !== undefined ? result : expression; return expressionResult; } +function deleteNulls(object) { + for (let key in object) { + if (object[key] === null) { + delete object[key]; + } + } +} + function isTransform(identifier) { return jexl.getTransform(identifier) !== (null || undefined); } diff --git a/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js b/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js index ad3811c25..e6d3c0f30 100644 --- a/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js +++ b/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js @@ -744,7 +744,6 @@ describe('Java expression language (JEXL) based transformations plugin', functio beforeEach(function () { nock.cleanAll(); - // logger.setLevel('DEBUG'); contextBrokerMock = nock('http://192.168.1.1:1026') .matchHeader('fiware-service', 'smartgondor') @@ -756,10 +755,6 @@ describe('Java expression language (JEXL) based transformations plugin', functio value: null, type: 'Number' }, - b: { - value: 0, - type: 'Number' - }, c: { value: true, type: 'Boolean' @@ -877,7 +872,7 @@ describe('Java expression language (JEXL) based transformations plugin', functio type: 'Number' }, b: { - value: 0, + value: null, type: 'Number' }, c: { @@ -925,7 +920,6 @@ describe('Java expression language (JEXL) based transformations plugin', functio beforeEach(function () { nock.cleanAll(); - logger.setLevel('DEBUG'); contextBrokerMock = nock('http://192.168.1.1:1026') .matchHeader('fiware-service', 'smartgondor') @@ -997,10 +991,6 @@ describe('Java expression language (JEXL) based transformations plugin', functio .post('/v2/entities?options=upsert', { id: 'testNullExplicit1', type: 'testNullExplicit', - b: { - value: 0, - type: 'Number' - }, c: { value: true, type: 'Boolean' @@ -1514,6 +1504,7 @@ describe('Java expression language (JEXL) based transformations plugin', functio }); }); }); + describe('When a measure arrives and there is not enough information to calculate an expression', function () { const values = [ { From 3ae8c9cf5170bde9bc42d2e960f20af8eaf29819 Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:11:32 +0200 Subject: [PATCH 4/4] Add CNR entry --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index b12cf170a..97aac23cf 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ +- Fix: null values arithmetics in JEXL expressions (#1440) - Fix: remove mongo `DeprecationWarning: current Server Discovery and Monitoring engine is deprecated` by setting `useUnifiedTopology = true` - Upgrade mongodb dev dep from 4.17.0 to 4.17.1