From db2a0014d3c29ca8f20e3a3c7dd913d35fbdb825 Mon Sep 17 00:00:00 2001 From: yshashix Date: Fri, 6 Oct 2023 14:01:21 +0200 Subject: [PATCH] oisp-mqtt-gw: removed postgres Database functions and running on node16 as oer DPT version --- Dockerfile | 2 +- README.md | 3 +- config.js | 10 +-- lib/cache/index.js | 73 +--------------------- package.json | 8 +-- test/unit/api_actuationTest.js | 22 ------- test/unit/api_spb_data_ingestionTest.js | 32 ---------- test/unit/config_Test.js | 22 ------- test/unit/lib_cacheTest.js | 82 ------------------------- 9 files changed, 8 insertions(+), 246 deletions(-) diff --git a/Dockerfile b/Dockerfile index 08b715bb..e28f2f70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:14-alpine +FROM node:16-alpine ADD / /app RUN apk update && apk upgrade && \ diff --git a/README.md b/README.md index 3c3a636d..e3fe3257 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ The Bridge also handles the auth, acl and load-balancing for MQTT and MQTT with ``` bash "sparkplug": { - "spBKafkaProduce": true, + "spBKafkaProduce": false, "spBkafkaTopic": "sparkplugB", "ngsildKafkaProduce": true, "ngsildKafkaTopic": "ngsildSpB" @@ -86,7 +86,6 @@ The Bridge also handles the auth, acl and load-balancing for MQTT and MQTT with | true | true | SpB NGSI-LD format metric on "ngsildSpB" topic; Rest on "sparkplugB" | | true | false | All message on "sparkplugB" | | false | true | SpB NGSI-LD format metric on "ngsildSpB" topic; Rest ignored | -| false | false | All message on "metric" | 2. NGSI-LD message format converted from received SpB message format diff --git a/config.js b/config.js index 0a882cae..c12026e2 100644 --- a/config.js +++ b/config.js @@ -87,13 +87,6 @@ var config = { "linger": parsedConfig.kafkaConfig.linger, "partitioner": parsedConfig.kafkaConfig.partitioner }, - "postgres": { - "host": parsedConfig.postgresConfig.hostname, - "dbname": parsedConfig.postgresConfig.dbname, - "port": parsedConfig.postgresConfig.port, - "username": parsedConfig.postgresConfig.username, - "password": parsedConfig.postgresConfig.password - }, "topics": { "prefix": parsedConfig.topicsPrefix || "server", "publish": { @@ -118,7 +111,8 @@ var config = { "sparkplugb_data_ingestion": "spBv1.0/+/+/+/+" }, "publish": { - "error": "server/error/{accountId}/{deviceId}" + "error": "server/error/{accountId}/{deviceId}", + "actuation": "/{realmId}/DCMD/{gatewayId}/{deviceId}" } } }, diff --git a/lib/cache/index.js b/lib/cache/index.js index 56aeec89..13128862 100644 --- a/lib/cache/index.js +++ b/lib/cache/index.js @@ -18,10 +18,6 @@ "use strict"; const redis = require("redis"); -const { Sequelize } = require('sequelize'); -const { QueryTypes } = require('sequelize'); -var uuidValidate = require('uuid-validate'); - var me; class CacheFactory { @@ -33,7 +29,6 @@ class CacheFactory { getInstance() { return CacheFactory.instance; } - } class Cache { @@ -41,44 +36,14 @@ class Cache { constructor(conf, log) { this.redisClient = redis.createClient({port: conf.cache.port, host: conf.cache.host}); - if (process.env.PGSSLMODE === "require") { - this.sequelize = new Sequelize(conf.postgres.dbname, conf.postgres.username, conf.postgres.password, { - host: conf.postgres.host, - port: conf.postgres.port, - dialect: 'postgres', - dialectOptions: { - ssl: { - rejectUnauthorized: false - } - }, - }); - } else { - this.sequelize = new Sequelize(conf.postgres.dbname, conf.postgres.username, conf.postgres.password, { - host: conf.postgres.host, - port: conf.postgres.port, - dialect: 'postgres' - }); - } - - this.sequelize.authenticate() - .then(() => { - console.log('DB connection has been established.'); - }) - .catch(error => { - console.error('Unable to connect to DB:', error); - }); - - this.config = conf; + this.config = conf; this.logger = log; me = this; this.redisClient.on("error", function(err) { me.logger.info("Error in Redis client: " + err); }); } - async initialize() { - - - } + async setValue(key, valueType, value) { return new Promise((resolve, reject) => { this.redisClient.hmset(key, valueType, value, (err, result) => { @@ -117,39 +82,5 @@ class Cache { }); }); } - async getDidAndDataType(item) { - var cid = item.componentId; - //check whether cid = uuid - if (!uuidValidate(cid)) { - throw new Error("cid not UUID. Rejected!"); - } - - var value = await this.getValues(cid); - var didAndDataType; - if (value === null || (Array.isArray(value) && value.length === 1 && value[0] === null)) { - me.logger.debug("Could not find " + cid + "in cache. Now trying sql query."); - // no cached value found => make db lookup and store in cache - var sqlquery='SELECT devices.id,"dataType" FROM dashboard.device_components,dashboard.devices,dashboard.component_types WHERE "componentId"::text=\'' + cid + '\' and "deviceUID"::text=devices.uid::text and device_components."componentTypeId"::text=component_types.id::text'; - me.logger.debug("Applying SQL query: " + sqlquery); - didAndDataType = await this.sequelize.query(sqlquery, { type: QueryTypes.SELECT }); - me.logger.debug("Result of sql query: " + JSON.stringify(didAndDataType)); - } else { - me.logger.debug("Found in cache: " + JSON.stringify(value)); - didAndDataType = [value]; - } - - if (didAndDataType === undefined || didAndDataType === null) { - throw new Error("DB lookup failed!"); - } - var redisResult = await this.setValue(cid, "id", didAndDataType[0].id) && this.setValue(cid, "dataType", didAndDataType[0].dataType); - didAndDataType[0].dataElement = item; - if (redisResult) { - return didAndDataType[0]; - } else { - me.logger.warn("Could not store db value in redis. This will significantly reduce performance"); - return didAndDataType[0]; - } - } - } module.exports = CacheFactory; diff --git a/package.json b/package.json index 4f2a9810..f86ed562 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "iotkit-gateway", "version": "0.12.1", - "date": "2015-01-26T13:46:41.332Z", + "date": "2023-10-09T13:46:41.332Z", "description": "Authotization gateway and kafka bridge for MQTTGW", "main": "app.js", "dependencies": { @@ -24,10 +24,6 @@ "test": "node gateway-agent", "start": "node gateway-agent" }, - "repository": { - "type": "git", - "url": "git://github.com/enableiot/iotkit-analytics.git" - }, "keywords": [ "iotkit", "authorization" @@ -40,7 +36,7 @@ "homepage": "https://github.com/enableiot/iotkit-analytics", "devDependencies": { "asserts": "^4.0.0", - "chai": "^4.2.0", + "chai": "^4.3.6", "grunt": "^1.3.0", "grunt-cli": "^1.3.2", "grunt-contrib-jshint": "^3.0.0", diff --git a/test/unit/api_actuationTest.js b/test/unit/api_actuationTest.js index b4181d1f..f1bb4380 100644 --- a/test/unit/api_actuationTest.js +++ b/test/unit/api_actuationTest.js @@ -50,7 +50,6 @@ describe(fileToTest, function() { "authServicePort": "2345", \ "redisConf": "@@OISP_REDIS_CONFIG", \ "kafkaConfig": "@@OISP_KAFKA_CONFIG", \ - "postgresConfig": "@@OISP_POSTGRES_CONFIG", \ "keycloakConfig": "@@OISP_KEYCLOAK_CONFIG", \ "aesKey": "/app/keys/mqtt/mqtt_gw_secret.key" \ }'; @@ -61,17 +60,6 @@ describe(fileToTest, function() { "password": "password" \ }'; - process.env.OISP_POSTGRES_CONFIG = '{\ - "dbname": "oisp",\ - "hostname": "postgres-ro",\ - "writeHostname": "postgres",\ - "port": "5432",\ - "su_username": "su_username",\ - "su_password": "su_password",\ - "username": "username",\ - "password": "password"\ - }'; - var config = { "mqttBrokerUrl": "brokerUrl", "mqttBrokerLocalPort": "1234", @@ -105,16 +93,6 @@ describe(fileToTest, function() { "requestTimeout": 4, "maxRetryTime": 10 }, - "postgres": { - "dbname": "oisp", - "hostname": "postgres-ro", - "writeHostname": "postgres", - "port": "5432", - "su_username": "su_username", - "su_password": "su_password", - "username": "username", - "password": "password" - }, "aesKey": "/app/keys/mqtt/mqtt_gw_secret.key" }; var ToTest = rewire(fileToTest); diff --git a/test/unit/api_spb_data_ingestionTest.js b/test/unit/api_spb_data_ingestionTest.js index 8994bf16..9a914f4c 100644 --- a/test/unit/api_spb_data_ingestionTest.js +++ b/test/unit/api_spb_data_ingestionTest.js @@ -47,7 +47,6 @@ describe(fileToTest, function() { "authServicePort": "2345", \ "redisConf": "@@OISP_REDIS_CONFIG", \ "kafkaConfig": "@@OISP_KAFKA_CONFIG", \ - "postgresConfig": "@@OISP_POSTGRES_CONFIG", \ "keycloakConfig": "@@OISP_KEYCLOAK_CONFIG", \ "aesKey": "/app/keys/mqtt/mqtt_gw_secret.key" \ }'; @@ -58,17 +57,6 @@ describe(fileToTest, function() { "password": "password" \ }'; - process.env.OISP_POSTGRES_CONFIG = '{\ - "dbname": "oisp",\ - "hostname": "postgres-ro",\ - "writeHostname": "postgres",\ - "port": "5432",\ - "su_username": "su_username",\ - "su_password": "su_password",\ - "username": "username",\ - "password": "password"\ - }'; - var config = { "mqttBrokerUrl": "brokerUrl", "mqttBrokerLocalPort": "1234", @@ -109,16 +97,6 @@ describe(fileToTest, function() { "requestTimeout": 4, "maxRetryTime": 10 }, - "postgres": { - "dbname": "oisp", - "hostname": "postgres-ro", - "writeHostname": "postgres", - "port": "5432", - "su_username": "su_username", - "su_password": "su_password", - "username": "username", - "password": "password" - }, "aesKey": "/app/keys/mqtt/mqtt_gw_secret.key" }; @@ -162,16 +140,6 @@ describe(fileToTest, function() { "requestTimeout": 4, "maxRetryTime": 10 }, - "postgres": { - "dbname": "oisp", - "hostname": "postgres-ro", - "writeHostname": "postgres", - "port": "5432", - "su_username": "su_username", - "su_password": "su_password", - "username": "username", - "password": "password" - }, "aesKey": "/app/keys/mqtt/mqtt_gw_secret.key" }; diff --git a/test/unit/config_Test.js b/test/unit/config_Test.js index d9b35ec6..87c3fc62 100644 --- a/test/unit/config_Test.js +++ b/test/unit/config_Test.js @@ -47,7 +47,6 @@ describe(fileToTest, function() { "authServicePort": "2345", \ "redisConf": "@@OISP_REDIS_CONFIG", \ "kafkaConfig": "@@OISP_KAFKA_CONFIG", \ - "postgresConfig": "@@OISP_POSTGRES_CONFIG", \ "keycloakConfig": "@@OISP_KEYCLOAK_CONFIG", \ "aesKey": "/app/keys/mqtt/mqtt_gw_secret.key" \ }'; @@ -58,17 +57,6 @@ describe(fileToTest, function() { "password": "password" \ }'; - process.env.OISP_POSTGRES_CONFIG = '{\ - "dbname": "oisp",\ - "hostname": "postgres-ro",\ - "writeHostname": "postgres",\ - "port": "5432",\ - "su_username": "su_username",\ - "su_password": "su_password",\ - "username": "username",\ - "password": "password"\ - }'; - var parseResult = { "mqttBrokerUrl": "brokerUrl", "mqttBrokerLocalPort": "1234", @@ -95,16 +83,6 @@ describe(fileToTest, function() { "requestTimeout": 4, "maxRetryTime": 10 }, - "postgresConfig": { - "dbname": "oisp", - "hostname": "postgres-ro", - "writeHostname": "postgres", - "port": "5432", - "su_username": "su_username", - "su_password": "su_password", - "username": "username", - "password": "password" - }, "keycloakConfig": { "auth-server-url": "keycloak", "listenerPort": 4080 diff --git a/test/unit/lib_cacheTest.js b/test/unit/lib_cacheTest.js index b2f6675e..7dd15a7e 100644 --- a/test/unit/lib_cacheTest.js +++ b/test/unit/lib_cacheTest.js @@ -17,8 +17,6 @@ var assert = require('chai').assert, rewire = require('rewire'); -var expect = require('chai').expect; - var fileToTest = "../../lib/cache/index.js"; @@ -31,13 +29,6 @@ describe(fileToTest, function(){ port: 1234, host: "redishost" }, - postgres: { - dbname: "dbname", - username: "username", - password: "password", - host: "postgres", - port: 1235 - } }; var logger = { debug: function() {}, @@ -50,15 +41,7 @@ describe(fileToTest, function(){ }; } }; - var Sequelize = class Sequelize { - constructor() { - } - authenticate(){ - return Promise.resolve(); - } - }; ToTest.__set__("redis", redis); - ToTest.__set__("Sequelize", Sequelize); var CacheFactory = new ToTest(config, logger); var cache = CacheFactory.getInstance(); @@ -67,13 +50,6 @@ describe(fileToTest, function(){ cache: { port: 1234, host: "redishost2" - }, - postgres: { - dbname: "dbname2", - username: "username2", - password: "password2", - host: "postgres2", - port: 1235 } }; var CacheFactory2 = new ToTest(config2, logger); @@ -83,62 +59,4 @@ describe(fileToTest, function(){ } done(); }); - it('Shall retrieve did and dataType from redisCache', function(done){ - ToTest = rewire(fileToTest); // to reset the singleton - var cid = "895cac7d-49d1-4650-9b63-a6c7c0c9c4c7"; - var config = { - cache: { - port: 1234, - host: "redishost" - }, - postgres: { - dbname: "dbname", - username: "username", - password: "password", - host: "postgres", - port: 1235 - } - }; - var logger = { - debug: function() {}, - info: function() {} - }; - var redisValue = { - id: "did", - dataType: "dataType" - }; - var redis = { - createClient: function() { - return { - hgetall: function(key, callback) { - callback(null, redisValue); - }, - on: function(){}, - hmset: function(key, valueType, value, callback){ - assert.equal(key, cid, "wrong key received"); - expect(valueType).to.be.oneOf(["id", "dataType"]); - expect(value).to.be.oneOf(["did", "dataType"]); - callback(null, true); - } - }; - } - }; - var Sequelize = class Sequelize { - constructor() { - } - authenticate(){ - return Promise.resolve(); - } - }; - ToTest.__set__("redis", redis); - ToTest.__set__("Sequelize", Sequelize); - var CacheFactory = new ToTest(config, logger); - var cache = CacheFactory.getInstance(); - cache.getDidAndDataType({componentId: "895cac7d-49d1-4650-9b63-a6c7c0c9c4c7"}) - .then((result) => { - assert.deepEqual(result, redisValue, "Wrong redis value received"); - done(); - }) - .catch((e) => done(e)); - }); }); \ No newline at end of file