From a4c9f45e47a2e1f1514c56f8e9908e3a1df52b96 Mon Sep 17 00:00:00 2001 From: George Raduta Date: Thu, 29 Aug 2024 10:52:17 +0200 Subject: [PATCH 1/4] Rename and move service to new structure --- .../QueryService.js} | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) rename InfoLogger/lib/{SQLDataSource.js => services/QueryService.js} (92%) diff --git a/InfoLogger/lib/SQLDataSource.js b/InfoLogger/lib/services/QueryService.js similarity index 92% rename from InfoLogger/lib/SQLDataSource.js rename to InfoLogger/lib/services/QueryService.js index b5b32cc9e..642dcaa82 100644 --- a/InfoLogger/lib/SQLDataSource.js +++ b/InfoLogger/lib/services/QueryService.js @@ -15,12 +15,10 @@ const logger = require('@aliceo2/web-ui').LogManager .getLogger(`${process.env.npm_config_log_label ?? 'ilg'}/sql`); -module.exports = class SQLDataSource { +class QueryService { /** - * Instantiate SQL data source and connect to database - * MySQL options: https://github.com/mysqljs/mysql#connection-options - * Limit option - * @param {object} connection - mysql connection + * Query service that is to be used to map the InfoLogger parameters to SQL query and retrieve data + * @param {MySql} connection - mysql connection * @param {object} configMySql - mysql config */ constructor(connection, configMySql) { @@ -30,19 +28,12 @@ module.exports = class SQLDataSource { /** * Method to check if mysql driver connection is up - * @returns {Promise} with the results + * @returns {Promise} - resolves/rejects */ async isConnectionUpAndRunning() { - return await this.connection - .query('select timestamp from messages LIMIT 1000;') - .then(() => { - const url = `${this.configMySql.host}:${this.configMySql.port}/${this.configMySql.database}`; - logger.info(`Connected to infoLogger database ${url}`); - }) - .catch((error) => { - logger.error(error); - throw error; - }); + await this.connection.query('select timestamp from messages LIMIT 1000;'); + const url = `${this.configMySql.host}:${this.configMySql.port}/${this.configMySql.database}`; + logger.infoMessage(`Connected to infoLogger database ${url}`); } /** @@ -271,3 +262,5 @@ module.exports = class SQLDataSource { .then((data) => data); } }; + +module.exports.QueryService = QueryService; From cc8a0d1275e04d50346bcde66ce0a0ef5736a271 Mon Sep 17 00:00:00 2001 From: George Raduta Date: Thu, 29 Aug 2024 10:52:51 +0200 Subject: [PATCH 2/4] Rename test service file and move as per new struct --- .../mocha-QueryService.js} | 176 +++++++++++------- 1 file changed, 104 insertions(+), 72 deletions(-) rename InfoLogger/test/lib/{mocha-sqldatasource.js => services/mocha-QueryService.js} (64%) diff --git a/InfoLogger/test/lib/mocha-sqldatasource.js b/InfoLogger/test/lib/services/mocha-QueryService.js similarity index 64% rename from InfoLogger/test/lib/mocha-sqldatasource.js rename to InfoLogger/test/lib/services/mocha-QueryService.js index f486d40fc..50f6759ea 100644 --- a/InfoLogger/test/lib/mocha-sqldatasource.js +++ b/InfoLogger/test/lib/services/mocha-QueryService.js @@ -10,23 +10,21 @@ * In applying this license CERN does not waive the privileges and immunities * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. -*/ -/* eslint-disable max-len */ + */ const assert = require('assert'); const sinon = require('sinon'); -const config = require('../../config-default.js'); -const SQLDataSource = require('../../lib/SQLDataSource.js'); -const {MySQL} = require('@aliceo2/web-ui'); +const config = require('../../../config-default.js'); +const { QueryService } = require('./../../../lib/services/QueryService.js'); +const { MySQL } = require('@aliceo2/web-ui'); - -describe('SQLDataSource', () => { +describe('QueryService', () => { const filters = { timestamp: { since: -5, until: -1, $since: '2019-07-22T11:23:21.351Z', - $until: '2019-07-22T11:24:21.354Z' + $until: '2019-07-22T11:24:21.354Z', }, hostname: { match: 'test', @@ -43,12 +41,12 @@ describe('SQLDataSource', () => { $max: 21, // 0, 1, 6, 11, 21 }, pid: { - $maxWrong: 22 + $maxWrong: 22, }, userId: { min: 10, - $min: 10 - } + $min: 10, + }, }; const realFilters = { @@ -56,7 +54,7 @@ describe('SQLDataSource', () => { since: -5, until: -1, $since: '2019-07-22T11:23:21.351Z', - $until: '2019-07-22T11:24:21.354Z' + $until: '2019-07-22T11:24:21.354Z', }, hostname: { match: 'test', @@ -71,30 +69,33 @@ describe('SQLDataSource', () => { level: { max: null, // 0, 1, 6, 11, 21 $max: null, // 0, 1, 6, 11, 21 - } + }, }; - const emptySqlDataSource = new SQLDataSource(undefined, {}); + const emptySqlDataSource = new QueryService(undefined, {}); describe('Should check connection to mysql driver', () => { - it('should throw error when checking connection with mysql driver and driver returns rejected Promise', async () => { - const stub = sinon.createStubInstance(MySQL, + it('should reject with error when connection with mysql driver fails', async () => { + const stub = sinon.createStubInstance( + MySQL, { - query: sinon.stub().rejects(new Error('Unable to connect')) - } + query: sinon.stub().rejects(new Error('Unable to connect')), + }, ); - const sqlDataSource = new SQLDataSource(stub, config.mysql); + const sqlDataSource = new QueryService(stub, config.mysql); await assert.rejects(async () => { await sqlDataSource.isConnectionUpAndRunning(); }, new Error('Unable to connect')); }); + it('should do nothing when checking connection with mysql driver and driver returns resolved Promise', async () => { - const stub = sinon.createStubInstance(MySQL, + const stub = sinon.createStubInstance( + MySQL, { - query: sinon.stub().resolves('Connection is fine') - } + query: sinon.stub().resolves('Connection is fine'), + }, ); - const sqlDataSource = new SQLDataSource(stub, config.mysql); + const sqlDataSource = new QueryService(stub, config.mysql); await assert.doesNotReject(async () => { await sqlDataSource.isConnectionUpAndRunning(); @@ -104,36 +105,59 @@ describe('SQLDataSource', () => { describe('Filter to SQL Conditions', () => { it('should successfully return empty values & criteria when translating empty filters from client', () => { - assert.deepStrictEqual(emptySqlDataSource._filtersToSqlConditions({}), {values: [], criteria: []}); + assert.deepStrictEqual(emptySqlDataSource._filtersToSqlConditions({}), { values: [], criteria: [] }); }); it('should successfully return values & criteria when translating filters from client', () => { const expectedValues = [1563794601.351, 1563794661.354, 'test', 'testEx', ['D', 'W'], 21, 22, 10]; - const expectedCriteria = ['`timestamp`>=?', '`timestamp`<=?', - '`hostname` = ?', 'NOT(`hostname` = ? AND `hostname` IS NOT NULL)', - '`severity` IN (?)', '`level`<=?', '`userId`>=?']; - assert.deepStrictEqual(emptySqlDataSource._filtersToSqlConditions(filters), - {values: expectedValues, criteria: expectedCriteria}); + const expectedCriteria = [ + '`timestamp`>=?', + '`timestamp`<=?', + '`hostname` = ?', + 'NOT(`hostname` = ? AND `hostname` IS NOT NULL)', + '`severity` IN (?)', + '`level`<=?', + '`userId`>=?', + ]; + assert.deepStrictEqual( + emptySqlDataSource._filtersToSqlConditions(filters), + { values: expectedValues, criteria: expectedCriteria }, + ); }); it('should successfully return values & criteria when translating filters from client (2)', () => { - let likeFilters = filters; - likeFilters.hostname = {match: 'test%', exclude: 'testEx', $match: 'test%', $exclude: 'testEx'}; + const likeFilters = filters; + likeFilters.hostname = { match: 'test%', exclude: 'testEx', $match: 'test%', $exclude: 'testEx' }; const expectedValues = [1563794601.351, 1563794661.354, 'test%', 'testEx', ['D', 'W'], 21, 22, 10]; - const expectedCriteria = ['`timestamp`>=?', '`timestamp`<=?', - '`hostname` LIKE (?)', 'NOT(`hostname` = ? AND `hostname` IS NOT NULL)', - '`severity` IN (?)', '`level`<=?', '`userId`>=?']; - assert.deepStrictEqual(emptySqlDataSource._filtersToSqlConditions(likeFilters), - {values: expectedValues, criteria: expectedCriteria}); + const expectedCriteria = [ + '`timestamp`>=?', + '`timestamp`<=?', + '`hostname` LIKE (?)', + 'NOT(`hostname` = ? AND `hostname` IS NOT NULL)', + '`severity` IN (?)', + '`level`<=?', + '`userId`>=?', + ]; + assert.deepStrictEqual( + emptySqlDataSource._filtersToSqlConditions(likeFilters), + { values: expectedValues, criteria: expectedCriteria }, + ); }); it('should successfully build query when excluding multiple hostnames', () => { - let likeFilters = filters; - likeFilters.hostname = {$exclude: 'test testEx', exclude: 'test testEx'}; + const likeFilters = filters; + likeFilters.hostname = { $exclude: 'test testEx', exclude: 'test testEx' }; const expectedValues = [1563794601.351, 1563794661.354, 'test', 'testEx', ['D', 'W'], 21, 22, 10]; - const expectedCriteria = ['`timestamp`>=?', '`timestamp`<=?', + const expectedCriteria = [ + '`timestamp`>=?', + '`timestamp`<=?', 'NOT(`hostname` = ? AND `hostname` IS NOT NULL OR `hostname` = ? AND `hostname` IS NOT NULL)', - '`severity` IN (?)', '`level`<=?', '`userId`>=?']; - assert.deepStrictEqual(emptySqlDataSource._filtersToSqlConditions(likeFilters), - {values: expectedValues, criteria: expectedCriteria}); + '`severity` IN (?)', + '`level`<=?', + '`userId`>=?', + ]; + assert.deepStrictEqual( + emptySqlDataSource._filtersToSqlConditions(likeFilters), + { values: expectedValues, criteria: expectedCriteria }, + ); }); }); @@ -151,9 +175,13 @@ describe('SQLDataSource', () => { }); it('should successfully return SQL format criteria if array contains values', () => { - const criteria = ['`timestamp`>=?', '`timestamp`<=?', - '`hostname` = ?', 'NOT(`hostname` = ? AND `hostname` IS NOT NULL)', - '`severity` IN (?)']; + const criteria = [ + '`timestamp`>=?', + '`timestamp`<=?', + '`hostname` = ?', + 'NOT(`hostname` = ? AND `hostname` IS NOT NULL)', + '`severity` IN (?)', + ]; const expectedCriteriaString = 'WHERE `timestamp`>=? AND `timestamp`<=? AND ' + '`hostname` = ? AND NOT(`hostname` = ? AND `hostname` IS NOT NULL) AND `severity` IN (?)'; assert.deepStrictEqual(emptySqlDataSource._getCriteriaAsString(criteria), expectedCriteriaString); @@ -161,25 +189,30 @@ describe('SQLDataSource', () => { }); it('should successfully return messages when querying mysql driver', async () => { - const stub = sinon.createStubInstance(MySQL, {query: sinon.stub().resolves([{severity: 'W'}, {severity: 'I'}])}); - const sqlDataSource = new SQLDataSource(stub, config.mysql); + const stub = sinon.createStubInstance( + MySQL, + { + query: sinon.stub().resolves([{ severity: 'W' }, { severity: 'I' }]), + }, + ); + const sqlDataSource = new QueryService(stub, config.mysql); const queryResult = await sqlDataSource._queryMessagesOnOptions('criteriaString', []); - assert.deepStrictEqual(queryResult, [{severity: 'W'}, {severity: 'I'}]); + assert.deepStrictEqual(queryResult, [{ severity: 'W' }, { severity: 'I' }]); }); it('should throw an error when unable to query within private method due to rejected promise', async () => { - const stub = sinon.createStubInstance(MySQL, {query: sinon.stub().rejects()}); - const sqlDataSource = new SQLDataSource(stub, config.mysql); + const stub = sinon.createStubInstance(MySQL, { query: sinon.stub().rejects() }); + const sqlDataSource = new QueryService(stub, config.mysql); return assert.rejects(async () => { await sqlDataSource._queryMessagesOnOptions('criteriaString', []); }, new Error('Error')); }); it('should throw an error when unable to query(API) due to rejected promise', async () => { - const stub = sinon.createStubInstance(MySQL, {query: sinon.stub().rejects()}); - const sqlDataSource = new SQLDataSource(stub, config.mysql); + const stub = sinon.createStubInstance(MySQL, { query: sinon.stub().rejects() }); + const sqlDataSource = new QueryService(stub, config.mysql); return assert.rejects(async () => { - await sqlDataSource.queryFromFilters(realFilters, {limit: 10}); + await sqlDataSource.queryFromFilters(realFilters, { limit: 10 }); }, new Error('Error')); }); @@ -197,16 +230,16 @@ describe('SQLDataSource', () => { const query = 'SELECT * FROM `messages` WHERE `timestamp`>=? AND `timestamp`<=? AND `hostname` = ? AND NOT(`hostname` = ? AND `hostname` IS NOT NULL) AND `severity` IN (?) ORDER BY `TIMESTAMP` LIMIT 10'; const queryStub = sinon.stub(); queryStub.withArgs(requestRows, values).resolves([]); - const stub = sinon.createStubInstance(MySQL, {query: queryStub}); + const stub = sinon.createStubInstance(MySQL, { query: queryStub }); - const sqlDataSource = new SQLDataSource(stub, config.mysql); - const result = await sqlDataSource.queryFromFilters(realFilters, {limit: 10}); + const sqlDataSource = new QueryService(stub, config.mysql); + const result = await sqlDataSource.queryFromFilters(realFilters, { limit: 10 }); const expectedResult = { rows: [], count: 0, limit: 10, - queryAsString: query + queryAsString: query, }; delete result.time; assert.deepStrictEqual(result, expectedResult); @@ -218,9 +251,9 @@ describe('SQLDataSource', () => { query: sinon.stub().resolves([ { severity: 'E', 'COUNT(*)': 102 }, { severity: 'F', 'COUNT(*)': 1 }, - ]) - } - const dataService = new SQLDataSource(sqlStub, config.mysql); + ]), + }; + const dataService = new QueryService(sqlStub, config.mysql); const data = await dataService.queryGroupCountLogsBySeverity(51234); assert.deepStrictEqual(data, { D: 0, @@ -228,26 +261,25 @@ describe('SQLDataSource', () => { W: 0, E: 102, F: 1, - }) + }); }); it('should throw error if data service throws error', async () => { const sqlStub = { - query: sinon.stub().rejects(new Error('Data Service went bad')) - } - const dataService = new SQLDataSource(sqlStub, config.mysql); - - await assert.rejects(dataService.queryGroupCountLogsBySeverity(51234), new Error('Data Service went bad')) + query: sinon.stub().rejects(new Error('Data Service went bad')), + }; + const dataService = new QueryService(sqlStub, config.mysql); + + await assert.rejects(dataService.queryGroupCountLogsBySeverity(51234), new Error('Data Service went bad')); }); it('should throw error if data service throws error', async () => { const sqlStub = { - query: sinon.stub().throws(new Error('Data Service went bad')) - } - const dataService = new SQLDataSource(sqlStub, config.mysql); - - await assert.rejects(dataService.queryGroupCountLogsBySeverity(51234), new Error('Data Service went bad')) - }); + query: sinon.stub().throws(new Error('Data Service went bad')), + }; + const dataService = new QueryService(sqlStub, config.mysql); + await assert.rejects(dataService.queryGroupCountLogsBySeverity(51234), new Error('Data Service went bad')); + }); }); }); From 35c4bfa68b8d6f97f1e8e09418798b3acbcc868a Mon Sep 17 00:00:00 2001 From: George Raduta Date: Thu, 29 Aug 2024 10:53:03 +0200 Subject: [PATCH 3/4] Update controller to use new service --- InfoLogger/lib/controller/QueryController.mjs | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/InfoLogger/lib/controller/QueryController.mjs b/InfoLogger/lib/controller/QueryController.mjs index ac03a9ff8..ed30d9836 100644 --- a/InfoLogger/lib/controller/QueryController.mjs +++ b/InfoLogger/lib/controller/QueryController.mjs @@ -12,7 +12,7 @@ * or submit itself to any jurisdiction. */ -import { LogManager } from '@aliceo2/web-ui'; +import { LogManager, updateAndSendExpressResponseFromNativeError } from '@aliceo2/web-ui'; /** * Gateway for all calls that are to query InfoLogger database @@ -30,15 +30,38 @@ export class QueryController { this._logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'ilg'}/query-ctrl`); } + /** + * Given InfoLogger parameters, use the query service to retrieve logs requested + * @param {Request} req - HTTP request object with "query" information on object + * @param {Response} res - HTTP response object to provide information on request + * @returns {void} + */ + async getLogs(req, res) { + if (this._queryService) { + try { + const { body: { criterias, options } } = req; + const logs = await this._queryService.queryFromFilters(criterias, options); + res.status(200).json(logs); + } catch (error) { + updateAndSendExpressResponseFromNativeError(res, error); + } + } else { + res.status(503).json({ message: 'Query Service was not configured' }); + } + } + /** * API endpoint for retrieving total number of logs grouped by severity for a given runNumber + * (Used within FLP) * @param {Request} req - HTTP request object with "query" information on object * @param {Response} res - HTTP response object to provide information on request * @returns {void} */ async getQueryStats(req, res) { const { runNumber } = req.query; - if (!runNumber || isNaN(runNumber)) { + if (this._queryService) { + res.status(503).json({ message: 'Query Service was not configured' }); + } else if (!runNumber || isNaN(runNumber)) { res.status(400).json({ error: 'Invalid runNumber provided' }); } else { try { @@ -50,20 +73,4 @@ export class QueryController { } } } - - /** - * Setter for updating the queryService - * @param {SQLDataSource} queryService - service to be used to query information on the logs - */ - set queryService(queryService) { - this._queryService = queryService; - } - - /** - * Getter for the queryService instance currently being used - * @returns {SQLDataSource} - instance of queryService - */ - get queryService() { - return this._queryService; - } } From 0e412be5deff3540a9d8defcfb40af93a6fcdbaa Mon Sep 17 00:00:00 2001 From: George Raduta Date: Thu, 29 Aug 2024 10:53:21 +0200 Subject: [PATCH 4/4] Update API to use new service --- InfoLogger/lib/api.js | 98 ++++++++----------------------------------- 1 file changed, 17 insertions(+), 81 deletions(-) diff --git a/InfoLogger/lib/api.js b/InfoLogger/lib/api.js index 7044e1480..0eae29be7 100644 --- a/InfoLogger/lib/api.js +++ b/InfoLogger/lib/api.js @@ -13,30 +13,36 @@ */ const { LogManager, WebSocketMessage, InfoLoggerReceiver, MySQL } = require('@aliceo2/web-ui'); -const SQLDataSource = require('./SQLDataSource.js'); +const { QueryService } = require('./services/QueryService.js'); const ProfileService = require('./ProfileService.js'); const JsonFileConnector = require('./JSONFileConnector.js'); const StatusService = require('./StatusService.js'); -const projPackage = require('../package.json'); - -const logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'ilg'}/api`); +const projPackage = require('./../package.json'); const config = require('./configProvider.js'); -let querySource = null; let liveSource = null; - -const jsonDb = new JsonFileConnector(config.dbFile || `${__dirname}/../db.json`); - -const profileService = new ProfileService(jsonDb); +let sqlService = null; +let queryService = null; module.exports.attachTo = async (http, ws) => { + const logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'ilg'}/api`); + const { QueryController } = await import('./controller/QueryController.mjs'); - const queryController = new QueryController(); + + if (config.mysql) { + sqlService = new MySQL(config.mysql); + queryService = new QueryService(sqlService, config.mysql); + } + const queryController = new QueryController(queryService); const statusService = new StatusService(config, projPackage, ws); + statusService.setQuerySource(queryService); + + const jsonDb = new JsonFileConnector(config.dbFile || `${__dirname}/../db.json`); + const profileService = new ProfileService(jsonDb); - http.post('/query', query); + http.post('/query', queryController.getLogs.bind(queryController)); http.get('/query/stats', queryController.getQueryStats.bind(queryController), { public: true }); http.get('/status/gui', statusService.getILGStatus.bind(statusService), { public: true }); @@ -46,18 +52,6 @@ module.exports.attachTo = async (http, ws) => { http.get('/getProfile', (req, res) => profileService.getProfile(req, res)); http.post('/saveUserProfile', (req, res) => profileService.saveUserProfile(req, res)); - if (config.mysql) { - logger.info('[API] Detected InfoLogger database configuration'); - setupMySQLConnectors(); - setInterval(() => { - if (!querySource) { - setupMySQLConnectors(); - } - }, config.mysql.retryMs || 5000); - } else { - logger.warn('[API] InfoLogger database config not found, Query mode not available'); - } - if (config.infoLoggerServer) { logger.info('[API] InfoLogger server config found'); liveSource = new InfoLoggerReceiver(); @@ -87,62 +81,4 @@ module.exports.attachTo = async (http, ws) => { ws.unfilteredBroadcast(new WebSocketMessage().setCommand('il-server-close')); }); } - - /** - * Method to attempt creating a connection to the InfoLogger SQL DB - */ - function setupMySQLConnectors() { - const connector = new MySQL(config.mysql); - connector.testConnection().then(() => { - querySource = new SQLDataSource(connector, config.mysql); - querySource.isConnectionUpAndRunning() - .then(() => { - ws.unfilteredBroadcast(new WebSocketMessage().setCommand('il-sql-server-status').setPayload({ ok: true })); - statusService.setQuerySource(querySource); - queryController.queryService = querySource; - }).catch((error) => { - logger.error(`[API] Unable to instantiate data source due to ${error}`); - ws.unfilteredBroadcast(new WebSocketMessage().setCommand('il-sql-server-status') - .setPayload({ ok: false, message: 'Query service is unavailable' })); - querySource = null; - statusService.setQuerySource(querySource); - }); - }).catch((error) => { - logger.error(`[API] Unable to connect to mysql due to ${error}`); - querySource = null; - ws.unfilteredBroadcast(new WebSocketMessage().setCommand('il-sql-server-status') - .setPayload({ ok: false, message: 'Query service is unavailable' })); - statusService.setQuerySource(querySource); - }); - } - - /** - * Method to perform a query on the SQL Data Source - * @param {Request} req - HTTP Request object - * @param {Response} res - HTTP Response object - * @returns {void} - */ - function query(req, res) { - if (querySource) { - querySource.queryFromFilters(req.body.criterias, req.body.options) - .then((result) => res.json(result)) - .catch((error) => { - setupMySQLConnectors(); - handleError(res, error); - }); - } else { - handleError(res, '[API] MySQL Data Source is currently not available'); - } - } - - /** - * Catch all HTTP errors - * @param {Response} res - HTTP Response object - * @param {Error} error - Error object to handle and send - * @param {number} status - HTTP status code to send - */ - function handleError(res, error, status = 500) { - logger.trace(error); - res.status(status).json({ message: error.message }); - } };