From bdc565cacb93e30b83e9382da5bfb10c216893c2 Mon Sep 17 00:00:00 2001 From: Anthony Rimet Date: Mon, 25 Sep 2023 15:14:38 +0200 Subject: [PATCH] Feat(tenant): Create a logger for each tenant --- globalConfig.json | 2 +- src/api/controller/api/dataset.js | 8 ++-- src/api/controller/api/publish.js | 5 ++- src/api/controller/api/publishFacets.js | 6 ++- src/api/controller/customPage.js | 5 ++- src/api/index.js | 6 ++- src/api/services/enrichment/enrichment.js | 28 +++++++------- .../services/enrichment/enrichment.spec.js | 3 ++ src/api/services/logger.js | 37 +++++++++++++------ src/api/services/publishCharacteristics.js | 4 +- src/api/services/publishDocuments.js | 7 ++-- src/api/services/publishDocuments.spec.js | 3 ++ src/api/workers/tools.js | 4 +- 13 files changed, 76 insertions(+), 42 deletions(-) diff --git a/globalConfig.json b/globalConfig.json index f53601ce28..9febe54eaf 100644 --- a/globalConfig.json +++ b/globalConfig.json @@ -1 +1 @@ -{"mongoUri":"mongodb://127.0.0.1:36595/jest?","mongoDBName":"jest"} \ No newline at end of file +{"mongoUri":"mongodb://127.0.0.1:39569/jest?","mongoDBName":"jest"} \ No newline at end of file diff --git a/src/api/controller/api/dataset.js b/src/api/controller/api/dataset.js index 7703d7b14f..a78d4cfda0 100644 --- a/src/api/controller/api/dataset.js +++ b/src/api/controller/api/dataset.js @@ -1,7 +1,7 @@ import Koa from 'koa'; import route from 'koa-route'; -import logger from '../../services/logger'; import koaBodyParser from 'koa-bodyparser'; +import getLogger from '../../services/logger'; const app = new Koa(); @@ -17,7 +17,8 @@ export const clearDataset = async ctx => { } ctx.body = { status: 'success' }; } catch (error) { - logger.error('clear dataset error', { + const logger = getLogger(ctx.tenant); + logger.error(`Clear dataset error`, { error, }); ctx.body = { status: 'error', error }; @@ -94,7 +95,8 @@ export const deleteDatasetRow = async (ctx, id) => { await ctx.dataset.deleteOne(id); ctx.body = { status: 'deleted' }; } catch (error) { - logger.error('delete dataset row error', { + const logger = getLogger(ctx.tenant); + logger.error(`Delete dataset row error`, { error, }); ctx.body = { status: 'error', error }; diff --git a/src/api/controller/api/publish.js b/src/api/controller/api/publish.js index 7077a7e42b..bfd2e9fcfb 100644 --- a/src/api/controller/api/publish.js +++ b/src/api/controller/api/publish.js @@ -3,9 +3,9 @@ import route from 'koa-route'; import { v1 as uuid } from 'uuid'; import clearPublished from '../../services/clearPublished'; -import logger from '../../services/logger'; import { workerQueues } from '../../workers'; import { PUBLISHER } from '../../workers/publisher'; +import getLogger from '../../services/logger'; const app = new Koa(); @@ -28,7 +28,8 @@ export const handleClearPublished = async ctx => { status: 'success', }; } catch (error) { - logger.error(`handle clear published error - ${ctx.tenant}`, { + const logger = getLogger(ctx.tenant); + logger.error(`Handle clear published error`, { error, }); ctx.body = { diff --git a/src/api/controller/api/publishFacets.js b/src/api/controller/api/publishFacets.js index ec1f7b658c..d9c1efa683 100644 --- a/src/api/controller/api/publishFacets.js +++ b/src/api/controller/api/publishFacets.js @@ -15,7 +15,8 @@ export default async (ctx, fields, withProgress = false) => { label: 'publishing', type: 'publisher', }); - jobLogger.info(ctx.job, `Publishing facets - ${ctx.tenant}`); + + jobLogger.info(ctx.job, `Publishing facets`); const names = fields.map(({ name }) => name); await ctx.publishedFacet.remove({ field: { $in: names } }); @@ -42,5 +43,6 @@ export default async (ctx, fields, withProgress = false) => { progress.throw(ctx.tenant, error); }); withProgress && progress.finish(ctx.tenant); - jobLogger.info(ctx.job, `Facets published - ${ctx.tenant}`); + + jobLogger.info(ctx.job, `Facets published`); }; diff --git a/src/api/controller/customPage.js b/src/api/controller/customPage.js index 536278f8ab..4c25ef9df2 100644 --- a/src/api/controller/customPage.js +++ b/src/api/controller/customPage.js @@ -1,7 +1,7 @@ import path from 'path'; import { getFileStatsIfExists, readFile } from '../services/fsHelpers'; -import logger from '../services/logger'; +import getLogger from '../services/logger'; const scriptRegEx = new RegExp('.*?', 'gm'); @@ -46,7 +46,8 @@ export default async ctx => { try { html = (await readFile(pathname)).toString(); } catch (error) { - logger.error('Unable to read custom page file', { + const logger = getLogger(ctx.tenant); + logger.error(`Unable to read custom page file`, { pathname, error, }); diff --git a/src/api/index.js b/src/api/index.js index 3a8df79abc..78db412284 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -9,7 +9,6 @@ import { KoaAdapter } from '@bull-board/koa'; import { createBullBoard } from '@bull-board/api'; import { BullAdapter } from '@bull-board/api/bullAdapter'; -import logger from './services/logger'; import controller from './controller'; import testController from './controller/testController'; import indexSearchableFields from './services/indexSearchableFields'; @@ -25,6 +24,7 @@ import Meter from '@uswitch/koa-prometheus'; import MeterConfig from '@uswitch/koa-prometheus/build/koa-prometheus.defaults.json'; import tracer, { eventTrace, eventError } from '@uswitch/koa-tracer'; import access, { eventAccess } from '@uswitch/koa-access'; +import getLogger from './services/logger'; // KoaQs use qs to parse query string. There is an default limit of 20 items in an array. Above this limit, qs will transform the array into an key/value object. // We need to increase this limit to 1000 to be able to handle the facets array in the query string. @@ -86,7 +86,8 @@ app.use(async (ctx, next) => { ); ctx.job = filteredActiveJobs[0]; } catch (e) { - logger.error('An error occured on loading running job', e); + const logger = getLogger(ctx.tenant); + logger.error(`An error occured on loading running job`, e); } await next(); }); @@ -104,6 +105,7 @@ app.use(async (ctx, next) => { } await next(); ctx.httpLog.status = ctx.status; + const logger = getLogger(ctx.tenant); logger.info(ctx.request.url, ctx.httpLog); }); diff --git a/src/api/services/enrichment/enrichment.js b/src/api/services/enrichment/enrichment.js index 76745de819..f1affffe8e 100644 --- a/src/api/services/enrichment/enrichment.js +++ b/src/api/services/enrichment/enrichment.js @@ -16,7 +16,7 @@ import { import { ENRICHING, PENDING } from '../../../common/progressStatus'; import { jobLogger } from '../../workers/tools'; import { CancelWorkerError } from '../../workers'; -import logger from '../logger'; +import getLogger from '../logger'; const { enrichmentBatchSize: BATCH_SIZE = 10 } = localConfig; @@ -96,7 +96,8 @@ export const getEnrichmentDataPreview = async ctx => { result.push(...values.map(v => v.value)); } } catch (error) { - logger.error('Error while processing enrichment preview', error); + const logger = getLogger(ctx.tenant); + logger.error(`Error while processing enrichment preview`, error); return []; } return result; @@ -193,7 +194,8 @@ const processEzsEnrichment = (entries, commands, ctx, preview = false) => { try { sourceChunk = JSON.parse(error.sourceChunk); } catch (e) { - logger.error('Error while parsing sourceChunk', e); + const logger = getLogger(ctx.tenant); + logger.error(`Error while parsing sourceChunk`, e); } } return values.push({ @@ -220,7 +222,7 @@ export const processEnrichment = async (enrichment, ctx) => { { $set: { ['status']: IN_PROGRESS } }, ); let errorCount = 0; - + const room = `${ctx.tenant}-enrichment-job-${ctx.job.id}`; const commands = createEzsRuleCommands(enrichment.rule); const dataSetSize = await ctx.dataset.count(); @@ -239,7 +241,7 @@ export const processEnrichment = async (enrichment, ctx) => { if (!entry.uri) { const logData = JSON.stringify({ level: 'error', - message: `Unable to enrich row with no URI, see object _id#${entry._id}`, + message: `[Instance: ${ctx.tenant}] Unable to enrich row with no URI, see object _id#${entry._id}`, timestamp: new Date(), status: IN_PROGRESS, }); @@ -247,7 +249,7 @@ export const processEnrichment = async (enrichment, ctx) => { } else { const logData = JSON.stringify({ level: 'info', - message: `Started enriching #${entry.uri}`, + message: `[Instance: ${ctx.tenant}] Started enriching #${entry.uri}`, timestamp: new Date(), status: IN_PROGRESS, }); @@ -281,8 +283,8 @@ export const processEnrichment = async (enrichment, ctx) => { const logData = JSON.stringify({ level: enrichedValue.error ? 'error' : 'info', message: enrichedValue.error - ? `Error enriching #${id}: ${value}` - : `Finished enriching #${id} (output: ${value})`, + ? `[Instance: ${ctx.tenant}] Error enriching #${id}: ${value}` + : `[Instance: ${ctx.tenant}] Finished enriching #${id} (output: ${value})`, timestamp: new Date(), status: IN_PROGRESS, }); @@ -312,7 +314,7 @@ export const processEnrichment = async (enrichment, ctx) => { const logData = JSON.stringify({ level: 'error', - message: e.message, + message: `[Instance: ${ctx.tenant}] ${e.message}`, timestamp: new Date(), status: IN_PROGRESS, }); @@ -335,7 +337,7 @@ export const processEnrichment = async (enrichment, ctx) => { progress.finish(ctx.tenant); const logData = JSON.stringify({ level: 'ok', - message: `Enrichement finished - ${ctx.tenant}`, + message: `[Instance: ${ctx.tenant}] Enrichement finished`, timestamp: new Date(), status: FINISHED, }); @@ -369,7 +371,7 @@ export const startEnrichment = async ctx => { const room = `enrichment-job-${ctx.job.id}`; const logData = JSON.stringify({ level: 'ok', - message: `Enrichement started`, + message: `[Instance: ${ctx.tenant}] Enrichement started`, timestamp: new Date(), status: IN_PROGRESS, }); @@ -397,8 +399,8 @@ export const setEnrichmentError = async (ctx, err) => { level: 'error', message: err instanceof CancelWorkerError - ? `${err?.message}` - : `Enrichement errored : ${err?.message}`, + ? `[Instance: ${ctx.tenant}] ${err?.message}` + : `[Instance: ${ctx.tenant}] Enrichement errored : ${err?.message}`, timestamp: new Date(), status: err instanceof CancelWorkerError ? CANCELED : ERROR, }); diff --git a/src/api/services/enrichment/enrichment.spec.js b/src/api/services/enrichment/enrichment.spec.js index a11baee9bb..9103e22534 100644 --- a/src/api/services/enrichment/enrichment.spec.js +++ b/src/api/services/enrichment/enrichment.spec.js @@ -372,6 +372,9 @@ describe('enrichment', () => { id: 1, log: jest.fn(), isActive: jest.fn().mockReturnValue(true), + data: { + tenant: 'lodex_test', + }, }, enrichment: { updateOne: jest.fn(), diff --git a/src/api/services/logger.js b/src/api/services/logger.js index 9a7096784e..7106a50fcc 100644 --- a/src/api/services/logger.js +++ b/src/api/services/logger.js @@ -2,14 +2,29 @@ import winston, { createLogger, format } from 'winston'; import config from 'config'; import { MESSAGE } from 'triple-beam'; -export default createLogger({ - level: process.env.NODE_ENV === 'development' ? 'debug' : 'info', - transports: [new winston.transports.Console()], - format: format.combine( - format.timestamp(), - format.colorize(), - format.simple(), - format.printf(info => `${info.timestamp} ${info[MESSAGE]}`), - ), - silent: config.logger.disabled, -}); +const loggers = new Map(); + +const getLogger = tenant => { + if (loggers.has(tenant)) { + return loggers.get(tenant); + } + + const logger = createLogger({ + level: process.env.NODE_ENV === 'development' ? 'debug' : 'info', + transports: [new winston.transports.Console()], + format: format.combine( + format.timestamp(), + format.colorize(), + format.simple(), + format.printf( + info => ` [${tenant}] ${info.timestamp} ${info[MESSAGE]}`, + ), + ), + silent: config.logger.disabled, + }); + + loggers.set(tenant, logger); + return logger; +}; + +export default getLogger; diff --git a/src/api/services/publishCharacteristics.js b/src/api/services/publishCharacteristics.js index 467eaf8937..b9f29d3bca 100644 --- a/src/api/services/publishCharacteristics.js +++ b/src/api/services/publishCharacteristics.js @@ -7,7 +7,7 @@ export const publishCharacteristicsFactory = ({ getDocumentTransformer }) => return; } - jobLogger.info(ctx.job, 'Publishing characteristics'); + jobLogger.info(ctx.job, `Publishing characteristics`); const getPublishedCharacteristics = getDocumentTransformer( ctx.dataset.findBy, datasetScopeFields, @@ -34,7 +34,7 @@ export const publishCharacteristicsFactory = ({ getDocumentTransformer }) => publishedCharacteristics, ); } - jobLogger.info(ctx.job, 'Characteristics published'); + jobLogger.info(ctx.job, `Characteristics published`); }; export default publishCharacteristicsFactory({ getDocumentTransformer }); diff --git a/src/api/services/publishDocuments.js b/src/api/services/publishDocuments.js index 9c222c684c..58be273a9d 100644 --- a/src/api/services/publishDocuments.js +++ b/src/api/services/publishDocuments.js @@ -8,8 +8,8 @@ import { PUBLISH_DOCUMENT } from '../../common/progressStatus'; import { URI_FIELD_NAME } from '../../common/uris'; import { SCOPE_COLLECTION, SCOPE_DOCUMENT } from '../../common/scope'; import parseValue from '../../common/tools/parseValue'; -import logger from './logger'; import { jobLogger } from '../workers/tools'; +import getLogger from './logger'; export const versionTransformerDecorator = ( transformDocument, @@ -88,10 +88,11 @@ export const publishDocumentsFactory = ({ transformAllDocuments, }) => async (ctx, count, fields) => { if (!ctx.job) { - logger.error('Job is not defined'); + const logger = getLogger(ctx.tenant); + logger.error(`Job is not defined`); return; } - jobLogger.info(ctx.job, 'Publishing documents'); + jobLogger.info(ctx.job, `Publishing documents`); const mainResourceFields = fields .filter( c => diff --git a/src/api/services/publishDocuments.spec.js b/src/api/services/publishDocuments.spec.js index 5fe39bbeef..0c084f138e 100644 --- a/src/api/services/publishDocuments.spec.js +++ b/src/api/services/publishDocuments.spec.js @@ -23,6 +23,9 @@ const fields = [ const job = { log: jest.fn(), isActive: () => true, + data: { + tenant: 'lodex_test', + }, }; const getCtx = ({ subresources } = {}) => ({ tenant: 'lodex_test', diff --git a/src/api/workers/tools.js b/src/api/workers/tools.js index b40f0ad2a1..9cc73cf513 100644 --- a/src/api/workers/tools.js +++ b/src/api/workers/tools.js @@ -1,5 +1,5 @@ import { cleanWaitingJobsOfType, workerQueues } from '.'; -import logger from '../services/logger'; +import getLogger from '../services/logger'; import progress from '../services/progress'; export const jobLogger = { @@ -7,6 +7,7 @@ export const jobLogger = { if (!job) { return; } + const logger = getLogger(job.data.tenant); logger.info(message); job.log(message); }, @@ -14,6 +15,7 @@ export const jobLogger = { if (!job) { return; } + const logger = getLogger(job.data.tenant); logger.error(message); job.log(message); },