From edc6a16c51720d570a05d1f91324b96a81d8b8ac Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 15:28:59 +0100 Subject: [PATCH 1/5] Allow JSON HTTP Request logging from the FileServer Matching the behaviour of the forge app --- forge/config.js | 89 +++++++++++++++++++++++++++++++++++++++++++++ forge/fileServer.js | 55 +++++++++++++++++++--------- 2 files changed, 126 insertions(+), 18 deletions(-) diff --git a/forge/config.js b/forge/config.js index d2ff7e1..cc8807b 100644 --- a/forge/config.js +++ b/forge/config.js @@ -3,6 +3,95 @@ const fp = require('fastify-plugin') const path = require('path') const YAML = require('yaml') +let config = {} + +module.exports = { + init: (opts) => { + if (opts.config) { + // A custom config has been passed in. This means we're running + // programmatically rather than manually. At this stage, that + // means its our test framework. + process.env.NODE_ENV = 'development' + process.env.FLOWFORGE_HOME = process.cwd() + } else if (!process.env.FLOWFORGE_HOME) { + if (process.env.NODE_ENV === 'development') { + process.env.FLOWFORGE_HOME = path.resolve(__dirname, '../..') + } else { + process.env.FLOWFORGE_HOME = process.cwd() + if (fs.existsSync('/opt/flowforge-file-storage')) { + process.env.FLOWFORGE_HOME = '/opt/flowforge-file-storage' + } else { + process.env.FLOWFORGE_HOME = process.cwd() + } + } + } + + let ffVersion + if (process.env.npm_package_version) { + ffVersion = process.env.npm_package_version + // npm start + } else { + // everything else + const { version } = require(path.join(module.parent.path, '..', 'package.json')) + ffVersion = version + } + try { + fs.statSync(path.join(__dirname, '..', '..', '.git')) + ffVersion += '-git' + } catch (err) { + // No git directory + } + + if (opts.config !== undefined) { + // Programmatically provided config - eg tests + config = { ...opts.config } + } else { + let configFile = path.join(process.env.FLOWFORGE_HOME, '/etc/flowforge-storage.yml') + if (fs.existsSync(path.join(process.env.FLOWFORGE_HOME, '/etc/flowforge-storage.local.yml'))) { + configFile = path.join(process.env.FLOWFORGE_HOME, '/etc/flowforge-storage.local.yml') + } + try { + const configFileContent = fs.readFileSync(configFile, 'utf-8') + config = YAML.parse(configFileContent) + config.configFile = configFile + } catch (err) { + app.log.error(`Failed to read config file ${configFile}: ${err.message}`) + } + } + + config.version = ffVersion + config.home = process.env.FLOWFORGE_HOME + config.port = process.env.PORT || config.port || 3001 + config.host = config.host || 'localhost' + + config.version = ffVersion + const defaultLogging = { + level: 'info', + http: 'warn', + pretty: process.env.NODE_ENV === 'development' + } + config.logging = { ...defaultLogging, ...config.logging } + + return config + }, + attach: fp(async function (app, opts, next){ + Object.freeze(config) + app.decorate('config', config) + + if (process.env.NODE_ENV === 'development') { + app.log.info('Development mode') + } + app.log.info(`FlowForge File Storage v${config.ffVersion}`) + app.log.info(`FlowForge File Storage running with NodeJS ${process.version}`) + app.log.info(`FlowForge File Storage HOME Directory: ${process.env.FLOWFORGE_HOME}`) + if (!opts.config) { + app.log.info(`Config File: ${config.configFile}`) + } + + next() + }) +} + module.exports = fp(async function (app, opts, next) { if (!opts.config && process.env.FF_FS_TEST_CONFIG) { opts.config = YAML.parse(process.env.FF_FS_TEST_CONFIG) diff --git a/forge/fileServer.js b/forge/fileServer.js index c0e08d0..12fb3d9 100644 --- a/forge/fileServer.js +++ b/forge/fileServer.js @@ -6,26 +6,48 @@ const routes = require('./routes') const helmet = require('@fastify/helmet') module.exports = async (options = {}) => { - let loggerLevel = 'info' - if (options.config?.logging?.level) { - loggerLevel = options.config.logging.level + const runtimeConfig = config.init(options) + const loggerConfig = { + formatters: { + level: (label) => { + return { level: label.toUpperCase() } + }, + bindings: (bindings) => { + return { } + } + }, + timestamp: require('pino').stdTimeFunctions.isoTime, + level: runtimeConfig.logging.level, + serializers: { + res (reply) { + return { + statusCode: reply.statusCode, + request: { + url: reply.request?.raw?.url, + method: reply.request?.method, + remoteAddress: reply.request?.ip, + remotePort: reply.request?.socket.remotePort + } + } + } + } + } + if (runtimeConfig.logging.pretty !== false) { + loggerConfig.transport = { + target: 'pino-pretty', + options: { + translateTime: "UTC:yyyy-mm-dd'T'HH:MM:ss.l'Z'", + ignore: 'pid,hostname', + singleLine: true + } + } } const server = fastify({ bodyLimit: 10 * 1024 * 1024, // Limit set to 10MB, maxParamLength: 500, trustProxy: true, - logger: { - transport: { - target: 'pino-pretty', - options: { - translateTime: "UTC:yyyy-mm-dd'T'HH:MM:ss.l'Z'", - ignore: 'pid,hostname', - singleLine: true - } - }, - level: loggerLevel - } + logger: loggerConfig }) server.addHook('onError', async (request, reply, error) => { @@ -35,10 +57,7 @@ module.exports = async (options = {}) => { try { // Config - await server.register(config, options) - if (server.config.logging?.level) { - server.log.level = server.config.logging.level - } + await server.register(config.attach, options) // // Setup DB // await server.register(db, {}) From 44b0815d4cf29ed1af75a3f8d5953585556db443 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 15:45:42 +0100 Subject: [PATCH 2/5] Add pino package --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 91e4272..275158f 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "fastify-plugin": "^4.5.1", "got": "^11.8.6", "pg": "^8.11.2", + "pino": "^8.15.1", "pino-pretty": "^10.2.0", "semver": "^7.5.4", "sequelize": "^6.33.0", From b156f94b79319670e208e3193233b09d8930e2fe Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 15:47:14 +0100 Subject: [PATCH 3/5] Add package-lock.json --- package-lock.json | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5036d52..6f053df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "fastify-plugin": "^4.5.1", "got": "^11.8.6", "pg": "^8.11.2", + "pino": "^8.15.1", "pino-pretty": "^10.2.0", "semver": "^7.5.4", "sequelize": "^6.33.0", @@ -5397,14 +5398,14 @@ } }, "node_modules/pino": { - "version": "8.14.1", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.14.1.tgz", - "integrity": "sha512-8LYNv7BKWXSfS+k6oEc6occy5La+q2sPwU3q2ljTX5AZk7v+5kND2o5W794FyRaqha6DJajmkNRsWtPpFyMUdw==", + "version": "8.15.4", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.15.4.tgz", + "integrity": "sha512-3s+SfSxeugMt8QeBVXprIJAgXuGDeGuHBfquXKEXKnpghlXzMGMjoa8tOSyzz00iBfQX3xlZvm2yJQ+d6SrVsg==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", - "pino-abstract-transport": "v1.0.0", + "pino-abstract-transport": "v1.1.0", "pino-std-serializers": "^6.0.0", "process-warning": "^2.0.0", "quick-format-unescaped": "^4.0.3", @@ -5418,9 +5419,9 @@ } }, "node_modules/pino-abstract-transport": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz", - "integrity": "sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", + "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", "dependencies": { "readable-stream": "^4.0.0", "split2": "^4.0.0" @@ -10841,14 +10842,14 @@ "dev": true }, "pino": { - "version": "8.14.1", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.14.1.tgz", - "integrity": "sha512-8LYNv7BKWXSfS+k6oEc6occy5La+q2sPwU3q2ljTX5AZk7v+5kND2o5W794FyRaqha6DJajmkNRsWtPpFyMUdw==", + "version": "8.15.4", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.15.4.tgz", + "integrity": "sha512-3s+SfSxeugMt8QeBVXprIJAgXuGDeGuHBfquXKEXKnpghlXzMGMjoa8tOSyzz00iBfQX3xlZvm2yJQ+d6SrVsg==", "requires": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", - "pino-abstract-transport": "v1.0.0", + "pino-abstract-transport": "v1.1.0", "pino-std-serializers": "^6.0.0", "process-warning": "^2.0.0", "quick-format-unescaped": "^4.0.3", @@ -10859,9 +10860,9 @@ } }, "pino-abstract-transport": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz", - "integrity": "sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", + "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", "requires": { "readable-stream": "^4.0.0", "split2": "^4.0.0" From 4c65321efed819f41f9219a759640a9f2c43095c Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 15:52:12 +0100 Subject: [PATCH 4/5] Fix lint --- forge/config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/forge/config.js b/forge/config.js index cc8807b..96fcef0 100644 --- a/forge/config.js +++ b/forge/config.js @@ -55,7 +55,7 @@ module.exports = { config = YAML.parse(configFileContent) config.configFile = configFile } catch (err) { - app.log.error(`Failed to read config file ${configFile}: ${err.message}`) + throw new Error(`Failed to read config file ${configFile}: ${err}`) } } @@ -74,7 +74,7 @@ module.exports = { return config }, - attach: fp(async function (app, opts, next){ + attach: fp(async function (app, opts, next) { Object.freeze(config) app.decorate('config', config) From d3453da711362ba36d8beddc75953105ea373642 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 3 Oct 2023 16:08:11 +0100 Subject: [PATCH 5/5] Update forge/config.js Co-authored-by: Nick O'Leary --- forge/config.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/forge/config.js b/forge/config.js index 96fcef0..030a940 100644 --- a/forge/config.js +++ b/forge/config.js @@ -81,9 +81,9 @@ module.exports = { if (process.env.NODE_ENV === 'development') { app.log.info('Development mode') } - app.log.info(`FlowForge File Storage v${config.ffVersion}`) - app.log.info(`FlowForge File Storage running with NodeJS ${process.version}`) - app.log.info(`FlowForge File Storage HOME Directory: ${process.env.FLOWFORGE_HOME}`) + app.log.info(`FlowFuse File Storage v${config.ffVersion}`) + app.log.info(`FlowFuse File Storage running with NodeJS ${process.version}`) + app.log.info(`FlowFuse File Storage HOME Directory: ${process.env.FLOWFORGE_HOME}`) if (!opts.config) { app.log.info(`Config File: ${config.configFile}`) }