From be1877d390000ac693836ffb98f864d03d797490 Mon Sep 17 00:00:00 2001 From: Henry8192 <50559854+Henry8192@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:35:07 -0400 Subject: [PATCH] clp-package: Add the log-viewer and serve it using the log-viewer-webui. (#490) --- .gitmodules | 3 + Taskfile.yml | 81 ++++++++++++++++--- .../clp_package_utils/scripts/start_clp.py | 1 + .../log-viewer-webui/server/settings.json | 3 +- components/log-viewer-webui/server/src/app.js | 32 +------- .../server/src/routes/static.js | 56 +++++++++++++ components/log-viewer-webui/yscope-log-viewer | 1 + 7 files changed, 134 insertions(+), 43 deletions(-) create mode 100644 components/log-viewer-webui/server/src/routes/static.js create mode 160000 components/log-viewer-webui/yscope-log-viewer diff --git a/.gitmodules b/.gitmodules index 5eb798197..3935b0101 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,3 +26,6 @@ [submodule "components/core/submodules/outcome"] path = components/core/submodules/outcome url = https://github.com/ned14/outcome.git +[submodule "components/log-viewer-webui/yscope-log-viewer"] + path = components/log-viewer-webui/yscope-log-viewer + url = https://github.com/y-scope/yscope-log-viewer.git diff --git a/Taskfile.yml b/Taskfile.yml index e547dd68b..f0c466e31 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -47,6 +47,7 @@ tasks: - "rm -rf 'components/log-viewer-webui/client/node_modules'" - "rm -rf 'components/log-viewer-webui/node_modules'" - "rm -rf 'components/log-viewer-webui/server/node_modules'" + - "rm -rf 'components/log-viewer-webui/yscope-log-viewer/node_modules'" clean-webui: cmds: @@ -74,7 +75,7 @@ tasks: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" sources: - - "{{.G_BUILD_DIR}}/log-viewer-webui-client.md5" + - "{{.G_BUILD_DIR}}/log-viewer-webui-clients.md5" - "{{.G_BUILD_DIR}}/package-venv.md5" - "{{.G_BUILD_DIR}}/webui.md5" - "{{.G_BUILD_DIR}}/webui-nodejs.md5" @@ -100,7 +101,7 @@ tasks: - "clp-py-utils" - "init" - "job-orchestration" - - "log-viewer-webui-client" + - "log-viewer-webui-clients" - "nodejs-14" - "package-venv" - task: "utils:validate-checksum" @@ -150,6 +151,7 @@ tasks: - >- rsync -a "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/client" + "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/yscope-log-viewer" "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/" - |- cd components/log-viewer-webui/server/ @@ -206,17 +208,23 @@ tasks: vars: COMPONENT: "{{.TASK}}" - log-viewer-webui-client: + log-viewer-webui-clients: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}" sources: - - "{{.G_BUILD_DIR}}/log-viewer-modules.md5" + - "{{.G_BUILD_DIR}}/log-viewer-webui-node-modules.md5" - "{{.TASKFILE}}" - "client/package.json" - "client/src/**/*.css" - "client/src/**/*.jsx" - "client/src/webpack.config.js" + - "yscope-log-viewer/.babelrc" + - "yscope-log-viewer/customized-packages/**/*" + - "yscope-log-viewer/package.json" + - "yscope-log-viewer/src/**/*" + - "yscope-log-viewer/webpack.common.js" + - "yscope-log-viewer/webpack.prod.js" dir: "components/log-viewer-webui" generates: ["{{.CHECKSUM_FILE}}"] deps: @@ -228,10 +236,13 @@ tasks: DATA_DIR: "{{.OUTPUT_DIR}}" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - - |- - cd client - PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build -- \ - --output-path "{{.OUTPUT_DIR}}/client" + - for: + - "client" + - "yscope-log-viewer" + cmd: |- + cd "{{.ITEM}}" + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build -- \ + --output-path "{{.OUTPUT_DIR}}/{{.ITEM}}" - task: "utils:compute-checksum" vars: DATA_DIR: "{{.OUTPUT_DIR}}" @@ -364,25 +375,31 @@ tasks: DATA_DIR: "{{.OUTPUT_DIR}}" OUTPUT_FILE: "{{.CHECKSUM_FILE}}" - # NOTE: The log-viewer-webui has three different node_modules directories (client, server, and the - # top-level one we call "package"), meaning we have to create three different checksums. To allow - # tasks which depend on this task to only have to check one checksum file, we concatenate the - # three checksum files into one. + # NOTE: The log-viewer-webui has four different node_modules directories: + # * client + # * server + # * log-viewer submodule + # * the top-level one we call "package" + # This means we have to create four different checksums. To allow tasks which depend on this task + # to only have to check one checksum file, we concatenate the four checksum files into one. log-viewer-webui-node-modules: internal: true vars: # Checksum files CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" CLIENT_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-client-node-modules.md5" + LOG_VIEWER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-log-viewer-node-modules.md5" PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-package-node-modules.md5" SERVER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-server-node-modules.md5" # Directories SRC_DIR: "{{.TASKFILE_DIR}}/components/log-viewer-webui" CLIENT_OUTPUT_DIR: "{{.SRC_DIR}}/client/node_modules" + LOG_VIEWER_OUTPUT_DIR: "{{.SRC_DIR}}/yscope-log-viewer/node_modules" PACKAGE_OUTPUT_DIR: "{{.SRC_DIR}}/node_modules" SERVER_OUTPUT_DIR: "{{.SRC_DIR}}/server/node_modules" sources: + - "{{.G_BUILD_DIR}}/log-viewer-webui-submodules.md5" - "{{.G_BUILD_DIR}}/nodejs-22.md5" - "{{.TASKFILE}}" - "client/package.json" @@ -391,13 +408,17 @@ tasks: - "package-lock.json" - "server/package.json" - "server/package-lock.json" + - "yscope-log-viewer/package.json" + - "yscope-log-viewer/package-lock.json" dir: "{{.SRC_DIR}}" generates: - "{{.CHECKSUM_FILE}}" - "{{.CLIENT_CHECKSUM_FILE}}" + - "{{.LOG_VIEWER_CHECKSUM_FILE}}" - "{{.PACKAGE_CHECKSUM_FILE}}" - "{{.SERVER_CHECKSUM_FILE}}" deps: + - "log-viewer-webui-submodules" - "nodejs-22" - task: "utils:validate-checksum" vars: @@ -411,15 +432,26 @@ tasks: vars: CHECKSUM_FILE: "{{.PACKAGE_CHECKSUM_FILE}}" DATA_DIR: "{{.PACKAGE_OUTPUT_DIR}}" + - task: "utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}" + DATA_DIR: "{{.LOG_VIEWER_OUTPUT_DIR}}" cmds: - "rm -f {{.CHECKSUM_FILE}}" - task: "clean-log-viewer-webui" - "PATH='{{.G_NODEJS_22_BIN_DIR}}':$PATH npm run init" + - |- + cd yscope-log-viewer + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm install # These commands must be last - task: "utils:compute-checksum" vars: DATA_DIR: "{{.CLIENT_OUTPUT_DIR}}" OUTPUT_FILE: "{{.CLIENT_CHECKSUM_FILE}}" + - task: "utils:compute-checksum" + vars: + DATA_DIR: "{{.LOG_VIEWER_OUTPUT_DIR}}" + OUTPUT_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}" - task: "utils:compute-checksum" vars: DATA_DIR: "{{.PACKAGE_OUTPUT_DIR}}" @@ -432,10 +464,35 @@ tasks: - >- cat "{{.CLIENT_CHECKSUM_FILE}}" + "{{.LOG_VIEWER_CHECKSUM_FILE}}" "{{.PACKAGE_CHECKSUM_FILE}}" "{{.SERVER_CHECKSUM_FILE}}" > "{{.CHECKSUM_FILE}}" + log-viewer-webui-submodules: + internal: true + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "yscope-log-viewer" + sources: + - "{{.TASKFILE}}" + - ".gitmodules" + dir: "components/log-viewer-webui" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - "init" + - task: "utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.OUTPUT_DIR}}" + cmds: + - "git submodule update --init --recursive yscope-log-viewer" + # This command must be last + - task: "utils:compute-checksum" + vars: + DATA_DIR: "{{.OUTPUT_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + meteor: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 74bb5f018..5e5965bf5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -864,6 +864,7 @@ def start_log_viewer_webui( "MongoDbIrFilesCollectionName": clp_config.results_cache.ir_collection_name, "ClientDir": str(container_log_viewer_webui_dir / "client"), "IrFilesDir": str(container_clp_config.ir_output.directory), + "LogViewerDir": str(container_log_viewer_webui_dir / "yscope-log-viewer"), } settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates) with open(settings_json_path, "w") as settings_json_file: diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index 76183a879..bb1aac48a 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -9,5 +9,6 @@ "MongoDbIrFilesCollectionName": "ir-files", "ClientDir": "../client/dist", - "IrFilesDir": "../../../build/clp-package/var/data/ir" + "IrFilesDir": "../../../build/clp-package/var/data/ir", + "LogViewerDir": "../yscope-log-viewer/dist" } diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 28c7fdfe5..69918ea58 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -1,14 +1,11 @@ import fastify from "fastify"; -import * as path from "node:path"; import process from "node:process"; -import {fileURLToPath} from "node:url"; - -import {fastifyStatic} from "@fastify/static"; import settings from "../settings.json" with {type: "json"}; import DbManager from "./DbManager.js"; import exampleRoutes from "./routes/example.js"; import queryRoutes from "./routes/query.js"; +import staticRoutes from "./routes/static.js"; /** @@ -26,20 +23,8 @@ const app = async ({ sqlDbPass, }) => { const server = fastify(fastifyOptions); - const filename = fileURLToPath(import.meta.url); - const dirname = path.dirname(filename); - const parentDirname = path.resolve(dirname, ".."); if ("test" !== process.env.NODE_ENV) { - let irFilesDir = settings.IrFilesDir; - if (false === path.isAbsolute(irFilesDir)) { - irFilesDir = path.resolve(parentDirname, irFilesDir); - } - await server.register(fastifyStatic, { - prefix: "/ir", - root: irFilesDir, - }); - await server.register(DbManager, { mysqlConfig: { database: settings.SqlDbName, @@ -58,20 +43,7 @@ const app = async ({ }); } - if ("production" === process.env.NODE_ENV) { - // In the development environment, we expect the client to use a separate webserver that - // supports live reloading. - if (false === path.isAbsolute(settings.ClientDir)) { - throw new Error("`clientDir` must be an absolute path."); - } - - await server.register(fastifyStatic, { - prefix: "/", - root: settings.ClientDir, - decorateReply: false, - }); - } - + await server.register(staticRoutes); await server.register(exampleRoutes); await server.register(queryRoutes); diff --git a/components/log-viewer-webui/server/src/routes/static.js b/components/log-viewer-webui/server/src/routes/static.js new file mode 100644 index 000000000..42d9048f0 --- /dev/null +++ b/components/log-viewer-webui/server/src/routes/static.js @@ -0,0 +1,56 @@ +import path from "node:path"; +import process from "node:process"; +import {fileURLToPath} from "node:url"; + +import {fastifyStatic} from "@fastify/static"; + +import settings from "../../settings.json" with {type: "json"}; + + +/** + * Creates static files serving routes. + * + * @param {import("fastify").FastifyInstance} fastify + * @param {import("fastify").FastifyPluginOptions} options + */ +const routes = async (fastify, options) => { + const filename = fileURLToPath(import.meta.url); + const dirname = path.dirname(filename); + const rootDirname = path.resolve(dirname, "../.."); + + let irFilesDir = settings.IrFilesDir; + if (false === path.isAbsolute(irFilesDir)) { + irFilesDir = path.resolve(rootDirname, irFilesDir); + } + await fastify.register(fastifyStatic, { + prefix: "/ir", + root: irFilesDir, + }); + + let logViewerDir = settings.LogViewerDir; + if (false === path.isAbsolute(logViewerDir)) { + logViewerDir = path.resolve(rootDirname, logViewerDir); + } + await fastify.register(fastifyStatic, { + prefix: "/log-viewer", + root: logViewerDir, + decorateReply: false, + }); + + if ("production" === process.env.NODE_ENV) { + // In the development environment, we expect the client to use a separate webserver that + // supports live reloading. + let clientDir = settings.ClientDir; + if (false === path.isAbsolute(clientDir)) { + clientDir = path.resolve(rootDirname, settings.ClientDir); + } + + await fastify.register(fastifyStatic, { + prefix: "/", + root: clientDir, + decorateReply: false, + }); + } +}; + +export default routes; diff --git a/components/log-viewer-webui/yscope-log-viewer b/components/log-viewer-webui/yscope-log-viewer new file mode 160000 index 000000000..df996eac0 --- /dev/null +++ b/components/log-viewer-webui/yscope-log-viewer @@ -0,0 +1 @@ +Subproject commit df996eac000823d02f9cd0b9eb4bb732dd634ae5