Skip to content

Commit

Permalink
clp-package: Add the log-viewer and serve it using the log-viewer-web…
Browse files Browse the repository at this point in the history
…ui. (y-scope#490)
  • Loading branch information
Henry8192 authored Jul 24, 2024
1 parent 9f85a88 commit be1877d
Show file tree
Hide file tree
Showing 7 changed files with 134 additions and 43 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,6 @@
[submodule "components/core/submodules/outcome"]
path = components/core/submodules/outcome
url = https://github.com/ned14/outcome.git
[submodule "components/log-viewer-webui/yscope-log-viewer"]
path = components/log-viewer-webui/yscope-log-viewer
url = https://github.com/y-scope/yscope-log-viewer.git
81 changes: 69 additions & 12 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ tasks:
- "rm -rf 'components/log-viewer-webui/client/node_modules'"
- "rm -rf 'components/log-viewer-webui/node_modules'"
- "rm -rf 'components/log-viewer-webui/server/node_modules'"
- "rm -rf 'components/log-viewer-webui/yscope-log-viewer/node_modules'"

clean-webui:
cmds:
Expand Down Expand Up @@ -74,7 +75,7 @@ tasks:
CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}"
sources:
- "{{.G_BUILD_DIR}}/log-viewer-webui-client.md5"
- "{{.G_BUILD_DIR}}/log-viewer-webui-clients.md5"
- "{{.G_BUILD_DIR}}/package-venv.md5"
- "{{.G_BUILD_DIR}}/webui.md5"
- "{{.G_BUILD_DIR}}/webui-nodejs.md5"
Expand All @@ -100,7 +101,7 @@ tasks:
- "clp-py-utils"
- "init"
- "job-orchestration"
- "log-viewer-webui-client"
- "log-viewer-webui-clients"
- "nodejs-14"
- "package-venv"
- task: "utils:validate-checksum"
Expand Down Expand Up @@ -150,6 +151,7 @@ tasks:
- >-
rsync -a
"{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/client"
"{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/yscope-log-viewer"
"{{.OUTPUT_DIR}}/var/www/log_viewer_webui/"
- |-
cd components/log-viewer-webui/server/
Expand Down Expand Up @@ -206,17 +208,23 @@ tasks:
vars:
COMPONENT: "{{.TASK}}"

log-viewer-webui-client:
log-viewer-webui-clients:
vars:
CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}"
sources:
- "{{.G_BUILD_DIR}}/log-viewer-modules.md5"
- "{{.G_BUILD_DIR}}/log-viewer-webui-node-modules.md5"
- "{{.TASKFILE}}"
- "client/package.json"
- "client/src/**/*.css"
- "client/src/**/*.jsx"
- "client/src/webpack.config.js"
- "yscope-log-viewer/.babelrc"
- "yscope-log-viewer/customized-packages/**/*"
- "yscope-log-viewer/package.json"
- "yscope-log-viewer/src/**/*"
- "yscope-log-viewer/webpack.common.js"
- "yscope-log-viewer/webpack.prod.js"
dir: "components/log-viewer-webui"
generates: ["{{.CHECKSUM_FILE}}"]
deps:
Expand All @@ -228,10 +236,13 @@ tasks:
DATA_DIR: "{{.OUTPUT_DIR}}"
cmds:
- "rm -rf '{{.OUTPUT_DIR}}'"
- |-
cd client
PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build -- \
--output-path "{{.OUTPUT_DIR}}/client"
- for:
- "client"
- "yscope-log-viewer"
cmd: |-
cd "{{.ITEM}}"
PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build -- \
--output-path "{{.OUTPUT_DIR}}/{{.ITEM}}"
- task: "utils:compute-checksum"
vars:
DATA_DIR: "{{.OUTPUT_DIR}}"
Expand Down Expand Up @@ -364,25 +375,31 @@ tasks:
DATA_DIR: "{{.OUTPUT_DIR}}"
OUTPUT_FILE: "{{.CHECKSUM_FILE}}"

# NOTE: The log-viewer-webui has three different node_modules directories (client, server, and the
# top-level one we call "package"), meaning we have to create three different checksums. To allow
# tasks which depend on this task to only have to check one checksum file, we concatenate the
# three checksum files into one.
# NOTE: The log-viewer-webui has four different node_modules directories:
# * client
# * server
# * log-viewer submodule
# * the top-level one we call "package"
# This means we have to create four different checksums. To allow tasks which depend on this task
# to only have to check one checksum file, we concatenate the four checksum files into one.
log-viewer-webui-node-modules:
internal: true
vars:
# Checksum files
CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
CLIENT_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-client-node-modules.md5"
LOG_VIEWER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-log-viewer-node-modules.md5"
PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-package-node-modules.md5"
SERVER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/log-viewer-webui-server-node-modules.md5"

# Directories
SRC_DIR: "{{.TASKFILE_DIR}}/components/log-viewer-webui"
CLIENT_OUTPUT_DIR: "{{.SRC_DIR}}/client/node_modules"
LOG_VIEWER_OUTPUT_DIR: "{{.SRC_DIR}}/yscope-log-viewer/node_modules"
PACKAGE_OUTPUT_DIR: "{{.SRC_DIR}}/node_modules"
SERVER_OUTPUT_DIR: "{{.SRC_DIR}}/server/node_modules"
sources:
- "{{.G_BUILD_DIR}}/log-viewer-webui-submodules.md5"
- "{{.G_BUILD_DIR}}/nodejs-22.md5"
- "{{.TASKFILE}}"
- "client/package.json"
Expand All @@ -391,13 +408,17 @@ tasks:
- "package-lock.json"
- "server/package.json"
- "server/package-lock.json"
- "yscope-log-viewer/package.json"
- "yscope-log-viewer/package-lock.json"
dir: "{{.SRC_DIR}}"
generates:
- "{{.CHECKSUM_FILE}}"
- "{{.CLIENT_CHECKSUM_FILE}}"
- "{{.LOG_VIEWER_CHECKSUM_FILE}}"
- "{{.PACKAGE_CHECKSUM_FILE}}"
- "{{.SERVER_CHECKSUM_FILE}}"
deps:
- "log-viewer-webui-submodules"
- "nodejs-22"
- task: "utils:validate-checksum"
vars:
Expand All @@ -411,15 +432,26 @@ tasks:
vars:
CHECKSUM_FILE: "{{.PACKAGE_CHECKSUM_FILE}}"
DATA_DIR: "{{.PACKAGE_OUTPUT_DIR}}"
- task: "utils:validate-checksum"
vars:
CHECKSUM_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}"
DATA_DIR: "{{.LOG_VIEWER_OUTPUT_DIR}}"
cmds:
- "rm -f {{.CHECKSUM_FILE}}"
- task: "clean-log-viewer-webui"
- "PATH='{{.G_NODEJS_22_BIN_DIR}}':$PATH npm run init"
- |-
cd yscope-log-viewer
PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm install
# These commands must be last
- task: "utils:compute-checksum"
vars:
DATA_DIR: "{{.CLIENT_OUTPUT_DIR}}"
OUTPUT_FILE: "{{.CLIENT_CHECKSUM_FILE}}"
- task: "utils:compute-checksum"
vars:
DATA_DIR: "{{.LOG_VIEWER_OUTPUT_DIR}}"
OUTPUT_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}"
- task: "utils:compute-checksum"
vars:
DATA_DIR: "{{.PACKAGE_OUTPUT_DIR}}"
Expand All @@ -432,10 +464,35 @@ tasks:
- >-
cat
"{{.CLIENT_CHECKSUM_FILE}}"
"{{.LOG_VIEWER_CHECKSUM_FILE}}"
"{{.PACKAGE_CHECKSUM_FILE}}"
"{{.SERVER_CHECKSUM_FILE}}"
> "{{.CHECKSUM_FILE}}"
log-viewer-webui-submodules:
internal: true
vars:
CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
OUTPUT_DIR: "yscope-log-viewer"
sources:
- "{{.TASKFILE}}"
- ".gitmodules"
dir: "components/log-viewer-webui"
generates: ["{{.CHECKSUM_FILE}}"]
deps:
- "init"
- task: "utils:validate-checksum"
vars:
CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
DATA_DIR: "{{.OUTPUT_DIR}}"
cmds:
- "git submodule update --init --recursive yscope-log-viewer"
# This command must be last
- task: "utils:compute-checksum"
vars:
DATA_DIR: "{{.OUTPUT_DIR}}"
OUTPUT_FILE: "{{.CHECKSUM_FILE}}"

meteor:
vars:
CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,7 @@ def start_log_viewer_webui(
"MongoDbIrFilesCollectionName": clp_config.results_cache.ir_collection_name,
"ClientDir": str(container_log_viewer_webui_dir / "client"),
"IrFilesDir": str(container_clp_config.ir_output.directory),
"LogViewerDir": str(container_log_viewer_webui_dir / "yscope-log-viewer"),
}
settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates)
with open(settings_json_path, "w") as settings_json_file:
Expand Down
3 changes: 2 additions & 1 deletion components/log-viewer-webui/server/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@
"MongoDbIrFilesCollectionName": "ir-files",

"ClientDir": "../client/dist",
"IrFilesDir": "../../../build/clp-package/var/data/ir"
"IrFilesDir": "../../../build/clp-package/var/data/ir",
"LogViewerDir": "../yscope-log-viewer/dist"
}
32 changes: 2 additions & 30 deletions components/log-viewer-webui/server/src/app.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
import fastify from "fastify";
import * as path from "node:path";
import process from "node:process";
import {fileURLToPath} from "node:url";

import {fastifyStatic} from "@fastify/static";

import settings from "../settings.json" with {type: "json"};
import DbManager from "./DbManager.js";
import exampleRoutes from "./routes/example.js";
import queryRoutes from "./routes/query.js";
import staticRoutes from "./routes/static.js";


/**
Expand All @@ -26,20 +23,8 @@ const app = async ({
sqlDbPass,
}) => {
const server = fastify(fastifyOptions);
const filename = fileURLToPath(import.meta.url);
const dirname = path.dirname(filename);
const parentDirname = path.resolve(dirname, "..");

if ("test" !== process.env.NODE_ENV) {
let irFilesDir = settings.IrFilesDir;
if (false === path.isAbsolute(irFilesDir)) {
irFilesDir = path.resolve(parentDirname, irFilesDir);
}
await server.register(fastifyStatic, {
prefix: "/ir",
root: irFilesDir,
});

await server.register(DbManager, {
mysqlConfig: {
database: settings.SqlDbName,
Expand All @@ -58,20 +43,7 @@ const app = async ({
});
}

if ("production" === process.env.NODE_ENV) {
// In the development environment, we expect the client to use a separate webserver that
// supports live reloading.
if (false === path.isAbsolute(settings.ClientDir)) {
throw new Error("`clientDir` must be an absolute path.");
}

await server.register(fastifyStatic, {
prefix: "/",
root: settings.ClientDir,
decorateReply: false,
});
}

await server.register(staticRoutes);
await server.register(exampleRoutes);
await server.register(queryRoutes);

Expand Down
56 changes: 56 additions & 0 deletions components/log-viewer-webui/server/src/routes/static.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import path from "node:path";
import process from "node:process";
import {fileURLToPath} from "node:url";

import {fastifyStatic} from "@fastify/static";

import settings from "../../settings.json" with {type: "json"};


/**
* Creates static files serving routes.
*
* @param {import("fastify").FastifyInstance} fastify
* @param {import("fastify").FastifyPluginOptions} options
*/
const routes = async (fastify, options) => {
const filename = fileURLToPath(import.meta.url);
const dirname = path.dirname(filename);
const rootDirname = path.resolve(dirname, "../..");

let irFilesDir = settings.IrFilesDir;
if (false === path.isAbsolute(irFilesDir)) {
irFilesDir = path.resolve(rootDirname, irFilesDir);
}
await fastify.register(fastifyStatic, {
prefix: "/ir",
root: irFilesDir,
});

let logViewerDir = settings.LogViewerDir;
if (false === path.isAbsolute(logViewerDir)) {
logViewerDir = path.resolve(rootDirname, logViewerDir);
}
await fastify.register(fastifyStatic, {
prefix: "/log-viewer",
root: logViewerDir,
decorateReply: false,
});

if ("production" === process.env.NODE_ENV) {
// In the development environment, we expect the client to use a separate webserver that
// supports live reloading.
let clientDir = settings.ClientDir;
if (false === path.isAbsolute(clientDir)) {
clientDir = path.resolve(rootDirname, settings.ClientDir);
}

await fastify.register(fastifyStatic, {
prefix: "/",
root: clientDir,
decorateReply: false,
});
}
};

export default routes;
1 change: 1 addition & 0 deletions components/log-viewer-webui/yscope-log-viewer
Submodule yscope-log-viewer added at df996e

0 comments on commit be1877d

Please sign in to comment.