From 5a751bd3af4119941d2d0d0d4e3eb3b7ee5d9b4c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 03:25:44 -0500 Subject: [PATCH 1/9] update prototype. --- src/services/LogFileManager/index.ts | 10 +-- src/services/MainWorker.ts | 2 + src/services/decoders/ClpIrDecoder.ts | 76 ++++++++++++++++++--- src/services/decoders/JsonlDecoder/index.ts | 8 +-- src/services/decoders/JsonlDecoder/utils.ts | 5 +- src/typings/config.ts | 4 +- src/typings/decoders.ts | 23 +++++-- src/typings/worker.ts | 4 +- src/utils/config.ts | 4 +- 9 files changed, 105 insertions(+), 31 deletions(-) diff --git a/src/services/LogFileManager/index.ts b/src/services/LogFileManager/index.ts index 67b1c0e45..fc1faabac 100644 --- a/src/services/LogFileManager/index.ts +++ b/src/services/LogFileManager/index.ts @@ -1,7 +1,7 @@ /* eslint max-lines: ["error", 400] */ import { Decoder, - DecoderOptionsType, + DecoderOptions, } from "../../typings/decoders"; import {MAX_V8_STRING_LENGTH} from "../../typings/js"; import {LogLevelFilter} from "../../typings/logs"; @@ -110,7 +110,7 @@ class LogFileManager { static async create ( fileSrc: FileSrcType, pageSize: number, - decoderOptions: DecoderOptionsType, + decoderOptions: DecoderOptions, onQueryResults: (queryResults: QueryResults) => void, ): Promise { const {fileName, fileData} = await loadFile(fileSrc); @@ -138,13 +138,13 @@ class LogFileManager { static async #initDecoder ( fileName: string, fileData: Uint8Array, - decoderOptions: DecoderOptionsType + decoderOptions: DecoderOptions ): Promise { let decoder: Decoder; if (fileName.endsWith(".jsonl")) { decoder = new JsonlDecoder(fileData, decoderOptions); } else if (fileName.endsWith(".clp.zst")) { - decoder = await ClpIrDecoder.create(fileData); + decoder = await ClpIrDecoder.create(fileData, decoderOptions); } else { throw new Error(`No decoder supports ${fileName}`); } @@ -161,7 +161,7 @@ class LogFileManager { /* Sets any formatter options that exist in the decoder's options. * @param options */ - setFormatterOptions (options: DecoderOptionsType) { + setFormatterOptions (options: DecoderOptions) { this.#decoder.setFormatterOptions(options); } diff --git a/src/services/MainWorker.ts b/src/services/MainWorker.ts index 73b2f5da2..021353f37 100644 --- a/src/services/MainWorker.ts +++ b/src/services/MainWorker.ts @@ -1,4 +1,5 @@ import dayjs from "dayjs"; +import dayjsBigIntSupport from "dayjs/plugin/bigIntSupport"; import dayjsTimezone from "dayjs/plugin/timezone"; import dayjsUtc from "dayjs/plugin/utc"; @@ -17,6 +18,7 @@ import LogFileManager from "./LogFileManager"; /* eslint-disable import/no-named-as-default-member */ dayjs.extend(dayjsUtc); dayjs.extend(dayjsTimezone); +dayjs.extend(dayjsBigIntSupport); /* eslint-enable import/no-named-as-default-member */ /** diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index 5222715f1..6d5b151a3 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -1,32 +1,61 @@ -import clpFfiJsModuleInit, {ClpIrStreamReader} from "clp-ffi-js"; - +import clpFfiJsModuleInit, {ClpStreamReader} from "../../../deps/ClpFfiJs-worker"; import {Nullable} from "../../typings/common"; import { + ClpIrDecoderOptions, Decoder, DecodeResultType, FilteredLogEventMap, LogEventCount, } from "../../typings/decoders"; -import {LogLevelFilter} from "../../typings/logs"; +import {JsonObject} from "../../typings/js"; +import { + LOG_LEVEL, + LogLevelFilter, +} from "../../typings/logs"; +import LogbackFormatter from "../formatters/LogbackFormatter"; +import { + convertToDayjsTimestamp, + isJsonObject, +} from "./JsonlDecoder/utils"; class ClpIrDecoder implements Decoder { - #streamReader: ClpIrStreamReader; + #streamReader: ClpStreamReader; + + #formatter: Nullable; + + #logLevelKey: Nullable; - constructor (streamReader: ClpIrStreamReader) { + constructor (streamReader: ClpStreamReader, formatter: Nullable, logLevelKey: Nullable) { this.#streamReader = streamReader; + this.#formatter = formatter; + this.#logLevelKey = logLevelKey; } /** * Creates a new ClpIrDecoder instance. * * @param dataArray The input data array to be passed to the decoder. + * @param decoderOptions * @return The created ClpIrDecoder instance. */ - static async create (dataArray: Uint8Array): Promise { + static async create ( + dataArray: Uint8Array, + decoderOptions: ClpIrDecoderOptions + ): Promise { const module = await clpFfiJsModuleInit(); - const streamReader = new module.ClpIrStreamReader(dataArray); - return new ClpIrDecoder(streamReader); + const streamReader = new module.ClpStreamReader(dataArray, decoderOptions); + + let formatter: Nullable = null; + let logLevelKey: Nullable = null; + if ( + module.IRProtocolErrorCode.SUPPORTED === streamReader.getIrProtocolErrorCode() + ) { + formatter = new LogbackFormatter({formatString: decoderOptions.formatString}); + ({logLevelKey} = decoderOptions); + } + + return new ClpIrDecoder(streamReader, formatter, logLevelKey); } getEstimatedNumEvents (): number { @@ -60,7 +89,36 @@ class ClpIrDecoder implements Decoder { endIdx: number, useFilter: boolean ): Nullable { - return this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); + const results = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); + if (null === this.#formatter) { + return results; + } + results.forEach((result) => { + const [message, timestamp] = result; + let fields: JsonObject = {}; + try { + fields = JSON.parse(message) as JsonObject; + if (false === isJsonObject(fields)) { + throw new Error("Unexpected non-object."); + } else if (null !== this.#logLevelKey) { + const logLevel = fields[this.#logLevelKey]; + fields[this.#logLevelKey] = LOG_LEVEL[logLevel as LOG_LEVEL]; + } + } catch (e) { + if (0 === message.length) { + return; + } + console.error(e, message); + } + + // @ts-expect-error `this.#formatter` is certainly non-null + result[0] = this.#formatter.formatLogEvent({ + fields: fields, + timestamp: convertToDayjsTimestamp(timestamp), + }); + }); + + return results; } } diff --git a/src/services/decoders/JsonlDecoder/index.ts b/src/services/decoders/JsonlDecoder/index.ts index 8fba66a68..f21343ed8 100644 --- a/src/services/decoders/JsonlDecoder/index.ts +++ b/src/services/decoders/JsonlDecoder/index.ts @@ -5,7 +5,7 @@ import { Decoder, DecodeResultType, FilteredLogEventMap, - JsonlDecoderOptionsType, + JsonlDecoderOptions, LogEventCount, } from "../../../typings/decoders"; import {Formatter} from "../../../typings/formatters"; @@ -25,7 +25,7 @@ import { /** - * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for + * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptions` for * properties that are specific to log events (compared to generic JSON records). */ class JsonlDecoder implements Decoder { @@ -49,7 +49,7 @@ class JsonlDecoder implements Decoder { * @param dataArray * @param decoderOptions */ - constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { + constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptions) { this.#dataArray = dataArray; this.#logLevelKey = decoderOptions.logLevelKey; this.#timestampKey = decoderOptions.timestampKey; @@ -81,7 +81,7 @@ class JsonlDecoder implements Decoder { }; } - setFormatterOptions (options: JsonlDecoderOptionsType): boolean { + setFormatterOptions (options: JsonlDecoderOptions): boolean { this.#formatter = new LogbackFormatter({formatString: options.formatString}); return true; diff --git a/src/services/decoders/JsonlDecoder/utils.ts b/src/services/decoders/JsonlDecoder/utils.ts index e76ebb300..717e51fe6 100644 --- a/src/services/decoders/JsonlDecoder/utils.ts +++ b/src/services/decoders/JsonlDecoder/utils.ts @@ -53,12 +53,13 @@ const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { * - the timestamp's value is an unsupported type. * - the timestamp's value is not a valid dayjs timestamp. */ -const convertToDayjsTimestamp = (field: JsonValue | undefined): dayjs.Dayjs => { +const convertToDayjsTimestamp = (field: JsonValue | bigint | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. // NOTE: dayjs surprisingly thinks `undefined` is a valid date. See // https://day.js.org/docs/en/parse/now#docsNav if (("string" !== typeof field && - "number" !== typeof field) || + "number" !== typeof field && + "bigint" !== typeof field) || "undefined" === typeof field ) { // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is diff --git a/src/typings/config.ts b/src/typings/config.ts index c70b475ae..6c66c8436 100644 --- a/src/typings/config.ts +++ b/src/typings/config.ts @@ -1,4 +1,4 @@ -import {JsonlDecoderOptionsType} from "./decoders"; +import {JsonlDecoderOptions} from "./decoders"; import {TAB_NAME} from "./tab"; @@ -27,7 +27,7 @@ enum LOCAL_STORAGE_KEY { /* eslint-enable @typescript-eslint/prefer-literal-enum-member */ type ConfigMap = { - [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptionsType, + [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptions, [CONFIG_KEY.INITIAL_TAB_NAME]: TAB_NAME, [CONFIG_KEY.THEME]: THEME_NAME, [CONFIG_KEY.PAGE_SIZE]: number, diff --git a/src/typings/decoders.ts b/src/typings/decoders.ts index 065f00fe0..8d6d64d99 100644 --- a/src/typings/decoders.ts +++ b/src/typings/decoders.ts @@ -14,13 +14,25 @@ interface LogEventCount { * @property logLevelKey The key of the kv-pair that contains the log level in every record. * @property timestampKey The key of the kv-pair that contains the timestamp in every record. */ -interface JsonlDecoderOptionsType { +interface JsonlDecoderOptions { formatString: string, logLevelKey: string, timestampKey: string, } -type DecoderOptionsType = JsonlDecoderOptionsType; +/** + * Options for the Structured CLP IR decoder. + */ +interface StructuredClpIrDecoderOptions extends JsonlDecoderOptions { +} + +/** + * Options for the CLP IR decoder. Currently, those options are only effective if the stream is + * Structured IR. + */ +type ClpIrDecoderOptions = StructuredClpIrDecoderOptions; + +type DecoderOptions = JsonlDecoderOptions | ClpIrDecoderOptions; /** * Type of the decoded log event. We use an array rather than object so that it's easier to return @@ -85,7 +97,7 @@ interface Decoder { * @param options * @return Whether the options were successfully set. */ - setFormatterOptions(options: DecoderOptionsType): boolean; + setFormatterOptions(options: DecoderOptions): boolean; /** * Decodes log events in the range `[beginIdx, endIdx)` of the filtered or unfiltered @@ -106,10 +118,11 @@ interface Decoder { export type { ActiveLogCollectionEventIdx, + ClpIrDecoderOptions, Decoder, DecodeResultType, - DecoderOptionsType, + DecoderOptions, FilteredLogEventMap, - JsonlDecoderOptionsType, + JsonlDecoderOptions, LogEventCount, }; diff --git a/src/typings/worker.ts b/src/typings/worker.ts index 8e64e2837..f589c7060 100644 --- a/src/typings/worker.ts +++ b/src/typings/worker.ts @@ -1,7 +1,7 @@ import {Nullable} from "./common"; import { ActiveLogCollectionEventIdx, - DecoderOptionsType, + DecoderOptions, } from "./decoders"; import { LOG_LEVEL, @@ -89,7 +89,7 @@ type WorkerReqMap = { fileSrc: FileSrcType, pageSize: number, cursor: CursorType, - decoderOptions: DecoderOptionsType + decoderOptions: DecoderOptions }, [WORKER_REQ_CODE.LOAD_PAGE]: { cursor: CursorType, diff --git a/src/utils/config.ts b/src/utils/config.ts index 8283b516a..9ca3c3240 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -6,7 +6,7 @@ import { LOCAL_STORAGE_KEY, THEME_NAME, } from "../typings/config"; -import {DecoderOptionsType} from "../typings/decoders"; +import {DecoderOptions} from "../typings/decoders"; import {TAB_NAME} from "../typings/tab"; @@ -131,7 +131,7 @@ const getConfig = (key: T): ConfigMap[T] => { timestampKey: window.localStorage.getItem( LOCAL_STORAGE_KEY.DECODER_OPTIONS_TIMESTAMP_KEY ), - } as DecoderOptionsType; + } as DecoderOptions; break; case CONFIG_KEY.INITIAL_TAB_NAME: value = window.localStorage.getItem(LOCAL_STORAGE_KEY.INITIAL_TAB_NAME); From 11e16d5fc4b8aeeda55532a6e5520c34c089aa32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Nov 2024 19:44:05 -0500 Subject: [PATCH 2/9] Update clp-ffi-js to v0.3.0 and refactor ClpIrDecoder. --- package-lock.json | 9 ++-- package.json | 2 +- src/services/decoders/ClpIrDecoder.ts | 74 +++++++++++++++------------ 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca5fa8b1b..db70105c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@mui/icons-material": "^6.1.0", "@mui/joy": "^5.0.0-beta.48", "axios": "^1.7.2", - "clp-ffi-js": "^0.2.0", + "clp-ffi-js": "^0.3.0", "dayjs": "^1.11.11", "monaco-editor": "^0.50.0", "react": "^18.3.1", @@ -4811,10 +4811,9 @@ } }, "node_modules/clp-ffi-js": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/clp-ffi-js/-/clp-ffi-js-0.2.0.tgz", - "integrity": "sha512-4zGQ6Jh3y57LvO4dOD9YNi3LCpj+YWCPvqX1u8ugfp9RPnfIBjK7lilJa4jBWYsGq/Dhxe9mXb0i7WmqxuLotw==", - "license": "Apache-2.0" + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/clp-ffi-js/-/clp-ffi-js-0.3.0.tgz", + "integrity": "sha512-+jNULrIosKTSP9WbwDa9AcXzgHCN9W3/iZsQW6jjrzCRvqj9OKXwGzPKT1od6nDsSsKPNVqeBSmasGUWExtadA==" }, "node_modules/clsx": { "version": "2.1.1", diff --git a/package.json b/package.json index 886cf2a3f..63833fe4c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@mui/icons-material": "^6.1.0", "@mui/joy": "^5.0.0-beta.48", "axios": "^1.7.2", - "clp-ffi-js": "^0.2.0", + "clp-ffi-js": "^0.3.0", "dayjs": "^1.11.11", "monaco-editor": "^0.50.0", "react": "^18.3.1", diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index 6d5b151a3..39bb9832e 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -1,10 +1,12 @@ -import clpFfiJsModuleInit, {ClpStreamReader} from "../../../deps/ClpFfiJs-worker"; +import clpFfiJsModuleInit, {ClpStreamReader} from "clp-ffi-js"; + import {Nullable} from "../../typings/common"; import { ClpIrDecoderOptions, Decoder, DecodeResultType, FilteredLogEventMap, + JsonlDecoderOptions, LogEventCount, } from "../../typings/decoders"; import {JsonObject} from "../../typings/js"; @@ -19,17 +21,28 @@ import { } from "./JsonlDecoder/utils"; +enum CLP_IR_STREAM_TYPE { + STRUCTURED = "structured", + UNSTRUCTURED = "unstructured", +} + class ClpIrDecoder implements Decoder { #streamReader: ClpStreamReader; - #formatter: Nullable; + #streamType: CLP_IR_STREAM_TYPE; - #logLevelKey: Nullable; + #formatter: Nullable; - constructor (streamReader: ClpStreamReader, formatter: Nullable, logLevelKey: Nullable) { + constructor ( + streamType: CLP_IR_STREAM_TYPE, + streamReader: ClpStreamReader, + decoderOptions: ClpIrDecoderOptions + ) { + this.#streamType = streamType; this.#streamReader = streamReader; - this.#formatter = formatter; - this.#logLevelKey = logLevelKey; + this.#formatter = (streamType === CLP_IR_STREAM_TYPE.STRUCTURED) ? + new LogbackFormatter({formatString: decoderOptions.formatString}) : + null; } /** @@ -45,17 +58,11 @@ class ClpIrDecoder implements Decoder { ): Promise { const module = await clpFfiJsModuleInit(); const streamReader = new module.ClpStreamReader(dataArray, decoderOptions); + const streamType = streamReader.getIrStreamType() === module.IrStreamType.STRUCTURED ? + CLP_IR_STREAM_TYPE.STRUCTURED : + CLP_IR_STREAM_TYPE.UNSTRUCTURED; - let formatter: Nullable = null; - let logLevelKey: Nullable = null; - if ( - module.IRProtocolErrorCode.SUPPORTED === streamReader.getIrProtocolErrorCode() - ) { - formatter = new LogbackFormatter({formatString: decoderOptions.formatString}); - ({logLevelKey} = decoderOptions); - } - - return new ClpIrDecoder(streamReader, formatter, logLevelKey); + return new ClpIrDecoder(streamType, streamReader, decoderOptions); } getEstimatedNumEvents (): number { @@ -79,8 +86,9 @@ class ClpIrDecoder implements Decoder { }; } - // eslint-disable-next-line class-methods-use-this - setFormatterOptions (): boolean { + setFormatterOptions (options: JsonlDecoderOptions): boolean { + this.#formatter = new LogbackFormatter({formatString: options.formatString}); + return true; } @@ -90,30 +98,29 @@ class ClpIrDecoder implements Decoder { useFilter: boolean ): Nullable { const results = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); - if (null === this.#formatter) { + if (this.#streamType === CLP_IR_STREAM_TYPE.UNSTRUCTURED) { return results; } results.forEach((result) => { const [message, timestamp] = result; let fields: JsonObject = {}; - try { - fields = JSON.parse(message) as JsonObject; - if (false === isJsonObject(fields)) { - throw new Error("Unexpected non-object."); - } else if (null !== this.#logLevelKey) { - const logLevel = fields[this.#logLevelKey]; - fields[this.#logLevelKey] = LOG_LEVEL[logLevel as LOG_LEVEL]; - } - } catch (e) { - if (0 === message.length) { - return; + + if (0 < message.length) { + try { + fields = JSON.parse(message) as JsonObject; + if (false === isJsonObject(fields)) { + throw new Error("Unexpected non-object."); + } + } catch (e) { + console.error(e, message); } - console.error(e, message); } - // @ts-expect-error `this.#formatter` is certainly non-null - result[0] = this.#formatter.formatLogEvent({ + // `this.#formatter` has been null-checked at the method entry. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + result[0] = this.#formatter!.formatLogEvent({ fields: fields, + level: LOG_LEVEL.UNKNOWN, timestamp: convertToDayjsTimestamp(timestamp), }); }); @@ -122,4 +129,5 @@ class ClpIrDecoder implements Decoder { } } + export default ClpIrDecoder; From 826057fc6406573facd1210ee0ea5ddc8faccf96 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Nov 2024 19:45:34 -0500 Subject: [PATCH 3/9] Update ClpIrDecoder to use generic Formatter type. --- src/services/decoders/ClpIrDecoder.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index 39bb9832e..4888538d8 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -9,6 +9,7 @@ import { JsonlDecoderOptions, LogEventCount, } from "../../typings/decoders"; +import {Formatter} from "../../typings/formatters"; import {JsonObject} from "../../typings/js"; import { LOG_LEVEL, @@ -31,7 +32,7 @@ class ClpIrDecoder implements Decoder { #streamType: CLP_IR_STREAM_TYPE; - #formatter: Nullable; + #formatter: Nullable; constructor ( streamType: CLP_IR_STREAM_TYPE, From e68a907e4c5808a2a0f86c040c5ea3d775418c10 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Nov 2024 19:47:51 -0500 Subject: [PATCH 4/9] Add type annotation for `results` in `decodeRange` function - Apply suggestions from code review Co-authored-by: davemarco <83603688+davemarco@users.noreply.github.com> --- src/services/decoders/ClpIrDecoder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index 4888538d8..08fa77d67 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -98,7 +98,7 @@ class ClpIrDecoder implements Decoder { endIdx: number, useFilter: boolean ): Nullable { - const results = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); + const results: DecodeResultType[] = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); if (this.#streamType === CLP_IR_STREAM_TYPE.UNSTRUCTURED) { return results; } From 7aaedb5a113b05c144082c4f1a30ae7c62edb22d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Nov 2024 19:55:28 -0500 Subject: [PATCH 5/9] Update the JsonlDecoder and ClpIrDecoder to use a unified DecoderOptions interface. --- src/services/decoders/ClpIrDecoder.ts | 19 +++++++++++-------- src/services/decoders/JsonlDecoder/index.ts | 8 ++++---- src/typings/config.ts | 4 ++-- src/typings/decoders.ts | 20 +------------------- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index 08fa77d67..d73e72615 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -2,11 +2,10 @@ import clpFfiJsModuleInit, {ClpStreamReader} from "clp-ffi-js"; import {Nullable} from "../../typings/common"; import { - ClpIrDecoderOptions, Decoder, DecodeResultType, + DecoderOptions, FilteredLogEventMap, - JsonlDecoderOptions, LogEventCount, } from "../../typings/decoders"; import {Formatter} from "../../typings/formatters"; @@ -30,14 +29,14 @@ enum CLP_IR_STREAM_TYPE { class ClpIrDecoder implements Decoder { #streamReader: ClpStreamReader; - #streamType: CLP_IR_STREAM_TYPE; + readonly #streamType: CLP_IR_STREAM_TYPE; #formatter: Nullable; constructor ( streamType: CLP_IR_STREAM_TYPE, streamReader: ClpStreamReader, - decoderOptions: ClpIrDecoderOptions + decoderOptions: DecoderOptions ) { this.#streamType = streamType; this.#streamReader = streamReader; @@ -50,12 +49,13 @@ class ClpIrDecoder implements Decoder { * Creates a new ClpIrDecoder instance. * * @param dataArray The input data array to be passed to the decoder. - * @param decoderOptions + * @param decoderOptions The options are only effective if the stream type is + * {@link CLP_IR_STREAM_TYPE.STRUCTURED}. * @return The created ClpIrDecoder instance. */ static async create ( dataArray: Uint8Array, - decoderOptions: ClpIrDecoderOptions + decoderOptions: DecoderOptions ): Promise { const module = await clpFfiJsModuleInit(); const streamReader = new module.ClpStreamReader(dataArray, decoderOptions); @@ -87,7 +87,7 @@ class ClpIrDecoder implements Decoder { }; } - setFormatterOptions (options: JsonlDecoderOptions): boolean { + setFormatterOptions (options: DecoderOptions): boolean { this.#formatter = new LogbackFormatter({formatString: options.formatString}); return true; @@ -98,10 +98,13 @@ class ClpIrDecoder implements Decoder { endIdx: number, useFilter: boolean ): Nullable { - const results: DecodeResultType[] = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); + const results: DecodeResultType[] = + this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); + if (this.#streamType === CLP_IR_STREAM_TYPE.UNSTRUCTURED) { return results; } + results.forEach((result) => { const [message, timestamp] = result; let fields: JsonObject = {}; diff --git a/src/services/decoders/JsonlDecoder/index.ts b/src/services/decoders/JsonlDecoder/index.ts index f21343ed8..63ad2ce32 100644 --- a/src/services/decoders/JsonlDecoder/index.ts +++ b/src/services/decoders/JsonlDecoder/index.ts @@ -4,8 +4,8 @@ import {Nullable} from "../../../typings/common"; import { Decoder, DecodeResultType, + DecoderOptions, FilteredLogEventMap, - JsonlDecoderOptions, LogEventCount, } from "../../../typings/decoders"; import {Formatter} from "../../../typings/formatters"; @@ -25,7 +25,7 @@ import { /** - * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptions` for + * A decoder for JSONL (JSON lines) files that contain log events. See `DecoderOptions` for * properties that are specific to log events (compared to generic JSON records). */ class JsonlDecoder implements Decoder { @@ -49,7 +49,7 @@ class JsonlDecoder implements Decoder { * @param dataArray * @param decoderOptions */ - constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptions) { + constructor (dataArray: Uint8Array, decoderOptions: DecoderOptions) { this.#dataArray = dataArray; this.#logLevelKey = decoderOptions.logLevelKey; this.#timestampKey = decoderOptions.timestampKey; @@ -81,7 +81,7 @@ class JsonlDecoder implements Decoder { }; } - setFormatterOptions (options: JsonlDecoderOptions): boolean { + setFormatterOptions (options: DecoderOptions): boolean { this.#formatter = new LogbackFormatter({formatString: options.formatString}); return true; diff --git a/src/typings/config.ts b/src/typings/config.ts index 6c66c8436..9e5f21b2e 100644 --- a/src/typings/config.ts +++ b/src/typings/config.ts @@ -1,4 +1,4 @@ -import {JsonlDecoderOptions} from "./decoders"; +import {DecoderOptions} from "./decoders"; import {TAB_NAME} from "./tab"; @@ -27,7 +27,7 @@ enum LOCAL_STORAGE_KEY { /* eslint-enable @typescript-eslint/prefer-literal-enum-member */ type ConfigMap = { - [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptions, + [CONFIG_KEY.DECODER_OPTIONS]: DecoderOptions, [CONFIG_KEY.INITIAL_TAB_NAME]: TAB_NAME, [CONFIG_KEY.THEME]: THEME_NAME, [CONFIG_KEY.PAGE_SIZE]: number, diff --git a/src/typings/decoders.ts b/src/typings/decoders.ts index 8d6d64d99..fb7552a65 100644 --- a/src/typings/decoders.ts +++ b/src/typings/decoders.ts @@ -8,32 +8,16 @@ interface LogEventCount { } /** - * Options for the JSONL decoder. - * * @property formatString The format string to use to serialize records as plain text. * @property logLevelKey The key of the kv-pair that contains the log level in every record. * @property timestampKey The key of the kv-pair that contains the timestamp in every record. */ -interface JsonlDecoderOptions { +interface DecoderOptions { formatString: string, logLevelKey: string, timestampKey: string, } -/** - * Options for the Structured CLP IR decoder. - */ -interface StructuredClpIrDecoderOptions extends JsonlDecoderOptions { -} - -/** - * Options for the CLP IR decoder. Currently, those options are only effective if the stream is - * Structured IR. - */ -type ClpIrDecoderOptions = StructuredClpIrDecoderOptions; - -type DecoderOptions = JsonlDecoderOptions | ClpIrDecoderOptions; - /** * Type of the decoded log event. We use an array rather than object so that it's easier to return * results from WASM-based decoders. @@ -118,11 +102,9 @@ interface Decoder { export type { ActiveLogCollectionEventIdx, - ClpIrDecoderOptions, Decoder, DecodeResultType, DecoderOptions, FilteredLogEventMap, - JsonlDecoderOptions, LogEventCount, }; From 2df1e9c3b2268ad9dd8f37ae49dda19c8d64d4f9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Nov 2024 20:02:08 -0500 Subject: [PATCH 6/9] Simplify `decodeRange`. --- src/services/decoders/ClpIrDecoder.ts | 31 ++++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index d73e72615..c4895a5a5 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -1,4 +1,5 @@ import clpFfiJsModuleInit, {ClpStreamReader} from "clp-ffi-js"; +import {Dayjs} from "dayjs"; import {Nullable} from "../../typings/common"; import { @@ -10,10 +11,7 @@ import { } from "../../typings/decoders"; import {Formatter} from "../../typings/formatters"; import {JsonObject} from "../../typings/js"; -import { - LOG_LEVEL, - LogLevelFilter, -} from "../../typings/logs"; +import {LogLevelFilter} from "../../typings/logs"; import LogbackFormatter from "../formatters/LogbackFormatter"; import { convertToDayjsTimestamp, @@ -106,26 +104,29 @@ class ClpIrDecoder implements Decoder { } results.forEach((result) => { - const [message, timestamp] = result; + const [ + message, + timestamp, + level, + ] = result; + const dayJsTimestamp: Dayjs = convertToDayjsTimestamp(timestamp); let fields: JsonObject = {}; - if (0 < message.length) { - try { - fields = JSON.parse(message) as JsonObject; - if (false === isJsonObject(fields)) { - throw new Error("Unexpected non-object."); - } - } catch (e) { - console.error(e, message); + try { + fields = JSON.parse(message) as JsonObject; + if (false === isJsonObject(fields)) { + throw new Error("Unexpected non-object."); } + } catch (e) { + console.error(e, message); } // `this.#formatter` has been null-checked at the method entry. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion result[0] = this.#formatter!.formatLogEvent({ fields: fields, - level: LOG_LEVEL.UNKNOWN, - timestamp: convertToDayjsTimestamp(timestamp), + level: level, + timestamp: dayJsTimestamp, }); }); From 479fd71873d96fc22688c8588579ed5b2bf89605 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Nov 2024 20:39:28 -0500 Subject: [PATCH 7/9] Rename `DecodeResultType` -> `DecodeResult`; Update `DecodeResult` as per https://github.com/y-scope/clp-ffi-js/pull/34 --- src/services/decoders/ClpIrDecoder.ts | 6 +++--- src/services/decoders/JsonlDecoder/index.ts | 10 +++++----- src/typings/decoders.ts | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index c4895a5a5..df59c9f3d 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -4,7 +4,7 @@ import {Dayjs} from "dayjs"; import {Nullable} from "../../typings/common"; import { Decoder, - DecodeResultType, + DecodeResult, DecoderOptions, FilteredLogEventMap, LogEventCount, @@ -95,8 +95,8 @@ class ClpIrDecoder implements Decoder { beginIdx: number, endIdx: number, useFilter: boolean - ): Nullable { - const results: DecodeResultType[] = + ): Nullable { + const results: DecodeResult[] = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); if (this.#streamType === CLP_IR_STREAM_TYPE.UNSTRUCTURED) { diff --git a/src/services/decoders/JsonlDecoder/index.ts b/src/services/decoders/JsonlDecoder/index.ts index 63ad2ce32..5b873ed7d 100644 --- a/src/services/decoders/JsonlDecoder/index.ts +++ b/src/services/decoders/JsonlDecoder/index.ts @@ -3,7 +3,7 @@ import {Dayjs} from "dayjs"; import {Nullable} from "../../../typings/common"; import { Decoder, - DecodeResultType, + DecodeResult, DecoderOptions, FilteredLogEventMap, LogEventCount, @@ -91,7 +91,7 @@ class JsonlDecoder implements Decoder { beginIdx: number, endIdx: number, useFilter: boolean, - ): Nullable { + ): Nullable { if (useFilter && null === this.#filteredLogEventMap) { return null; } @@ -104,7 +104,7 @@ class JsonlDecoder implements Decoder { return null; } - const results: DecodeResultType[] = []; + const results: DecodeResult[] = []; for (let i = beginIdx; i < endIdx; i++) { // Explicit cast since typescript thinks `#filteredLogEventMap[i]` can be undefined, but // it shouldn't be since we performed a bounds check at the beginning of the method. @@ -204,12 +204,12 @@ class JsonlDecoder implements Decoder { } /** - * Decodes a log event into a `DecodeResultType`. + * Decodes a log event into a `DecodeResult`. * * @param logEventIdx * @return The decoded log event. */ - #decodeLogEvent = (logEventIdx: number): DecodeResultType => { + #decodeLogEvent = (logEventIdx: number): DecodeResult => { let timestamp: number; let message: string; let logLevel: LOG_LEVEL; diff --git a/src/typings/decoders.ts b/src/typings/decoders.ts index fb7552a65..1eaa0cb59 100644 --- a/src/typings/decoders.ts +++ b/src/typings/decoders.ts @@ -27,7 +27,7 @@ interface DecoderOptions { * @property level * @property number */ -type DecodeResultType = [string, number, number, number]; +type DecodeResult = [string, bigint, number, number]; /** * Mapping between an index in the filtered log events collection to an index in the unfiltered log @@ -97,13 +97,13 @@ interface Decoder { beginIdx: number, endIdx: number, useFilter: boolean - ): Nullable; + ): Nullable; } export type { ActiveLogCollectionEventIdx, Decoder, - DecodeResultType, + DecodeResult, DecoderOptions, FilteredLogEventMap, LogEventCount, From 29d85a60b437bb0bd4447ddcc337ea96a7ad0d1d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 15 Nov 2024 10:35:30 -0500 Subject: [PATCH 8/9] Update docs for ClpIrDecoder.create(). --- src/services/decoders/ClpIrDecoder.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index df59c9f3d..f0f634f0a 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -45,10 +45,11 @@ class ClpIrDecoder implements Decoder { /** * Creates a new ClpIrDecoder instance. + * NOTE: `decoderOptions` only affects decode results if the stream type is + * {@link CLP_IR_STREAM_TYPE.STRUCTURED}. * * @param dataArray The input data array to be passed to the decoder. - * @param decoderOptions The options are only effective if the stream type is - * {@link CLP_IR_STREAM_TYPE.STRUCTURED}. + * @param decoderOptions * @return The created ClpIrDecoder instance. */ static async create ( From 023e3451e5142c92d68984d44b7298ddc91fcc21 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 15 Nov 2024 10:49:18 -0500 Subject: [PATCH 9/9] Turn forEach into `for...of` in decodeRange(). --- src/services/decoders/ClpIrDecoder.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index f0f634f0a..0f801664a 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -100,16 +100,22 @@ class ClpIrDecoder implements Decoder { const results: DecodeResult[] = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter); - if (this.#streamType === CLP_IR_STREAM_TYPE.UNSTRUCTURED) { + if (null === this.#formatter) { + if (this.#streamType === CLP_IR_STREAM_TYPE.STRUCTURED) { + // eslint-disable-next-line no-warning-comments + // TODO: Revisit when we allow displaying structured logs without a formatter. + console.error("Formatter is not set for structured logs."); + } + return results; } - results.forEach((result) => { + for (const r of results) { const [ message, timestamp, level, - ] = result; + ] = r; const dayJsTimestamp: Dayjs = convertToDayjsTimestamp(timestamp); let fields: JsonObject = {}; @@ -122,14 +128,12 @@ class ClpIrDecoder implements Decoder { console.error(e, message); } - // `this.#formatter` has been null-checked at the method entry. - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - result[0] = this.#formatter!.formatLogEvent({ + r[0] = this.#formatter.formatLogEvent({ fields: fields, level: level, timestamp: dayJsTimestamp, }); - }); + } return results; }