From 62a69c72968bc99d0f2c664f47eb543729a3762a Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 18:42:48 +0000 Subject: [PATCH 01/23] first draft --- .../src/contexts/StateContextProvider.tsx | 6 +- new-log-viewer/src/services/LogFileManager.ts | 28 +- new-log-viewer/src/services/MainWorker.ts | 10 +- .../src/services/decoders/ClpIrDecoder.ts | 9 +- .../src/services/decoders/JsonlDecoder.ts | 242 ++++++++++++------ .../services/formatters/LogbackFormatter.ts | 39 +-- new-log-viewer/src/typings/config.ts | 6 +- new-log-viewer/src/typings/decoders.ts | 41 +-- new-log-viewer/src/typings/formatters.ts | 11 +- new-log-viewer/src/typings/worker.ts | 9 +- new-log-viewer/src/utils/config.ts | 4 +- 11 files changed, 239 insertions(+), 166 deletions(-) diff --git a/new-log-viewer/src/contexts/StateContextProvider.tsx b/new-log-viewer/src/contexts/StateContextProvider.tsx index c6e07803..24f37b25 100644 --- a/new-log-viewer/src/contexts/StateContextProvider.tsx +++ b/new-log-viewer/src/contexts/StateContextProvider.tsx @@ -210,7 +210,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { workerPostReq( mainWorkerRef.current, WORKER_REQ_CODE.EXPORT_LOG, - {decoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS)} + {DecoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS)} ); }, [ numEvents, @@ -232,7 +232,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { fileSrc: fileSrc, pageSize: getConfig(CONFIG_KEY.PAGE_SIZE), cursor: cursor, - decoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), + DecoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), }); setExportProgress(STATE_DEFAULT.exportProgress); @@ -248,7 +248,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { } workerPostReq(mainWorkerRef.current, WORKER_REQ_CODE.LOAD_PAGE, { cursor: {code: CURSOR_CODE.PAGE_NUM, args: {pageNum: newPageNum}}, - decoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), + DecoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), }); }; diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index 2ceded83..ab4c96a7 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -1,6 +1,6 @@ import { Decoder, - DecoderOptionsType, + DecoderOptions, LOG_EVENT_FILE_END_IDX, } from "../typings/decoders"; import {MAX_V8_STRING_LENGTH} from "../typings/js"; @@ -98,16 +98,17 @@ class LogFileManager { * @param fileSrc The source of the file to load. This can be a string representing a URL, or a * File object. * @param pageSize Page size for setting up pagination. - * @param decoderOptions Initial decoder options. + * @param DecoderOptions Initial decoder options. * @return A Promise that resolves to the created LogFileManager instance. */ static async create ( fileSrc: FileSrcType, pageSize: number, - decoderOptions: DecoderOptionsType + buildOptions: BuildOptions, + DecoderOptions: DecoderOptions, ): Promise { const {fileName, fileData} = await loadFile(fileSrc); - const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions); + const decoder = await LogFileManager.#initDecoder(fileName, fileData, buildOptions, DecoderOptions); return new LogFileManager(decoder, fileName, pageSize); } @@ -117,18 +118,18 @@ class LogFileManager { * * @param fileName * @param fileData - * @param decoderOptions Initial decoder options. + * @param DecoderOptions Initial decoder options. * @return The constructed decoder. * @throws {Error} if no decoder supports a file with the given extension. */ static async #initDecoder ( fileName: string, fileData: Uint8Array, - decoderOptions: DecoderOptionsType + DecoderOptions: DecoderOptions ): Promise { let decoder: Decoder; if (fileName.endsWith(".jsonl")) { - decoder = new JsonlDecoder(fileData, decoderOptions); + decoder = new JsonlDecoder(fileData, DecoderOptions); } else if (fileName.endsWith(".clp.zst")) { decoder = await ClpIrDecoder.create(fileData); } else { @@ -145,12 +146,12 @@ class LogFileManager { } /** - * Sets options for the decoder. + * Sets formatting options for the decoder. * * @param options */ - setDecoderOptions (options: DecoderOptionsType) { - this.#decoder.setDecoderOptions(options); + setFormatterOptions (options: DecoderOptions) { + this.#decoder.setFormatterOptions(options); } /** @@ -166,9 +167,10 @@ class LogFileManager { logs: string, } { const endLogEventIdx = Math.min(beginLogEventIdx + EXPORT_LOGS_CHUNK_SIZE, this.#numEvents); - const results = this.#decoder.decode( + const results = this.#decoder.decodeRange( beginLogEventIdx, - endLogEventIdx + endLogEventIdx, + false, ); if (null === results) { @@ -200,7 +202,7 @@ class LogFileManager { console.debug(`loadPage: cursor=${JSON.stringify(cursor)}`); const {beginLogEventNum, endLogEventNum} = this.#getCursorRange(cursor); - const results = this.#decoder.decode(beginLogEventNum - 1, endLogEventNum); + const results = this.#decoder.decodeRange(beginLogEventNum - 1, endLogEventNum, false); if (null === results) { throw new Error("Error occurred during decoding. " + `beginLogEventNum=${beginLogEventNum}, ` + diff --git a/new-log-viewer/src/services/MainWorker.ts b/new-log-viewer/src/services/MainWorker.ts index a04d134c..0f2dd839 100644 --- a/new-log-viewer/src/services/MainWorker.ts +++ b/new-log-viewer/src/services/MainWorker.ts @@ -49,8 +49,8 @@ onmessage = async (ev: MessageEvent) => { if (null === LOG_FILE_MANAGER) { throw new Error("Log file manager hasn't been initialized"); } - if ("undefined" !== typeof args.decoderOptions) { - LOG_FILE_MANAGER.setDecoderOptions(args.decoderOptions); + if ("undefined" !== typeof args.DecoderOptions) { + LOG_FILE_MANAGER.setDecoderOptions(args.DecoderOptions); } let decodedEventIdx = 0; @@ -67,7 +67,7 @@ onmessage = async (ev: MessageEvent) => { LOG_FILE_MANAGER = await LogFileManager.create( args.fileSrc, args.pageSize, - args.decoderOptions + args.DecoderOptions ); postResp(WORKER_RESP_CODE.LOG_FILE_INFO, { @@ -84,8 +84,8 @@ onmessage = async (ev: MessageEvent) => { if (null === LOG_FILE_MANAGER) { throw new Error("Log file manager hasn't been initialized"); } - if ("undefined" !== typeof args.decoderOptions) { - LOG_FILE_MANAGER.setDecoderOptions(args.decoderOptions); + if ("undefined" !== typeof args.DecoderOptions) { + LOG_FILE_MANAGER.setDecoderOptions(args.DecoderOptions); } postResp( WORKER_RESP_CODE.PAGE_DATA, diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index 4cab4590..bc58d543 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -4,6 +4,7 @@ import {Nullable} from "../../typings/common"; import { Decoder, DecodeResultType, + LOG_EVENT_FILE_END_IDX, LogEventCount, } from "../../typings/decoders"; @@ -31,19 +32,19 @@ class ClpIrDecoder implements Decoder { return this.#streamReader.getNumEventsBuffered(); } - buildIdx (beginIdx: number, endIdx: number): Nullable { + build (): LogEventCount { return { numInvalidEvents: 0, - numValidEvents: this.#streamReader.deserializeRange(beginIdx, endIdx), + numValidEvents: this.#streamReader.deserializeRange(0, LOG_EVENT_FILE_END_IDX), }; } // eslint-disable-next-line class-methods-use-this - setDecoderOptions (): boolean { + setFormatterOptions (): boolean { return true; } - decode (beginIdx: number, endIdx: number): Nullable { + decodeRange (beginIdx: number, endIdx: number, useFilteredIndices: boolean): Nullable { return this.#streamReader.decodeRange(beginIdx, endIdx); } } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index 62a5b21e..89626f80 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -2,9 +2,12 @@ import {Nullable} from "../../typings/common"; import { Decoder, DecodeResultType, - JsonlDecoderOptionsType, + JsonlBuildOptions, + JsonlDecoderOptions, LogEventCount, } from "../../typings/decoders"; +import dayjs, { Dayjs } from "dayjs"; +import utc from "dayjs/plugin/utc" import {Formatter} from "../../typings/formatters"; import { JsonObject, @@ -16,69 +19,84 @@ import { } from "../../typings/logs"; import LogbackFormatter from "../formatters/LogbackFormatter"; +dayjs.extend(utc); /** - * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecodeOptionsType` for + * A log event parsed from a JSON log. + */ +interface JsonLogEvent { + timestamp: Dayjs, + level: LOG_LEVEL, + fields: JsonObject +} + +/** + * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for * properties that are specific to log events (compared to generic JSON records). */ class JsonlDecoder implements Decoder { static #textDecoder = new TextDecoder(); - #logLevelKey: string = "level"; + #dataArray: Nullable; + + #logLevelKey: string; + + #timestampKey: string; - #logEvents: JsonObject[] = []; + #logEvents: JsonLogEvent[] = []; + + #filteredLogEventIndices: number[] = []; #invalidLogEventIdxToRawLine: Map = new Map(); - // @ts-expect-error #fomatter is set in the constructor by `setDecoderOptions()` #formatter: Formatter; /** * @param dataArray - * @param decoderOptions + * @param DecoderOptions * @throws {Error} if the initial decoder options are erroneous. */ - constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { - const isOptionSet = this.setDecoderOptions(decoderOptions); - if (false === isOptionSet) { - throw new Error( - `Initial decoder options are erroneous: ${JSON.stringify(decoderOptions)}` - ); - } - this.#deserialize(dataArray); + constructor (dataArray: Uint8Array, DecoderOptions: JsonlDecoderOptions) { + this.#dataArray = dataArray; + this.#logLevelKey = DecoderOptions.logLevelKey; + this.#timestampKey = DecoderOptions.timestampKey; + this.#formatter = new LogbackFormatter({formatString: DecoderOptions.formatString}); } getEstimatedNumEvents (): number { return this.#logEvents.length; } - buildIdx (beginIdx: number, endIdx: number): Nullable { - // This method is a dummy implementation since the actual deserialization is done in the - // constructor. + getFilteredLogEventIndices (): Nullable { + return this.#filteredLogEventIndices; + } - if (0 > beginIdx || this.#logEvents.length < endIdx) { - return null; - } + build (): LogEventCount { + this.#deserialize(); - const numInvalidEvents = Array.from(this.#invalidLogEventIdxToRawLine.keys()) - .filter((eventIdx) => (beginIdx <= eventIdx && eventIdx < endIdx)) - .length; + const numInvalidEvents = Array.from(this.#invalidLogEventIdxToRawLine.keys()).length; return { - numValidEvents: endIdx - beginIdx - numInvalidEvents, + numValidEvents: this.#logEvents.length - numInvalidEvents, numInvalidEvents: numInvalidEvents, }; } - setDecoderOptions (options: JsonlDecoderOptionsType): boolean { - this.#formatter = new LogbackFormatter(options); - this.#logLevelKey = options.logLevelKey; - + setFormatterOptions (options: JsonlDecoderOptions): boolean { + this.#formatter = new LogbackFormatter({formatString: options.formatString}); return true; } - decode (beginIdx: number, endIdx: number): Nullable { - if (0 > beginIdx || this.#logEvents.length < endIdx) { + decodeRange ( + beginIdx: number, + endIdx: number, + useFilteredIndices: boolean, + ): Nullable { + const length: number = useFilteredIndices ? + this.#filteredLogEventIndices.length : + this.#logEvents.length; + + if (0 > beginIdx || length < endIdx) { return null; } @@ -86,23 +104,30 @@ class JsonlDecoder implements Decoder { // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on // every iteration. const results: DecodeResultType[] = []; - for (let logEventIdx = beginIdx; logEventIdx < endIdx; logEventIdx++) { + for (let cursorIdx = beginIdx; cursorIdx < endIdx; cursorIdx++) { let timestamp: number; let message: string; let logLevel: LOG_LEVEL; + + // Explicit cast since typescript thinks `#filteredLogEventIndices[filteredLogEventIdx]` + // can be undefined, but it shouldn't be since we performed a bounds check at the + // beginning of the method. + const logEventIdx: number = useFilteredIndices ? + (this.#filteredLogEventIndices[cursorIdx] as number) : + cursorIdx; + if (this.#invalidLogEventIdxToRawLine.has(logEventIdx)) { timestamp = INVALID_TIMESTAMP_VALUE; message = `${this.#invalidLogEventIdxToRawLine.get(logEventIdx)}\n`; logLevel = LOG_LEVEL.NONE; } else { - // Explicit cast since typescript thinks `#logEvents[logEventIdx]` can be undefined, - // but it shouldn't be since we performed a bounds check at the beginning of the - // method. - const logEvent = this.#logEvents[logEventIdx] as JsonObject; - ( - {timestamp, message} = this.#formatter.formatLogEvent(logEvent) - ); - logLevel = this.#parseLogLevel(logEvent); + // Explicit cast since typescript thinks `#logEvents[filteredIdx]` can be undefined, + // but it shouldn't be since the index comes from a class-internal filter. + const logEvent: JsonLogEvent = this.#logEvents[logEventIdx] as JsonLogEvent; + + logLevel = logEvent.level; + message = this.#formatter.formatLogEvent(logEvent); + timestamp = logEvent.timestamp.valueOf(); } results.push([ @@ -117,14 +142,18 @@ class JsonlDecoder implements Decoder { } /** - * Parses each line from the given data array as a JSON object and buffers it internally. If a + * Parses each line from the data array as a JSON object and buffers it internally. If a * line cannot be parsed as a JSON object, an error is logged and the line is skipped. * - * @param dataArray + * NOTE: The data array is freed after the very first run of this method. */ - #deserialize (dataArray: Uint8Array) { - const text = JsonlDecoder.#textDecoder.decode(dataArray); - let beginIdx = 0; + #deserialize () { + if (null === this.#dataArray) { + return; + } + + const text = JsonlDecoder.#textDecoder.decode(this.#dataArray); + let beginIdx: number = 0; while (beginIdx < text.length) { const endIdx = text.indexOf("\n", beginIdx); const line = (-1 === endIdx) ? @@ -135,52 +164,115 @@ class JsonlDecoder implements Decoder { text.length : endIdx + 1; - try { - const logEvent = JSON.parse(line) as JsonValue; - if ("object" !== typeof logEvent) { - throw new Error("Unexpected non-object."); - } - this.#logEvents.push(logEvent as JsonObject); - } catch (e) { - if (0 === line.length) { - continue; - } - console.error(e, line); - const currentLogEventIdx = this.#logEvents.length; - this.#invalidLogEventIdxToRawLine.set(currentLogEventIdx, line); - this.#logEvents.push({}); + this.#parseJson(line); + } + + this.#dataArray = null; + } + + /** + * Parse line into a json log event and add to log events array. If the line contains invalid + * json, an entry is added to invalid log event map. + * + * @param line + */ + #parseJson (line: string) { + try { + const fields = JSON.parse(line) as JsonValue; + if (!this.#isJsonObject(fields)) { + throw new Error("Unexpected non-object."); + } + this.#logEvents.push({ + fields: fields, + level: this.#parseLogLevel(fields[this.#logLevelKey]), + timestamp: this.#parseTimestamp(fields[this.#timestampKey]), + }); + } catch (e) { + if (0 === line.length) { + return; } + console.error(e, line); + const currentLogEventIdx = this.#logEvents.length; + this.#invalidLogEventIdxToRawLine.set(currentLogEventIdx, line); + this.#logEvents.push({ + fields: {}, + level: LOG_LEVEL.NONE, + timestamp: dayjs.utc(INVALID_TIMESTAMP_VALUE), + }); } } /** - * Parses the log level from the given log event. + * Narrows input to JsonObject if valid type. + * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates * - * @param logEvent - * @return The parsed log level. + * @param fields + * @return Whether type is JsonObject. */ - #parseLogLevel (logEvent: JsonObject): number { - let logLevel = LOG_LEVEL.NONE; + #isJsonObject(fields: JsonValue): fields is JsonObject { + return "object" === typeof fields; + } - const parsedLogLevel = logEvent[this.#logLevelKey]; - if ("undefined" === typeof parsedLogLevel) { - console.error(`${this.#logLevelKey} doesn't exist in log event.`); + /** + * Maps the log level field to a log level value. + * + * @param logLevelField Field in log event indexed by log level key. + * @return Log level value. + */ + #parseLogLevel (logLevelField: JsonValue | undefined): number { + let logLevelValue = LOG_LEVEL.NONE; - return logLevel; + if ("undefined" === typeof logLevelField) { + return logLevelValue; } - const logLevelStr = "object" === typeof parsedLogLevel ? - JSON.stringify(parsedLogLevel) : - String(parsedLogLevel); + const logLevelName = "object" === typeof logLevelField ? + JSON.stringify(logLevelField) : + String(logLevelField); - if (false === (logLevelStr.toUpperCase() in LOG_LEVEL)) { - console.error(`${logLevelStr} doesn't match any known log level.`); - } else { - logLevel = LOG_LEVEL[logLevelStr as (keyof typeof LOG_LEVEL)]; + if (logLevelName.toUpperCase() in LOG_LEVEL) { + logLevelValue = LOG_LEVEL[logLevelName.toUpperCase() as keyof typeof LOG_LEVEL]; } - return logLevel; + return logLevelValue; + } + + /** + * Parses timestamp into dayjs timestamp. + * + * @param timestampField + * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: + * - the timestamp key doesn't exist in the log, or + * - the timestamp's value is not a number. + */ + #parseTimestamp (timestampField: JsonValue | undefined): dayjs.Dayjs { + + // If the field is an invalid type, then set the timestamp to INVALID_TIMESTAMP_VALUE. + if (typeof timestampField !== "string" && + typeof timestampField !== "number" || + // Dayjs library surprisingly thinks undefined is valid date... + // Reference: https://day.js.org/docs/en/parse/now#docsNav + typeof timestampField === undefined + ) { + // INVALID_TIMESTAMP_VALUE is a valid dayjs date. Another potential option is daysjs(null) + // to show `Invalid Date` in UI. + timestampField = INVALID_TIMESTAMP_VALUE; + } + + const dayjsTimestamp: Dayjs = dayjs.utc(timestampField); + + // Note if input is not valid (timestampField = "deadbeef"), this can produce a non-valid + // timestamp and will show up in UI as `Invalid Date`. Here we modify invalid dates to + // INVALID_TIMESTAMP_VALUE. + if (false === dayjsTimestamp.isValid()) { + dayjsTimestamp == dayjs.utc(INVALID_TIMESTAMP_VALUE) + } + + return dayjsTimestamp; } } export default JsonlDecoder; +export type { + JsonLogEvent, +} diff --git a/new-log-viewer/src/services/formatters/LogbackFormatter.ts b/new-log-viewer/src/services/formatters/LogbackFormatter.ts index 5d310df1..82b36e25 100644 --- a/new-log-viewer/src/services/formatters/LogbackFormatter.ts +++ b/new-log-viewer/src/services/formatters/LogbackFormatter.ts @@ -6,8 +6,9 @@ import { FormatterOptionsType, TimestampAndMessageType, } from "../../typings/formatters"; -import {JsonObject} from "../../typings/js"; +import {JsonLogEvent} from "../decoders/JsonlDecoder"; import {INVALID_TIMESTAMP_VALUE} from "../../typings/logs"; +import {JsonObject} from "../../typings/js"; /** @@ -39,14 +40,11 @@ class LogbackFormatter implements Formatter { #dateFormat: string = ""; - #timestampKey: string; - #keys: string[] = []; constructor (options: FormatterOptionsType) { // NOTE: It's safe for these values to be empty strings. this.#formatString = options.formatString; - this.#timestampKey = options.timestampKey; // Remove new line this.#formatString = this.#formatString.replace("%n", ""); @@ -59,17 +57,13 @@ class LogbackFormatter implements Formatter { * Formats the given log event. * * @param logEvent - * @return The log event's timestamp and the formatted string. + * @return The formatted message. */ - formatLogEvent (logEvent: JsonObject): TimestampAndMessageType { - const timestamp = this.#parseTimestamp(logEvent); - let formatted = this.#formatTimestamp(timestamp, this.#formatString); - formatted = this.#formatVariables(formatted, logEvent); - - return { - timestamp: timestamp.valueOf(), - message: formatted, - }; + formatLogEvent (logEvent: JsonLogEvent): string { + const {fields, timestamp} = logEvent; + let formatStringWithTimestamp: string = this.#formatTimestamp(timestamp, this.#formatString); + let message = this.#formatVariables(formatStringWithTimestamp, fields); + return message } /** @@ -110,23 +104,6 @@ class LogbackFormatter implements Formatter { } } - /** - * Gets the timestamp from the log event. - * - * @param logEvent - * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: - * - the timestamp key doesn't exist in the log, or - * - the timestamp's value is not a number. - */ - #parseTimestamp (logEvent: JsonObject): dayjs.Dayjs { - let timestamp = logEvent[this.#timestampKey]; - if ("number" !== typeof timestamp && "string" !== typeof timestamp) { - timestamp = INVALID_TIMESTAMP_VALUE; - } - - return dayjs.utc(timestamp); - } - /** * Replaces the timestamp specifier in `formatString` with `timestamp`, formatted with * `#dateFormat`. diff --git a/new-log-viewer/src/typings/config.ts b/new-log-viewer/src/typings/config.ts index 7f80b3c5..f6e9471c 100644 --- a/new-log-viewer/src/typings/config.ts +++ b/new-log-viewer/src/typings/config.ts @@ -1,4 +1,4 @@ -import {JsonlDecoderOptionsType} from "./decoders"; +import {JsonlDecoderOptions} from "./decoders"; enum THEME_NAME { @@ -8,7 +8,7 @@ enum THEME_NAME { } enum CONFIG_KEY { - DECODER_OPTIONS = "decoderOptions", + DECODER_OPTIONS = "DecoderOptions", THEME = "theme", PAGE_SIZE = "pageSize", } @@ -24,7 +24,7 @@ enum LOCAL_STORAGE_KEY { /* eslint-enable @typescript-eslint/prefer-literal-enum-member */ type ConfigMap = { - [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptionsType, + [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptions, [CONFIG_KEY.THEME]: THEME_NAME, [CONFIG_KEY.PAGE_SIZE]: number, }; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 53408184..042bc5eb 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -13,13 +13,21 @@ interface LogEventCount { * @property logLevelKey The key of the kv-pair that contains the log level in every record. * @property timestampKey The key of the kv-pair that contains the timestamp in every record. */ -interface JsonlDecoderOptionsType { +interface JsonlDecoderOptions { formatString: string, logLevelKey: string, timestampKey: string, } -type DecoderOptionsType = JsonlDecoderOptionsType; +interface JsonlBuildOptions { + formatString: string, + logLevelKey: string, + timestampKey: string, +} + +type DecoderOptions = JsonlDecoderOptions; + +type BuildOptions = JsonlBuildOptions; /** * Type of the decoded log event. We use an array rather than object so that it's easier to return @@ -28,7 +36,7 @@ type DecoderOptionsType = JsonlDecoderOptionsType; * @property message * @property timestamp * @property level - * @property number + * @property number The log event number is always the unfiltered number. */ type DecodeResultType = [string, number, number, number]; @@ -42,33 +50,32 @@ interface Decoder { getEstimatedNumEvents(): number; /** - * When applicable, deserializes log events in the range `[beginIdx, endIdx)`. - * - * @param beginIdx - * @param endIdx End index. To deserialize to the end of the file, use `LOG_EVENT_FILE_END_IDX`. + * Deserializes all log events in the file. * @return Count of the successfully deserialized ("valid") log events and count of any - * un-deserializable ("invalid") log events within the range; or null if any log event in the - * range doesn't exist (e.g., the range exceeds the number of log events in the file). + * un-deserializable ("invalid") log events within the range; */ - buildIdx(beginIdx: number, endIdx: number): Nullable; + build(): LogEventCount; /** - * Sets options for the decoder. + * Sets formatting options. Changes are efficient and do not require rebuilding existing + * log events. * * @param options * @return Whether the options were successfully set. */ - setDecoderOptions(options: DecoderOptionsType): boolean; + setFormatterOptions(options: DecoderOptions): boolean; /** - * Decodes the log events in the range `[beginIdx, endIdx)`. + * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log event + * indices or filtered log event indices based on the flag `useFilteredIndices`. * * @param beginIdx * @param endIdx + * @param useFilteredIndices Whether to decode from the filtered or unfiltered log events array. * @return The decoded log events on success or null if any log event in the range doesn't exist * (e.g., the range exceeds the number of log events in the file). */ - decode(beginIdx: number, endIdx: number): Nullable; + decodeRange(BeginIdx: number, EndIdx: number, useFilteredIndices: boolean): Nullable; } /** @@ -81,7 +88,9 @@ export {LOG_EVENT_FILE_END_IDX}; export type { Decoder, DecodeResultType, - DecoderOptionsType, - JsonlDecoderOptionsType, + BuildOptions, + DecoderOptions, + JsonlDecoderOptions, + JsonlBuildOptions, LogEventCount, }; diff --git a/new-log-viewer/src/typings/formatters.ts b/new-log-viewer/src/typings/formatters.ts index 0f559d82..49397cd2 100644 --- a/new-log-viewer/src/typings/formatters.ts +++ b/new-log-viewer/src/typings/formatters.ts @@ -21,28 +21,19 @@ import {JsonObject} from "./js"; * inserts a newline as the last character. * - `` - Any specifier besides those above indicate a key for a kv-pair, if said kv-pair * exists in a given log event. - * @property timestampKey The key of the kv-pair that contains the authoritative timestamp for - * every log event. */ interface LogbackFormatterOptionsType { formatString: string, - timestampKey: string, } type FormatterOptionsType = LogbackFormatterOptionsType; -interface TimestampAndMessageType { - timestamp: number, - message: string, -} - interface Formatter { - formatLogEvent: (jsonObject: JsonObject) => TimestampAndMessageType + formatLogEvent: (logEvent: JsonLogEvent) => string } export type { Formatter, FormatterOptionsType, LogbackFormatterOptionsType, - TimestampAndMessageType, }; diff --git a/new-log-viewer/src/typings/worker.ts b/new-log-viewer/src/typings/worker.ts index 82b186c0..cdf3f1f3 100644 --- a/new-log-viewer/src/typings/worker.ts +++ b/new-log-viewer/src/typings/worker.ts @@ -1,4 +1,4 @@ -import {DecoderOptionsType} from "./decoders"; +import {DecoderOptions} from "./decoders"; import {LOG_LEVEL} from "./logs"; @@ -52,17 +52,18 @@ enum WORKER_RESP_CODE { type WorkerReqMap = { [WORKER_REQ_CODE.EXPORT_LOG]: { - decoderOptions: DecoderOptionsType + DecoderOptions: DecoderOptions } [WORKER_REQ_CODE.LOAD_FILE]: { fileSrc: FileSrcType, pageSize: number, cursor: CursorType, - decoderOptions: DecoderOptionsType + buildOptions: BuildOptions + DecoderOptions: DecoderOptions }, [WORKER_REQ_CODE.LOAD_PAGE]: { cursor: CursorType, - decoderOptions?: DecoderOptionsType + DecoderOptions?: DecoderOptions }, }; diff --git a/new-log-viewer/src/utils/config.ts b/new-log-viewer/src/utils/config.ts index 3dd16001..995a486d 100644 --- a/new-log-viewer/src/utils/config.ts +++ b/new-log-viewer/src/utils/config.ts @@ -6,7 +6,7 @@ import { LOCAL_STORAGE_KEY, THEME_NAME, } from "../typings/config"; -import {DecoderOptionsType} from "../typings/decoders"; +import {DecoderOptions} from "../typings/decoders"; const MAX_PAGE_SIZE = 1_000_000; @@ -122,7 +122,7 @@ const getConfig = (key: T): ConfigMap[T] => { timestampKey: window.localStorage.getItem( LOCAL_STORAGE_KEY.DECODER_OPTIONS_TIMESTAMP_KEY ), - } as DecoderOptionsType; + } as DecoderOptions; break; case CONFIG_KEY.THEME: throw new Error(`"${key}" cannot be managed using these utilities.`); From f409855bbb18a1e73380d0916bd966149cf426db Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 18:56:05 +0000 Subject: [PATCH 02/23] remove uneccesary format changes --- .../src/contexts/StateContextProvider.tsx | 6 ++--- new-log-viewer/src/services/LogFileManager.ts | 22 +++++++++---------- new-log-viewer/src/services/MainWorker.ts | 8 +++---- .../src/services/decoders/JsonlDecoder.ts | 11 +++++----- new-log-viewer/src/typings/config.ts | 6 ++--- new-log-viewer/src/typings/decoders.ts | 10 ++++----- new-log-viewer/src/typings/worker.ts | 9 ++++---- new-log-viewer/src/utils/config.ts | 4 ++-- 8 files changed, 36 insertions(+), 40 deletions(-) diff --git a/new-log-viewer/src/contexts/StateContextProvider.tsx b/new-log-viewer/src/contexts/StateContextProvider.tsx index 24f37b25..c6e07803 100644 --- a/new-log-viewer/src/contexts/StateContextProvider.tsx +++ b/new-log-viewer/src/contexts/StateContextProvider.tsx @@ -210,7 +210,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { workerPostReq( mainWorkerRef.current, WORKER_REQ_CODE.EXPORT_LOG, - {DecoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS)} + {decoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS)} ); }, [ numEvents, @@ -232,7 +232,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { fileSrc: fileSrc, pageSize: getConfig(CONFIG_KEY.PAGE_SIZE), cursor: cursor, - DecoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), + decoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), }); setExportProgress(STATE_DEFAULT.exportProgress); @@ -248,7 +248,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { } workerPostReq(mainWorkerRef.current, WORKER_REQ_CODE.LOAD_PAGE, { cursor: {code: CURSOR_CODE.PAGE_NUM, args: {pageNum: newPageNum}}, - DecoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), + decoderOptions: getConfig(CONFIG_KEY.DECODER_OPTIONS), }); }; diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index ab4c96a7..6ebb4033 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -1,7 +1,6 @@ import { Decoder, - DecoderOptions, - LOG_EVENT_FILE_END_IDX, + DecoderOptionsType, } from "../typings/decoders"; import {MAX_V8_STRING_LENGTH} from "../typings/js"; import { @@ -74,10 +73,10 @@ class LogFileManager { this.#pageSize = pageSize; this.#decoder = decoder; - // Build index for the entire file - const buildIdxResult = decoder.buildIdx(0, LOG_EVENT_FILE_END_IDX); - if (null !== buildIdxResult && 0 < buildIdxResult.numInvalidEvents) { - console.error("Invalid events found in decoder.buildIdx():", buildIdxResult); + // Build index for the entire file. + const buildResult = decoder.build(); + if (null !== buildResult && 0 < buildResult.numInvalidEvents) { + console.error("Invalid events found in decoder.buildIdx():", buildResult); } this.#numEvents = decoder.getEstimatedNumEvents(); @@ -104,11 +103,10 @@ class LogFileManager { static async create ( fileSrc: FileSrcType, pageSize: number, - buildOptions: BuildOptions, - DecoderOptions: DecoderOptions, + decoderOptions: DecoderOptionsType, ): Promise { const {fileName, fileData} = await loadFile(fileSrc); - const decoder = await LogFileManager.#initDecoder(fileName, fileData, buildOptions, DecoderOptions); + const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions); return new LogFileManager(decoder, fileName, pageSize); } @@ -125,11 +123,11 @@ class LogFileManager { static async #initDecoder ( fileName: string, fileData: Uint8Array, - DecoderOptions: DecoderOptions + decoderOptions: DecoderOptionsType ): Promise { let decoder: Decoder; if (fileName.endsWith(".jsonl")) { - decoder = new JsonlDecoder(fileData, DecoderOptions); + decoder = new JsonlDecoder(fileData, decoderOptions); } else if (fileName.endsWith(".clp.zst")) { decoder = await ClpIrDecoder.create(fileData); } else { @@ -150,7 +148,7 @@ class LogFileManager { * * @param options */ - setFormatterOptions (options: DecoderOptions) { + setFormatterOptions (options: DecoderOptionsType) { this.#decoder.setFormatterOptions(options); } diff --git a/new-log-viewer/src/services/MainWorker.ts b/new-log-viewer/src/services/MainWorker.ts index 0f2dd839..2435c8ce 100644 --- a/new-log-viewer/src/services/MainWorker.ts +++ b/new-log-viewer/src/services/MainWorker.ts @@ -49,8 +49,8 @@ onmessage = async (ev: MessageEvent) => { if (null === LOG_FILE_MANAGER) { throw new Error("Log file manager hasn't been initialized"); } - if ("undefined" !== typeof args.DecoderOptions) { - LOG_FILE_MANAGER.setDecoderOptions(args.DecoderOptions); + if ("undefined" !== typeof args.decoderOptions) { + LOG_FILE_MANAGER.setFormatterOptions(args.decoderOptions); } let decodedEventIdx = 0; @@ -84,8 +84,8 @@ onmessage = async (ev: MessageEvent) => { if (null === LOG_FILE_MANAGER) { throw new Error("Log file manager hasn't been initialized"); } - if ("undefined" !== typeof args.DecoderOptions) { - LOG_FILE_MANAGER.setDecoderOptions(args.DecoderOptions); + if ("undefined" !== typeof args.decoderOptions) { + LOG_FILE_MANAGER.setFormatterOptions(args.decoderOptions); } postResp( WORKER_RESP_CODE.PAGE_DATA, diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index 89626f80..b4f062bb 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -2,7 +2,6 @@ import {Nullable} from "../../typings/common"; import { Decoder, DecodeResultType, - JsonlBuildOptions, JsonlDecoderOptions, LogEventCount, } from "../../typings/decoders"; @@ -53,14 +52,14 @@ class JsonlDecoder implements Decoder { /** * @param dataArray - * @param DecoderOptions + * @param decoderOptions * @throws {Error} if the initial decoder options are erroneous. */ - constructor (dataArray: Uint8Array, DecoderOptions: JsonlDecoderOptions) { + constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptions) { this.#dataArray = dataArray; - this.#logLevelKey = DecoderOptions.logLevelKey; - this.#timestampKey = DecoderOptions.timestampKey; - this.#formatter = new LogbackFormatter({formatString: DecoderOptions.formatString}); + this.#logLevelKey = decoderOptions.logLevelKey; + this.#timestampKey = decoderOptions.timestampKey; + this.#formatter = new LogbackFormatter({formatString: decoderOptions.formatString}); } getEstimatedNumEvents (): number { diff --git a/new-log-viewer/src/typings/config.ts b/new-log-viewer/src/typings/config.ts index f6e9471c..7f80b3c5 100644 --- a/new-log-viewer/src/typings/config.ts +++ b/new-log-viewer/src/typings/config.ts @@ -1,4 +1,4 @@ -import {JsonlDecoderOptions} from "./decoders"; +import {JsonlDecoderOptionsType} from "./decoders"; enum THEME_NAME { @@ -8,7 +8,7 @@ enum THEME_NAME { } enum CONFIG_KEY { - DECODER_OPTIONS = "DecoderOptions", + DECODER_OPTIONS = "decoderOptions", THEME = "theme", PAGE_SIZE = "pageSize", } @@ -24,7 +24,7 @@ enum LOCAL_STORAGE_KEY { /* eslint-enable @typescript-eslint/prefer-literal-enum-member */ type ConfigMap = { - [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptions, + [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptionsType, [CONFIG_KEY.THEME]: THEME_NAME, [CONFIG_KEY.PAGE_SIZE]: number, }; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 042bc5eb..815babb0 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -13,7 +13,7 @@ interface LogEventCount { * @property logLevelKey The key of the kv-pair that contains the log level in every record. * @property timestampKey The key of the kv-pair that contains the timestamp in every record. */ -interface JsonlDecoderOptions { +interface JsonlDecoderOptionsType { formatString: string, logLevelKey: string, timestampKey: string, @@ -25,7 +25,7 @@ interface JsonlBuildOptions { timestampKey: string, } -type DecoderOptions = JsonlDecoderOptions; +type DecoderOptionsType = JsonlDecoderOptionsType; type BuildOptions = JsonlBuildOptions; @@ -63,7 +63,7 @@ interface Decoder { * @param options * @return Whether the options were successfully set. */ - setFormatterOptions(options: DecoderOptions): boolean; + setFormatterOptions(options: DecoderOptionsType): boolean; /** * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log event @@ -89,8 +89,8 @@ export type { Decoder, DecodeResultType, BuildOptions, - DecoderOptions, - JsonlDecoderOptions, + DecoderOptionsType, + JsonlDecoderOptionsType, JsonlBuildOptions, LogEventCount, }; diff --git a/new-log-viewer/src/typings/worker.ts b/new-log-viewer/src/typings/worker.ts index cdf3f1f3..83ef5ef3 100644 --- a/new-log-viewer/src/typings/worker.ts +++ b/new-log-viewer/src/typings/worker.ts @@ -1,4 +1,4 @@ -import {DecoderOptions} from "./decoders"; +import {DecoderOptionsType} from "./decoders"; import {LOG_LEVEL} from "./logs"; @@ -52,18 +52,17 @@ enum WORKER_RESP_CODE { type WorkerReqMap = { [WORKER_REQ_CODE.EXPORT_LOG]: { - DecoderOptions: DecoderOptions + DecoderOptions: DecoderOptionsType } [WORKER_REQ_CODE.LOAD_FILE]: { fileSrc: FileSrcType, pageSize: number, cursor: CursorType, - buildOptions: BuildOptions - DecoderOptions: DecoderOptions + DecoderOptions: DecoderOptionsType }, [WORKER_REQ_CODE.LOAD_PAGE]: { cursor: CursorType, - DecoderOptions?: DecoderOptions + DecoderOptions?: DecoderOptionsType }, }; diff --git a/new-log-viewer/src/utils/config.ts b/new-log-viewer/src/utils/config.ts index 995a486d..3dd16001 100644 --- a/new-log-viewer/src/utils/config.ts +++ b/new-log-viewer/src/utils/config.ts @@ -6,7 +6,7 @@ import { LOCAL_STORAGE_KEY, THEME_NAME, } from "../typings/config"; -import {DecoderOptions} from "../typings/decoders"; +import {DecoderOptionsType} from "../typings/decoders"; const MAX_PAGE_SIZE = 1_000_000; @@ -122,7 +122,7 @@ const getConfig = (key: T): ConfigMap[T] => { timestampKey: window.localStorage.getItem( LOCAL_STORAGE_KEY.DECODER_OPTIONS_TIMESTAMP_KEY ), - } as DecoderOptions; + } as DecoderOptionsType; break; case CONFIG_KEY.THEME: throw new Error(`"${key}" cannot be managed using these utilities.`); From 3343510ab37ecf0173598b7552b7f0d0b77dbfbb Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 19:01:08 +0000 Subject: [PATCH 03/23] more unneceesary changes --- new-log-viewer/src/services/LogFileManager.ts | 4 ++-- new-log-viewer/src/services/MainWorker.ts | 2 +- new-log-viewer/src/services/decoders/JsonlDecoder.ts | 4 ++-- new-log-viewer/src/typings/decoders.ts | 9 --------- new-log-viewer/src/typings/worker.ts | 6 +++--- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index 6ebb4033..b38c7a83 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -97,7 +97,7 @@ class LogFileManager { * @param fileSrc The source of the file to load. This can be a string representing a URL, or a * File object. * @param pageSize Page size for setting up pagination. - * @param DecoderOptions Initial decoder options. + * @param decoderOptions Initial decoder options. * @return A Promise that resolves to the created LogFileManager instance. */ static async create ( @@ -116,7 +116,7 @@ class LogFileManager { * * @param fileName * @param fileData - * @param DecoderOptions Initial decoder options. + * @param decoderOptions Initial decoder options. * @return The constructed decoder. * @throws {Error} if no decoder supports a file with the given extension. */ diff --git a/new-log-viewer/src/services/MainWorker.ts b/new-log-viewer/src/services/MainWorker.ts index 2435c8ce..4fe43c8d 100644 --- a/new-log-viewer/src/services/MainWorker.ts +++ b/new-log-viewer/src/services/MainWorker.ts @@ -67,7 +67,7 @@ onmessage = async (ev: MessageEvent) => { LOG_FILE_MANAGER = await LogFileManager.create( args.fileSrc, args.pageSize, - args.DecoderOptions + args.decoderOptions ); postResp(WORKER_RESP_CODE.LOG_FILE_INFO, { diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index b4f062bb..508bc81c 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -2,7 +2,7 @@ import {Nullable} from "../../typings/common"; import { Decoder, DecodeResultType, - JsonlDecoderOptions, + JsonlDecoderOptionsType, LogEventCount, } from "../../typings/decoders"; import dayjs, { Dayjs } from "dayjs"; @@ -55,7 +55,7 @@ class JsonlDecoder implements Decoder { * @param decoderOptions * @throws {Error} if the initial decoder options are erroneous. */ - constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptions) { + constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { this.#dataArray = dataArray; this.#logLevelKey = decoderOptions.logLevelKey; this.#timestampKey = decoderOptions.timestampKey; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 815babb0..491304e1 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -19,16 +19,8 @@ interface JsonlDecoderOptionsType { timestampKey: string, } -interface JsonlBuildOptions { - formatString: string, - logLevelKey: string, - timestampKey: string, -} - type DecoderOptionsType = JsonlDecoderOptionsType; -type BuildOptions = JsonlBuildOptions; - /** * Type of the decoded log event. We use an array rather than object so that it's easier to return * results from WASM-based decoders. @@ -91,6 +83,5 @@ export type { BuildOptions, DecoderOptionsType, JsonlDecoderOptionsType, - JsonlBuildOptions, LogEventCount, }; diff --git a/new-log-viewer/src/typings/worker.ts b/new-log-viewer/src/typings/worker.ts index 83ef5ef3..82b186c0 100644 --- a/new-log-viewer/src/typings/worker.ts +++ b/new-log-viewer/src/typings/worker.ts @@ -52,17 +52,17 @@ enum WORKER_RESP_CODE { type WorkerReqMap = { [WORKER_REQ_CODE.EXPORT_LOG]: { - DecoderOptions: DecoderOptionsType + decoderOptions: DecoderOptionsType } [WORKER_REQ_CODE.LOAD_FILE]: { fileSrc: FileSrcType, pageSize: number, cursor: CursorType, - DecoderOptions: DecoderOptionsType + decoderOptions: DecoderOptionsType }, [WORKER_REQ_CODE.LOAD_PAGE]: { cursor: CursorType, - DecoderOptions?: DecoderOptionsType + decoderOptions?: DecoderOptionsType }, }; From 434f2f8f0cf0a19fe4e0dd4aadfd1d3c6dfdb404 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 19:58:11 +0000 Subject: [PATCH 04/23] add missing functions --- new-log-viewer/src/services/LogFileManager.ts | 4 +-- .../src/services/decoders/ClpIrDecoder.ts | 15 +++++++++- .../src/services/decoders/JsonlDecoder.ts | 29 +++++++++++++++++-- new-log-viewer/src/typings/decoders.ts | 17 +++++++++++ new-log-viewer/src/typings/logs.ts | 5 ++++ 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index b38c7a83..cee5a8fd 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -75,7 +75,7 @@ class LogFileManager { // Build index for the entire file. const buildResult = decoder.build(); - if (null !== buildResult && 0 < buildResult.numInvalidEvents) { + if (0 < buildResult.numInvalidEvents) { console.error("Invalid events found in decoder.buildIdx():", buildResult); } @@ -103,7 +103,7 @@ class LogFileManager { static async create ( fileSrc: FileSrcType, pageSize: number, - decoderOptions: DecoderOptionsType, + decoderOptions: DecoderOptionsType ): Promise { const {fileName, fileData} = await loadFile(fileSrc); const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions); diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index bc58d543..891aed7d 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -7,7 +7,7 @@ import { LOG_EVENT_FILE_END_IDX, LogEventCount, } from "../../typings/decoders"; - +import {LogLevelFilter} from "../../typings/logs"; class ClpIrDecoder implements Decoder { #streamReader: ClpIrStreamReader; @@ -32,6 +32,19 @@ class ClpIrDecoder implements Decoder { return this.#streamReader.getNumEventsBuffered(); } + getFilteredLogEventIndices (): Nullable { + // eslint-disable-next-line no-warning-comments + // TODO: Update this after log level filtering is implemented in clp-ffi-js + return Array.from({length: this.#streamReader.getNumEventsBuffered()}, (_, index) => index); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars, class-methods-use-this + setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { + // eslint-disable-next-line no-warning-comments + // TODO fix this after log level filtering is implemented in clp-ffi-js + return false; + } + build (): LogEventCount { return { numInvalidEvents: 0, diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index 508bc81c..51ce149c 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -15,6 +15,7 @@ import { import { INVALID_TIMESTAMP_VALUE, LOG_LEVEL, + LogLevelFilter, } from "../../typings/logs"; import LogbackFormatter from "../formatters/LogbackFormatter"; @@ -44,7 +45,7 @@ class JsonlDecoder implements Decoder { #logEvents: JsonLogEvent[] = []; - #filteredLogEventIndices: number[] = []; + #filteredLogEventIndices: Nullable = null; #invalidLogEventIdxToRawLine: Map = new Map(); @@ -70,6 +71,11 @@ class JsonlDecoder implements Decoder { return this.#filteredLogEventIndices; } + setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { + this.#filterLogs(logLevelFilter); + return true; + } + build (): LogEventCount { this.#deserialize(); @@ -81,7 +87,7 @@ class JsonlDecoder implements Decoder { }; } - setFormatterOptions (options: JsonlDecoderOptions): boolean { + setFormatterOptions (options: JsonlDecoderOptionsType): boolean { this.#formatter = new LogbackFormatter({formatString: options.formatString}); return true; } @@ -269,6 +275,25 @@ class JsonlDecoder implements Decoder { return dayjsTimestamp; } + + /** + * Computes and saves the indices of the log events that match the log level filter. + * + * @param logLevelFilter + */ + #filterLogs (logLevelFilter: LogLevelFilter) { + this.#filteredLogEventIndices = null; + + if (null === logLevelFilter) { + return; + } + + this.#logEvents.forEach((logEvent, index) => { + if (logLevelFilter.includes(logEvent.level)) { + (this.#filteredLogEventIndices as number[]).push(index); + } + }); + } } export default JsonlDecoder; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 491304e1..a99999df 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -1,5 +1,9 @@ import {Nullable} from "./common"; +import { + LogLevelFilter, +} from "./logs" + interface LogEventCount { numValidEvents: number, @@ -41,6 +45,19 @@ interface Decoder { */ getEstimatedNumEvents(): number; + /** + * @return Indices of the filtered events. + */ + getFilteredLogEventIndices(): Nullable; + + /** + * Sets the log level filter for the decoder. + * + * @param logLevelFilter + * @return Whether the filter was successfully set. + */ + setLogLevelFilter(logLevelFilter: LogLevelFilter): boolean + /** * Deserializes all log events in the file. * @return Count of the successfully deserialized ("valid") log events and count of any diff --git a/new-log-viewer/src/typings/logs.ts b/new-log-viewer/src/typings/logs.ts index c3154b29..2dacafff 100644 --- a/new-log-viewer/src/typings/logs.ts +++ b/new-log-viewer/src/typings/logs.ts @@ -1,3 +1,5 @@ +import {Nullable} from "./common"; + enum LOG_LEVEL { NONE = 0, TRACE, @@ -8,8 +10,11 @@ enum LOG_LEVEL { FATAL } +type LogLevelFilter = Nullable; + const INVALID_TIMESTAMP_VALUE = 0; +export type {LogLevelFilter}; export { INVALID_TIMESTAMP_VALUE, LOG_LEVEL, From 33da060fbfbc6bb143160e1ce466b76396450268 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 20:17:07 +0000 Subject: [PATCH 05/23] latest --- .../src/services/decoders/ClpIrDecoder.ts | 4 ++-- .../src/services/decoders/JsonlDecoder.ts | 17 +++++++++++++---- .../src/services/formatters/LogbackFormatter.ts | 2 -- new-log-viewer/src/typings/formatters.ts | 1 + 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index 891aed7d..c88b8b13 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -35,13 +35,13 @@ class ClpIrDecoder implements Decoder { getFilteredLogEventIndices (): Nullable { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js - return Array.from({length: this.#streamReader.getNumEventsBuffered()}, (_, index) => index); + return null } // eslint-disable-next-line @typescript-eslint/no-unused-vars, class-methods-use-this setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { // eslint-disable-next-line no-warning-comments - // TODO fix this after log level filtering is implemented in clp-ffi-js + // TODO: Update this after log level filtering is implemented in clp-ffi-js return false; } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index 51ce149c..94a38e06 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -97,8 +97,16 @@ class JsonlDecoder implements Decoder { endIdx: number, useFilteredIndices: boolean, ): Nullable { + + if (useFilteredIndices && this.#filteredLogEventIndices === null) { + return null; + } + + // Prevents typescript potential null warning. + const filteredLogEventIndices: number[] = this.#filteredLogEventIndices as number[]; + const length: number = useFilteredIndices ? - this.#filteredLogEventIndices.length : + filteredLogEventIndices.length : this.#logEvents.length; if (0 > beginIdx || length < endIdx) { @@ -109,7 +117,7 @@ class JsonlDecoder implements Decoder { // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on // every iteration. const results: DecodeResultType[] = []; - for (let cursorIdx = beginIdx; cursorIdx < endIdx; cursorIdx++) { + for (let i = beginIdx; i < endIdx; i++) { let timestamp: number; let message: string; let logLevel: LOG_LEVEL; @@ -118,8 +126,8 @@ class JsonlDecoder implements Decoder { // can be undefined, but it shouldn't be since we performed a bounds check at the // beginning of the method. const logEventIdx: number = useFilteredIndices ? - (this.#filteredLogEventIndices[cursorIdx] as number) : - cursorIdx; + (filteredLogEventIndices[i] as number) : + i; if (this.#invalidLogEventIdxToRawLine.has(logEventIdx)) { timestamp = INVALID_TIMESTAMP_VALUE; @@ -288,6 +296,7 @@ class JsonlDecoder implements Decoder { return; } + this.#filteredLogEventIndices = []; this.#logEvents.forEach((logEvent, index) => { if (logLevelFilter.includes(logEvent.level)) { (this.#filteredLogEventIndices as number[]).push(index); diff --git a/new-log-viewer/src/services/formatters/LogbackFormatter.ts b/new-log-viewer/src/services/formatters/LogbackFormatter.ts index 82b36e25..89f79a00 100644 --- a/new-log-viewer/src/services/formatters/LogbackFormatter.ts +++ b/new-log-viewer/src/services/formatters/LogbackFormatter.ts @@ -4,10 +4,8 @@ import {Nullable} from "../../typings/common"; import { Formatter, FormatterOptionsType, - TimestampAndMessageType, } from "../../typings/formatters"; import {JsonLogEvent} from "../decoders/JsonlDecoder"; -import {INVALID_TIMESTAMP_VALUE} from "../../typings/logs"; import {JsonObject} from "../../typings/js"; diff --git a/new-log-viewer/src/typings/formatters.ts b/new-log-viewer/src/typings/formatters.ts index 49397cd2..572276a7 100644 --- a/new-log-viewer/src/typings/formatters.ts +++ b/new-log-viewer/src/typings/formatters.ts @@ -1,4 +1,5 @@ import {JsonObject} from "./js"; +import {JsonLogEvent} from "../services/decoders/JsonlDecoder"; /** From e5b6abef2f8f2f4571f3ba26f62bb4dec78fdc91 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 20:28:48 +0000 Subject: [PATCH 06/23] small changes --- .../src/services/decoders/JsonlDecoder.ts | 24 +++++++++---------- new-log-viewer/src/typings/decoders.ts | 5 ++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index 94a38e06..9313d9f8 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -158,7 +158,7 @@ class JsonlDecoder implements Decoder { * Parses each line from the data array as a JSON object and buffers it internally. If a * line cannot be parsed as a JSON object, an error is logged and the line is skipped. * - * NOTE: The data array is freed after the very first run of this method. + * NOTE: The data array(file) is freed after the very first run of this method. */ #deserialize () { if (null === this.#dataArray) { @@ -184,7 +184,7 @@ class JsonlDecoder implements Decoder { } /** - * Parse line into a json log event and add to log events array. If the line contains invalid + * Parse line into a json log event and buffer internally. If the line contains invalid * json, an entry is added to invalid log event map. * * @param line @@ -251,32 +251,32 @@ class JsonlDecoder implements Decoder { } /** - * Parses timestamp into dayjs timestamp. + * Parses timestamp field in log event into dayjs timestamp. * * @param timestampField * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: - * - the timestamp key doesn't exist in the log, or - * - the timestamp's value is not a number. + * 1. the timestamp key doesn't exist in the log + * 2. the timestamp's value is an unsupported type + * 3. the timestamp's value is not a valid dayjs timestamp */ #parseTimestamp (timestampField: JsonValue | undefined): dayjs.Dayjs { - - // If the field is an invalid type, then set the timestamp to INVALID_TIMESTAMP_VALUE. + // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. if (typeof timestampField !== "string" && typeof timestampField !== "number" || // Dayjs library surprisingly thinks undefined is valid date... // Reference: https://day.js.org/docs/en/parse/now#docsNav typeof timestampField === undefined ) { - // INVALID_TIMESTAMP_VALUE is a valid dayjs date. Another potential option is daysjs(null) - // to show `Invalid Date` in UI. + // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is + // `daysjs(null)` to show `Invalid Date` in UI. timestampField = INVALID_TIMESTAMP_VALUE; } const dayjsTimestamp: Dayjs = dayjs.utc(timestampField); - // Note if input is not valid (timestampField = "deadbeef"), this can produce a non-valid - // timestamp and will show up in UI as `Invalid Date`. Here we modify invalid dates to - // INVALID_TIMESTAMP_VALUE. + // Note if input is not valid (ex. timestampField = "deadbeef"), this can produce a + // non-valid timestamp and will show up in UI as `Invalid Date`. Current behaviour is to + // modify invalid dates to `INVALID_TIMESTAMP_VALUE`. if (false === dayjsTimestamp.isValid()) { dayjsTimestamp == dayjs.utc(INVALID_TIMESTAMP_VALUE) } diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index a99999df..b6c994e1 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -66,8 +66,8 @@ interface Decoder { build(): LogEventCount; /** - * Sets formatting options. Changes are efficient and do not require rebuilding existing - * log events. + * Sets formatting options. Decoders support changing formatting without rebuilding + * existing log events. * * @param options * @return Whether the options were successfully set. @@ -97,7 +97,6 @@ export {LOG_EVENT_FILE_END_IDX}; export type { Decoder, DecodeResultType, - BuildOptions, DecoderOptionsType, JsonlDecoderOptionsType, LogEventCount, From 2ecd5b14171d14f0eebe38eccada4bc78f085da2 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 21:01:52 +0000 Subject: [PATCH 07/23] fix linting --- .../src/services/decoders/ClpIrDecoder.ts | 12 +- .../src/services/decoders/JsonlDecoder.ts | 311 ------------------ .../services/formatters/LogbackFormatter.ts | 9 +- new-log-viewer/src/typings/decoders.ts | 15 +- new-log-viewer/src/typings/formatters.ts | 3 +- new-log-viewer/src/typings/logs.ts | 1 + 6 files changed, 25 insertions(+), 326 deletions(-) delete mode 100644 new-log-viewer/src/services/decoders/JsonlDecoder.ts diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index c88b8b13..60c50555 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -9,6 +9,7 @@ import { } from "../../typings/decoders"; import {LogLevelFilter} from "../../typings/logs"; + class ClpIrDecoder implements Decoder { #streamReader: ClpIrStreamReader; @@ -32,10 +33,11 @@ class ClpIrDecoder implements Decoder { return this.#streamReader.getNumEventsBuffered(); } + // eslint-disable-next-line class-methods-use-this getFilteredLogEventIndices (): Nullable { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js - return null + return null; } // eslint-disable-next-line @typescript-eslint/no-unused-vars, class-methods-use-this @@ -57,7 +59,13 @@ class ClpIrDecoder implements Decoder { return true; } - decodeRange (beginIdx: number, endIdx: number, useFilteredIndices: boolean): Nullable { + + decodeRange ( + beginIdx: number, + endIdx: number, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + useFilteredIndices: boolean + ): Nullable { return this.#streamReader.decodeRange(beginIdx, endIdx); } } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts deleted file mode 100644 index 9313d9f8..00000000 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ /dev/null @@ -1,311 +0,0 @@ -import {Nullable} from "../../typings/common"; -import { - Decoder, - DecodeResultType, - JsonlDecoderOptionsType, - LogEventCount, -} from "../../typings/decoders"; -import dayjs, { Dayjs } from "dayjs"; -import utc from "dayjs/plugin/utc" -import {Formatter} from "../../typings/formatters"; -import { - JsonObject, - JsonValue, -} from "../../typings/js"; -import { - INVALID_TIMESTAMP_VALUE, - LOG_LEVEL, - LogLevelFilter, -} from "../../typings/logs"; -import LogbackFormatter from "../formatters/LogbackFormatter"; - -dayjs.extend(utc); - -/** - * A log event parsed from a JSON log. - */ -interface JsonLogEvent { - timestamp: Dayjs, - level: LOG_LEVEL, - fields: JsonObject -} - -/** - * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for - * properties that are specific to log events (compared to generic JSON records). - */ -class JsonlDecoder implements Decoder { - static #textDecoder = new TextDecoder(); - - #dataArray: Nullable; - - #logLevelKey: string; - - #timestampKey: string; - - #logEvents: JsonLogEvent[] = []; - - #filteredLogEventIndices: Nullable = null; - - #invalidLogEventIdxToRawLine: Map = new Map(); - - #formatter: Formatter; - - /** - * @param dataArray - * @param decoderOptions - * @throws {Error} if the initial decoder options are erroneous. - */ - constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { - this.#dataArray = dataArray; - this.#logLevelKey = decoderOptions.logLevelKey; - this.#timestampKey = decoderOptions.timestampKey; - this.#formatter = new LogbackFormatter({formatString: decoderOptions.formatString}); - } - - getEstimatedNumEvents (): number { - return this.#logEvents.length; - } - - getFilteredLogEventIndices (): Nullable { - return this.#filteredLogEventIndices; - } - - setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { - this.#filterLogs(logLevelFilter); - return true; - } - - build (): LogEventCount { - this.#deserialize(); - - const numInvalidEvents = Array.from(this.#invalidLogEventIdxToRawLine.keys()).length; - - return { - numValidEvents: this.#logEvents.length - numInvalidEvents, - numInvalidEvents: numInvalidEvents, - }; - } - - setFormatterOptions (options: JsonlDecoderOptionsType): boolean { - this.#formatter = new LogbackFormatter({formatString: options.formatString}); - return true; - } - - decodeRange ( - beginIdx: number, - endIdx: number, - useFilteredIndices: boolean, - ): Nullable { - - if (useFilteredIndices && this.#filteredLogEventIndices === null) { - return null; - } - - // Prevents typescript potential null warning. - const filteredLogEventIndices: number[] = this.#filteredLogEventIndices as number[]; - - const length: number = useFilteredIndices ? - filteredLogEventIndices.length : - this.#logEvents.length; - - if (0 > beginIdx || length < endIdx) { - return null; - } - - // eslint-disable-next-line no-warning-comments - // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on - // every iteration. - const results: DecodeResultType[] = []; - for (let i = beginIdx; i < endIdx; i++) { - let timestamp: number; - let message: string; - let logLevel: LOG_LEVEL; - - // Explicit cast since typescript thinks `#filteredLogEventIndices[filteredLogEventIdx]` - // can be undefined, but it shouldn't be since we performed a bounds check at the - // beginning of the method. - const logEventIdx: number = useFilteredIndices ? - (filteredLogEventIndices[i] as number) : - i; - - if (this.#invalidLogEventIdxToRawLine.has(logEventIdx)) { - timestamp = INVALID_TIMESTAMP_VALUE; - message = `${this.#invalidLogEventIdxToRawLine.get(logEventIdx)}\n`; - logLevel = LOG_LEVEL.NONE; - } else { - // Explicit cast since typescript thinks `#logEvents[filteredIdx]` can be undefined, - // but it shouldn't be since the index comes from a class-internal filter. - const logEvent: JsonLogEvent = this.#logEvents[logEventIdx] as JsonLogEvent; - - logLevel = logEvent.level; - message = this.#formatter.formatLogEvent(logEvent); - timestamp = logEvent.timestamp.valueOf(); - } - - results.push([ - message, - timestamp, - logLevel, - logEventIdx + 1, - ]); - } - - return results; - } - - /** - * Parses each line from the data array as a JSON object and buffers it internally. If a - * line cannot be parsed as a JSON object, an error is logged and the line is skipped. - * - * NOTE: The data array(file) is freed after the very first run of this method. - */ - #deserialize () { - if (null === this.#dataArray) { - return; - } - - const text = JsonlDecoder.#textDecoder.decode(this.#dataArray); - let beginIdx: number = 0; - while (beginIdx < text.length) { - const endIdx = text.indexOf("\n", beginIdx); - const line = (-1 === endIdx) ? - text.substring(beginIdx) : - text.substring(beginIdx, endIdx); - - beginIdx = (-1 === endIdx) ? - text.length : - endIdx + 1; - - this.#parseJson(line); - } - - this.#dataArray = null; - } - - /** - * Parse line into a json log event and buffer internally. If the line contains invalid - * json, an entry is added to invalid log event map. - * - * @param line - */ - #parseJson (line: string) { - try { - const fields = JSON.parse(line) as JsonValue; - if (!this.#isJsonObject(fields)) { - throw new Error("Unexpected non-object."); - } - this.#logEvents.push({ - fields: fields, - level: this.#parseLogLevel(fields[this.#logLevelKey]), - timestamp: this.#parseTimestamp(fields[this.#timestampKey]), - }); - } catch (e) { - if (0 === line.length) { - return; - } - console.error(e, line); - const currentLogEventIdx = this.#logEvents.length; - this.#invalidLogEventIdxToRawLine.set(currentLogEventIdx, line); - this.#logEvents.push({ - fields: {}, - level: LOG_LEVEL.NONE, - timestamp: dayjs.utc(INVALID_TIMESTAMP_VALUE), - }); - } - } - - /** - * Narrows input to JsonObject if valid type. - * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates - * - * @param fields - * @return Whether type is JsonObject. - */ - #isJsonObject(fields: JsonValue): fields is JsonObject { - return "object" === typeof fields; - } - - /** - * Maps the log level field to a log level value. - * - * @param logLevelField Field in log event indexed by log level key. - * @return Log level value. - */ - #parseLogLevel (logLevelField: JsonValue | undefined): number { - let logLevelValue = LOG_LEVEL.NONE; - - if ("undefined" === typeof logLevelField) { - return logLevelValue; - } - - const logLevelName = "object" === typeof logLevelField ? - JSON.stringify(logLevelField) : - String(logLevelField); - - if (logLevelName.toUpperCase() in LOG_LEVEL) { - logLevelValue = LOG_LEVEL[logLevelName.toUpperCase() as keyof typeof LOG_LEVEL]; - } - - return logLevelValue; - } - - /** - * Parses timestamp field in log event into dayjs timestamp. - * - * @param timestampField - * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: - * 1. the timestamp key doesn't exist in the log - * 2. the timestamp's value is an unsupported type - * 3. the timestamp's value is not a valid dayjs timestamp - */ - #parseTimestamp (timestampField: JsonValue | undefined): dayjs.Dayjs { - // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. - if (typeof timestampField !== "string" && - typeof timestampField !== "number" || - // Dayjs library surprisingly thinks undefined is valid date... - // Reference: https://day.js.org/docs/en/parse/now#docsNav - typeof timestampField === undefined - ) { - // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is - // `daysjs(null)` to show `Invalid Date` in UI. - timestampField = INVALID_TIMESTAMP_VALUE; - } - - const dayjsTimestamp: Dayjs = dayjs.utc(timestampField); - - // Note if input is not valid (ex. timestampField = "deadbeef"), this can produce a - // non-valid timestamp and will show up in UI as `Invalid Date`. Current behaviour is to - // modify invalid dates to `INVALID_TIMESTAMP_VALUE`. - if (false === dayjsTimestamp.isValid()) { - dayjsTimestamp == dayjs.utc(INVALID_TIMESTAMP_VALUE) - } - - return dayjsTimestamp; - } - - /** - * Computes and saves the indices of the log events that match the log level filter. - * - * @param logLevelFilter - */ - #filterLogs (logLevelFilter: LogLevelFilter) { - this.#filteredLogEventIndices = null; - - if (null === logLevelFilter) { - return; - } - - this.#filteredLogEventIndices = []; - this.#logEvents.forEach((logEvent, index) => { - if (logLevelFilter.includes(logEvent.level)) { - (this.#filteredLogEventIndices as number[]).push(index); - } - }); - } -} - -export default JsonlDecoder; -export type { - JsonLogEvent, -} diff --git a/new-log-viewer/src/services/formatters/LogbackFormatter.ts b/new-log-viewer/src/services/formatters/LogbackFormatter.ts index 89f79a00..7761b1b3 100644 --- a/new-log-viewer/src/services/formatters/LogbackFormatter.ts +++ b/new-log-viewer/src/services/formatters/LogbackFormatter.ts @@ -5,8 +5,8 @@ import { Formatter, FormatterOptionsType, } from "../../typings/formatters"; -import {JsonLogEvent} from "../decoders/JsonlDecoder"; import {JsonObject} from "../../typings/js"; +import {JsonLogEvent} from "../decoders/JsonlDecoder/utils"; /** @@ -59,9 +59,10 @@ class LogbackFormatter implements Formatter { */ formatLogEvent (logEvent: JsonLogEvent): string { const {fields, timestamp} = logEvent; - let formatStringWithTimestamp: string = this.#formatTimestamp(timestamp, this.#formatString); - let message = this.#formatVariables(formatStringWithTimestamp, fields); - return message + const formatStringWithTimestamp: string = + this.#formatTimestamp(timestamp, this.#formatString); + const message = this.#formatVariables(formatStringWithTimestamp, fields); + return message; } /** diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index b6c994e1..f7e728a7 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -1,8 +1,5 @@ import {Nullable} from "./common"; - -import { - LogLevelFilter, -} from "./logs" +import {LogLevelFilter} from "./logs"; interface LogEventCount { @@ -60,6 +57,7 @@ interface Decoder { /** * Deserializes all log events in the file. + * * @return Count of the successfully deserialized ("valid") log events and count of any * un-deserializable ("invalid") log events within the range; */ @@ -75,8 +73,8 @@ interface Decoder { setFormatterOptions(options: DecoderOptionsType): boolean; /** - * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log event - * indices or filtered log event indices based on the flag `useFilteredIndices`. + * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log + * event indices or filtered log event indices based on the flag `useFilteredIndices`. * * @param beginIdx * @param endIdx @@ -84,7 +82,10 @@ interface Decoder { * @return The decoded log events on success or null if any log event in the range doesn't exist * (e.g., the range exceeds the number of log events in the file). */ - decodeRange(BeginIdx: number, EndIdx: number, useFilteredIndices: boolean): Nullable; + decodeRange(beginIdx: number, + endIdx: number, + useFilteredIndices: boolean + ): Nullable; } /** diff --git a/new-log-viewer/src/typings/formatters.ts b/new-log-viewer/src/typings/formatters.ts index 572276a7..95afaafe 100644 --- a/new-log-viewer/src/typings/formatters.ts +++ b/new-log-viewer/src/typings/formatters.ts @@ -1,5 +1,4 @@ -import {JsonObject} from "./js"; -import {JsonLogEvent} from "../services/decoders/JsonlDecoder"; +import {JsonLogEvent} from "../services/decoders/JsonlDecoder/utils"; /** diff --git a/new-log-viewer/src/typings/logs.ts b/new-log-viewer/src/typings/logs.ts index 2dacafff..11ef5703 100644 --- a/new-log-viewer/src/typings/logs.ts +++ b/new-log-viewer/src/typings/logs.ts @@ -1,5 +1,6 @@ import {Nullable} from "./common"; + enum LOG_LEVEL { NONE = 0, TRACE, From 477fe2340626fd1dd57ce0e9a093889c376494eb Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 21:03:29 +0000 Subject: [PATCH 08/23] add moved file --- .../services/decoders/JsonlDecoder/index.ts | 240 ++++++++++++++++++ .../services/decoders/JsonlDecoder/utils.ts | 100 ++++++++ 2 files changed, 340 insertions(+) create mode 100644 new-log-viewer/src/services/decoders/JsonlDecoder/index.ts create mode 100644 new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts new file mode 100644 index 00000000..346603d4 --- /dev/null +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -0,0 +1,240 @@ +import {Nullable} from "../../../typings/common"; +import { + Decoder, + DecodeResultType, + JsonlDecoderOptionsType, + LogEventCount, +} from "../../../typings/decoders"; +import {Formatter} from "../../../typings/formatters"; +import {JsonValue} from "../../../typings/js"; +import { + INVALID_TIMESTAMP_VALUE, + LOG_LEVEL, + LogLevelFilter, +} from "../../../typings/logs"; +import LogbackFormatter from "../../formatters/LogbackFormatter"; +import { + isJsonObject, + JsonLogEvent, + parseLogLevel, + parseTimestamp, +} from "./utils"; + + +/** + * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for + * properties that are specific to log events (compared to generic JSON records). + */ +class JsonlDecoder implements Decoder { + static #textDecoder = new TextDecoder(); + + #dataArray: Nullable; + + #logLevelKey: string; + + #timestampKey: string; + + #logEvents: JsonLogEvent[] = []; + + #filteredLogEventIndices: Nullable = null; + + #invalidLogEventIdxToRawLine: Map = new Map(); + + #formatter: Formatter; + + /** + * @param dataArray + * @param decoderOptions + * @throws {Error} if the initial decoder options are erroneous. + */ + constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { + this.#dataArray = dataArray; + this.#logLevelKey = decoderOptions.logLevelKey; + this.#timestampKey = decoderOptions.timestampKey; + this.#formatter = new LogbackFormatter({formatString: decoderOptions.formatString}); + } + + getEstimatedNumEvents (): number { + return this.#logEvents.length; + } + + getFilteredLogEventIndices (): Nullable { + return this.#filteredLogEventIndices; + } + + setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { + this.#filterLogs(logLevelFilter); + + return true; + } + + build (): LogEventCount { + this.#deserialize(); + + const numInvalidEvents = Array.from(this.#invalidLogEventIdxToRawLine.keys()).length; + + return { + numValidEvents: this.#logEvents.length - numInvalidEvents, + numInvalidEvents: numInvalidEvents, + }; + } + + setFormatterOptions (options: JsonlDecoderOptionsType): boolean { + this.#formatter = new LogbackFormatter({formatString: options.formatString}); + + return true; + } + + decodeRange ( + beginIdx: number, + endIdx: number, + useFilteredIndices: boolean, + ): Nullable { + if (useFilteredIndices && null === this.#filteredLogEventIndices) { + return null; + } + + // Prevents typescript potential null warning. + const filteredLogEventIndices: number[] = this.#filteredLogEventIndices as number[]; + + const length: number = useFilteredIndices ? + filteredLogEventIndices.length : + this.#logEvents.length; + + if (0 > beginIdx || length < endIdx) { + return null; + } + + const results: DecodeResultType[] = []; + for (let i = beginIdx; i < endIdx; i++) { + // Explicit cast since typescript thinks `#filteredLogEventIndices[filteredLogEventIdx]` + // can be undefined, but it shouldn't be since we performed a bounds check at the + // beginning of the method. + const logEventIdx: number = useFilteredIndices ? + (filteredLogEventIndices[i] as number) : + i; + + results.push(this.#getDecodeResult(logEventIdx)); + } + + return results; + } + + /** + * Retrieves log event using index then decodes into `DecodeResultType`. + * + * @param logEventIdx + * @return + */ + #getDecodeResult = (logEventIdx: number): DecodeResultType => { + let timestamp: number; + let message: string; + let logLevel: LOG_LEVEL; + + // eslint-disable-next-line no-warning-comments + // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on + // every iteration. + if (this.#invalidLogEventIdxToRawLine.has(logEventIdx)) { + timestamp = INVALID_TIMESTAMP_VALUE; + message = `${this.#invalidLogEventIdxToRawLine.get(logEventIdx)}\n`; + logLevel = LOG_LEVEL.NONE; + } else { + // Explicit cast since typescript thinks `#logEvents[filteredIdx]` can be undefined, + // but it shouldn't be since the index comes from a class-internal filter. + const logEvent = this.#logEvents[logEventIdx] as JsonLogEvent; + logLevel = logEvent.level; + message = this.#formatter.formatLogEvent(logEvent); + timestamp = logEvent.timestamp.valueOf(); + } + + return [ + message, + timestamp, + logLevel, + logEventIdx + 1, + ]; + }; + + /** + * Parses each line from the data array as a JSON object and buffers it internally. If a + * line cannot be parsed as a JSON object, an error is logged and the line is skipped. + * + * NOTE: The data array(file) is freed after the very first run of this method. + */ + #deserialize () { + if (null === this.#dataArray) { + return; + } + + const text = JsonlDecoder.#textDecoder.decode(this.#dataArray); + let beginIdx: number = 0; + while (beginIdx < text.length) { + const endIdx = text.indexOf("\n", beginIdx); + const line = (-1 === endIdx) ? + text.substring(beginIdx) : + text.substring(beginIdx, endIdx); + + beginIdx = (-1 === endIdx) ? + text.length : + endIdx + 1; + + this.#parseJson(line); + } + + this.#dataArray = null; + } + + /** + * Parse line into a json log event and buffer internally. If the line contains invalid + * json, an entry is added to invalid log event map. + * + * @param line + */ + #parseJson (line: string) { + try { + const fields = JSON.parse(line) as JsonValue; + if (!isJsonObject(fields)) { + throw new Error("Unexpected non-object."); + } + this.#logEvents.push({ + fields: fields, + level: parseLogLevel(fields[this.#logLevelKey]), + timestamp: parseTimestamp(fields[this.#timestampKey]), + }); + } catch (e) { + if (0 === line.length) { + return; + } + console.error(e, line); + const currentLogEventIdx = this.#logEvents.length; + this.#invalidLogEventIdxToRawLine.set(currentLogEventIdx, line); + this.#logEvents.push({ + fields: {}, + level: LOG_LEVEL.NONE, + timestamp: parseTimestamp(INVALID_TIMESTAMP_VALUE), + }); + } + } + + /** + * Computes and saves the indices of the log events that match the log level filter. + * + * @param logLevelFilter + */ + #filterLogs (logLevelFilter: LogLevelFilter) { + this.#filteredLogEventIndices = null; + + if (null === logLevelFilter) { + return; + } + + this.#filteredLogEventIndices = []; + this.#logEvents.forEach((logEvent, index) => { + if (logLevelFilter.includes(logEvent.level)) { + (this.#filteredLogEventIndices as number[]).push(index); + } + }); + } +} + +export default JsonlDecoder; diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts new file mode 100644 index 00000000..03bd126b --- /dev/null +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -0,0 +1,100 @@ +import dayjs, {Dayjs} from "dayjs"; +import utc from "dayjs/plugin/utc"; + +import { + JsonObject, + JsonValue, +} from "../../../typings/js"; +import { + INVALID_TIMESTAMP_VALUE, + LOG_LEVEL, +} from "../../../typings/logs"; + + +// eslint-disable-next-line import/no-named-as-default-member +dayjs.extend(utc); + +/** + * A log event parsed from a JSON log. + */ +interface JsonLogEvent { + timestamp: Dayjs, + level: LOG_LEVEL, + fields: JsonObject +} + +/** + * Narrows input to JsonObject if valid type. + * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates + * + * @param fields + * @return Whether type is JsonObject. + */ +const isJsonObject = (fields: JsonValue): fields is JsonObject => { + return "object" === typeof fields; +}; + +/** + * Maps the log level field to a log level value. + * + * @param logLevelField Field in log event indexed by log level key. + * @return Log level value. + */ +const parseLogLevel = (logLevelField: JsonValue | undefined): number => { + let logLevelValue = LOG_LEVEL.NONE; + + if ("undefined" === typeof logLevelField) { + return logLevelValue; + } + + const logLevelName = "object" === typeof logLevelField ? + JSON.stringify(logLevelField) : + String(logLevelField); + + if (logLevelName.toUpperCase() in LOG_LEVEL) { + logLevelValue = LOG_LEVEL[logLevelName.toUpperCase() as keyof typeof LOG_LEVEL]; + } + + return logLevelValue; +}; + +/** + * Parses timestamp field in log event into dayjs timestamp. + * + * @param timestampField + * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: + * 1. the timestamp key doesn't exist in the log + * 2. the timestamp's value is an unsupported type + * 3. the timestamp's value is not a valid dayjs timestamp + */ +const parseTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { + // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. + if (("string" !== typeof timestampField && + "number" !== typeof timestampField) || + + // Dayjs library surprisingly thinks undefined is valid date... + // Reference: https://day.js.org/docs/en/parse/now#docsNav + "undefined" === typeof timestampField + ) { + // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is + // `daysjs(null)` to show `Invalid Date` in UI. + timestampField = INVALID_TIMESTAMP_VALUE; + } + + const dayjsTimestamp: Dayjs = dayjs.utc(timestampField); + + // Note if input is not valid (ex. timestampField = "deadbeef"), this can produce a + // non-valid timestamp and will show up in UI as `Invalid Date`. Current behaviour is to + // modify invalid dates to `INVALID_TIMESTAMP_VALUE`. + if (false === dayjsTimestamp.isValid()) { + dayjsTimestamp = dayjs.utc(INVALID_TIMESTAMP_VALUE); + } + + return dayjsTimestamp; +}; +export { + isJsonObject, + parseLogLevel, + parseTimestamp, +}; +export type {JsonLogEvent}; From 17c281a9b813bec5d6d136c337c7f371228deb00 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 21:59:17 +0000 Subject: [PATCH 09/23] small changes --- .../src/services/decoders/ClpIrDecoder.ts | 5 +-- .../services/decoders/JsonlDecoder/index.ts | 36 ++++++++++--------- .../services/decoders/JsonlDecoder/utils.ts | 22 ++++++------ new-log-viewer/src/typings/decoders.ts | 27 +++++++++----- 4 files changed, 51 insertions(+), 39 deletions(-) diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index 60c50555..66513740 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -4,6 +4,7 @@ import {Nullable} from "../../typings/common"; import { Decoder, DecodeResultType, + FilteredLogEventMap, LOG_EVENT_FILE_END_IDX, LogEventCount, } from "../../typings/decoders"; @@ -34,7 +35,7 @@ class ClpIrDecoder implements Decoder { } // eslint-disable-next-line class-methods-use-this - getFilteredLogEventIndices (): Nullable { + getFilteredLogEventMap (): FilteredLogEventMap { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js return null; @@ -64,7 +65,7 @@ class ClpIrDecoder implements Decoder { beginIdx: number, endIdx: number, // eslint-disable-next-line @typescript-eslint/no-unused-vars - useFilteredIndices: boolean + useFilter: boolean ): Nullable { return this.#streamReader.decodeRange(beginIdx, endIdx); } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index 346603d4..671c30a3 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -2,6 +2,7 @@ import {Nullable} from "../../../typings/common"; import { Decoder, DecodeResultType, + FilteredLogEventMap, JsonlDecoderOptionsType, LogEventCount, } from "../../../typings/decoders"; @@ -16,8 +17,8 @@ import LogbackFormatter from "../../formatters/LogbackFormatter"; import { isJsonObject, JsonLogEvent, - parseLogLevel, - parseTimestamp, + LogLevelValue, + DayjsTimestamp, } from "./utils"; @@ -36,7 +37,7 @@ class JsonlDecoder implements Decoder { #logEvents: JsonLogEvent[] = []; - #filteredLogEventIndices: Nullable = null; + #filteredLogEventMap: FilteredLogEventMap = null; #invalidLogEventIdxToRawLine: Map = new Map(); @@ -58,8 +59,8 @@ class JsonlDecoder implements Decoder { return this.#logEvents.length; } - getFilteredLogEventIndices (): Nullable { - return this.#filteredLogEventIndices; + getFilteredLogEventMap (): FilteredLogEventMap { + return this.#filteredLogEventMap; } setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { @@ -88,16 +89,16 @@ class JsonlDecoder implements Decoder { decodeRange ( beginIdx: number, endIdx: number, - useFilteredIndices: boolean, + useFilter: boolean, ): Nullable { - if (useFilteredIndices && null === this.#filteredLogEventIndices) { + if (useFilter && null === this.#filteredLogEventMap) { return null; } // Prevents typescript potential null warning. - const filteredLogEventIndices: number[] = this.#filteredLogEventIndices as number[]; + const filteredLogEventIndices: number[] = this.#filteredLogEventMap as number[]; - const length: number = useFilteredIndices ? + const length: number = useFilter ? filteredLogEventIndices.length : this.#logEvents.length; @@ -110,7 +111,7 @@ class JsonlDecoder implements Decoder { // Explicit cast since typescript thinks `#filteredLogEventIndices[filteredLogEventIdx]` // can be undefined, but it shouldn't be since we performed a bounds check at the // beginning of the method. - const logEventIdx: number = useFilteredIndices ? + const logEventIdx: number = useFilter ? (filteredLogEventIndices[i] as number) : i; @@ -198,8 +199,8 @@ class JsonlDecoder implements Decoder { } this.#logEvents.push({ fields: fields, - level: parseLogLevel(fields[this.#logLevelKey]), - timestamp: parseTimestamp(fields[this.#timestampKey]), + level: LogLevelValue(fields[this.#logLevelKey]), + timestamp: DayjsTimestamp(fields[this.#timestampKey]), }); } catch (e) { if (0 === line.length) { @@ -211,27 +212,28 @@ class JsonlDecoder implements Decoder { this.#logEvents.push({ fields: {}, level: LOG_LEVEL.NONE, - timestamp: parseTimestamp(INVALID_TIMESTAMP_VALUE), + timestamp: DayjsTimestamp(INVALID_TIMESTAMP_VALUE), }); } } /** - * Computes and saves the indices of the log events that match the log level filter. + * Computes the indices of the log events that match the log level filter and + * buffers internally. Sets the indices to null if the filter is null. * * @param logLevelFilter */ #filterLogs (logLevelFilter: LogLevelFilter) { - this.#filteredLogEventIndices = null; + this.#filteredLogEventMap = null; if (null === logLevelFilter) { return; } - this.#filteredLogEventIndices = []; + this.#filteredLogEventMap = []; this.#logEvents.forEach((logEvent, index) => { if (logLevelFilter.includes(logEvent.level)) { - (this.#filteredLogEventIndices as number[]).push(index); + (this.#filteredLogEventMap as number[]).push(index); } }); } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index 03bd126b..7dbf1491 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -24,7 +24,7 @@ interface JsonLogEvent { } /** - * Narrows input to JsonObject if valid type. + * Narrow JSON value to JSON object if compatible. * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates * * @param fields @@ -35,12 +35,12 @@ const isJsonObject = (fields: JsonValue): fields is JsonObject => { }; /** - * Maps the log level field to a log level value. + * Converts JSON log level field into a log level value. * * @param logLevelField Field in log event indexed by log level key. * @return Log level value. */ -const parseLogLevel = (logLevelField: JsonValue | undefined): number => { +const LogLevelValue = (logLevelField: JsonValue | undefined): number => { let logLevelValue = LOG_LEVEL.NONE; if ("undefined" === typeof logLevelField) { @@ -59,7 +59,7 @@ const parseLogLevel = (logLevelField: JsonValue | undefined): number => { }; /** - * Parses timestamp field in log event into dayjs timestamp. + * Converts JSON timestamp field into a dayjs timestamp. * * @param timestampField * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: @@ -67,7 +67,7 @@ const parseLogLevel = (logLevelField: JsonValue | undefined): number => { * 2. the timestamp's value is an unsupported type * 3. the timestamp's value is not a valid dayjs timestamp */ -const parseTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { +const DayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. if (("string" !== typeof timestampField && "number" !== typeof timestampField) || @@ -81,11 +81,11 @@ const parseTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { timestampField = INVALID_TIMESTAMP_VALUE; } - const dayjsTimestamp: Dayjs = dayjs.utc(timestampField); + let dayjsTimestamp: Dayjs = dayjs.utc(timestampField); - // Note if input is not valid (ex. timestampField = "deadbeef"), this can produce a - // non-valid timestamp and will show up in UI as `Invalid Date`. Current behaviour is to - // modify invalid dates to `INVALID_TIMESTAMP_VALUE`. + // Sanitize invalid date to `INVALID_TIMESTAMP_VALUE`. Note if input is not valid + // (ex. timestampField = "deadbeef") and not sanitized, result will be produce a + // non-valid dayjs timestamp and will show up in UI as `Invalid Date`. if (false === dayjsTimestamp.isValid()) { dayjsTimestamp = dayjs.utc(INVALID_TIMESTAMP_VALUE); } @@ -94,7 +94,7 @@ const parseTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { }; export { isJsonObject, - parseLogLevel, - parseTimestamp, + LogLevelValue, + DayjsTimestamp, }; export type {JsonLogEvent}; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index f7e728a7..17d2cc63 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -33,6 +33,12 @@ type DecoderOptionsType = JsonlDecoderOptionsType; */ type DecodeResultType = [string, number, number, number]; +/** + * Mapping between filtered log event indices and log events indices. The array index refers to + * the `filtered log event index` and the value refers to the `log event index`. + */ +type FilteredLogEventMap = Nullable; + interface Decoder { /** @@ -43,9 +49,9 @@ interface Decoder { getEstimatedNumEvents(): number; /** - * @return Indices of the filtered events. + * @return Filtered log event map. */ - getFilteredLogEventIndices(): Nullable; + getFilteredLogEventMap(): FilteredLogEventMap; /** * Sets the log level filter for the decoder. @@ -64,8 +70,10 @@ interface Decoder { build(): LogEventCount; /** - * Sets formatting options. Decoders support changing formatting without rebuilding - * existing log events. + * Sets formatting options. + * + * NOTE: The decoder supports changing formatting without rebuilding existing log + * events; however, the front-end currently does not support this. * * @param options * @return Whether the options were successfully set. @@ -73,18 +81,18 @@ interface Decoder { setFormatterOptions(options: DecoderOptionsType): boolean; /** - * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log - * event indices or filtered log event indices based on the flag `useFilteredIndices`. - * + * Decode log events. The flag `useFilter` specifies whether the range boundaries `[BeginIdx, EndIdx)` + * refer to the log event index directly or a filtered index. The filtered index is based on a subset + * of log events that are included by the set filter. * @param beginIdx * @param endIdx - * @param useFilteredIndices Whether to decode from the filtered or unfiltered log events array. + * @param useFilter Whether index refers to filtered index or log event index. * @return The decoded log events on success or null if any log event in the range doesn't exist * (e.g., the range exceeds the number of log events in the file). */ decodeRange(beginIdx: number, endIdx: number, - useFilteredIndices: boolean + useFilter: boolean ): Nullable; } @@ -99,6 +107,7 @@ export type { Decoder, DecodeResultType, DecoderOptionsType, + FilteredLogEventMap, JsonlDecoderOptionsType, LogEventCount, }; From da61b98643ef7768d40d1ad8bf6ad5200ecadc7e Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 26 Sep 2024 22:01:39 +0000 Subject: [PATCH 10/23] small changes --- new-log-viewer/src/services/decoders/JsonlDecoder/index.ts | 2 +- new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts | 2 +- new-log-viewer/src/typings/decoders.ts | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index 671c30a3..25479e9b 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -15,10 +15,10 @@ import { } from "../../../typings/logs"; import LogbackFormatter from "../../formatters/LogbackFormatter"; import { + DayjsTimestamp, isJsonObject, JsonLogEvent, LogLevelValue, - DayjsTimestamp, } from "./utils"; diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index 7dbf1491..8aa5eff6 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -93,8 +93,8 @@ const DayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { return dayjsTimestamp; }; export { + DayjsTimestamp, isJsonObject, LogLevelValue, - DayjsTimestamp, }; export type {JsonLogEvent}; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 17d2cc63..84002bf8 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -84,6 +84,7 @@ interface Decoder { * Decode log events. The flag `useFilter` specifies whether the range boundaries `[BeginIdx, EndIdx)` * refer to the log event index directly or a filtered index. The filtered index is based on a subset * of log events that are included by the set filter. + * * @param beginIdx * @param endIdx * @param useFilter Whether index refers to filtered index or log event index. From f3cb5b4189d0da747e25f42e6c0e23b0f177f25b Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 27 Sep 2024 01:14:06 +0000 Subject: [PATCH 11/23] small change --- .../services/decoders/JsonlDecoder/index.ts | 22 +++++++++---------- .../services/decoders/JsonlDecoder/utils.ts | 18 +++++++-------- new-log-viewer/src/typings/decoders.ts | 16 +++++--------- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index 25479e9b..2d57c656 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -15,16 +15,16 @@ import { } from "../../../typings/logs"; import LogbackFormatter from "../../formatters/LogbackFormatter"; import { - DayjsTimestamp, + convertToDayjsTimestamp, + convertToLogLevelValue, isJsonObject, JsonLogEvent, - LogLevelValue, } from "./utils"; /** * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for - * properties that are specific to log events (compared to generic JSON records). + * properties that are specific to json log events (compared to generic JSON records). */ class JsonlDecoder implements Decoder { static #textDecoder = new TextDecoder(); @@ -46,7 +46,6 @@ class JsonlDecoder implements Decoder { /** * @param dataArray * @param decoderOptions - * @throws {Error} if the initial decoder options are erroneous. */ constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { this.#dataArray = dataArray; @@ -125,7 +124,7 @@ class JsonlDecoder implements Decoder { * Retrieves log event using index then decodes into `DecodeResultType`. * * @param logEventIdx - * @return + * @return Decoded result. */ #getDecodeResult = (logEventIdx: number): DecodeResultType => { let timestamp: number; @@ -158,7 +157,8 @@ class JsonlDecoder implements Decoder { /** * Parses each line from the data array as a JSON object and buffers it internally. If a - * line cannot be parsed as a JSON object, an error is logged and the line is skipped. + * line cannot be parsed as a JSON object, an error is logged and the line added to an + * invalid map. * * NOTE: The data array(file) is freed after the very first run of this method. */ @@ -199,8 +199,8 @@ class JsonlDecoder implements Decoder { } this.#logEvents.push({ fields: fields, - level: LogLevelValue(fields[this.#logLevelKey]), - timestamp: DayjsTimestamp(fields[this.#timestampKey]), + level: convertToLogLevelValue(fields[this.#logLevelKey]), + timestamp: convertToDayjsTimestamp(fields[this.#timestampKey]), }); } catch (e) { if (0 === line.length) { @@ -212,14 +212,14 @@ class JsonlDecoder implements Decoder { this.#logEvents.push({ fields: {}, level: LOG_LEVEL.NONE, - timestamp: DayjsTimestamp(INVALID_TIMESTAMP_VALUE), + timestamp: convertToDayjsTimestamp(INVALID_TIMESTAMP_VALUE), }); } } /** - * Computes the indices of the log events that match the log level filter and - * buffers internally. Sets the indices to null if the filter is null. + * Filters log events and generates an internal array which serves as a mapping between filtered + * events and the original log event index. Sets the map to null if the filter is null. * * @param logLevelFilter */ diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index 8aa5eff6..bebefcca 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -37,10 +37,10 @@ const isJsonObject = (fields: JsonValue): fields is JsonObject => { /** * Converts JSON log level field into a log level value. * - * @param logLevelField Field in log event indexed by log level key. - * @return Log level value. + * @param logLevelField + * @return Integer for log level. */ -const LogLevelValue = (logLevelField: JsonValue | undefined): number => { +const convertToLogLevelValue = (logLevelField: JsonValue | undefined): number => { let logLevelValue = LOG_LEVEL.NONE; if ("undefined" === typeof logLevelField) { @@ -63,11 +63,11 @@ const LogLevelValue = (logLevelField: JsonValue | undefined): number => { * * @param timestampField * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: - * 1. the timestamp key doesn't exist in the log - * 2. the timestamp's value is an unsupported type - * 3. the timestamp's value is not a valid dayjs timestamp + * 1. the timestamp key doesn't exist in the log. + * 2. the timestamp's value is an unsupported type. + * 3. the timestamp's value is not a valid dayjs timestamp. */ -const DayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { +const convertToDayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. if (("string" !== typeof timestampField && "number" !== typeof timestampField) || @@ -93,8 +93,8 @@ const DayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { return dayjsTimestamp; }; export { - DayjsTimestamp, + convertToDayjsTimestamp, + convertToLogLevelValue, isJsonObject, - LogLevelValue, }; export type {JsonLogEvent}; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 84002bf8..622d6da2 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -49,7 +49,7 @@ interface Decoder { getEstimatedNumEvents(): number; /** - * @return Filtered log event map. + * @return Indices of the filtered events. */ getFilteredLogEventMap(): FilteredLogEventMap; @@ -70,10 +70,8 @@ interface Decoder { build(): LogEventCount; /** - * Sets formatting options. - * - * NOTE: The decoder supports changing formatting without rebuilding existing log - * events; however, the front-end currently does not support this. + * Sets formatting options. Decoders support changing formatting without rebuilding + * existing log events. * * @param options * @return Whether the options were successfully set. @@ -81,13 +79,12 @@ interface Decoder { setFormatterOptions(options: DecoderOptionsType): boolean; /** - * Decode log events. The flag `useFilter` specifies whether the range boundaries `[BeginIdx, EndIdx)` - * refer to the log event index directly or a filtered index. The filtered index is based on a subset - * of log events that are included by the set filter. + * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log + * event indices or filtered log event indices based on the flag `useFilter`. * * @param beginIdx * @param endIdx - * @param useFilter Whether index refers to filtered index or log event index. + * @param useFilter Whether to decode from the filtered or unfiltered log events array. * @return The decoded log events on success or null if any log event in the range doesn't exist * (e.g., the range exceeds the number of log events in the file). */ @@ -108,7 +105,6 @@ export type { Decoder, DecodeResultType, DecoderOptionsType, - FilteredLogEventMap, JsonlDecoderOptionsType, LogEventCount, }; From 4159305ad8c7c65983df50518cff5d39bd7fed34 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:13:55 -0400 Subject: [PATCH 12/23] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- new-log-viewer/src/services/LogFileManager.ts | 2 +- .../src/services/decoders/ClpIrDecoder.ts | 1 - .../services/decoders/JsonlDecoder/index.ts | 29 ++++++------- .../services/decoders/JsonlDecoder/utils.ts | 43 +++++++++---------- .../services/formatters/LogbackFormatter.ts | 6 +-- new-log-viewer/src/typings/decoders.ts | 22 +++++----- 6 files changed, 49 insertions(+), 54 deletions(-) diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index cee5a8fd..7deb08d9 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -76,7 +76,7 @@ class LogFileManager { // Build index for the entire file. const buildResult = decoder.build(); if (0 < buildResult.numInvalidEvents) { - console.error("Invalid events found in decoder.buildIdx():", buildResult); + console.error("Invalid events found in decoder.build():", buildResult); } this.#numEvents = decoder.getEstimatedNumEvents(); diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index 66513740..eafb2efa 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -60,7 +60,6 @@ class ClpIrDecoder implements Decoder { return true; } - decodeRange ( beginIdx: number, endIdx: number, diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index 2d57c656..bb310240 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -24,7 +24,7 @@ import { /** * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for - * properties that are specific to json log events (compared to generic JSON records). + * properties that are specific to log events (compared to generic JSON records). */ class JsonlDecoder implements Decoder { static #textDecoder = new TextDecoder(); @@ -121,10 +121,10 @@ class JsonlDecoder implements Decoder { } /** - * Retrieves log event using index then decodes into `DecodeResultType`. + * Decodes a log event into a `DecodeResultType`. * * @param logEventIdx - * @return Decoded result. + * @return The decoded log event. */ #getDecodeResult = (logEventIdx: number): DecodeResultType => { let timestamp: number; @@ -139,7 +139,7 @@ class JsonlDecoder implements Decoder { message = `${this.#invalidLogEventIdxToRawLine.get(logEventIdx)}\n`; logLevel = LOG_LEVEL.NONE; } else { - // Explicit cast since typescript thinks `#logEvents[filteredIdx]` can be undefined, + // Explicit cast since typescript thinks `#logEvents[logEventIdx]` can be undefined, // but it shouldn't be since the index comes from a class-internal filter. const logEvent = this.#logEvents[logEventIdx] as JsonLogEvent; logLevel = logEvent.level; @@ -156,11 +156,9 @@ class JsonlDecoder implements Decoder { }; /** - * Parses each line from the data array as a JSON object and buffers it internally. If a - * line cannot be parsed as a JSON object, an error is logged and the line added to an - * invalid map. + * Parses each line from the data array and buffers it internally. * - * NOTE: The data array(file) is freed after the very first run of this method. + * NOTE: `#dataArray` is freed after the very first run of this method. */ #deserialize () { if (null === this.#dataArray) { @@ -168,7 +166,7 @@ class JsonlDecoder implements Decoder { } const text = JsonlDecoder.#textDecoder.decode(this.#dataArray); - let beginIdx: number = 0; + let beginIdx = 0; while (beginIdx < text.length) { const endIdx = text.indexOf("\n", beginIdx); const line = (-1 === endIdx) ? @@ -186,15 +184,15 @@ class JsonlDecoder implements Decoder { } /** - * Parse line into a json log event and buffer internally. If the line contains invalid - * json, an entry is added to invalid log event map. + * Parses a JSON line into a log event and buffers it internally. If the line isn't valid JSON, + * a default log event is buffered and the line is added to `#invalidLogEventIdxToRawLine`. * * @param line */ #parseJson (line: string) { try { const fields = JSON.parse(line) as JsonValue; - if (!isJsonObject(fields)) { + if (false === isJsonObject(fields)) { throw new Error("Unexpected non-object."); } this.#logEvents.push({ @@ -218,15 +216,14 @@ class JsonlDecoder implements Decoder { } /** - * Filters log events and generates an internal array which serves as a mapping between filtered - * events and the original log event index. Sets the map to null if the filter is null. + * Filters log events and generates `#filteredLogEventMap`. If `logLevelFilter` is `null`, + * `#filteredLogEventMap` will be set to `null`. * * @param logLevelFilter */ #filterLogs (logLevelFilter: LogLevelFilter) { - this.#filteredLogEventMap = null; - if (null === logLevelFilter) { + this.#filteredLogEventMap = null; return; } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index bebefcca..f9fed268 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -14,9 +14,6 @@ import { // eslint-disable-next-line import/no-named-as-default-member dayjs.extend(utc); -/** - * A log event parsed from a JSON log. - */ interface JsonLogEvent { timestamp: Dayjs, level: LOG_LEVEL, @@ -24,23 +21,25 @@ interface JsonLogEvent { } /** - * Narrow JSON value to JSON object if compatible. + * Determines whether the given value is a `JsonObject` and applies a TypeScript narrowing + * conversion if so. + * * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates * * @param fields - * @return Whether type is JsonObject. + * @return A TypeScript type predicate indicating whether `fields` is a `JsonObject`. */ const isJsonObject = (fields: JsonValue): fields is JsonObject => { - return "object" === typeof fields; + return "object" === typeof fields && null !== fields; }; /** - * Converts JSON log level field into a log level value. + * Converts a field into a log level if possible. * * @param logLevelField - * @return Integer for log level. + * @return The log level or `LOG_LEVEL.NONE` if the field couldn't be converted. */ -const convertToLogLevelValue = (logLevelField: JsonValue | undefined): number => { +const convertToLogLevelValue = (logLevelField: JsonValue | undefined): LOG_LEVEL => { let logLevelValue = LOG_LEVEL.NONE; if ("undefined" === typeof logLevelField) { @@ -51,41 +50,41 @@ const convertToLogLevelValue = (logLevelField: JsonValue | undefined): number => JSON.stringify(logLevelField) : String(logLevelField); - if (logLevelName.toUpperCase() in LOG_LEVEL) { - logLevelValue = LOG_LEVEL[logLevelName.toUpperCase() as keyof typeof LOG_LEVEL]; + const uppercaseLogLevelName = logLevelName.toUpperCase(); + if (uppercaseLogLevelName in LOG_LEVEL) { + logLevelValue = LOG_LEVEL[uppercaseLogLevelName as keyof typeof LOG_LEVEL]; } return logLevelValue; }; /** - * Converts JSON timestamp field into a dayjs timestamp. + * Converts a field into a dayjs timestamp if possible. * * @param timestampField - * @return The timestamp or `INVALID_TIMESTAMP_VALUE` if: - * 1. the timestamp key doesn't exist in the log. - * 2. the timestamp's value is an unsupported type. - * 3. the timestamp's value is not a valid dayjs timestamp. + * @return The field as a dayjs timestamp or `dayjs.utc(INVALID_TIMESTAMP_VALUE)` if: + * - the timestamp key doesn't exist in the log. + * - the timestamp's value is an unsupported type. + * - the timestamp's value is not a valid dayjs timestamp. */ const convertToDayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. if (("string" !== typeof timestampField && "number" !== typeof timestampField) || - // Dayjs library surprisingly thinks undefined is valid date... - // Reference: https://day.js.org/docs/en/parse/now#docsNav + // dayjs surprisingly thinks `undefined` is a valid date: + // https://day.js.org/docs/en/parse/now#docsNav "undefined" === typeof timestampField ) { // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is - // `daysjs(null)` to show `Invalid Date` in UI. + // `dayjs(null)` to show "Invalid Date" in the UI. timestampField = INVALID_TIMESTAMP_VALUE; } let dayjsTimestamp: Dayjs = dayjs.utc(timestampField); - // Sanitize invalid date to `INVALID_TIMESTAMP_VALUE`. Note if input is not valid - // (ex. timestampField = "deadbeef") and not sanitized, result will be produce a - // non-valid dayjs timestamp and will show up in UI as `Invalid Date`. + // Sanitize invalid (e.g., "deadbeef") timestamps to `INVALID_TIMESTAMP_VALUE`; otherwise + // they'll show up in UI as "Invalid Date". if (false === dayjsTimestamp.isValid()) { dayjsTimestamp = dayjs.utc(INVALID_TIMESTAMP_VALUE); } diff --git a/new-log-viewer/src/services/formatters/LogbackFormatter.ts b/new-log-viewer/src/services/formatters/LogbackFormatter.ts index 7761b1b3..63769984 100644 --- a/new-log-viewer/src/services/formatters/LogbackFormatter.ts +++ b/new-log-viewer/src/services/formatters/LogbackFormatter.ts @@ -55,14 +55,14 @@ class LogbackFormatter implements Formatter { * Formats the given log event. * * @param logEvent - * @return The formatted message. + * @return The formatted log event. */ formatLogEvent (logEvent: JsonLogEvent): string { const {fields, timestamp} = logEvent; const formatStringWithTimestamp: string = this.#formatTimestamp(timestamp, this.#formatString); - const message = this.#formatVariables(formatStringWithTimestamp, fields); - return message; + + return this.#formatVariables(formatStringWithTimestamp, fields); } /** diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index 622d6da2..dea63a39 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -29,13 +29,13 @@ type DecoderOptionsType = JsonlDecoderOptionsType; * @property message * @property timestamp * @property level - * @property number The log event number is always the unfiltered number. + * @property number */ type DecodeResultType = [string, number, number, number]; /** - * Mapping between filtered log event indices and log events indices. The array index refers to - * the `filtered log event index` and the value refers to the `log event index`. + * Mapping between an index in the filtered log events collection to an index in the unfiltered log + * events collection. */ type FilteredLogEventMap = Nullable; @@ -49,7 +49,7 @@ interface Decoder { getEstimatedNumEvents(): number; /** - * @return Indices of the filtered events. + * @return The filtered log events map. */ getFilteredLogEventMap(): FilteredLogEventMap; @@ -65,13 +65,12 @@ interface Decoder { * Deserializes all log events in the file. * * @return Count of the successfully deserialized ("valid") log events and count of any - * un-deserializable ("invalid") log events within the range; + * un-deserializable ("invalid") log events. */ build(): LogEventCount; /** - * Sets formatting options. Decoders support changing formatting without rebuilding - * existing log events. + * Sets any formatter options that exist in the decoder's options. * * @param options * @return Whether the options were successfully set. @@ -79,16 +78,17 @@ interface Decoder { setFormatterOptions(options: DecoderOptionsType): boolean; /** - * Decode log events. The range boundaries `[BeginIdx, EndIdx)` can refer to unfiltered log - * event indices or filtered log event indices based on the flag `useFilter`. + * Decodes log events in the range `[beginIdx, endIdx)` of the filtered or unfiltered + * (depending on the value of `useFilter`) log events collection. * * @param beginIdx * @param endIdx - * @param useFilter Whether to decode from the filtered or unfiltered log events array. + * @param useFilter Whether to decode from the filtered or unfiltered log events collection. * @return The decoded log events on success or null if any log event in the range doesn't exist * (e.g., the range exceeds the number of log events in the file). */ - decodeRange(beginIdx: number, + decodeRange( + beginIdx: number, endIdx: number, useFilter: boolean ): Nullable; From 01e4723a6f5ebc1fc189a7b61378c61c91d52646 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 30 Sep 2024 22:30:57 +0000 Subject: [PATCH 13/23] add filteredLogEventMap --- .../src/services/decoders/JsonlDecoder/index.ts | 9 +++++---- new-log-viewer/src/typings/decoders.ts | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index bb310240..290e1284 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -95,10 +95,11 @@ class JsonlDecoder implements Decoder { } // Prevents typescript potential null warning. - const filteredLogEventIndices: number[] = this.#filteredLogEventMap as number[]; + const filteredLogEventMap: number[] = this.#filteredLogEventMap as number[]; + const length: number = useFilter ? - filteredLogEventIndices.length : + filteredLogEventMap.length : this.#logEvents.length; if (0 > beginIdx || length < endIdx) { @@ -107,11 +108,11 @@ class JsonlDecoder implements Decoder { const results: DecodeResultType[] = []; for (let i = beginIdx; i < endIdx; i++) { - // Explicit cast since typescript thinks `#filteredLogEventIndices[filteredLogEventIdx]` + // Explicit cast since typescript thinks `#filteredLogEventMap[i]` // can be undefined, but it shouldn't be since we performed a bounds check at the // beginning of the method. const logEventIdx: number = useFilter ? - (filteredLogEventIndices[i] as number) : + (filteredLogEventMap[i] as number) : i; results.push(this.#getDecodeResult(logEventIdx)); diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index dea63a39..acff65a1 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -105,6 +105,7 @@ export type { Decoder, DecodeResultType, DecoderOptionsType, + FilteredLogEventMap, JsonlDecoderOptionsType, LogEventCount, }; From dc9e533016d0abb75fc22ebdba92343fd23d8fe8 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 30 Sep 2024 22:35:49 +0000 Subject: [PATCH 14/23] forgot this suggestion --- new-log-viewer/src/services/LogFileManager.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index 7deb08d9..675d3c9e 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -143,9 +143,7 @@ class LogFileManager { return decoder; } - /** - * Sets formatting options for the decoder. - * + /* Sets any formatter options that exist in the decoder's options. * @param options */ setFormatterOptions (options: DecoderOptionsType) { From 114af1dd433c168f33f986ecc18fdf6c5166c491 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 30 Sep 2024 23:22:11 +0000 Subject: [PATCH 15/23] remove 1 type cast --- .../services/decoders/JsonlDecoder/index.ts | 119 +++++++++--------- .../services/decoders/JsonlDecoder/utils.ts | 41 +++--- .../services/formatters/LogbackFormatter.ts | 2 +- new-log-viewer/src/typings/formatters.ts | 3 +- new-log-viewer/src/typings/logs.ts | 15 ++- 5 files changed, 93 insertions(+), 87 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index 290e1284..13022be4 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -6,9 +6,11 @@ import { JsonlDecoderOptionsType, LogEventCount, } from "../../../typings/decoders"; +import {Dayjs} from "dayjs"; import {Formatter} from "../../../typings/formatters"; import {JsonValue} from "../../../typings/js"; import { + JsonLogEvent, INVALID_TIMESTAMP_VALUE, LOG_LEVEL, LogLevelFilter, @@ -18,7 +20,6 @@ import { convertToDayjsTimestamp, convertToLogLevelValue, isJsonObject, - JsonLogEvent, } from "./utils"; @@ -63,7 +64,7 @@ class JsonlDecoder implements Decoder { } setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { - this.#filterLogs(logLevelFilter); + this.#filterLogEvents(logLevelFilter); return true; } @@ -90,16 +91,12 @@ class JsonlDecoder implements Decoder { endIdx: number, useFilter: boolean, ): Nullable { - if (useFilter && null === this.#filteredLogEventMap) { + if (null === this.#filteredLogEventMap && useFilter) { return null; } - // Prevents typescript potential null warning. - const filteredLogEventMap: number[] = this.#filteredLogEventMap as number[]; - - - const length: number = useFilter ? - filteredLogEventMap.length : + const length: number = (useFilter && null !== this.#filteredLogEventMap) ? + this.#filteredLogEventMap.length : this.#logEvents.length; if (0 > beginIdx || length < endIdx) { @@ -111,51 +108,16 @@ class JsonlDecoder implements Decoder { // Explicit cast since typescript thinks `#filteredLogEventMap[i]` // can be undefined, but it shouldn't be since we performed a bounds check at the // beginning of the method. - const logEventIdx: number = useFilter ? - (filteredLogEventMap[i] as number) : + const logEventIdx: number = (useFilter && null !== this.#filteredLogEventMap) ? + (this.#filteredLogEventMap[i] as number) : i; - results.push(this.#getDecodeResult(logEventIdx)); + results.push(this.#decodeLogEvent(logEventIdx)); } return results; } - /** - * Decodes a log event into a `DecodeResultType`. - * - * @param logEventIdx - * @return The decoded log event. - */ - #getDecodeResult = (logEventIdx: number): DecodeResultType => { - let timestamp: number; - let message: string; - let logLevel: LOG_LEVEL; - - // eslint-disable-next-line no-warning-comments - // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on - // every iteration. - if (this.#invalidLogEventIdxToRawLine.has(logEventIdx)) { - timestamp = INVALID_TIMESTAMP_VALUE; - message = `${this.#invalidLogEventIdxToRawLine.get(logEventIdx)}\n`; - logLevel = LOG_LEVEL.NONE; - } else { - // Explicit cast since typescript thinks `#logEvents[logEventIdx]` can be undefined, - // but it shouldn't be since the index comes from a class-internal filter. - const logEvent = this.#logEvents[logEventIdx] as JsonLogEvent; - logLevel = logEvent.level; - message = this.#formatter.formatLogEvent(logEvent); - timestamp = logEvent.timestamp.valueOf(); - } - - return [ - message, - timestamp, - logLevel, - logEventIdx + 1, - ]; - }; - /** * Parses each line from the data array and buffers it internally. * @@ -191,16 +153,16 @@ class JsonlDecoder implements Decoder { * @param line */ #parseJson (line: string) { + let fields: JsonValue; + let level: LOG_LEVEL; + let timestamp: Dayjs; try { - const fields = JSON.parse(line) as JsonValue; + fields = JSON.parse(line) as JsonValue; if (false === isJsonObject(fields)) { throw new Error("Unexpected non-object."); } - this.#logEvents.push({ - fields: fields, - level: convertToLogLevelValue(fields[this.#logLevelKey]), - timestamp: convertToDayjsTimestamp(fields[this.#timestampKey]), - }); + level = convertToLogLevelValue(fields[this.#logLevelKey]); + timestamp = convertToDayjsTimestamp(fields[this.#timestampKey]); } catch (e) { if (0 === line.length) { return; @@ -208,12 +170,15 @@ class JsonlDecoder implements Decoder { console.error(e, line); const currentLogEventIdx = this.#logEvents.length; this.#invalidLogEventIdxToRawLine.set(currentLogEventIdx, line); - this.#logEvents.push({ - fields: {}, - level: LOG_LEVEL.NONE, - timestamp: convertToDayjsTimestamp(INVALID_TIMESTAMP_VALUE), - }); + fields = {}; + level = LOG_LEVEL.NONE; + timestamp = convertToDayjsTimestamp(INVALID_TIMESTAMP_VALUE); } + this.#logEvents.push({ + fields, + level, + timestamp, + }); } /** @@ -222,7 +187,7 @@ class JsonlDecoder implements Decoder { * * @param logLevelFilter */ - #filterLogs (logLevelFilter: LogLevelFilter) { + #filterLogEvents (logLevelFilter: LogLevelFilter) { if (null === logLevelFilter) { this.#filteredLogEventMap = null; return; @@ -235,6 +200,42 @@ class JsonlDecoder implements Decoder { } }); } + + /** + * Decodes a log event into a `DecodeResultType`. + * + * @param logEventIdx + * @return The decoded log event. + */ + #decodeLogEvent = (logEventIdx: number): DecodeResultType => { + let timestamp: number; + let message: string; + let logLevel: LOG_LEVEL; + + // eslint-disable-next-line no-warning-comments + // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on + // every iteration. + if (this.#invalidLogEventIdxToRawLine.has(logEventIdx)) { + timestamp = INVALID_TIMESTAMP_VALUE; + message = `${this.#invalidLogEventIdxToRawLine.get(logEventIdx)}\n`; + logLevel = LOG_LEVEL.NONE; + } else { + // Explicit cast since typescript thinks `#logEvents[logEventIdx]` can be undefined, + // but it shouldn't be since the index comes from a class-internal filter. + const logEvent = this.#logEvents[logEventIdx] as JsonLogEvent; + logLevel = logEvent.level; + message = this.#formatter.formatLogEvent(logEvent); + timestamp = logEvent.timestamp.valueOf(); + } + + return [ + message, + timestamp, + logLevel, + logEventIdx + 1, + ]; + }; } + export default JsonlDecoder; diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index f9fed268..14b93ddf 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -14,41 +14,35 @@ import { // eslint-disable-next-line import/no-named-as-default-member dayjs.extend(utc); -interface JsonLogEvent { - timestamp: Dayjs, - level: LOG_LEVEL, - fields: JsonObject -} - /** * Determines whether the given value is a `JsonObject` and applies a TypeScript narrowing * conversion if so. * * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates * - * @param fields - * @return A TypeScript type predicate indicating whether `fields` is a `JsonObject`. + * @param value + * @return A TypeScript type predicate indicating whether `value` is a `JsonObject`. */ -const isJsonObject = (fields: JsonValue): fields is JsonObject => { - return "object" === typeof fields && null !== fields; +const isJsonObject = (value: JsonValue): value is JsonObject => { + return "object" === typeof value && null !== value; }; /** * Converts a field into a log level if possible. * - * @param logLevelField + * @param field * @return The log level or `LOG_LEVEL.NONE` if the field couldn't be converted. */ -const convertToLogLevelValue = (logLevelField: JsonValue | undefined): LOG_LEVEL => { +const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { let logLevelValue = LOG_LEVEL.NONE; - if ("undefined" === typeof logLevelField) { + if ("undefined" === typeof field) { return logLevelValue; } - const logLevelName = "object" === typeof logLevelField ? - JSON.stringify(logLevelField) : - String(logLevelField); + const logLevelName = "object" === typeof field ? + JSON.stringify(field) : + String(field); const uppercaseLogLevelName = logLevelName.toUpperCase(); if (uppercaseLogLevelName in LOG_LEVEL) { @@ -61,27 +55,27 @@ const convertToLogLevelValue = (logLevelField: JsonValue | undefined): LOG_LEVEL /** * Converts a field into a dayjs timestamp if possible. * - * @param timestampField + * @param field * @return The field as a dayjs timestamp or `dayjs.utc(INVALID_TIMESTAMP_VALUE)` if: * - the timestamp key doesn't exist in the log. * - the timestamp's value is an unsupported type. * - the timestamp's value is not a valid dayjs timestamp. */ -const convertToDayjsTimestamp = (timestampField: JsonValue | undefined): dayjs.Dayjs => { +const convertToDayjsTimestamp = (field: JsonValue | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. - if (("string" !== typeof timestampField && - "number" !== typeof timestampField) || + if (("string" !== typeof field && + "number" !== typeof field) || // dayjs surprisingly thinks `undefined` is a valid date: // https://day.js.org/docs/en/parse/now#docsNav - "undefined" === typeof timestampField + "undefined" === typeof field ) { // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is // `dayjs(null)` to show "Invalid Date" in the UI. - timestampField = INVALID_TIMESTAMP_VALUE; + field = INVALID_TIMESTAMP_VALUE; } - let dayjsTimestamp: Dayjs = dayjs.utc(timestampField); + let dayjsTimestamp: Dayjs = dayjs.utc(field); // Sanitize invalid (e.g., "deadbeef") timestamps to `INVALID_TIMESTAMP_VALUE`; otherwise // they'll show up in UI as "Invalid Date". @@ -96,4 +90,3 @@ export { convertToLogLevelValue, isJsonObject, }; -export type {JsonLogEvent}; diff --git a/new-log-viewer/src/services/formatters/LogbackFormatter.ts b/new-log-viewer/src/services/formatters/LogbackFormatter.ts index 63769984..93f23eb7 100644 --- a/new-log-viewer/src/services/formatters/LogbackFormatter.ts +++ b/new-log-viewer/src/services/formatters/LogbackFormatter.ts @@ -6,7 +6,7 @@ import { FormatterOptionsType, } from "../../typings/formatters"; import {JsonObject} from "../../typings/js"; -import {JsonLogEvent} from "../decoders/JsonlDecoder/utils"; +import {JsonLogEvent} from "../../typings/logs"; /** diff --git a/new-log-viewer/src/typings/formatters.ts b/new-log-viewer/src/typings/formatters.ts index 95afaafe..6ed90509 100644 --- a/new-log-viewer/src/typings/formatters.ts +++ b/new-log-viewer/src/typings/formatters.ts @@ -1,5 +1,4 @@ -import {JsonLogEvent} from "../services/decoders/JsonlDecoder/utils"; - +import {JsonLogEvent} from "./logs"; /** * Options for the LogbackFormatter. diff --git a/new-log-viewer/src/typings/logs.ts b/new-log-viewer/src/typings/logs.ts index 11ef5703..3c89e7c5 100644 --- a/new-log-viewer/src/typings/logs.ts +++ b/new-log-viewer/src/typings/logs.ts @@ -1,5 +1,10 @@ import {Nullable} from "./common"; +import {Dayjs} from "dayjs"; + +import { + JsonObject, +} from "./js"; enum LOG_LEVEL { NONE = 0, @@ -15,7 +20,15 @@ type LogLevelFilter = Nullable; const INVALID_TIMESTAMP_VALUE = 0; -export type {LogLevelFilter}; +interface JsonLogEvent { + timestamp: Dayjs, + level: LOG_LEVEL, + fields: JsonObject +} + +export type { + JsonLogEvent, + LogLevelFilter}; export { INVALID_TIMESTAMP_VALUE, LOG_LEVEL, From a2814e42dc3ecd1bf7946281162bca81516c03cc Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 1 Oct 2024 00:34:40 +0000 Subject: [PATCH 16/23] small changes --- .../src/services/decoders/JsonlDecoder/index.ts | 13 ++++++++----- new-log-viewer/src/typings/formatters.ts | 1 + new-log-viewer/src/typings/logs.ts | 11 +++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index 13022be4..b597c98c 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -1,3 +1,5 @@ +import {Dayjs} from "dayjs"; + import {Nullable} from "../../../typings/common"; import { Decoder, @@ -6,12 +8,11 @@ import { JsonlDecoderOptionsType, LogEventCount, } from "../../../typings/decoders"; -import {Dayjs} from "dayjs"; import {Formatter} from "../../../typings/formatters"; import {JsonValue} from "../../../typings/js"; import { - JsonLogEvent, INVALID_TIMESTAMP_VALUE, + JsonLogEvent, LOG_LEVEL, LogLevelFilter, } from "../../../typings/logs"; @@ -91,7 +92,7 @@ class JsonlDecoder implements Decoder { endIdx: number, useFilter: boolean, ): Nullable { - if (null === this.#filteredLogEventMap && useFilter) { + if (useFilter && null === this.#filteredLogEventMap) { return null; } @@ -190,15 +191,17 @@ class JsonlDecoder implements Decoder { #filterLogEvents (logLevelFilter: LogLevelFilter) { if (null === logLevelFilter) { this.#filteredLogEventMap = null; + return; } - this.#filteredLogEventMap = []; + const filteredLogEventMap: number[] = []; this.#logEvents.forEach((logEvent, index) => { if (logLevelFilter.includes(logEvent.level)) { - (this.#filteredLogEventMap as number[]).push(index); + filteredLogEventMap.push(index); } }); + this.#filteredLogEventMap = filteredLogEventMap; } /** diff --git a/new-log-viewer/src/typings/formatters.ts b/new-log-viewer/src/typings/formatters.ts index 6ed90509..f426af58 100644 --- a/new-log-viewer/src/typings/formatters.ts +++ b/new-log-viewer/src/typings/formatters.ts @@ -1,5 +1,6 @@ import {JsonLogEvent} from "./logs"; + /** * Options for the LogbackFormatter. * diff --git a/new-log-viewer/src/typings/logs.ts b/new-log-viewer/src/typings/logs.ts index 3c89e7c5..17b381b7 100644 --- a/new-log-viewer/src/typings/logs.ts +++ b/new-log-viewer/src/typings/logs.ts @@ -1,10 +1,8 @@ -import {Nullable} from "./common"; - import {Dayjs} from "dayjs"; -import { - JsonObject, -} from "./js"; +import {Nullable} from "./common"; +import {JsonObject} from "./js"; + enum LOG_LEVEL { NONE = 0, @@ -28,7 +26,8 @@ interface JsonLogEvent { export type { JsonLogEvent, - LogLevelFilter}; + LogLevelFilter, +}; export { INVALID_TIMESTAMP_VALUE, LOG_LEVEL, From b562c34ed382ab94cfcaf141c64e54b25e62f430 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:54:23 -0400 Subject: [PATCH 17/23] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Co-authored-by: Junhao Liao --- new-log-viewer/src/services/decoders/JsonlDecoder/index.ts | 7 +++---- new-log-viewer/src/typings/logs.ts | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index b597c98c..b1a54f4e 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -73,7 +73,7 @@ class JsonlDecoder implements Decoder { build (): LogEventCount { this.#deserialize(); - const numInvalidEvents = Array.from(this.#invalidLogEventIdxToRawLine.keys()).length; + const numInvalidEvents = this.#invalidLogEventIdxToRawLine.size; return { numValidEvents: this.#logEvents.length - numInvalidEvents, @@ -106,9 +106,8 @@ class JsonlDecoder implements Decoder { const results: DecodeResultType[] = []; for (let i = beginIdx; i < endIdx; i++) { - // Explicit cast since typescript thinks `#filteredLogEventMap[i]` - // can be undefined, but it shouldn't be since we performed a bounds check at the - // beginning of the method. + // Explicit cast since typescript thinks `#filteredLogEventMap[i]` can be undefined, but + // it shouldn't be since we performed a bounds check at the beginning of the method. const logEventIdx: number = (useFilter && null !== this.#filteredLogEventMap) ? (this.#filteredLogEventMap[i] as number) : i; diff --git a/new-log-viewer/src/typings/logs.ts b/new-log-viewer/src/typings/logs.ts index 17b381b7..97ce1e25 100644 --- a/new-log-viewer/src/typings/logs.ts +++ b/new-log-viewer/src/typings/logs.ts @@ -16,14 +16,14 @@ enum LOG_LEVEL { type LogLevelFilter = Nullable; -const INVALID_TIMESTAMP_VALUE = 0; - interface JsonLogEvent { timestamp: Dayjs, level: LOG_LEVEL, fields: JsonObject } +const INVALID_TIMESTAMP_VALUE = 0; + export type { JsonLogEvent, LogLevelFilter, From 5af877a6a8429154d8172a1245d89d7c2e8f1421 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 1 Oct 2024 22:29:26 +0000 Subject: [PATCH 18/23] junhao/kirk changes --- .../src/services/decoders/ClpIrDecoder.ts | 5 +++-- .../src/services/decoders/JsonlDecoder/index.ts | 6 +++--- .../src/services/decoders/JsonlDecoder/utils.ts | 14 ++++---------- .../src/services/formatters/LogbackFormatter.ts | 4 ++-- new-log-viewer/src/typings/formatters.ts | 4 ++-- new-log-viewer/src/typings/logs.ts | 4 ++-- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index eafb2efa..36cd4dd3 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -38,6 +38,7 @@ class ClpIrDecoder implements Decoder { getFilteredLogEventMap (): FilteredLogEventMap { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js + console.error("getFilteredLogEventMap not implemented for IR decoder."); return null; } @@ -45,6 +46,7 @@ class ClpIrDecoder implements Decoder { setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js + console.error("setLogLevelFilter not implemented for IR decoder."); return false; } @@ -63,8 +65,7 @@ class ClpIrDecoder implements Decoder { decodeRange ( beginIdx: number, endIdx: number, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - useFilter: boolean + _: boolean ): Nullable { return this.#streamReader.decodeRange(beginIdx, endIdx); } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index b597c98c..c6c4aa92 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -12,7 +12,7 @@ import {Formatter} from "../../../typings/formatters"; import {JsonValue} from "../../../typings/js"; import { INVALID_TIMESTAMP_VALUE, - JsonLogEvent, + LogEvent, LOG_LEVEL, LogLevelFilter, } from "../../../typings/logs"; @@ -37,7 +37,7 @@ class JsonlDecoder implements Decoder { #timestampKey: string; - #logEvents: JsonLogEvent[] = []; + #logEvents: LogEvent[] = []; #filteredLogEventMap: FilteredLogEventMap = null; @@ -225,7 +225,7 @@ class JsonlDecoder implements Decoder { } else { // Explicit cast since typescript thinks `#logEvents[logEventIdx]` can be undefined, // but it shouldn't be since the index comes from a class-internal filter. - const logEvent = this.#logEvents[logEventIdx] as JsonLogEvent; + const logEvent = this.#logEvents[logEventIdx] as LogEvent; logLevel = logEvent.level; message = this.#formatter.formatLogEvent(logEvent); timestamp = logEvent.timestamp.valueOf(); diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index 14b93ddf..e8734896 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -1,5 +1,4 @@ import dayjs, {Dayjs} from "dayjs"; -import utc from "dayjs/plugin/utc"; import { JsonObject, @@ -10,16 +9,10 @@ import { LOG_LEVEL, } from "../../../typings/logs"; - -// eslint-disable-next-line import/no-named-as-default-member -dayjs.extend(utc); - /** * Determines whether the given value is a `JsonObject` and applies a TypeScript narrowing * conversion if so. * - * Reference: https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates - * * @param value * @return A TypeScript type predicate indicating whether `value` is a `JsonObject`. */ @@ -40,6 +33,8 @@ const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { return logLevelValue; } + // Json stringify covers edge case where the field is an object with more than one key, e.g., + // `field = { "name": "INFO", "value": 20 }`. const logLevelName = "object" === typeof field ? JSON.stringify(field) : String(field); @@ -63,11 +58,10 @@ const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { */ const convertToDayjsTimestamp = (field: JsonValue | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. + // dayjs surprisingly thinks `undefined` is a valid date: + // https://day.js.org/docs/en/parse/now#docsNav if (("string" !== typeof field && "number" !== typeof field) || - - // dayjs surprisingly thinks `undefined` is a valid date: - // https://day.js.org/docs/en/parse/now#docsNav "undefined" === typeof field ) { // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is diff --git a/new-log-viewer/src/services/formatters/LogbackFormatter.ts b/new-log-viewer/src/services/formatters/LogbackFormatter.ts index 93f23eb7..8caf91d3 100644 --- a/new-log-viewer/src/services/formatters/LogbackFormatter.ts +++ b/new-log-viewer/src/services/formatters/LogbackFormatter.ts @@ -6,7 +6,7 @@ import { FormatterOptionsType, } from "../../typings/formatters"; import {JsonObject} from "../../typings/js"; -import {JsonLogEvent} from "../../typings/logs"; +import {LogEvent} from "../../typings/logs"; /** @@ -57,7 +57,7 @@ class LogbackFormatter implements Formatter { * @param logEvent * @return The formatted log event. */ - formatLogEvent (logEvent: JsonLogEvent): string { + formatLogEvent (logEvent: LogEvent): string { const {fields, timestamp} = logEvent; const formatStringWithTimestamp: string = this.#formatTimestamp(timestamp, this.#formatString); diff --git a/new-log-viewer/src/typings/formatters.ts b/new-log-viewer/src/typings/formatters.ts index f426af58..87a5d067 100644 --- a/new-log-viewer/src/typings/formatters.ts +++ b/new-log-viewer/src/typings/formatters.ts @@ -1,4 +1,4 @@ -import {JsonLogEvent} from "./logs"; +import {LogEvent} from "./logs"; /** @@ -29,7 +29,7 @@ interface LogbackFormatterOptionsType { type FormatterOptionsType = LogbackFormatterOptionsType; interface Formatter { - formatLogEvent: (logEvent: JsonLogEvent) => string + formatLogEvent: (logEvent: LogEvent) => string } export type { diff --git a/new-log-viewer/src/typings/logs.ts b/new-log-viewer/src/typings/logs.ts index 17b381b7..8f32e65f 100644 --- a/new-log-viewer/src/typings/logs.ts +++ b/new-log-viewer/src/typings/logs.ts @@ -18,14 +18,14 @@ type LogLevelFilter = Nullable; const INVALID_TIMESTAMP_VALUE = 0; -interface JsonLogEvent { +interface LogEvent { timestamp: Dayjs, level: LOG_LEVEL, fields: JsonObject } export type { - JsonLogEvent, + LogEvent, LogLevelFilter, }; export { From 4bf08dc50a64f6095852f41aa048a8a0c258cd6c Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 1 Oct 2024 22:33:38 +0000 Subject: [PATCH 19/23] fix lint --- new-log-viewer/src/services/decoders/ClpIrDecoder.ts | 5 ++++- new-log-viewer/src/services/decoders/JsonlDecoder/index.ts | 2 +- new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index 36cd4dd3..b61d5ebf 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -39,6 +39,7 @@ class ClpIrDecoder implements Decoder { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js console.error("getFilteredLogEventMap not implemented for IR decoder."); + return null; } @@ -47,6 +48,7 @@ class ClpIrDecoder implements Decoder { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js console.error("setLogLevelFilter not implemented for IR decoder."); + return false; } @@ -65,7 +67,8 @@ class ClpIrDecoder implements Decoder { decodeRange ( beginIdx: number, endIdx: number, - _: boolean + // eslint-disable-next-line @typescript-eslint/no-unused-vars + useFilter: boolean ): Nullable { return this.#streamReader.decodeRange(beginIdx, endIdx); } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts index fc92be3e..3b229fb3 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/index.ts @@ -12,8 +12,8 @@ import {Formatter} from "../../../typings/formatters"; import {JsonValue} from "../../../typings/js"; import { INVALID_TIMESTAMP_VALUE, - LogEvent, LOG_LEVEL, + LogEvent, LogLevelFilter, } from "../../../typings/logs"; import LogbackFormatter from "../../formatters/LogbackFormatter"; diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index e8734896..5b460e87 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -9,6 +9,7 @@ import { LOG_LEVEL, } from "../../../typings/logs"; + /** * Determines whether the given value is a `JsonObject` and applies a TypeScript narrowing * conversion if so. From 81389df5872282cac5d462aee635997424a96440 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 1 Oct 2024 23:50:41 +0000 Subject: [PATCH 20/23] change pageNum to useState to fix pageNum occasionaly not updating when manually chaning log event --- new-log-viewer/src/contexts/StateContextProvider.tsx | 9 +++++---- new-log-viewer/src/services/LogFileManager/index.ts | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/new-log-viewer/src/contexts/StateContextProvider.tsx b/new-log-viewer/src/contexts/StateContextProvider.tsx index bf65ca90..90c7c62a 100644 --- a/new-log-viewer/src/contexts/StateContextProvider.tsx +++ b/new-log-viewer/src/contexts/StateContextProvider.tsx @@ -186,6 +186,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { const [fileName, setFileName] = useState(STATE_DEFAULT.fileName); const [logData, setLogData] = useState(STATE_DEFAULT.logData); const [numEvents, setNumEvents] = useState(STATE_DEFAULT.numEvents); + const [pageNum, setPageNum] = useState(STATE_DEFAULT.pageNum); const beginLineNumToLogEventNumRef = useRef(STATE_DEFAULT.beginLineNumToLogEventNum); const [exportProgress, setExportProgress] = @@ -194,7 +195,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { // Refs const logEventNumRef = useRef(logEventNum); const numPagesRef = useRef(STATE_DEFAULT.numPages); - const pageNumRef = useRef(STATE_DEFAULT.pageNum); + const logExportManagerRef = useRef(null); const mainWorkerRef = useRef(null); @@ -220,7 +221,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { break; case WORKER_RESP_CODE.PAGE_DATA: { setLogData(args.logs); - pageNumRef.current = args.pageNum; + setPageNum(args.pageNum) beginLineNumToLogEventNumRef.current = args.beginLineNumToLogEventNum; updateWindowUrlHashParams({ logEventNum: args.logEventNum, @@ -289,7 +290,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { return; } - const cursor = getPageNumCursor(navAction, pageNumRef.current, numPagesRef.current); + const cursor = getPageNumCursor(navAction, pageNum, numPagesRef.current); if (null === cursor) { console.error(`Error with nav action ${navAction.code}.`); @@ -381,7 +382,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { logData: logData, numEvents: numEvents, numPages: numPagesRef.current, - pageNum: pageNumRef.current, + pageNum: pageNum, exportLogs: exportLogs, loadFile: loadFile, diff --git a/new-log-viewer/src/services/LogFileManager/index.ts b/new-log-viewer/src/services/LogFileManager/index.ts index f7cc887b..3767c92a 100644 --- a/new-log-viewer/src/services/LogFileManager/index.ts +++ b/new-log-viewer/src/services/LogFileManager/index.ts @@ -183,7 +183,12 @@ class LogFileManager { matchingLogEventNum, } = this.#getCursorData(cursor); - const results = this.#decoder.decodeRange(pageBeginLogEventNum - 1, pageEndLogEventNum - 1, false); + const results = this.#decoder.decodeRange( + pageBeginLogEventNum - 1, + pageEndLogEventNum - 1, + false + ); + if (null === results) { throw new Error("Error occurred during decoding. " + `pageBeginLogEventNum=${pageBeginLogEventNum}, ` + From 3777537567d3712e77e172cfe418d7830e52d99b Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Wed, 2 Oct 2024 00:02:58 +0000 Subject: [PATCH 21/23] revert changes will fix in another PR --- new-log-viewer/src/contexts/StateContextProvider.tsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/new-log-viewer/src/contexts/StateContextProvider.tsx b/new-log-viewer/src/contexts/StateContextProvider.tsx index 90c7c62a..bf65ca90 100644 --- a/new-log-viewer/src/contexts/StateContextProvider.tsx +++ b/new-log-viewer/src/contexts/StateContextProvider.tsx @@ -186,7 +186,6 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { const [fileName, setFileName] = useState(STATE_DEFAULT.fileName); const [logData, setLogData] = useState(STATE_DEFAULT.logData); const [numEvents, setNumEvents] = useState(STATE_DEFAULT.numEvents); - const [pageNum, setPageNum] = useState(STATE_DEFAULT.pageNum); const beginLineNumToLogEventNumRef = useRef(STATE_DEFAULT.beginLineNumToLogEventNum); const [exportProgress, setExportProgress] = @@ -195,7 +194,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { // Refs const logEventNumRef = useRef(logEventNum); const numPagesRef = useRef(STATE_DEFAULT.numPages); - + const pageNumRef = useRef(STATE_DEFAULT.pageNum); const logExportManagerRef = useRef(null); const mainWorkerRef = useRef(null); @@ -221,7 +220,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { break; case WORKER_RESP_CODE.PAGE_DATA: { setLogData(args.logs); - setPageNum(args.pageNum) + pageNumRef.current = args.pageNum; beginLineNumToLogEventNumRef.current = args.beginLineNumToLogEventNum; updateWindowUrlHashParams({ logEventNum: args.logEventNum, @@ -290,7 +289,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { return; } - const cursor = getPageNumCursor(navAction, pageNum, numPagesRef.current); + const cursor = getPageNumCursor(navAction, pageNumRef.current, numPagesRef.current); if (null === cursor) { console.error(`Error with nav action ${navAction.code}.`); @@ -382,7 +381,7 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { logData: logData, numEvents: numEvents, numPages: numPagesRef.current, - pageNum: pageNum, + pageNum: pageNumRef.current, exportLogs: exportLogs, loadFile: loadFile, From 5933c3726661f0caba86eef67ef58ae7c1b1b9c6 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Wed, 2 Oct 2024 15:17:37 +0000 Subject: [PATCH 22/23] kirk + junhao review --- .../src/services/decoders/ClpIrDecoder.ts | 4 ++-- .../src/services/decoders/JsonlDecoder/utils.ts | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts index b61d5ebf..e6b439a3 100644 --- a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -38,7 +38,7 @@ class ClpIrDecoder implements Decoder { getFilteredLogEventMap (): FilteredLogEventMap { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js - console.error("getFilteredLogEventMap not implemented for IR decoder."); + console.error("Not implemented."); return null; } @@ -47,7 +47,7 @@ class ClpIrDecoder implements Decoder { setLogLevelFilter (logLevelFilter: LogLevelFilter): boolean { // eslint-disable-next-line no-warning-comments // TODO: Update this after log level filtering is implemented in clp-ffi-js - console.error("setLogLevelFilter not implemented for IR decoder."); + console.error("Not implemented."); return false; } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index 5b460e87..fe7a53ef 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -18,7 +18,7 @@ import { * @return A TypeScript type predicate indicating whether `value` is a `JsonObject`. */ const isJsonObject = (value: JsonValue): value is JsonObject => { - return "object" === typeof value && null !== value; + return "object" === typeof value && null !== value && !Array.isArray(value); }; /** @@ -30,15 +30,13 @@ const isJsonObject = (value: JsonValue): value is JsonObject => { const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { let logLevelValue = LOG_LEVEL.NONE; - if ("undefined" === typeof field) { + // If the field is an object , e.g. `field = { "name": "INFO", "value": 20 }`. + // Then the user should specify a nested key. + if ("undefined" === typeof field || "object" === typeof field) { return logLevelValue; } - // Json stringify covers edge case where the field is an object with more than one key, e.g., - // `field = { "name": "INFO", "value": 20 }`. - const logLevelName = "object" === typeof field ? - JSON.stringify(field) : - String(field); + const logLevelName = String(field); const uppercaseLogLevelName = logLevelName.toUpperCase(); if (uppercaseLogLevelName in LOG_LEVEL) { @@ -59,7 +57,7 @@ const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { */ const convertToDayjsTimestamp = (field: JsonValue | undefined): dayjs.Dayjs => { // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`. - // dayjs surprisingly thinks `undefined` is a valid date: + // NOTE: dayjs surprisingly thinks `undefined` is a valid date. See // https://day.js.org/docs/en/parse/now#docsNav if (("string" !== typeof field && "number" !== typeof field) || From 637d4888e94b446dfe64eb70d49aa20153418454 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Wed, 2 Oct 2024 18:16:55 +0000 Subject: [PATCH 23/23] small changes --- new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts | 6 ++---- new-log-viewer/src/typings/decoders.ts | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts index fe7a53ef..3ba5f2a4 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder/utils.ts @@ -18,7 +18,7 @@ import { * @return A TypeScript type predicate indicating whether `value` is a `JsonObject`. */ const isJsonObject = (value: JsonValue): value is JsonObject => { - return "object" === typeof value && null !== value && !Array.isArray(value); + return "object" === typeof value && null !== value && false === Array.isArray(value); }; /** @@ -30,9 +30,7 @@ const isJsonObject = (value: JsonValue): value is JsonObject => { const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => { let logLevelValue = LOG_LEVEL.NONE; - // If the field is an object , e.g. `field = { "name": "INFO", "value": 20 }`. - // Then the user should specify a nested key. - if ("undefined" === typeof field || "object" === typeof field) { + if ("undefined" === typeof field || isJsonObject(field)) { return logLevelValue; } diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index acff65a1..d1b1eeae 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -85,7 +85,7 @@ interface Decoder { * @param endIdx * @param useFilter Whether to decode from the filtered or unfiltered log events collection. * @return The decoded log events on success or null if any log event in the range doesn't exist - * (e.g., the range exceeds the number of log events in the file). + * (e.g., the range exceeds the number of log events in the collection). */ decodeRange( beginIdx: number,