diff --git a/package-lock.json b/package-lock.json index a973cfe29..208135c91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@mui/icons-material": "^6.4.1", "@mui/joy": "^5.0.0-beta.51", "axios": "^1.7.9", - "clp-ffi-js": "^0.4.0", + "clp-ffi-js": "^0.5.0", "dayjs": "^1.11.13", "monaco-editor": "0.50.0", "react": "^19.0.0", @@ -33,7 +33,7 @@ "babel-loader": "^9.2.1", "copy-webpack-plugin": "^12.0.2", "css-loader": "^7.1.2", - "eslint-config-yscope": "latest", + "eslint-config-yscope": "^1.1.0", "globals": "^15.14.0", "html-webpack-plugin": "^5.6.3", "jest": "^29.7.0", @@ -6322,9 +6322,9 @@ } }, "node_modules/clp-ffi-js": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/clp-ffi-js/-/clp-ffi-js-0.4.0.tgz", - "integrity": "sha512-pL/YSRwok83gvweMx2DjWchHYu7Xl/lSxcszI4g0Guw55CQlxeQEDI/PDiT3b14zbaig3tJrtxE706Jm0FHvTA==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/clp-ffi-js/-/clp-ffi-js-0.5.0.tgz", + "integrity": "sha512-uXn5smxgHWv6WF/6Q8KUsyKscMLQk939C6MxxHnn5A1MPvHE9U2A+/p3Mkf+1TC0z6z3fu/liunMCx9eswo13Q==" }, "node_modules/clsx": { "version": "2.1.1", @@ -7716,7 +7716,6 @@ "resolved": "https://registry.npmjs.org/eslint-config-yscope/-/eslint-config-yscope-1.1.0.tgz", "integrity": "sha512-uDalIcAh3Qke95gtqDQjzA7zWDQ03yb/Li3DMiBd8Nj/XuAfntkyd1z68IcsY0DjEZv6tpwlQ94eFzXvqOrjkQ==", "dev": true, - "license": "Apache-2.0", "peerDependencies": { "@stylistic/eslint-plugin": "^2.12.1", "eslint": "^9.18.0", diff --git a/package.json b/package.json index 54b92083c..d00e42357 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@mui/icons-material": "^6.4.1", "@mui/joy": "^5.0.0-beta.51", "axios": "^1.7.9", - "clp-ffi-js": "^0.4.0", + "clp-ffi-js": "^0.5.0", "dayjs": "^1.11.13", "monaco-editor": "0.50.0", "react": "^19.0.0", diff --git a/src/services/decoders/ClpIrDecoder/index.ts b/src/services/decoders/ClpIrDecoder/index.ts index 0acf24399..13927ea08 100644 --- a/src/services/decoders/ClpIrDecoder/index.ts +++ b/src/services/decoders/ClpIrDecoder/index.ts @@ -21,6 +21,7 @@ import { convertToDayjsTimestamp, isJsonObject, } from "../JsonlDecoder/utils"; +import {parseFilterKeys} from "../utils"; import { CLP_IR_STREAM_TYPE, getStructuredIrNamespaceKeys, @@ -42,7 +43,8 @@ class ClpIrDecoder implements Decoder { dataArray: Uint8Array, decoderOptions: DecoderOptions ) { - this.#streamReader = new ffiModule.ClpStreamReader(dataArray, decoderOptions); + const readerOptions = parseFilterKeys(decoderOptions, true); + this.#streamReader = new ffiModule.ClpStreamReader(dataArray, readerOptions); this.#streamType = this.#streamReader.getIrStreamType() === ffiModule.IrStreamType.STRUCTURED ? CLP_IR_STREAM_TYPE.STRUCTURED : diff --git a/src/services/decoders/JsonlDecoder/index.ts b/src/services/decoders/JsonlDecoder/index.ts index 1ff3f4d75..fb6688560 100644 --- a/src/services/decoders/JsonlDecoder/index.ts +++ b/src/services/decoders/JsonlDecoder/index.ts @@ -16,8 +16,10 @@ import { LogEvent, LogLevelFilter, } from "../../../typings/logs"; +import {getNestedJsonValue} from "../../../utils/js"; import YscopeFormatter from "../../formatters/YscopeFormatter"; import {postFormatPopup} from "../../MainWorker"; +import {parseFilterKeys} from "../utils"; import { convertToDayjsTimestamp, convertToLogLevelValue, @@ -34,9 +36,9 @@ class JsonlDecoder implements Decoder { #dataArray: Nullable; - #logLevelKey: string; + #logLevelKeyParts: string[]; - #timestampKey: string; + #timestampKeyParts: string[]; #logEvents: LogEvent[] = []; @@ -52,8 +54,11 @@ class JsonlDecoder implements Decoder { */ constructor (dataArray: Uint8Array, decoderOptions: DecoderOptions) { this.#dataArray = dataArray; - this.#logLevelKey = decoderOptions.logLevelKey; - this.#timestampKey = decoderOptions.timestampKey; + + const filterKeys = parseFilterKeys(decoderOptions, false); + this.#logLevelKeyParts = filterKeys.logLevelKey.parts; + this.#timestampKeyParts = filterKeys.timestampKey.parts; + this.#formatter = new YscopeFormatter({formatString: decoderOptions.formatString}); if (0 === decoderOptions.formatString.length) { postFormatPopup(); @@ -165,8 +170,14 @@ class JsonlDecoder implements Decoder { if (false === isJsonObject(fields)) { throw new Error("Unexpected non-object."); } - level = convertToLogLevelValue(fields[this.#logLevelKey]); - timestamp = convertToDayjsTimestamp(fields[this.#timestampKey]); + level = convertToLogLevelValue(getNestedJsonValue( + fields, + this.#logLevelKeyParts + )); + timestamp = convertToDayjsTimestamp(getNestedJsonValue( + fields, + this.#timestampKeyParts + )); } catch (e) { if (0 === line.length) { return; diff --git a/src/services/decoders/utils.ts b/src/services/decoders/utils.ts new file mode 100644 index 000000000..1e5315f46 --- /dev/null +++ b/src/services/decoders/utils.ts @@ -0,0 +1,58 @@ +import {DecoderOptions} from "../../typings/decoders"; +import { + ParsedKey, + REPLACEMENT_CHARACTER, +} from "../../typings/formatters"; +import { + EXISTING_REPLACEMENT_CHARACTER_WARNING, + parseKey, + replaceDoubleBacklash, + UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE, +} from "../formatters/YscopeFormatter/utils"; + + +/** + * Preprocesses filter key by removing escaped backlash to facilitate simpler parsing, then parses + * the key. + * + * @param filterKey + * @return The parsed key. + */ +const preprocessThenParseFilterKey = (filterKey: string): ParsedKey => { + if (filterKey.includes(REPLACEMENT_CHARACTER)) { + console.warn(EXISTING_REPLACEMENT_CHARACTER_WARNING); + } + + return parseKey(replaceDoubleBacklash(filterKey)); +}; + +/** + * Parses the log level key and timestamp key from the decoder options. + * + * @param decoderOptions + * @param supportsAutoGeneratedKeys + * @return An object containing the parsed log level key and timestamp key. + * @throws {Error} If the keys contain reserved symbols. + */ +const parseFilterKeys = (decoderOptions: DecoderOptions, supportsAutoGeneratedKeys: boolean): { + logLevelKey: ParsedKey; + timestampKey: ParsedKey; +} => { + const parsedLogLevelKey = preprocessThenParseFilterKey(decoderOptions.logLevelKey); + const parsedTimestampKey = preprocessThenParseFilterKey(decoderOptions.timestampKey); + + if (false === supportsAutoGeneratedKeys && + (parsedLogLevelKey.isAutoGenerated || parsedTimestampKey.isAutoGenerated)) { + throw new Error(UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE); + } + + return { + logLevelKey: parsedLogLevelKey, + timestampKey: parsedTimestampKey, + }; +}; + +export { + parseFilterKeys, + preprocessThenParseFilterKey, +}; diff --git a/src/services/formatters/YscopeFormatter/index.ts b/src/services/formatters/YscopeFormatter/index.ts index 950f9e257..294261288 100644 --- a/src/services/formatters/YscopeFormatter/index.ts +++ b/src/services/formatters/YscopeFormatter/index.ts @@ -3,6 +3,7 @@ import { FIELD_PLACEHOLDER_REGEX, Formatter, FormatterOptionsType, + ParsedKey, REPLACEMENT_CHARACTER, YscopeFieldFormatter, YscopeFieldPlaceholder, @@ -11,10 +12,13 @@ import {LogEvent} from "../../../typings/logs"; import {jsonValueToString} from "../../../utils/js"; import {StructuredIrNamespaceKeys} from "../../decoders/ClpIrDecoder/utils"; import { + EXISTING_REPLACEMENT_CHARACTER_WARNING, getFormattedField, + parseKey, removeEscapeCharacters, replaceDoubleBacklash, splitFieldPlaceholder, + UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE, YSCOPE_FIELD_FORMATTER_MAP, } from "./utils"; @@ -34,8 +38,7 @@ class YscopeFormatter implements Formatter { this.#structuredIrNamespaceKeys = options.structuredIrNamespaceKeys ?? null; if (options.formatString.includes(REPLACEMENT_CHARACTER)) { - console.warn("Unicode replacement character `U+FFFD` found in format string; " + - "it will be replaced with \"\\\""); + console.warn(EXISTING_REPLACEMENT_CHARACTER_WARNING); } this.#processedFormatString = replaceDoubleBacklash(options.formatString); @@ -90,8 +93,13 @@ class YscopeFormatter implements Formatter { throw Error("Field placeholder regex is invalid and does not have a capture group"); } - const {parsedFieldName, formatterName, formatterOptions} = - splitFieldPlaceholder(groupMatch, this.#structuredIrNamespaceKeys); + const {fieldName, formatterName, formatterOptions} = + splitFieldPlaceholder(groupMatch); + + const parsedFieldName: ParsedKey = parseKey(fieldName); + if (null === this.#structuredIrNamespaceKeys && parsedFieldName.isAutoGenerated) { + throw new Error(UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE); + } let fieldFormatter: Nullable = null; if (null !== formatterName) { diff --git a/src/services/formatters/YscopeFormatter/utils.ts b/src/services/formatters/YscopeFormatter/utils.ts index d5d5d3d97..c6bfdde5d 100644 --- a/src/services/formatters/YscopeFormatter/utils.ts +++ b/src/services/formatters/YscopeFormatter/utils.ts @@ -3,7 +3,7 @@ import { AUTO_GENERATED_KEY_PREFIX, COLON_REGEX, DOUBLE_BACKSLASH, - ParsedFieldName, + ParsedKey, PERIOD_REGEX, REPLACEMENT_CHARACTER, SINGLE_BACKSLASH, @@ -22,6 +22,22 @@ import RoundFormatter from "./FieldFormatters/RoundFormatter"; import TimestampFormatter from "./FieldFormatters/TimestampFormatter"; +/** + * Error message for when a key is prefixed with the `@` symbol in the format string or + * filter key for JSON logs. + */ +const UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE = + "`@` is a reserved symbol for CLP IR logs and must be escaped with `\\` for JSONL logs."; + + +/** + * Warning message for when Unicode replacement character is found in format string or filter + * key. + */ +const EXISTING_REPLACEMENT_CHARACTER_WARNING = + "Unicode replacement character `U+FFFD` found in format string or filter key; " + + "it will be replaced with a double backlash"; + /** * List of currently supported field formatters. */ @@ -81,7 +97,7 @@ const replaceDoubleBacklash = (str: string): string => { /** * Retrieves fields from auto-generated or user-generated namespace of a structured IR log - * event based on the prefix of the parsed key. + * event based on the prefix of the parsed field name. * * @param logEvent * @param structuredIrNamespaceKeys @@ -93,7 +109,7 @@ const replaceDoubleBacklash = (str: string): string => { const getFieldsByNamespace = ( logEvent: LogEvent, structuredIrNamespaceKeys: StructuredIrNamespaceKeys, - parsedFieldName: ParsedFieldName + parsedFieldName: ParsedKey ): JsonObject => { const namespaceKey = parsedFieldName.isAutoGenerated ? structuredIrNamespaceKeys.autoGenerated : @@ -162,7 +178,7 @@ const validateComponent = (component: string | undefined): Nullable => { * @param key The key to be parsed. * @return The parsed key. */ -const parseKey = (key: string): ParsedFieldName => { +const parseKey = (key: string): ParsedKey => { const isAutoGenerated = AUTO_GENERATED_KEY_PREFIX === key.charAt(0); const keyWithoutAutoPrefix = isAutoGenerated ? key.substring(1) : @@ -176,22 +192,20 @@ const parseKey = (key: string): ParsedFieldName => { }; /** - * Splits a field placeholder string into its components: parsed field name, formatter name, and + * Splits a field placeholder string into its components: field name, formatter name, and * formatter options. * * @param placeholderString - * @param structuredIrNamespaceKeys * @return - An object containing: - * - parsedFieldName: The parsed field name. + * - fieldName: The field name. * - formatterName: The formatter name, or `null` if not provided. * - formatterOptions: The formatter options, or `null` if not provided. * @throws {Error} If the field name could not be parsed. */ const splitFieldPlaceholder = ( placeholderString: string, - structuredIrNamespaceKeys: Nullable ): { - parsedFieldName: ParsedFieldName; + fieldName: string; formatterName: Nullable; formatterOptions: Nullable; } => { @@ -207,14 +221,6 @@ const splitFieldPlaceholder = ( throw Error("Field name could not be parsed"); } - const parsedFieldName: ParsedFieldName = parseKey(fieldName); - if (null === structuredIrNamespaceKeys && parsedFieldName.isAutoGenerated) { - throw new Error( - "`@` is a reserved symbol and must be escaped with `\\` " + - "for JSONL logs." - ); - } - formatterName = validateComponent(formatterName); if (null !== formatterName) { formatterName = removeEscapeCharacters(formatterName); @@ -225,15 +231,17 @@ const splitFieldPlaceholder = ( formatterOptions = removeEscapeCharacters(formatterOptions); } - return {parsedFieldName, formatterName, formatterOptions}; + return {fieldName, formatterName, formatterOptions}; }; export { + EXISTING_REPLACEMENT_CHARACTER_WARNING, getFormattedField, parseKey, removeEscapeCharacters, replaceDoubleBacklash, splitFieldPlaceholder, + UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE, YSCOPE_FIELD_FORMATTER_MAP, }; diff --git a/src/typings/formatters.ts b/src/typings/formatters.ts index 45cf630cd..c0fd27e11 100644 --- a/src/typings/formatters.ts +++ b/src/typings/formatters.ts @@ -60,12 +60,12 @@ interface YscopeFieldFormatterMap { /** - * Parsed field name from YScope format string. + * Parsed key from YScope format string or filter keys. * * @property isAutoGenerated whether the key is prefixed with `AUTO_GENERATED_KEY_PREFIX`. * @property parts The key split into its hierarchical components. */ -type ParsedFieldName = { +type ParsedKey = { isAutoGenerated: boolean; parts: string[]; }; @@ -74,7 +74,7 @@ type ParsedFieldName = { * Parsed field placeholder from a YScope format string. */ type YscopeFieldPlaceholder = { - parsedFieldName: ParsedFieldName; + parsedFieldName: ParsedKey; fieldFormatter: Nullable; // Location of field placeholder in format string including braces. @@ -124,7 +124,7 @@ const PERIOD_REGEX = Object.freeze(/(?