diff --git a/new-log-viewer/src/contexts/StateContextProvider.tsx b/new-log-viewer/src/contexts/StateContextProvider.tsx index dec854c2..b8c7f981 100644 --- a/new-log-viewer/src/contexts/StateContextProvider.tsx +++ b/new-log-viewer/src/contexts/StateContextProvider.tsx @@ -20,6 +20,7 @@ import { EVENT_POSITION_ON_PAGE, FileSrcType, MainWorkerRespMessage, + QueryResults, WORKER_REQ_CODE, WORKER_RESP_CODE, WorkerReq, @@ -55,11 +56,13 @@ interface StateContextType { numPages: number, onDiskFileSizeInBytes: number, pageNum: number, + queryResults: QueryResults, exportLogs: () => void, loadFile: (fileSrc: FileSrcType, cursor: CursorType) => void, loadPageByAction: (navAction: NavigationAction) => void, setLogLevelFilter: (newLogLevelFilter: LogLevelFilter) => void, + startQuery: (queryString: string, isRegex: boolean, isCaseSensitive: boolean) => void, } const StateContext = createContext({} as StateContextType); @@ -75,11 +78,13 @@ const STATE_DEFAULT: Readonly = Object.freeze({ numPages: 0, onDiskFileSizeInBytes: 0, pageNum: 0, + queryResults: new Map(), exportLogs: () => null, loadFile: () => null, loadPageByAction: () => null, setLogLevelFilter: () => null, + startQuery: () => null, }); interface StateContextProviderProps { @@ -226,6 +231,8 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { const {filePath, logEventNum} = useContext(UrlContext); // States + const [exportProgress, setExportProgress] = + useState>(STATE_DEFAULT.exportProgress); const [fileName, setFileName] = useState(STATE_DEFAULT.fileName); const [logData, setLogData] = useState(STATE_DEFAULT.logData); const [numEvents, setNumEvents] = useState(STATE_DEFAULT.numEvents); @@ -233,10 +240,9 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { const [onDiskFileSizeInBytes, setOnDiskFileSizeInBytes] = useState(STATE_DEFAULT.onDiskFileSizeInBytes); const [pageNum, setPageNum] = useState(STATE_DEFAULT.pageNum); + const [queryResults, setQueryResults] = useState(STATE_DEFAULT.queryResults); const beginLineNumToLogEventNumRef = useRef(STATE_DEFAULT.beginLineNumToLogEventNum); - const [exportProgress, setExportProgress] = - useState>(STATE_DEFAULT.exportProgress); // Refs const logEventNumRef = useRef(logEventNum); @@ -276,12 +282,42 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { }); break; } + case WORKER_RESP_CODE.QUERY_RESULT: + setQueryResults((v) => { + args.results.forEach((resultsPerPage, queryPageNum) => { + if (false === v.has(queryPageNum)) { + v.set(queryPageNum, []); + } + v.get(queryPageNum)?.push(...resultsPerPage); + }); + + return v; + }); + break; default: console.error(`Unexpected ev.data: ${JSON.stringify(ev.data)}`); break; } }, []); + const startQuery = useCallback(( + queryString: string, + isRegex: boolean, + isCaseSensitive: boolean + ) => { + setQueryResults(STATE_DEFAULT.queryResults); + if (null === mainWorkerRef.current) { + console.error("Unexpected null mainWorkerRef.current"); + + return; + } + workerPostReq(mainWorkerRef.current, WORKER_REQ_CODE.START_QUERY, { + queryString: queryString, + isRegex: isRegex, + isCaseSensitive: isCaseSensitive, + }); + }, []); + const exportLogs = useCallback(() => { if (null === mainWorkerRef.current) { console.error("Unexpected null mainWorkerRef.current"); @@ -437,11 +473,13 @@ const StateContextProvider = ({children}: StateContextProviderProps) => { numPages: numPages, onDiskFileSizeInBytes: onDiskFileSizeInBytes, pageNum: pageNum, + queryResults: queryResults, exportLogs: exportLogs, loadFile: loadFile, loadPageByAction: loadPageByAction, setLogLevelFilter: setLogLevelFilter, + startQuery: startQuery, }} > {children} diff --git a/new-log-viewer/src/services/LogFileManager/index.ts b/new-log-viewer/src/services/LogFileManager/index.ts index b0012b27..67b1c0e4 100644 --- a/new-log-viewer/src/services/LogFileManager/index.ts +++ b/new-log-viewer/src/services/LogFileManager/index.ts @@ -1,3 +1,4 @@ +/* eslint max-lines: ["error", 400] */ import { Decoder, DecoderOptionsType, @@ -11,11 +12,16 @@ import { CursorType, EMPTY_PAGE_RESP, FileSrcType, + QueryResults, WORKER_RESP_CODE, WorkerResp, } from "../../typings/worker"; -import {EXPORT_LOGS_CHUNK_SIZE} from "../../utils/config"; +import { + EXPORT_LOGS_CHUNK_SIZE, + QUERY_CHUNK_SIZE, +} from "../../utils/config"; import {getChunkNum} from "../../utils/math"; +import {defer} from "../../utils/time"; import {formatSizeInBytes} from "../../utils/units"; import ClpIrDecoder from "../decoders/ClpIrDecoder"; import JsonlDecoder from "../decoders/JsonlDecoder"; @@ -31,35 +37,43 @@ import { * Class to manage the retrieval and decoding of a given log file. */ class LogFileManager { + readonly #fileName: string; + + readonly #numEvents: number = 0; + readonly #pageSize: number; - readonly #fileName: string; + #queryId: number = 0; readonly #onDiskFileSizeInBytes: number; - #decoder: Decoder; + readonly #onQueryResults: (queryResults: QueryResults) => void; - #numEvents: number = 0; + #decoder: Decoder; /** * Private constructor for LogFileManager. This is not intended to be invoked publicly. * Instead, use LogFileManager.create() to create a new instance of the class. * - * @param decoder - * @param fileName - * @param onDiskFileSizeInBytes - * @param pageSize Page size for setting up pagination. + * @param params + * @param params.decoder + * @param params.fileName + * @param params.onDiskFileSizeInBytes + * @param params.pageSize Page size for setting up pagination. + * @param params.onQueryResults */ - constructor ( + constructor ({decoder, fileName, onDiskFileSizeInBytes, pageSize, onQueryResults}: { decoder: Decoder, fileName: string, onDiskFileSizeInBytes: number, pageSize: number, - ) { + onQueryResults: (queryResults: QueryResults) => void, + }) { + this.#decoder = decoder; this.#fileName = fileName; - this.#onDiskFileSizeInBytes = onDiskFileSizeInBytes; this.#pageSize = pageSize; - this.#decoder = decoder; + this.#onDiskFileSizeInBytes = onDiskFileSizeInBytes; + this.#onQueryResults = onQueryResults; // Build index for the entire file. const buildResult = decoder.build(); @@ -90,17 +104,26 @@ class LogFileManager { * File object. * @param pageSize Page size for setting up pagination. * @param decoderOptions Initial decoder options. + * @param onQueryResults * @return A Promise that resolves to the created LogFileManager instance. */ static async create ( fileSrc: FileSrcType, pageSize: number, - decoderOptions: DecoderOptionsType + decoderOptions: DecoderOptionsType, + onQueryResults: (queryResults: QueryResults) => void, ): Promise { const {fileName, fileData} = await loadFile(fileSrc); const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions); - return new LogFileManager(decoder, fileName, fileData.length, pageSize); + return new LogFileManager({ + decoder: decoder, + fileName: fileName, + onDiskFileSizeInBytes: fileData.length, + pageSize: pageSize, + + onQueryResults: onQueryResults, + }); } /** @@ -254,6 +277,86 @@ class LogFileManager { }; } + /** + * Creates a RegExp object based on the given query string and options, and starts querying the + * first log chunk. + * + * @param queryString + * @param isRegex + * @param isCaseSensitive + */ + startQuery (queryString: string, isRegex: boolean, isCaseSensitive: boolean): void { + this.#queryId++; + + // If the query string is empty, or there are no logs, return + if ("" === queryString || 0 === this.#numEvents) { + return; + } + + // Construct query RegExp + const regexPattern = isRegex ? + queryString : + queryString.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const regexFlags = isCaseSensitive ? + "" : + "i"; + const queryRegex = new RegExp(regexPattern, regexFlags); + + this.#queryChunkAndScheduleNext(this.#queryId, 0, queryRegex); + } + + /** + * Queries a chunk of log events, sends the results, and schedules the next chunk query if more + * log events remain. + * + * @param queryId + * @param chunkBeginIdx + * @param queryRegex + */ + #queryChunkAndScheduleNext ( + queryId: number, + chunkBeginIdx: number, + queryRegex: RegExp + ): void { + if (queryId !== this.#queryId) { + // Current task no longer corresponds to the latest query in the LogFileManager. + return; + } + const chunkEndIdx = Math.min(chunkBeginIdx + QUERY_CHUNK_SIZE, this.#numEvents); + const results: QueryResults = new Map(); + const decodedEvents = this.#decoder.decodeRange( + chunkBeginIdx, + chunkEndIdx, + null !== this.#decoder.getFilteredLogEventMap() + ); + + decodedEvents?.forEach(([message, , , logEventNum]) => { + const matchResult = message.match(queryRegex); + if (null !== matchResult && "number" === typeof matchResult.index) { + const pageNum = Math.ceil(logEventNum / this.#pageSize); + if (false === results.has(pageNum)) { + results.set(pageNum, []); + } + results.get(pageNum)?.push({ + logEventNum: logEventNum, + message: message, + matchRange: [ + matchResult.index, + (matchResult.index + matchResult[0].length), + ], + }); + } + }); + + this.#onQueryResults(results); + + if (chunkEndIdx < this.#numEvents) { + defer(() => { + this.#queryChunkAndScheduleNext(queryId, chunkEndIdx, queryRegex); + }); + } + } + /** * Gets the data that corresponds to the cursor. * diff --git a/new-log-viewer/src/services/MainWorker.ts b/new-log-viewer/src/services/MainWorker.ts index 751d4cfc..73b2f5da 100644 --- a/new-log-viewer/src/services/MainWorker.ts +++ b/new-log-viewer/src/services/MainWorker.ts @@ -5,6 +5,7 @@ import dayjsUtc from "dayjs/plugin/utc"; import {LOG_LEVEL} from "../typings/logs"; import { MainWorkerReqMessage, + QueryResults, WORKER_REQ_CODE, WORKER_RESP_CODE, WorkerResp, @@ -36,6 +37,16 @@ const postResp = ( postMessage({code, args}); }; + +/** + * Post a response for a chunk of query results. + * + * @param queryResults + */ +const onQueryResults = (queryResults: QueryResults) => { + postResp(WORKER_RESP_CODE.QUERY_RESULT, {results: queryResults}); +}; + // eslint-disable-next-line no-warning-comments // TODO: Break this function up into smaller functions. // eslint-disable-next-line max-lines-per-function,max-statements @@ -63,7 +74,8 @@ onmessage = async (ev: MessageEvent) => { LOG_FILE_MANAGER = await LogFileManager.create( args.fileSrc, args.pageSize, - args.decoderOptions + args.decoderOptions, + onQueryResults ); postResp(WORKER_RESP_CODE.LOG_FILE_INFO, { @@ -97,6 +109,23 @@ onmessage = async (ev: MessageEvent) => { LOG_FILE_MANAGER.loadPage(args.cursor) ); break; + case WORKER_REQ_CODE.START_QUERY: + if (null === LOG_FILE_MANAGER) { + throw new Error("Log file manager hasn't been initialized"); + } + if ( + "string" !== typeof args.queryString || + "boolean" !== typeof args.isRegex || + "boolean" !== typeof args.isCaseSensitive + ) { + throw new Error("Invalid arguments for QUERY_LOG"); + } + LOG_FILE_MANAGER.startQuery( + args.queryString, + args.isRegex, + args.isCaseSensitive + ); + break; default: console.error(`Unexpected ev.data: ${JSON.stringify(ev.data)}`); break; diff --git a/new-log-viewer/src/typings/worker.ts b/new-log-viewer/src/typings/worker.ts index 9ceeea4c..8e64e283 100644 --- a/new-log-viewer/src/typings/worker.ts +++ b/new-log-viewer/src/typings/worker.ts @@ -72,13 +72,15 @@ enum WORKER_REQ_CODE { LOAD_FILE = "loadFile", LOAD_PAGE = "loadPage", SET_FILTER = "setFilter", + START_QUERY = "startQuery", } enum WORKER_RESP_CODE { CHUNK_DATA = "chunkData", LOG_FILE_INFO = "fileInfo", - PAGE_DATA = "pageData", NOTIFICATION = "notification", + PAGE_DATA = "pageData", + QUERY_RESULT = "queryResult", } type WorkerReqMap = { @@ -96,8 +98,23 @@ type WorkerReqMap = { cursor: CursorType, logLevelFilter: LogLevelFilter, }, + [WORKER_REQ_CODE.START_QUERY]: { + queryString: string, + isRegex: boolean, + isCaseSensitive: boolean, + }, }; +type TextRange = [number, number]; + +interface QueryResultsType { + logEventNum: number; + message: string; + matchRange: TextRange; +} + +type QueryResults = Map; + type WorkerRespMap = { [WORKER_RESP_CODE.CHUNK_DATA]: { logs: string @@ -107,18 +124,19 @@ type WorkerRespMap = { numEvents: number, onDiskFileSizeInBytes: number, }, + [WORKER_RESP_CODE.NOTIFICATION]: { + logLevel: LOG_LEVEL, + message: string, + }, [WORKER_RESP_CODE.PAGE_DATA]: { beginLineNumToLogEventNum: BeginLineNumToLogEventNumMap, - cursorLineNum: number - logEventNum: Nullable + cursorLineNum: number, + logEventNum: Nullable, logs: string, - numPages: number - pageNum: number - }, - [WORKER_RESP_CODE.NOTIFICATION]: { - logLevel: LOG_LEVEL, - message: string + numPages: number, + pageNum: number, }, + [WORKER_RESP_CODE.QUERY_RESULT]: { results: QueryResults }, }; type WorkerReq = T extends keyof WorkerReqMap ? @@ -164,6 +182,7 @@ export type { FileSrcType, MainWorkerReqMessage, MainWorkerRespMessage, + QueryResults, WorkerReq, WorkerResp, }; diff --git a/new-log-viewer/src/utils/config.ts b/new-log-viewer/src/utils/config.ts index 3dd16001..f46167fe 100644 --- a/new-log-viewer/src/utils/config.ts +++ b/new-log-viewer/src/utils/config.ts @@ -9,8 +9,9 @@ import { import {DecoderOptionsType} from "../typings/decoders"; -const MAX_PAGE_SIZE = 1_000_000; const EXPORT_LOGS_CHUNK_SIZE = 10_000; +const MAX_PAGE_SIZE = 1_000_000; +const QUERY_CHUNK_SIZE = 10_000; /** * The default configuration values. @@ -154,6 +155,7 @@ export { CONFIG_DEFAULT, EXPORT_LOGS_CHUNK_SIZE, getConfig, + QUERY_CHUNK_SIZE, setConfig, testConfig, }; diff --git a/new-log-viewer/src/utils/time.ts b/new-log-viewer/src/utils/time.ts new file mode 100644 index 00000000..471a0020 --- /dev/null +++ b/new-log-viewer/src/utils/time.ts @@ -0,0 +1,10 @@ +/** + * Defers the execution of a callback function until the call stack is clear. + * + * @param callbackFn The callback function to be executed. + */ +const defer = (callbackFn: () => void) => { + setTimeout(callbackFn, 0); +}; + +export {defer};