Skip to content

Commit

Permalink
new-log-viewer: Add log query support in StateContextProvider. (#80)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Henry8192
Henry8192 authored Oct 19, 2024
1 parent 31786da commit 10599f4
Show file tree
Hide file tree
Showing 6 changed files with 228 additions and 27 deletions.
42 changes: 40 additions & 2 deletions new-log-viewer/src/contexts/StateContextProvider.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
EVENT_POSITION_ON_PAGE,
FileSrcType,
MainWorkerRespMessage,
QueryResults,
WORKER_REQ_CODE,
WORKER_RESP_CODE,
WorkerReq,
Expand Down Expand Up @@ -57,11 +58,13 @@ interface StateContextType {
numPages: number,
onDiskFileSizeInBytes: number,
pageNum: number,
queryResults: QueryResults,

exportLogs: () => void,
loadFile: (fileSrc: FileSrcType, cursor: CursorType) => void,
loadPageByAction: (navAction: NavigationAction) => void,
setLogLevelFilter: (newLogLevelFilter: LogLevelFilter) => void,
startQuery: (queryString: string, isRegex: boolean, isCaseSensitive: boolean) => void,
}
const StateContext = createContext<StateContextType>({} as StateContextType);

Expand All @@ -77,11 +80,13 @@ const STATE_DEFAULT: Readonly<StateContextType> = Object.freeze({
numPages: 0,
onDiskFileSizeInBytes: 0,
pageNum: 0,
queryResults: new Map(),

exportLogs: () => null,
loadFile: () => null,
loadPageByAction: () => null,
setLogLevelFilter: () => null,
startQuery: () => null,
});

interface StateContextProviderProps {
Expand Down Expand Up @@ -229,17 +234,18 @@ const StateContextProvider = ({children}: StateContextProviderProps) => {
const {filePath, logEventNum} = useContext(UrlContext);

// States
const [exportProgress, setExportProgress] =
useState<Nullable<number>>(STATE_DEFAULT.exportProgress);
const [fileName, setFileName] = useState<string>(STATE_DEFAULT.fileName);
const [logData, setLogData] = useState<string>(STATE_DEFAULT.logData);
const [numEvents, setNumEvents] = useState<number>(STATE_DEFAULT.numEvents);
const [numPages, setNumPages] = useState<number>(STATE_DEFAULT.numPages);
const [onDiskFileSizeInBytes, setOnDiskFileSizeInBytes] =
useState(STATE_DEFAULT.onDiskFileSizeInBytes);
const [pageNum, setPageNum] = useState<number>(STATE_DEFAULT.pageNum);
const [queryResults, setQueryResults] = useState<QueryResults>(STATE_DEFAULT.queryResults);
const beginLineNumToLogEventNumRef =
useRef<BeginLineNumToLogEventNumMap>(STATE_DEFAULT.beginLineNumToLogEventNum);
const [exportProgress, setExportProgress] =
useState<Nullable<number>>(STATE_DEFAULT.exportProgress);

// Refs
const logEventNumRef = useRef(logEventNum);
Expand Down Expand Up @@ -281,12 +287,42 @@ const StateContextProvider = ({children}: StateContextProviderProps) => {
});
break;
}
case WORKER_RESP_CODE.QUERY_RESULT:
setQueryResults((v) => {
args.results.forEach((resultsPerPage, queryPageNum) => {
if (false === v.has(queryPageNum)) {
v.set(queryPageNum, []);
}
v.get(queryPageNum)?.push(...resultsPerPage);
});

return v;
});
break;
default:
console.error(`Unexpected ev.data: ${JSON.stringify(ev.data)}`);
break;
}
}, [postPopUp]);

const startQuery = useCallback((
queryString: string,
isRegex: boolean,
isCaseSensitive: boolean
) => {
setQueryResults(STATE_DEFAULT.queryResults);
if (null === mainWorkerRef.current) {
console.error("Unexpected null mainWorkerRef.current");

return;
}
workerPostReq(mainWorkerRef.current, WORKER_REQ_CODE.START_QUERY, {
queryString: queryString,
isRegex: isRegex,
isCaseSensitive: isCaseSensitive,
});
}, []);

const exportLogs = useCallback(() => {
if (null === mainWorkerRef.current) {
console.error("Unexpected null mainWorkerRef.current");
Expand Down Expand Up @@ -442,11 +478,13 @@ const StateContextProvider = ({children}: StateContextProviderProps) => {
numPages: numPages,
onDiskFileSizeInBytes: onDiskFileSizeInBytes,
pageNum: pageNum,
queryResults: queryResults,

exportLogs: exportLogs,
loadFile: loadFile,
loadPageByAction: loadPageByAction,
setLogLevelFilter: setLogLevelFilter,
startQuery: startQuery,
}}
>
{children}
Expand Down
131 changes: 117 additions & 14 deletions new-log-viewer/src/services/LogFileManager/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint max-lines: ["error", 400] */
import {
Decoder,
DecoderOptionsType,
Expand All @@ -11,11 +12,16 @@ import {
CursorType,
EMPTY_PAGE_RESP,
FileSrcType,
QueryResults,
WORKER_RESP_CODE,
WorkerResp,
} from "../../typings/worker";
import {EXPORT_LOGS_CHUNK_SIZE} from "../../utils/config";
import {
EXPORT_LOGS_CHUNK_SIZE,
QUERY_CHUNK_SIZE,
} from "../../utils/config";
import {getChunkNum} from "../../utils/math";
import {defer} from "../../utils/time";
import {formatSizeInBytes} from "../../utils/units";
import ClpIrDecoder from "../decoders/ClpIrDecoder";
import JsonlDecoder from "../decoders/JsonlDecoder";
Expand All @@ -31,35 +37,43 @@ import {
* Class to manage the retrieval and decoding of a given log file.
*/
class LogFileManager {
readonly #fileName: string;

readonly #numEvents: number = 0;

readonly #pageSize: number;

readonly #fileName: string;
#queryId: number = 0;

readonly #onDiskFileSizeInBytes: number;

#decoder: Decoder;
readonly #onQueryResults: (queryResults: QueryResults) => void;

#numEvents: number = 0;
#decoder: Decoder;

/**
* Private constructor for LogFileManager. This is not intended to be invoked publicly.
* Instead, use LogFileManager.create() to create a new instance of the class.
*
* @param decoder
* @param fileName
* @param onDiskFileSizeInBytes
* @param pageSize Page size for setting up pagination.
* @param params
* @param params.decoder
* @param params.fileName
* @param params.onDiskFileSizeInBytes
* @param params.pageSize Page size for setting up pagination.
* @param params.onQueryResults
*/
constructor (
constructor ({decoder, fileName, onDiskFileSizeInBytes, pageSize, onQueryResults}: {
decoder: Decoder,
fileName: string,
onDiskFileSizeInBytes: number,
pageSize: number,
) {
onQueryResults: (queryResults: QueryResults) => void,
}) {
this.#decoder = decoder;
this.#fileName = fileName;
this.#onDiskFileSizeInBytes = onDiskFileSizeInBytes;
this.#pageSize = pageSize;
this.#decoder = decoder;
this.#onDiskFileSizeInBytes = onDiskFileSizeInBytes;
this.#onQueryResults = onQueryResults;

// Build index for the entire file.
const buildResult = decoder.build();
Expand Down Expand Up @@ -90,17 +104,26 @@ class LogFileManager {
* File object.
* @param pageSize Page size for setting up pagination.
* @param decoderOptions Initial decoder options.
* @param onQueryResults
* @return A Promise that resolves to the created LogFileManager instance.
*/
static async create (
fileSrc: FileSrcType,
pageSize: number,
decoderOptions: DecoderOptionsType
decoderOptions: DecoderOptionsType,
onQueryResults: (queryResults: QueryResults) => void,
): Promise<LogFileManager> {
const {fileName, fileData} = await loadFile(fileSrc);
const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions);

return new LogFileManager(decoder, fileName, fileData.length, pageSize);
return new LogFileManager({
decoder: decoder,
fileName: fileName,
onDiskFileSizeInBytes: fileData.length,
pageSize: pageSize,

onQueryResults: onQueryResults,
});
}

/**
Expand Down Expand Up @@ -254,6 +277,86 @@ class LogFileManager {
};
}

/**
* Creates a RegExp object based on the given query string and options, and starts querying the
* first log chunk.
*
* @param queryString
* @param isRegex
* @param isCaseSensitive
*/
startQuery (queryString: string, isRegex: boolean, isCaseSensitive: boolean): void {
this.#queryId++;

// If the query string is empty, or there are no logs, return
if ("" === queryString || 0 === this.#numEvents) {
return;
}

// Construct query RegExp
const regexPattern = isRegex ?
queryString :
queryString.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const regexFlags = isCaseSensitive ?
"" :
"i";
const queryRegex = new RegExp(regexPattern, regexFlags);

this.#queryChunkAndScheduleNext(this.#queryId, 0, queryRegex);
}

/**
* Queries a chunk of log events, sends the results, and schedules the next chunk query if more
* log events remain.
*
* @param queryId
* @param chunkBeginIdx
* @param queryRegex
*/
#queryChunkAndScheduleNext (
queryId: number,
chunkBeginIdx: number,
queryRegex: RegExp
): void {
if (queryId !== this.#queryId) {
// Current task no longer corresponds to the latest query in the LogFileManager.
return;
}
const chunkEndIdx = Math.min(chunkBeginIdx + QUERY_CHUNK_SIZE, this.#numEvents);
const results: QueryResults = new Map();
const decodedEvents = this.#decoder.decodeRange(
chunkBeginIdx,
chunkEndIdx,
null !== this.#decoder.getFilteredLogEventMap()
);

decodedEvents?.forEach(([message, , , logEventNum]) => {
const matchResult = message.match(queryRegex);
if (null !== matchResult && "number" === typeof matchResult.index) {
const pageNum = Math.ceil(logEventNum / this.#pageSize);
if (false === results.has(pageNum)) {
results.set(pageNum, []);
}
results.get(pageNum)?.push({
logEventNum: logEventNum,
message: message,
matchRange: [
matchResult.index,
(matchResult.index + matchResult[0].length),
],
});
}
});

this.#onQueryResults(results);

if (chunkEndIdx < this.#numEvents) {
defer(() => {
this.#queryChunkAndScheduleNext(queryId, chunkEndIdx, queryRegex);
});
}
}

/**
* Gets the data that corresponds to the cursor.
*
Expand Down
31 changes: 30 additions & 1 deletion new-log-viewer/src/services/MainWorker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import dayjsUtc from "dayjs/plugin/utc";
import {LOG_LEVEL} from "../typings/logs";
import {
MainWorkerReqMessage,
QueryResults,
WORKER_REQ_CODE,
WORKER_RESP_CODE,
WorkerResp,
Expand Down Expand Up @@ -36,6 +37,16 @@ const postResp = <T extends WORKER_RESP_CODE>(
postMessage({code, args});
};


/**
* Post a response for a chunk of query results.
*
* @param queryResults
*/
const onQueryResults = (queryResults: QueryResults) => {
postResp(WORKER_RESP_CODE.QUERY_RESULT, {results: queryResults});
};

// eslint-disable-next-line no-warning-comments
// TODO: Break this function up into smaller functions.
// eslint-disable-next-line max-lines-per-function,max-statements
Expand Down Expand Up @@ -63,7 +74,8 @@ onmessage = async (ev: MessageEvent<MainWorkerReqMessage>) => {
LOG_FILE_MANAGER = await LogFileManager.create(
args.fileSrc,
args.pageSize,
args.decoderOptions
args.decoderOptions,
onQueryResults
);

postResp(WORKER_RESP_CODE.LOG_FILE_INFO, {
Expand Down Expand Up @@ -97,6 +109,23 @@ onmessage = async (ev: MessageEvent<MainWorkerReqMessage>) => {
LOG_FILE_MANAGER.loadPage(args.cursor)
);
break;
case WORKER_REQ_CODE.START_QUERY:
if (null === LOG_FILE_MANAGER) {
throw new Error("Log file manager hasn't been initialized");
}
if (
"string" !== typeof args.queryString ||
"boolean" !== typeof args.isRegex ||
"boolean" !== typeof args.isCaseSensitive
) {
throw new Error("Invalid arguments for QUERY_LOG");
}
LOG_FILE_MANAGER.startQuery(
args.queryString,
args.isRegex,
args.isCaseSensitive
);
break;
default:
console.error(`Unexpected ev.data: ${JSON.stringify(ev.data)}`);
break;
Expand Down
Loading

0 comments on commit 10599f4

Please sign in to comment.