From 510cf1738946e12ea44a040ad195a4d362b3861e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 31 Jan 2024 01:17:09 -0500 Subject: [PATCH] Support downloading decompressed Single-file CLP, zst, zip, tar.gz, gz, gzip archives --- src/Viewer/components/MenuBar/MenuBar.js | 76 +-------------------- src/Viewer/services/WorkerPool.js | 7 +- src/Viewer/services/decoder/FileManager.js | 45 ++++++++---- src/Viewer/services/decoder/decodeWorker.js | 55 ++++++++------- 4 files changed, 68 insertions(+), 115 deletions(-) diff --git a/src/Viewer/components/MenuBar/MenuBar.js b/src/Viewer/components/MenuBar/MenuBar.js index 1ac08a54..5152bff9 100644 --- a/src/Viewer/components/MenuBar/MenuBar.js +++ b/src/Viewer/components/MenuBar/MenuBar.js @@ -9,13 +9,10 @@ import { ChevronRight, FileText, Keyboard, - Moon, - Sun } from "react-bootstrap-icons"; import {THEME_STATES} from "../../../ThemeContext/THEME_STATES"; import {ThemeContext} from "../../../ThemeContext/ThemeContext"; -import LOCAL_STORAGE_KEYS from "../../services/LOCAL_STORAGE_KEYS"; import MODIFY_PAGE_ACTION from "../../services/MODIFY_PAGE_ACTION"; import STATE_CHANGE_TYPE from "../../services/STATE_CHANGE_TYPE"; import {EditableInput} from "./EditableInput/EditableInput"; @@ -47,22 +44,15 @@ MenuBar.propTypes = { * @return {JSX.Element} */ export function MenuBar ({ - logFileState, fileInfo, loadingLogs, changeStateCallback, loadFileCallback, + logFileState, fileInfo, loadingLogs, changeStateCallback, }) { - const {theme, switchTheme} = useContext(ThemeContext); + const {theme} = useContext(ThemeContext); - const [eventsPerPage, setEventsPerPage] = useState(logFileState.pages); - const [showSettings, setShowSettings] = useState(false); const [showHelp, setShowHelp] = useState(false); - const handleCloseSettings = () => setShowSettings(false); - const handleShowSettings = () => setShowSettings(true); - const handleCloseHelp = () => setShowHelp(false); const handleShowHelp = () => setShowHelp(true); - const downloadWorker = useRef(null); - const goToFirstPage = () => { if (logFileState.page !== 1) { changeStateCallback(STATE_CHANGE_TYPE.page, {action: MODIFY_PAGE_ACTION.firstPage}); @@ -95,38 +85,6 @@ export function MenuBar ({ return (THEME_STATES.LIGHT === theme)?"modal-light":"modal-dark"; }; - const saveModalChanges = (e) => { - // TODO Can't backspace 0 from the number input - // TODO What is the maximum number of events monaco can support? - e.preventDefault(); - handleCloseSettings(); - changeStateCallback(STATE_CHANGE_TYPE.pageSize, {pageSize: eventsPerPage}); - localStorage.setItem(LOCAL_STORAGE_KEYS.PAGE_SIZE, String(eventsPerPage)); - }; - - const closeModal = () => { - handleCloseSettings(); - }; - - const openModal = () => { - handleShowSettings(); - setEventsPerPage(logFileState.pageSize); - }; - - const getThemeIcon = () => { - if (THEME_STATES.LIGHT === theme) { - return ( - switchTheme(THEME_STATES.DARK)}/> - ); - } else if (THEME_STATES.DARK === theme) { - return ( - switchTheme(THEME_STATES.LIGHT)}/> - ); - } - }; - const getPageNav = () => { return ( <> @@ -160,7 +118,6 @@ export function MenuBar ({ }; const fileName = fileInfo.name.split("?")[0]; - // TODO make file icon a button to open modal with file info // TODO Move modals into their own component return ( @@ -185,35 +142,6 @@ export function MenuBar ({ {getLoadingBar()} - - -
- App Settings -
-
- {getThemeIcon()} -
- - - -
- setEventsPerPage(Number(e.target.value))} - className="input-sm num-event-input" /> - -
- - - - - - diff --git a/src/Viewer/services/WorkerPool.js b/src/Viewer/services/WorkerPool.js index b7c883bc..007eae11 100644 --- a/src/Viewer/services/WorkerPool.js +++ b/src/Viewer/services/WorkerPool.js @@ -54,7 +54,12 @@ class WorkerPool { const worker = this.getWorker(); if (worker) { const task = this.taskQueue.shift(); - worker.postMessage(task, [task.inputStream]); + if (null === task.pageLogs) { + worker.postMessage(task, [task.inputStream]); + } else { + // FIXME: dirty hack to get download working + worker.postMessage(task); + } worker.onmessage = () => { this.freeWorker(worker); this.processQueue(); diff --git a/src/Viewer/services/decoder/FileManager.js b/src/Viewer/services/decoder/FileManager.js index 0943cd32..ef0df1e9 100644 --- a/src/Viewer/services/decoder/FileManager.js +++ b/src/Viewer/services/decoder/FileManager.js @@ -86,7 +86,7 @@ class FileManager { this._logs = ""; this._logsArray = null; // for Single-file CLP Archive only - this._logsLineOffsetsArray = []; + this._logsPageLineOffsetsArray = []; this._loadState = { prevCheckTime: null, @@ -260,12 +260,15 @@ class FileManager { this.state.numberOfEvents = this._logsArray.length; this.computePageNumFromLogEventIdx(); this.createPages(); - this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state); this.decodePage(); // FIXME: dirty hack to get search working this._logEventOffsetsFiltered.length = this._logsArray.length; + this._logEventOffsets.length = this._logsArray.length; + this._setupDecodingPagesToDatabase(); + + this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state); } /** @@ -418,13 +421,14 @@ class FileManager { * @private */ async _decodeClpArchiveLogAndUpdate (decompressedLogFile) { - // Update decompression status - this.state.decompressedHumanSize = formatSizeInBytes(decompressedLogFile.byteLength, false); - this._loadingMessageCallback(`Decompressed ${this.state.decompressedHumanSize}.`); - const clpArchiveDecoder = new ClpArchiveDecoder(decompressedLogFile); this._logsArray = await clpArchiveDecoder.decode(); + // Update decompression status + this.state.decompressedHumanSize = + formatSizeInBytes(this._logsArray.join("\n").length, false); + this._loadingMessageCallback(`Decompressed ${this.state.decompressedHumanSize}.`); + // Update state this.state.verbosity = -1; this.state.lineNumber = 1; @@ -432,12 +436,14 @@ class FileManager { this.state.numberOfEvents = this._logsArray.length; this.computePageNumFromLogEventIdx(); this.createPages(); - this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state); - this.decodePage(); // FIXME: dirty hack to get search working this._logEventOffsetsFiltered.length = this._logsArray.length; + this._logEventOffsets.length = this._logsArray.length; + this._setupDecodingPagesToDatabase(); + + this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state); } /** @@ -531,6 +537,17 @@ class FileManager { ?numEventsAtLevel - targetEvent :pageSize; + if (null !== this._logsArray) { + // FIXME: dirty hack to get download working + this._workerPool.assignTask({ + fileName: this._fileInfo.name, + page: page, + pageLogs: + this._logsArray?.slice(targetEvent, targetEvent + numberOfEvents).join("\n"), + }); + return; + } + const pageData = this._arrayBuffer.slice( this._logEventOffsets[targetEvent].startIndex, this._logEventOffsets[targetEvent + numberOfEvents - 1].endIndex + 1 @@ -543,6 +560,7 @@ class FileManager { page: page, logEvents: logEvents, inputStream: inputStream, + pageLogs: null, // FIXME: dirty hack to get download working }); } @@ -558,15 +576,12 @@ class FileManager { this.state.pageSize * (this.state.page)); let offset = 0; - this._logsLineOffsetsArray.length = 0; + this._logsPageLineOffsetsArray.length = 0; this._logs = ""; - console.log(this._logsLineOffsetsArray); - console.log(this._logs); - console.log(startingEventIdx); - console.log(endingEventIdx); + for (let i = startingEventIdx; i< endingEventIdx; i++) { this._logs += this._logsArray[i] + "\n"; - this._logsLineOffsetsArray.push(offset); + this._logsPageLineOffsetsArray.push(offset); offset += this._logsArray[i].split("\n").length; } @@ -870,7 +885,7 @@ class FileManager { // FIXME: dirty hack for Single-file CLP Archive this.state.columnNumber = 1; this.state.lineNumber = - this._logsLineOffsetsArray[ + this._logsPageLineOffsetsArray[ this.state.logEventIdx - this.state.pageSize * (this.state.page - 1) ]; return; diff --git a/src/Viewer/services/decoder/decodeWorker.js b/src/Viewer/services/decoder/decodeWorker.js index 4aee641d..bd1a0e34 100644 --- a/src/Viewer/services/decoder/decodeWorker.js +++ b/src/Viewer/services/decoder/decodeWorker.js @@ -3,38 +3,42 @@ import {DataInputStream, DataInputStreamEOFError} from "./DataInputStream"; import FourByteClpIrStreamReader from "./FourByteClpIrStreamReader"; import ResizableUint8Array from "./ResizableUint8Array"; -const decodePage = async (fileName, logEvents, inputStream, page) => { - const dataInputStream = new DataInputStream(inputStream); - const _outputResizableBuffer = new ResizableUint8Array(inputStream.byteLength); - const _irStreamReader = new FourByteClpIrStreamReader(dataInputStream, null); +const decodePage = async (fileName, logEvents, inputStream, page, pageLogs) => { + let _logs = pageLogs; - const _logEventMetadata = []; - for (let i = 0; i < logEvents.length; i++) { - const decoder = _irStreamReader._streamProtocolDecoder; - decoder._setTimestamp(logEvents[i].prevTs); + if (null === _logs) { + const dataInputStream = new DataInputStream(inputStream); + const _outputResizableBuffer = new ResizableUint8Array(inputStream.byteLength); + const _irStreamReader = new FourByteClpIrStreamReader(dataInputStream, null); - try { - _irStreamReader.readAndDecodeLogEvent( - _outputResizableBuffer, - _logEventMetadata - ); - } catch (error) { + const _logEventMetadata = []; + for (let i = 0; i < logEvents.length; i++) { + const decoder = _irStreamReader._streamProtocolDecoder; + decoder._setTimestamp(logEvents[i].prevTs); + + try { + _irStreamReader.readAndDecodeLogEvent( + _outputResizableBuffer, + _logEventMetadata + ); + } catch (error) { // Ignore EOF errors since we should still be able // to print the decoded messages - if (error instanceof DataInputStreamEOFError) { + if (error instanceof DataInputStreamEOFError) { // TODO Give visual indication that the stream is truncated - console.error("Stream truncated."); - } else { - console.log("random error"); - throw error; + console.error("Stream truncated."); + } else { + console.log("random error"); + throw error; + } } } - } - // Decode the text - const _textDecoder = new TextDecoder(); - const logs = _textDecoder.decode(_outputResizableBuffer.getUint8Array()); - const _logs = logs.trim(); + // Decode the text + const _textDecoder = new TextDecoder(); + const logs = _textDecoder.decode(_outputResizableBuffer.getUint8Array()); + _logs = logs.trim(); + } const db = new Database(fileName); db.addPage(page, _logs).then(() => { @@ -51,5 +55,6 @@ onmessage = (e) => { const logEvents = e.data.logEvents; const inputStream = e.data.inputStream; const page = e.data.page; - decodePage(fileName, logEvents, inputStream, page); + const pageLogs = e.data.pageLogs; + decodePage(fileName, logEvents, inputStream, page, pageLogs); };