Skip to content

Commit

Permalink
Support downloading decompressed Single-file CLP, zst, zip, tar.gz, g…
Browse files Browse the repository at this point in the history 
…z, gzip archives
  • Loading branch information
@junhaoliao
junhaoliao committed Jan 31, 2024
1 parent 2e74a49 commit 510cf17
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 115 deletions.
76 changes: 2 additions & 74 deletions src/Viewer/components/MenuBar/MenuBar.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,10 @@ import {
ChevronRight,
FileText,
Keyboard,
Moon,
Sun
} from "react-bootstrap-icons";

import {THEME_STATES} from "../../../ThemeContext/THEME_STATES";
import {ThemeContext} from "../../../ThemeContext/ThemeContext";
import LOCAL_STORAGE_KEYS from "../../services/LOCAL_STORAGE_KEYS";
import MODIFY_PAGE_ACTION from "../../services/MODIFY_PAGE_ACTION";
import STATE_CHANGE_TYPE from "../../services/STATE_CHANGE_TYPE";
import {EditableInput} from "./EditableInput/EditableInput";
Expand Down Expand Up @@ -47,22 +44,15 @@ MenuBar.propTypes = {
* @return {JSX.Element}
*/
export function MenuBar ({
logFileState, fileInfo, loadingLogs, changeStateCallback, loadFileCallback,
logFileState, fileInfo, loadingLogs, changeStateCallback,
}) {
const {theme, switchTheme} = useContext(ThemeContext);
const {theme} = useContext(ThemeContext);

const [eventsPerPage, setEventsPerPage] = useState(logFileState.pages);
const [showSettings, setShowSettings] = useState(false);
const [showHelp, setShowHelp] = useState(false);

const handleCloseSettings = () => setShowSettings(false);
const handleShowSettings = () => setShowSettings(true);

const handleCloseHelp = () => setShowHelp(false);
const handleShowHelp = () => setShowHelp(true);

const downloadWorker = useRef(null);

const goToFirstPage = () => {
if (logFileState.page !== 1) {
changeStateCallback(STATE_CHANGE_TYPE.page, {action: MODIFY_PAGE_ACTION.firstPage});
Expand Down Expand Up @@ -95,38 +85,6 @@ export function MenuBar ({
return (THEME_STATES.LIGHT === theme)?"modal-light":"modal-dark";
};

const saveModalChanges = (e) => {
// TODO Can't backspace 0 from the number input
// TODO What is the maximum number of events monaco can support?
e.preventDefault();
handleCloseSettings();
changeStateCallback(STATE_CHANGE_TYPE.pageSize, {pageSize: eventsPerPage});
localStorage.setItem(LOCAL_STORAGE_KEYS.PAGE_SIZE, String(eventsPerPage));
};

const closeModal = () => {
handleCloseSettings();
};

const openModal = () => {
handleShowSettings();
setEventsPerPage(logFileState.pageSize);
};

const getThemeIcon = () => {
if (THEME_STATES.LIGHT === theme) {
return (
<Moon className="cursor-pointer" title="Set Light Mode"
onClick={() => switchTheme(THEME_STATES.DARK)}/>
);
} else if (THEME_STATES.DARK === theme) {
return (
<Sun className="cursor-pointer" title="Set Dark Mode"
onClick={() => switchTheme(THEME_STATES.LIGHT)}/>
);
}
};

const getPageNav = () => {
return (
<>
Expand Down Expand Up @@ -160,7 +118,6 @@ export function MenuBar ({
};

const fileName = fileInfo.name.split("?")[0];

// TODO make file icon a button to open modal with file info
// TODO Move modals into their own component
return (
Expand All @@ -185,35 +142,6 @@ export function MenuBar ({
</div>
{getLoadingBar()}
</div>
<Modal show={showSettings} className="border-0" onHide={handleCloseSettings}
contentClassName={getModalClass()}>
<Modal.Header className="modal-background border-0" >
<div className="float-left">
App Settings
</div>
<div className="float-right">
{getThemeIcon()}
</div>
</Modal.Header>
<Modal.Body className="modal-background p-3 pt-1" >
<label className="mb-2">Log Events per Page</label>
<Form onSubmit={saveModalChanges}>
<Form.Control type="number"
value={eventsPerPage}
onChange={(e) => setEventsPerPage(Number(e.target.value))}
className="input-sm num-event-input" />
</Form>
</Modal.Body>
<Modal.Footer className="modal-background border-0" >
<Button className="btn-sm" variant="success" onClick={saveModalChanges}>
Save Changes
</Button>
<Button className="btn-sm" variant="secondary" onClick={closeModal}>
Close
</Button>
</Modal.Footer>
</Modal>

<Modal show={showHelp} className="help-modal border-0" onHide={handleCloseHelp}
contentClassName={getModalClass()} data-theme={theme}>
<Modal.Header className="modal-background" >
Expand Down
7 changes: 6 additions & 1 deletion src/Viewer/services/WorkerPool.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,12 @@ class WorkerPool {
const worker = this.getWorker();
if (worker) {
const task = this.taskQueue.shift();
worker.postMessage(task, [task.inputStream]);
if (null === task.pageLogs) {
worker.postMessage(task, [task.inputStream]);
} else {
// FIXME: dirty hack to get download working
worker.postMessage(task);
}
worker.onmessage = () => {
this.freeWorker(worker);
this.processQueue();
Expand Down
45 changes: 30 additions & 15 deletions src/Viewer/services/decoder/FileManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ class FileManager {

this._logs = "";
this._logsArray = null; // for Single-file CLP Archive only
this._logsLineOffsetsArray = [];
this._logsPageLineOffsetsArray = [];

this._loadState = {
prevCheckTime: null,
Expand Down Expand Up @@ -260,12 +260,15 @@ class FileManager {
this.state.numberOfEvents = this._logsArray.length;
this.computePageNumFromLogEventIdx();
this.createPages();
this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state);

this.decodePage();

// FIXME: dirty hack to get search working
this._logEventOffsetsFiltered.length = this._logsArray.length;
this._logEventOffsets.length = this._logsArray.length;
this._setupDecodingPagesToDatabase();

this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state);
}

/**
Expand Down Expand Up @@ -418,26 +421,29 @@ class FileManager {
* @private
*/
async _decodeClpArchiveLogAndUpdate (decompressedLogFile) {
// Update decompression status
this.state.decompressedHumanSize = formatSizeInBytes(decompressedLogFile.byteLength, false);
this._loadingMessageCallback(`Decompressed ${this.state.decompressedHumanSize}.`);

const clpArchiveDecoder = new ClpArchiveDecoder(decompressedLogFile);
this._logsArray = await clpArchiveDecoder.decode();

// Update decompression status
this.state.decompressedHumanSize =
formatSizeInBytes(this._logsArray.join("\n").length, false);
this._loadingMessageCallback(`Decompressed ${this.state.decompressedHumanSize}.`);

// Update state
this.state.verbosity = -1;
this.state.lineNumber = 1;
this.state.columnNumber = 1;
this.state.numberOfEvents = this._logsArray.length;
this.computePageNumFromLogEventIdx();
this.createPages();
this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state);

this.decodePage();

// FIXME: dirty hack to get search working
this._logEventOffsetsFiltered.length = this._logsArray.length;
this._logEventOffsets.length = this._logsArray.length;
this._setupDecodingPagesToDatabase();

this._updateStateCallback(CLP_WORKER_PROTOCOL.UPDATE_STATE, this.state);
}

/**
Expand Down Expand Up @@ -531,6 +537,17 @@ class FileManager {
?numEventsAtLevel - targetEvent
:pageSize;

if (null !== this._logsArray) {
// FIXME: dirty hack to get download working
this._workerPool.assignTask({
fileName: this._fileInfo.name,
page: page,
pageLogs:
this._logsArray?.slice(targetEvent, targetEvent + numberOfEvents).join("\n"),
});
return;
}

const pageData = this._arrayBuffer.slice(
this._logEventOffsets[targetEvent].startIndex,
this._logEventOffsets[targetEvent + numberOfEvents - 1].endIndex + 1
Expand All @@ -543,6 +560,7 @@ class FileManager {
page: page,
logEvents: logEvents,
inputStream: inputStream,
pageLogs: null, // FIXME: dirty hack to get download working
});
}

Expand All @@ -558,15 +576,12 @@ class FileManager {
this.state.pageSize * (this.state.page));

let offset = 0;
this._logsLineOffsetsArray.length = 0;
this._logsPageLineOffsetsArray.length = 0;
this._logs = "";
console.log(this._logsLineOffsetsArray);
console.log(this._logs);
console.log(startingEventIdx);
console.log(endingEventIdx);

for (let i = startingEventIdx; i< endingEventIdx; i++) {
this._logs += this._logsArray[i] + "\n";
this._logsLineOffsetsArray.push(offset);
this._logsPageLineOffsetsArray.push(offset);
offset += this._logsArray[i].split("\n").length;
}

Expand Down Expand Up @@ -870,7 +885,7 @@ class FileManager {
// FIXME: dirty hack for Single-file CLP Archive
this.state.columnNumber = 1;
this.state.lineNumber =
this._logsLineOffsetsArray[
this._logsPageLineOffsetsArray[
this.state.logEventIdx - this.state.pageSize * (this.state.page - 1)
];
return;
Expand Down
55 changes: 30 additions & 25 deletions src/Viewer/services/decoder/decodeWorker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,42 @@ import {DataInputStream, DataInputStreamEOFError} from "./DataInputStream";
import FourByteClpIrStreamReader from "./FourByteClpIrStreamReader";
import ResizableUint8Array from "./ResizableUint8Array";

const decodePage = async (fileName, logEvents, inputStream, page) => {
const dataInputStream = new DataInputStream(inputStream);
const _outputResizableBuffer = new ResizableUint8Array(inputStream.byteLength);
const _irStreamReader = new FourByteClpIrStreamReader(dataInputStream, null);
const decodePage = async (fileName, logEvents, inputStream, page, pageLogs) => {
let _logs = pageLogs;

const _logEventMetadata = [];
for (let i = 0; i < logEvents.length; i++) {
const decoder = _irStreamReader._streamProtocolDecoder;
decoder._setTimestamp(logEvents[i].prevTs);
if (null === _logs) {
const dataInputStream = new DataInputStream(inputStream);
const _outputResizableBuffer = new ResizableUint8Array(inputStream.byteLength);
const _irStreamReader = new FourByteClpIrStreamReader(dataInputStream, null);

try {
_irStreamReader.readAndDecodeLogEvent(
_outputResizableBuffer,
_logEventMetadata
);
} catch (error) {
const _logEventMetadata = [];
for (let i = 0; i < logEvents.length; i++) {
const decoder = _irStreamReader._streamProtocolDecoder;
decoder._setTimestamp(logEvents[i].prevTs);

try {
_irStreamReader.readAndDecodeLogEvent(
_outputResizableBuffer,
_logEventMetadata
);
} catch (error) {
// Ignore EOF errors since we should still be able
// to print the decoded messages
if (error instanceof DataInputStreamEOFError) {
if (error instanceof DataInputStreamEOFError) {
// TODO Give visual indication that the stream is truncated
console.error("Stream truncated.");
} else {
console.log("random error");
throw error;
console.error("Stream truncated.");
} else {
console.log("random error");
throw error;
}
}
}
}

// Decode the text
const _textDecoder = new TextDecoder();
const logs = _textDecoder.decode(_outputResizableBuffer.getUint8Array());
const _logs = logs.trim();
// Decode the text
const _textDecoder = new TextDecoder();
const logs = _textDecoder.decode(_outputResizableBuffer.getUint8Array());
_logs = logs.trim();
}

const db = new Database(fileName);
db.addPage(page, _logs).then(() => {
Expand All @@ -51,5 +55,6 @@ onmessage = (e) => {
const logEvents = e.data.logEvents;
const inputStream = e.data.inputStream;
const page = e.data.page;
decodePage(fileName, logEvents, inputStream, page);
const pageLogs = e.data.pageLogs;
decodePage(fileName, logEvents, inputStream, page, pageLogs);
};

0 comments on commit 510cf17

Please sign in to comment.