Merge pull request #170 from kaitai-io/partial-parsing-tree

Display a partial object tree if a parsing error occurred, indicate errors

css/app.css

@@ -1,4 +1,4 @@
-/*.lm_content>div { height: 100%; width: 100% }*/
+/*.lm_content>div { height: 100%; width: 100% }*/
 .lm_splitter.lm_horizontal .lm_drag_handle { left:-5px; width:15px }
 .lm_splitter.lm_vertical .lm_drag_handle { top:-5px; height:15px }
 .errorWindow { background: white; font-family: Courier,monospace; font-size: 12px; white-space: pre-wrap; overflow-y: scroll; }
@@ -9,6 +9,9 @@
 #parsedDataTree.jstree-default>.jstree-container-ul>.jstree-node { margin-left: 0 }
 #parsedDataTree.jstree-default .jstree-ocl { background-position-y: -8px; width:18px; }
 #parsedDataTree .missing { font-family: Arial; color: #c00000; }
+.alert-color { color: #FFC20A; }
+.fail-color { color: #FF4136; }
+.instance-fail-color { color: #0074D9; }
 #fileDrop { display:none; font-family: Arial; position: fixed; top: 0; left: 0; bottom: 0; right: 0; background: rgba(0,0,0,0.8); z-index: 9999 }
 #fileDrop>div { border:2px dashed white; border-radius:25px; width:500px; margin-left:-250px; height:180px; margin-top:-100px; position:fixed; top:50%; left:50%; color:white; text-align:center; font-size:22px; padding-top:65px }
 #fileTree { font-family:Arial; font-size:12px }
@@ -55,4 +58,4 @@ body { color:#333 }
 #welcomeModalLabel { text-align:center }
 #welcomeModal .licenses { font-size:12px; margin-bottom:10px }
 #converterPanel { display:none }
-/*.marker_match { background:rgba(80, 255, 80, 0.30); }*/
+/*.marker_match { background:rgba(80, 255, 80, 0.30); }*/

src/entities.ts

@@ -1,4 +1,4 @@
-class ObjectType {
+class ObjectType {
     public static Primitive = "Primitive";
     public static Array = "Array";
     public static TypedArray = "TypedArray";
@@ -16,6 +16,11 @@ interface IWorkerMessage {
 }
 /* tslint:enable */
 
+interface IWorkerResponse {
+    result: any;
+    error: any;
+}
+
 interface IInstance {
     path: string[];
     offset: number;
@@ -34,6 +39,9 @@ interface IExportedValue {
     ioOffset: number;
     start: number;
     end: number;
+    incomplete: boolean;
+    validationError?: Error;
+    instanceError?: Error;
 
     primitiveValue?: any;
     arrayItems?: IExportedValue[];

src/v1/ExportToJson.ts

@@ -1,4 +1,4 @@
-import { workerMethods } from "./app.worker";
+import { workerMethods } from "./app.worker";
 
 export function exportToJson(useHex: boolean = false) {
     var indentLen = 2;
@@ -44,9 +44,16 @@ export function exportToJson(useHex: boolean = false) {
         }
     }
 
-    return workerMethods.reparse(true).then(exportedRoot => {
-        console.log("exported", exportedRoot);
-        expToNative(exportedRoot);
-        return result;
-    });
-}
+    return workerMethods.reparse(true)
+        .then(response => {
+            if (response.error) {
+                throw response.error;
+            }
+            return response.result;
+        })
+        .then(exportedRoot => {
+            console.log("exported", exportedRoot);
+            expToNative(exportedRoot);
+            return result;
+        });
+}

src/v1/app.ts

@@ -1,4 +1,4 @@
-import * as localforage from "localforage";
+import * as localforage from "localforage";
 import * as Vue from "vue";
 
 import { UI } from "./app.layout";
@@ -118,21 +118,29 @@ class AppController {
             let jsClassName = this.compilerService.ksySchema.meta.id.split("_").map((x: string) => x.ucFirst()).join("");
             await workerMethods.initCode(debugCode, jsClassName, this.compilerService.ksyTypes);
 
-            let exportedRoot = await workerMethods.reparse(this.vm.disableLazyParsing);
+            const { result: exportedRoot, error: parseError } = await workerMethods.reparse(this.vm.disableLazyParsing);
             kaitaiIde.root = exportedRoot;
             //console.log("reparse exportedRoot", exportedRoot);
 
             this.ui.parsedDataTreeHandler = new ParsedTreeHandler(this.ui.parsedDataTreeCont.getElement(), exportedRoot, this.compilerService.ksyTypes);
 
             this.ui.parsedDataTreeHandler.jstree.on("state_ready.jstree", () => {
                 this.ui.parsedDataTreeHandler.jstree.on("select_node.jstree", (e, selectNodeArgs) => {
-                    var node = <IParsedTreeNode>selectNodeArgs.node;
+                    const node = <IParsedTreeNode>selectNodeArgs.node;
                     //console.log("node", node);
-                    var exp = this.ui.parsedDataTreeHandler.getNodeData(node).exported;
+                    const exp = this.ui.parsedDataTreeHandler.getNodeData(node).exported;
 
                     if (exp && exp.path)
                         $("#parsedPath").text(exp.path.join("/"));
 
+                    if (exp) {
+                        if (exp.instanceError !== undefined) {
+                            app.errors.handle(exp.instanceError);
+                        } else if (exp.validationError !== undefined) {
+                            app.errors.handle(exp.validationError);
+                        }
+                    }
+
                     if (!this.blockRecursive && exp && exp.start < exp.end) {
                         this.selectedInTree = true;
                         //console.log("setSelection", exp.ioOffset, exp.start);
@@ -142,7 +150,7 @@ class AppController {
                 });
             });
 
-            this.errors.handle(null);
+            this.errors.handle(parseError);
         } catch(error) {
             this.errors.handle(error);
         }

src/v1/app.worker.ts

@@ -1,4 +1,4 @@
-var worker = new Worker("js/v1/kaitaiWorker.js");
+var worker = new Worker("js/v1/kaitaiWorker.js");
 
 var msgHandlers: { [msgId: number]: (msg: IWorkerMessage) => void } = {};
 
@@ -10,16 +10,20 @@ worker.onmessage = (ev: MessageEvent) => {
 };
 
 var lastMsgId = 0;
-function workerCall(request: IWorkerMessage) {
-    return new Promise<any>((resolve, reject) => {
+function workerCall(request: IWorkerMessage): Promise<IWorkerResponse> {
+    return new Promise((resolve, reject) => {
         request.msgId = ++lastMsgId;
         msgHandlers[request.msgId] = response => {
             if (response.error) {
                 console.log("error", response.error);
+            }
+
+            if (response.error && (response.result === undefined || response.result === null)) {
                 reject(response.error);
+            } else {
+                const { result, error } = response;
+                resolve({ result, error });
             }
-            else
-                resolve(response.result);
 
             //console.info(`[performance] [${(new Date()).format("H:i:s.u")}] Got worker response: ${Date.now()}.`);
         };
@@ -29,15 +33,15 @@ function workerCall(request: IWorkerMessage) {
 
 export var workerMethods = {
     initCode: (sourceCode: string, mainClassName: string, ksyTypes: IKsyTypes) => {
-        return <Promise<void>>workerCall({ type: "initCode", args: [sourceCode, mainClassName, ksyTypes] });
+        return workerCall({ type: "initCode", args: [sourceCode, mainClassName, ksyTypes] });
     },
     setInput: (inputBuffer: ArrayBuffer) => {
-        return <Promise<void>>workerCall({ type: "setInput", args: [inputBuffer] });
+        return workerCall({ type: "setInput", args: [inputBuffer] });
     },
     reparse: (eagerMode: boolean) => {
-        return <Promise<IExportedValue>>workerCall({ type: "reparse", args: [eagerMode] });
+        return workerCall({ type: "reparse", args: [eagerMode] });
     },
     get: (path: string[]) => {
-        return <Promise<IExportedValue>>workerCall({ type: "get", args: [path] });
+        return workerCall({ type: "get", args: [path] });
     }
-};
+};

src/v1/kaitaiWorker.ts

@@ -1,4 +1,4 @@
-// issue: https://github.com/Microsoft/TypeScript/issues/582
+// issue: https://github.com/Microsoft/TypeScript/issues/582
 var myself = <Worker><any>self;
 
 var wi = {
@@ -18,6 +18,8 @@ interface IDebugInfo {
     start: number;
     end: number;
     ioOffset: number;
+    validationError?: Error;
+    incomplete: boolean;
     arr?: IDebugInfo[];
     enumName?: string;
 }
@@ -35,10 +37,13 @@ function getObjectType(obj: any) {
         return ObjectType.Object;
 }
 
-function exportValue(obj: any, debug: IDebugInfo, path: string[], noLazy?: boolean): IExportedValue {
-    var result = <IExportedValue>{
+function exportValue(obj: any, debug: IDebugInfo, hasRawAttr: boolean, path: string[], noLazy: boolean): IExportedValue {
+    adjustDebug(debug);
+    var result: IExportedValue = {
         start: debug && debug.start,
         end: debug && debug.end,
+        incomplete: debug && debug.incomplete,
+        validationError: (debug && debug.validationError) || undefined,
         ioOffset: debug && debug.ioOffset,
         path: path,
         type: getObjectType(obj)
@@ -78,21 +83,33 @@ function exportValue(obj: any, debug: IDebugInfo, path: string[], noLazy?: boole
         }
     }
     else if (result.type === ObjectType.Array) {
-        result.arrayItems = (<any[]>obj).map((item, i) => exportValue(item, debug && debug.arr && debug.arr[i], path.concat(i.toString()), noLazy));
+        result.arrayItems = (<any[]>obj).map((item, i) => exportValue(item, debug && debug.arr && debug.arr[i], hasRawAttr, path.concat(i.toString()), noLazy));
+        if (result.incomplete && debug && debug.arr) {
+            debug.end = inferDebugEnd(debug.arr);
+            result.end = debug.end;
+        }
     }
     else if (result.type === ObjectType.Object) {
-        var childIoOffset = obj._io ? obj._io._byteOffset : 0;
-
-        if (result.start === childIoOffset) { // new KaitaiStream was used, fix start position
-            result.ioOffset = childIoOffset;
-            result.start -= childIoOffset;
-            result.end -= childIoOffset;
+        const hasSubstream = hasRawAttr && obj._io;
+        if (result.incomplete && hasSubstream) {
+            debug.end = debug.start + obj._io.size;
+            result.end = debug.end;
         }
-
         result.object = { class: obj.constructor.name, instances: {}, fields: {} };
-        var ksyType = wi.ksyTypes[result.object.class];
+        const ksyType = wi.ksyTypes[result.object.class];
 
-        Object.keys(obj).filter(x => x[0] !== "_").forEach(key => result.object.fields[key] = exportValue(obj[key], obj._debug && obj._debug[key], path.concat(key), noLazy));
+        const fieldNames = new Set<string>(Object.keys(obj));
+        if (obj._debug) {
+            Object.keys(obj._debug).forEach(k => fieldNames.add(k));
+        }
+        const fieldNamesArr = Array.from(fieldNames).filter(x => x[0] !== "_");
+        fieldNamesArr
+            .forEach(key => result.object.fields[key] = exportValue(obj[key], obj._debug && obj._debug[key], fieldNames.has(`_raw_${key}`), path.concat(key), noLazy));
+
+        if (result.incomplete && !hasSubstream && obj._debug) {
+            debug.end = inferDebugEnd(fieldNamesArr.map(key => <IDebugInfo>obj._debug[key]));
+            result.end = debug.end;
+        }
 
         const propNames = obj.constructor !== Object ?
             Object.getOwnPropertyNames(obj.constructor.prototype).filter(x => x[0] !== "_" && x !== "constructor") : [];
@@ -102,8 +119,8 @@ function exportValue(obj: any, debug: IDebugInfo, path: string[], noLazy?: boole
             const parseMode = ksyInstanceData["-webide-parse-mode"];
             const eagerLoad = parseMode === "eager" || (parseMode !== "lazy" && ksyInstanceData.value);
 
-            if (eagerLoad || noLazy)
-                result.object.fields[propName] = exportValue(obj[propName], obj._debug["_m_" + propName], path.concat(propName), noLazy);
+            if (Object.prototype.hasOwnProperty.call(obj, `_m_${propName}`) || eagerLoad || noLazy)
+                result.object.fields[propName] = fetchInstance(obj, propName, path, noLazy);
             else
                 result.object.instances[propName] = <IInstance>{ path: path.concat(propName), offset: 0 };
         }
@@ -114,6 +131,46 @@ function exportValue(obj: any, debug: IDebugInfo, path: string[], noLazy?: boole
     return result;
 }
 
+function inferDebugEnd(debugs: IDebugInfo[]): number {
+    const inferredEnd = debugs
+        .reduce((acc, debug) => debug && debug.end > acc ? debug.end : acc, Number.NEGATIVE_INFINITY);
+    if (inferredEnd === Number.NEGATIVE_INFINITY) {
+        return;
+    }
+    return inferredEnd;
+}
+
+function fetchInstance(obj: any, propName: string, objPath: string[], noLazy: boolean): IExportedValue {
+    let value;
+    let instanceError: Error;
+    try {
+        value = obj[propName];
+    } catch (e) {
+        instanceError = e;
+    }
+    if (instanceError !== undefined) {
+        try {
+            // retry once (important for validation errors)
+            value = obj[propName];
+        } catch (e) {}
+    }
+
+    const instHasRawAttr = Object.prototype.hasOwnProperty.call(obj, `_raw__m_${propName}`);
+    const debugInfo = <IDebugInfo>obj._debug[`_m_${propName}`];
+    const exported = exportValue(value, debugInfo, instHasRawAttr, objPath.concat(propName), noLazy);
+    if (instanceError !== undefined) {
+        exported.instanceError = instanceError;
+    }
+    return exported;
+}
+
+function adjustDebug(debug: IDebugInfo): void {
+    if (!debug || Object.prototype.hasOwnProperty.call(debug, 'incomplete')) {
+        return;
+    }
+    debug.incomplete = (debug.start != null && debug.end == null);
+}
+
 importScripts("../entities.js");
 importScripts("../../lib/_npm/kaitai-struct/KaitaiStream.js");
 
@@ -130,21 +187,30 @@ var apiMethods = {
         //var start = performance.now();
         wi.ioInput = new KaitaiStream(wi.inputBuffer, 0);
         wi.root = new wi.MainClass(wi.ioInput);
-        wi.root._read();
+        let error;
+        try {
+            wi.root._read();
+        } catch (e) {
+            error = e;
+        }
         if (hooks.nodeFilter)
             wi.root = hooks.nodeFilter(wi.root);
-        wi.exported = exportValue(wi.root, <IDebugInfo>{ start: 0, end: wi.inputBuffer.byteLength }, [], eagerMode);
+        wi.exported = exportValue(wi.root, { start: 0, end: wi.inputBuffer.byteLength, ioOffset: 0, incomplete: error !== undefined }, false, [], eagerMode);
         //console.log("parse before return", performance.now() - start, "date", Date.now());
-        return wi.exported;
+        return {
+            result: wi.exported,
+            error,
+        };
     },
     get: (path: string[]) => {
-        var obj = wi.root;
-        var parent: any = null;
-        path.forEach(key => { parent = obj; obj = obj[key]; });
-
-        var debug = <IDebugInfo>parent._debug["_m_" + path[path.length - 1]];
-        wi.exported = exportValue(obj, debug, path, false); //
-        return wi.exported;
+        let parent = wi.root;
+        const parentPath = path.slice(0, -1);
+        parentPath.forEach(key => parent = parent[key]);
+        const propName = path[path.length - 1];
+
+        return {
+            result: fetchInstance(parent, propName, parentPath, false),
+        };
     }
 };
 
@@ -154,7 +220,12 @@ myself.onmessage = (ev: MessageEvent) => {
 
     if (apiMethods.hasOwnProperty(msg.type)) {
         try {
-            msg.result = apiMethods[msg.type].apply(self, msg.args);
+            const ret = apiMethods[msg.type].apply(self, msg.args);
+            if (ret) {
+                const { result, error } =  ret;
+                msg.result = result;
+                msg.error = error;
+            }
         } catch (error) {
             console.log("[Worker] Error", error);
             msg.error = error;