Improve File Import performance

src/hub/dataSources/wpilog/wpilogWorker.ts

@@ -20,12 +20,13 @@ self.onmessage = (event) => {
   // MAIN LOGIC
 
   // Run worker
-  let log = new Log(false); // No timestamp set cache for efficiency
+  let log = new Log(false, false); // No timestamp set cache for efficiency, disable live sorting
   let reader = new WPILOGDecoder(payload[0]);
   let totalBytes = (payload[0] as Uint8Array).byteLength;
   let entryIds: { [id: number]: string } = {};
   let entryTypes: { [id: number]: string } = {};
   let lastProgressTimestamp = new Date().getTime();
+  let customSchemaRecords: { key: string; timestamp: number; value: Uint8Array; type: string }[] = [];
   try {
     reader.forEach((record, byteCount) => {
       if (record.isControl()) {
@@ -130,12 +131,7 @@ self.onmessage = (event) => {
                 } else {
                   log.putRaw(key, timestamp, record.getRaw());
                   if (CustomSchemas.has(type)) {
-                    try {
-                      CustomSchemas.get(type)!(log, key, timestamp, record.getRaw());
-                    } catch {
-                      console.error('Failed to decode custom schema "' + type + '"');
-                    }
-                    log.setGeneratedParent(key);
+                    customSchemaRecords.push({ key: key, timestamp: timestamp, value: record.getRaw(), type: type });
                   }
                 }
                 break;
@@ -150,17 +146,38 @@ self.onmessage = (event) => {
       let now = new Date().getTime();
       if (now - lastProgressTimestamp > 1000 / 60) {
         lastProgressTimestamp = now;
-        progress(byteCount / totalBytes);
+        progress((byteCount / totalBytes) * 0.2); // Show progress of 0-20% for file reading
       }
     });
   } catch (exception) {
     console.error(exception);
     reject();
     return;
   }
-  progress(1);
   setTimeout(() => {
     // Allow progress message to get through first
-    resolve(log.toSerialized());
+    log.sortAndProcess((x) => {
+      progress(0.2 + x * 0.5); // Show progress of 20-70% for log processing/sorting
+    });
+
+    // Process custom schemas
+    for (let i = 0; i < customSchemaRecords.length; i++) {
+      let record = customSchemaRecords[i];
+      try {
+        CustomSchemas.get(record.type)!(log, record.key, record.timestamp, record.value);
+      } catch {
+        console.error('Failed to decode custom schema "' + record.type + '"');
+      }
+      log.setGeneratedParent(record.key);
+    }
+
+    resolve(
+      log.toSerialized((x) => {
+        progress(0.2 + x * 0.8); // Show progress of 20-100% for log processing/sorting
+      })
+    );
+    progress(1);
   }, 0);
 };
+
+// torun();

src/shared/log/Log.ts

@@ -29,20 +29,22 @@ export default class Log {
   private generatedParents: Set<string> = new Set(); // Children of these fields are generated
   private timestampRange: [number, number] | null = null;
   private enableTimestampSetCache: boolean;
+  private enableLiveSorting: boolean;
   private timestampSetCache: { [id: string]: { keys: string[]; timestamps: number[] } } = {};
 
   private queuedStructs: QueuedStructure[] = [];
   private queuedStructArrays: QueuedStructure[] = [];
   private queuedProtos: QueuedStructure[] = [];
 
-  constructor(enableTimestampSetCache = true) {
+  constructor(enableTimestampSetCache = true, enableLiveSorting = true) {
     this.enableTimestampSetCache = enableTimestampSetCache;
+    this.enableLiveSorting = enableLiveSorting;
   }
 
   /** Checks if the field exists and registers it if necessary. */
   public createBlankField(key: string, type: LoggableType) {
     if (key in this.fields) return;
-    this.fields[key] = new LogField(type);
+    this.fields[key] = new LogField(type, this.enableLiveSorting);
   }
 
   /** Clears all data before the provided timestamp. */
@@ -673,7 +675,11 @@ export default class Log {
   }
 
   /** Returns a serialized version of the data from this log. */
-  toSerialized(): any {
+  toSerialized(progressCallback: ((progress: number) => void) | undefined = undefined): any {
+    if (!this.enableLiveSorting) {
+      this.sortAndProcess(progressCallback); // Enable live sorting after first serialization
+      this.enableLiveSorting = true;
+    }
     let result: any = {
       fields: {},
       generatedParents: Array.from(this.generatedParents),
@@ -684,11 +690,23 @@ export default class Log {
       queuedStructArrays: this.queuedStructArrays,
       queuedProtos: this.queuedProtos
     };
+    let totalFields = Object.keys(this.fields).length;
     Object.entries(this.fields).forEach(([key, value]) => {
       result.fields[key] = value.toSerialized();
     });
     return result;
   }
+  sortAndProcess(progressCallback: ((progress: number) => void) | undefined = undefined) {
+    if (!this.enableLiveSorting) {
+      let entires = Object.values(this.fields);
+      for (let i = 0; i < entires.length; i++) {
+        entires[i].sortAndProcess();
+        if (progressCallback != undefined) {
+          progressCallback(i / entires.length);
+        }
+      }
+    }
+  }
 
   /** Creates a new log based on the data from `toSerialized()` */
   static fromSerialized(serializedData: any): Log {

src/shared/log/LogField.ts

@@ -10,7 +10,10 @@ import {
   LogValueSetString,
   LogValueSetStringArray
 } from "./LogValueSets";
-
+type LogRecord = {
+  timestamp: number;
+  value: any;
+};
 /** A full log field that contains data. */
 export default class LogField {
   private type: LoggableType;
@@ -19,12 +22,13 @@ export default class LogField {
   public wpilibType: string | null = null; // Original type from WPILOG & NT4
   public metadataString = "";
   public typeWarning = false; // Flag that there was an attempt to write a conflicting type
-
+  private enableLiveSorting: boolean;
   // Toggles when first value is removed, useful for creating striping effects that persist as data is updated
   private stripingReference = false;
-
-  constructor(type: LoggableType) {
+  private rawData: LogRecord[] = [];
+  constructor(type: LoggableType, enableLiveSorting = true) {
     this.type = type;
+    this.enableLiveSorting = enableLiveSorting;
   }
 
   /** Returns the constant field type. */
@@ -115,41 +119,45 @@ export default class LogField {
 
   /** Inserts a new value at the correct index. */
   private putData(timestamp: number, value: any) {
-    if (value === null) return;
+    if (this.enableLiveSorting) {
+      if (value === null) return;
 
-    // Find position to insert based on timestamp
-    let insertIndex: number;
-    if (this.data.timestamps.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) {
-      // There's a good chance this data is at the end of the log, so check that first
-      insertIndex = this.data.timestamps.length;
-    } else {
-      // Adding in the middle, find where to insert it
-      let alreadyExists = false;
-      insertIndex =
-        this.data.timestamps.findLastIndex((x) => {
-          if (alreadyExists) return;
-          if (x === timestamp) alreadyExists = true;
-          return x < timestamp;
-        }) + 1;
-      if (alreadyExists) {
-        this.data.values[this.data.timestamps.indexOf(timestamp)] = value;
-        return;
+      // Find position to insert based on timestamp
+      let insertIndex: number;
+      if (this.data.timestamps.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) {
+        // There's a good chance this data is at the end of the log, so check that first
+        insertIndex = this.data.timestamps.length;
+      } else {
+        // Adding in the middle, find where to insert it
+        let alreadyExists = false;
+        insertIndex =
+          this.data.timestamps.findLastIndex((x) => {
+            if (alreadyExists) return;
+            if (x === timestamp) alreadyExists = true;
+            return x < timestamp;
+          }) + 1;
+        if (alreadyExists) {
+          this.data.values[this.data.timestamps.indexOf(timestamp)] = value;
+          return;
+        }
       }
-    }
 
-    // Compare to adjacent values
-    if (insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) {
-      // Same as the previous value
-    } else if (
-      insertIndex < this.data.values.length &&
-      logValuesEqual(this.type, value, this.data.values[insertIndex])
-    ) {
-      // Same as the next value
-      this.data.timestamps[insertIndex] = timestamp;
+      // Compare to adjacent values
+      if (insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) {
+        // Same as the previous value
+      } else if (
+        insertIndex < this.data.values.length &&
+        logValuesEqual(this.type, value, this.data.values[insertIndex])
+      ) {
+        // Same as the next value
+        this.data.timestamps[insertIndex] = timestamp;
+      } else {
+        // New value
+        this.data.timestamps.splice(insertIndex, 0, timestamp);
+        this.data.values.splice(insertIndex, 0, value);
+      }
     } else {
-      // New value
-      this.data.timestamps.splice(insertIndex, 0, timestamp);
-      this.data.values.splice(insertIndex, 0, value);
+      this.rawData.push({ timestamp: timestamp, value: value });
     }
   }
 
@@ -204,6 +212,10 @@ export default class LogField {
 
   /** Returns a serialized version of the data from this field. */
   toSerialized(): any {
+    if (!this.enableLiveSorting) {
+      this.sortAndProcess();
+      this.enableLiveSorting = true; // After first serilization enable live sorting
+    }
     return {
       type: this.type,
       timestamps: this.data.timestamps,
@@ -230,4 +242,34 @@ export default class LogField {
     field.typeWarning = serializedData.typeWarning;
     return field;
   }
+  /** Sorts and processes data all at once */
+  sortAndProcess() {
+    if (!this.enableLiveSorting) {
+      this.rawData.sort((a: LogRecord, b: LogRecord) => {
+        return a.timestamp - b.timestamp;
+      });
+      if (this.rawData.length > 0) {
+        // Bootstrap first value
+        this.data.timestamps.push(this.rawData[0].timestamp);
+        this.data.values.push(this.rawData[0].value);
+      }
+      for (let i = 1; i < this.rawData.length; i++) {
+        // Check if the timestamp is the same as the last one
+        if (this.rawData[i].timestamp == this.data.timestamps[this.data.values.length - 1]) {
+          // Overwrite the last value
+          this.data.values[this.data.values.length - 1] = this.rawData[i].value;
+          // If the values are equal do not add the value
+        } else if (
+          logValuesEqual(this.type, this.rawData[i].value, this.data.values[this.data.values.length - 1]) &&
+          i < this.rawData.length
+        ) {
+        } else {
+          // add the value
+          this.data.timestamps.push(this.rawData[i].timestamp);
+          this.data.values.push(this.rawData[i].value);
+        }
+      }
+      this.rawData = [];
+    }
+  }
 }