Improve syntax macro performance

We were taking a massive performance hit by using shift. I've added a custom FastShiftArray to to bypass those limitations.

Syntax macros processing of 10k lines went from 2216 ms to 156ms

Now compiling 200k lines in 52 seconds (M1 Pro 16gb).

Likely would have taken hours before these changes

Author: drew-y

src/lib/fast-shift-array.mts

@@ -0,0 +1,88 @@
+// TODO: Add map, filter, reduce, amd findIndex
+export class FastShiftArray<T> {
+  private items: T[];
+  private headIndex: number;
+
+  constructor(...args: T[]) {
+    this.items = args;
+    this.headIndex = 0;
+  }
+
+  shift(): T | undefined {
+    if (this.headIndex >= this.items.length) {
+      return undefined;
+    }
+    const value = this.items[this.headIndex];
+    this.items[this.headIndex] = undefined as any; // Optional: clear the reference for garbage collection
+    this.headIndex++;
+    return value;
+  }
+
+  unshift(...items: T[]): number {
+    if (items.length === 0) return this.length;
+    this.headIndex = Math.max(this.headIndex - items.length, 0);
+    for (let i = 0; i < items.length; i++) {
+      this.items[this.headIndex + i] = items[i];
+    }
+    return this.length;
+  }
+
+  push(...items: T[]): number {
+    this.items.push(...items);
+    return this.length;
+  }
+
+  pop(): T | undefined {
+    if (this.length === 0) return undefined;
+    return this.items.pop();
+  }
+
+  at(index: number): T | undefined {
+    if (index < 0) {
+      index = this.length + index;
+    }
+    return this.items[this.headIndex + index];
+  }
+
+  set(index: number, value: T): boolean {
+    const targetIndex = index < 0 ? this.length + index : index;
+    if (targetIndex < 0 || targetIndex >= this.length) {
+      return false; // Index out of bounds
+    }
+    this.items[this.headIndex + targetIndex] = value;
+    return true;
+  }
+
+  get length(): number {
+    return this.items.length - this.headIndex;
+  }
+
+  slice(start?: number, end?: number): T[] {
+    const actualStart =
+      start !== undefined
+        ? this.headIndex + (start < 0 ? this.length + start : start)
+        : this.headIndex;
+    const actualEnd =
+      end !== undefined
+        ? this.headIndex + (end < 0 ? this.length + end : end)
+        : this.items.length;
+    return this.items.slice(actualStart, actualEnd);
+  }
+
+  splice(start: number, deleteCount: number = 0, ...items: T[]): T[] {
+    const actualStart =
+      this.headIndex + (start < 0 ? this.length + start : start);
+    return this.items.splice(actualStart, deleteCount, ...items);
+  }
+
+  // Converts the FastShiftArray back to a normal array
+  toArray(): T[] {
+    return this.items.slice(this.headIndex);
+  }
+
+  // Optional: Method to reset the headIndex
+  resetShift(): void {
+    this.items.splice(0, this.headIndex);
+    this.headIndex = 0;
+  }
+}

src/syntax-objects/list.mts

@@ -1,3 +1,4 @@
+import { FastShiftArray } from "../lib/fast-shift-array.mjs";
 import { Expr } from "./expr.mjs";
 import { Float } from "./float.mjs";
 import { getIdStr } from "./get-id-str.mjs";
@@ -10,7 +11,7 @@ export class List extends Syntax {
   readonly syntaxType = "list";
   /** True when the list was defined by the user using parenthesis i.e. (hey, there) */
   mayBeTuple?: boolean;
-  value: Expr[] = [];
+  store: FastShiftArray<Expr> = new FastShiftArray();
 
   constructor(
     opts: SyntaxMetadata & {
@@ -29,20 +30,24 @@ export class List extends Syntax {
     }
   }
 
+  get value() {
+    return this.store.toArray();
+  }
+
   get hasChildren() {
-    return !!this.value.length;
+    return !!this.store.length;
   }
 
   get length() {
-    return this.value.length;
+    return this.store.length;
   }
 
   at(index: number): Expr | undefined {
-    return this.value.at(index);
+    return this.store.at(index);
   }
 
   exprAt(index: number): Expr {
-    const expr = this.value.at(index);
+    const expr = this.store.at(index);
     if (!expr) {
       throw new Error(`No expr at ${index}`);
     }
@@ -68,7 +73,7 @@ export class List extends Syntax {
   set(index: number, expr: Expr | string) {
     const result = typeof expr === "string" ? Identifier.from(expr) : expr;
     result.parent = this;
-    this.value[index] = result;
+    this.store.set(index, result);
     return this;
   }
 
@@ -82,48 +87,42 @@ export class List extends Syntax {
   }
 
   consume(): Expr {
-    const next = this.value.shift();
+    const next = this.store.shift();
     if (!next) throw new Error("No remaining expressions");
     return next;
   }
 
-  consumeRest(): List {
-    const newVal = this.slice(0);
-    this.value = [];
-    return newVal;
-  }
-
   first(): Expr | undefined {
-    return this.value[0];
+    return this.store.at(0);
   }
 
   last(): Expr | undefined {
-    return this.value.at(-1);
+    return this.store.at(-1);
   }
 
   /** Returns all but the first element in an array */
   rest(): Expr[] {
-    return this.value.slice(1);
+    return this.store.toArray().slice(1);
   }
 
   pop(): Expr | undefined {
-    return this.value.pop();
+    return this.store.pop();
   }
 
   push(...expr: ListValue[]) {
     expr.forEach((ex) => {
       if (typeof ex === "string") {
-        this.value.push(new Identifier({ value: ex, parent: this }));
+        this.store.push(new Identifier({ value: ex, parent: this }));
         return;
       }
 
       if (typeof ex === "number" && Number.isInteger(ex)) {
-        this.value.push(new Int({ value: ex, parent: this }));
+        this.store.push(new Int({ value: ex, parent: this }));
         return;
       }
 
       if (typeof ex === "number") {
-        this.value.push(new Float({ value: ex, parent: this }));
+        this.store.push(new Float({ value: ex, parent: this }));
         return;
       }
 
@@ -139,11 +138,11 @@ export class List extends Syntax {
       }
 
       if (ex.isList() && ex.calls("splice_quote")) {
-        this.value.push(...ex.rest());
+        this.store.push(...ex.rest());
         return;
       }
 
-      this.value.push(ex);
+      this.store.push(ex);
     });
 
     return this;
@@ -156,12 +155,12 @@ export class List extends Syntax {
   insert(expr: Expr | string, at = 0) {
     const result = typeof expr === "string" ? Identifier.from(expr) : expr;
     result.parent = this;
-    this.value.splice(at, 0, result);
+    this.store.splice(at, 0, result);
     return this;
   }
 
   remove(index: number, count = 1) {
-    this.value.splice(index, count);
+    this.store.splice(index, count);
     return this;
   }
 
@@ -200,16 +199,16 @@ export class List extends Syntax {
   slice(start?: number, end?: number): List {
     return new List({
       ...super.getCloneOpts(),
-      value: this.value.slice(start, end),
+      value: this.store.slice(start, end),
     });
   }
 
   sliceAsArray(start?: number, end?: number) {
-    return this.value.slice(start, end);
+    return this.store.slice(start, end);
   }
 
   toArray(): Expr[] {
-    return [...this.value];
+    return this.value;
   }
 
   toJSON() {