Address specification deficiency around expression bytes width

packages/pointers/src/data.ts

@@ -58,4 +58,30 @@ export class Data extends Uint8Array {
   toHex(): string {
     return `0x${toHex(this)}`;
   }
+
+  padUntilAtLeast(length: number): Data {
+    if (this.length >= length) {
+      return this;
+    }
+
+    const padded = new Uint8Array(length);
+    padded.set(this, length - this.length);
+    return Data.fromBytes(padded);
+  }
+
+  resizeTo(length: number): Data {
+    if (this.length === length) {
+      return this;
+    }
+
+    const resized = new Uint8Array(length);
+
+    if (this.length < length) {
+      resized.set(this, length - this.length);
+    } else {
+      resized.set(this.slice(this.length - length));
+    }
+
+    return Data.fromBytes(resized);
+  }
 }

packages/pointers/src/dereference/index.test.ts

@@ -123,7 +123,7 @@ describe("dereference", () => {
         location: "memory",
         offset: Data.fromUint(
           Data.fromNumber(index).asUint() * 32n
-        ),
+        ).padUntilAtLeast(1),
         length: Data.fromNumber(32),
       })
     }

packages/pointers/src/dereference/process.ts

@@ -32,11 +32,6 @@ export async function* processPointer(
   pointer: Pointer,
   options: ProcessOptions
 ): Process {
-  const {
-    regions: oldRegions,
-    variables: oldVariables,
-  } = options;
-
   if (Pointer.isRegion(pointer)) {
     const region = pointer;
 

packages/pointers/src/evaluate.test.ts

@@ -193,4 +193,38 @@ describe("evaluate", () => {
     expect(await evaluate(expression, options))
       .toEqual(Data.fromNumber(42));
   });
+
+  describe("resulting bytes widths", () => {
+    it("uses the fewest bytes necessary for a literal", async () => {
+      expect(await evaluate(0, options)).toHaveLength(0);
+      expect(await evaluate("0x00", options)).toHaveLength(1);
+      expect(await evaluate("0x0000", options)).toHaveLength(2);
+      expect(await evaluate(0xffff, options)).toHaveLength(2);
+    });
+
+    it("uses at least the largest bytes width amongst arithmetic operands", async () => {
+      expect(await evaluate({ $sum: [0, 0] }, options)).toHaveLength(0);
+
+      expect(await evaluate({ $difference: ["0x00", "0x00"] }, options))
+        .toHaveLength(1);
+
+      expect(await evaluate({ $remainder: ["0x0001", "0x01"] }, options))
+        .toHaveLength(2);
+    });
+
+    it("uses exactly as many bytes necessary to avoid arithmetic overflow", async () => {
+      expect(await evaluate({ $product: ["0xffff", "0xff"] }, options))
+        .toHaveLength(3);
+    });
+  });
+
+  it("evaluates resize expressions", async () => {
+    expect(await evaluate({ $sized1: 0 }, options)).toHaveLength(1);
+
+    {
+      const data = await evaluate({ $sized1: "0xabcd" }, options);
+      expect(data).toHaveLength(1);
+      expect(data).toEqual(Data.fromNumber(0xcd));
+    }
+  });
 });

packages/pointers/src/evaluate.ts

@@ -58,6 +58,10 @@ export async function evaluate(
     return evaluateKeccak256(expression, options);
   }
 
+  if (Pointer.Expression.isResize(expression)) {
+    return evaluateResize(expression, options);
+  }
+
   if (Pointer.Expression.isLookup(expression)) {
     if (Pointer.Expression.Lookup.isOffset(expression)) {
       return evaluateLookup(".offset", expression, options);
@@ -116,76 +120,93 @@ async function evaluateArithmeticSum(
   options: EvaluateOptions
 ): Promise<Data> {
   const operands = await Promise.all(expression.$sum.map(
-    async expression => (await evaluate(expression, options)).asUint()
+    async expression => await evaluate(expression, options)
   ));
 
-  return Data.fromUint(
-    operands.reduce((sum, data) => sum + data, 0n)
-  );
+  const maxLength = operands
+    .reduce((max, { length }) => length > max ? length : max, 0);
+
+  const data = Data
+    .fromUint(operands.reduce((sum, data) => sum + data.asUint(), 0n))
+    .padUntilAtLeast(maxLength);
+
+  return data;
 }
 
 async function evaluateArithmeticDifference(
   expression: Pointer.Expression.Arithmetic.Difference,
   options: EvaluateOptions
 ): Promise<Data> {
   const [a, b] = await Promise.all(expression.$difference.map(
-    async expression => (await evaluate(expression, options)).asUint()
+    async expression => await evaluate(expression, options)
   ));
 
-  if (b > a) {
-    return Data.fromNumber(0);
-  }
+  const maxLength = a.length > b.length ? a.length : b.length;
 
-  return Data.fromUint(a - b);
+  const unpadded = a.asUint() > b.asUint()
+    ? Data.fromUint(a.asUint() - b.asUint())
+    : Data.fromNumber(0);
+
+  const data = unpadded.padUntilAtLeast(maxLength);
+  return data;
 }
 
 async function evaluateArithmeticProduct(
   expression: Pointer.Expression.Arithmetic.Product,
   options: EvaluateOptions
 ): Promise<Data> {
   const operands = await Promise.all(expression.$product.map(
-    async expression => (await evaluate(expression, options)).asUint()
+    async expression => await evaluate(expression, options)
   ));
 
-  return Data.fromUint(
-    operands.reduce((product, data) => product * data, 1n)
-  );
+  const maxLength = operands
+    .reduce((max, { length }) => length > max ? length : max, 0);
+
+  return Data
+    .fromUint(operands.reduce((product, data) => product * data.asUint(), 1n))
+    .padUntilAtLeast(maxLength);
 }
 
 async function evaluateArithmeticQuotient(
   expression: Pointer.Expression.Arithmetic.Quotient,
   options: EvaluateOptions
 ): Promise<Data> {
   const [a, b] = await Promise.all(expression.$quotient.map(
-    async expression => (await evaluate(expression, options)).asUint()
+    async expression => (await evaluate(expression, options))
   ));
 
-  return Data.fromUint(a / b);
+  const maxLength = a.length > b.length ? a.length : b.length;
+
+  const data = Data
+    .fromUint(a.asUint() / b.asUint())
+    .padUntilAtLeast(maxLength);
+
+  return data;
 }
 
 async function evaluateArithmeticRemainder(
   expression: Pointer.Expression.Arithmetic.Remainder,
   options: EvaluateOptions
 ): Promise<Data> {
   const [a, b] = await Promise.all(expression.$remainder.map(
-    async expression => (await evaluate(expression, options)).asUint()
+    async expression => await evaluate(expression, options)
   ));
 
-  return Data.fromUint(a % b);
+  const maxLength = a.length > b.length ? a.length : b.length;
+
+  const data = Data
+    .fromUint(a.asUint() % b.asUint())
+    .padUntilAtLeast(maxLength);
+
+  return data;
 }
 
 async function evaluateKeccak256(
   expression: Pointer.Expression.Keccak256,
   options: EvaluateOptions
 ): Promise<Data> {
   const operands = await Promise.all(expression.$keccak256.map(
-    async expression => {
-      const unpaddedData = await evaluate(expression, options);
-      const data = new Data(32);
-      data.set(unpaddedData, 32 - unpaddedData.length);
-
-      return data;
-    }
+    async expression => await evaluate(expression, options)
   ));
 
   // HACK concatenate via string representation
@@ -200,6 +221,17 @@ async function evaluateKeccak256(
   return Data.fromBytes(hash);
 }
 
+async function evaluateResize(
+  expression: Pointer.Expression.Resize,
+  options: EvaluateOptions
+): Promise<Data> {
+  const [[operation, subexpression]] = Object.entries(expression);
+
+  const newLength = Number(operation.match(/^\$sized([1-9]+[0-9]*)$/)![1]);
+
+  return (await evaluate(subexpression, options)).resizeTo(newLength);
+}
+
 async function evaluateLookup<O extends Pointer.Expression.Lookup.Operation>(
   operation: O,
   lookup: Pointer.Expression.Lookup.ForOperation<O>,

packages/pointers/src/pointer.test.ts

@@ -105,6 +105,11 @@ describe("type guards", () => {
       pointer: "#/$defs/Keccak256",
       guard: Pointer.Expression.isKeccak256
     },
+    {
+      schema: expressionSchema,
+      pointer: "#/$defs/Resize",
+      guard: Pointer.Expression.isResize
+    },
     {
       schema: {
         id: "schema:ethdebug/format/pointer/region"

packages/pointers/src/pointer.ts

@@ -211,7 +211,8 @@ export namespace Pointer {
     | Expression.Arithmetic
     | Expression.Lookup
     | Expression.Read
-    | Expression.Keccak256;
+    | Expression.Keccak256
+    | Expression.Resize;
 
   export const isExpression = (value: unknown): value is Expression =>
     [
@@ -221,7 +222,8 @@ export namespace Pointer {
       Expression.isArithmetic,
       Expression.isLookup,
       Expression.isRead,
-      Expression.isKeccak256
+      Expression.isKeccak256,
+      Expression.isResize
     ].some(guard => guard(value));
 
   export namespace Expression {
@@ -378,5 +380,23 @@ export namespace Pointer {
     }
     export const isKeccak256 =
       makeIsOperation<"$keccak256", Keccak256>("$keccak256", isOperands);
+
+    export type Resize<N extends number = number> = {
+      [K in `$sized${N}`]: Expression;
+    }
+    export const isResize = <N extends number>(
+      value: unknown
+    ): value is Resize<N> => {
+      if (
+        !value ||
+          typeof value !== "object" ||
+          Object.keys(value).length !== 1
+      ) {
+        return false;
+      }
+      const [key] = Object.keys(value);
+
+      return typeof key === "string" && /^\$sized([1-9]+[0-9]*)$/.test(key);
+    }
   }
 }

packages/pointers/test/ganache.ts

@@ -144,16 +144,15 @@ function toMachineState(
   ): Machine.State.Words => {
     return {
       async read({
-        slot: unpaddedSlot,
+        slot,
         slice: {
           offset = 0n,
           length = 32n
         } = {}
       }) {
-        const slot = new Data(32);
-        slot.set(unpaddedSlot, 32 - unpaddedSlot.length);
-
-        const rawHex = slots[slot.toHex().slice(2) as keyof typeof slots];
+        const rawHex = slots[
+          slot.resizeTo(32).toHex().slice(2) as keyof typeof slots
+        ];
 
         const data = Data.fromHex(`0x${rawHex}`);
 

packages/web/docs/implementation-guides/pointers/evaluating-expressions.mdx

@@ -126,6 +126,20 @@ Evaluating remainders:
     sourceFile => sourceFile.getFunction("evaluateArithmeticRemainder")
   } />
 
+## Evaluating resize expressions
+
+This schema provides the `{ "$sized<N>": <expression> }` construct to allow
+explicitly resizing a subexpression. This implementation uses the
+[`Data.prototype.resizeTo()`](/docs/implementation-guides/pointers/types/data-and-machines)
+method to perform this operation.
+
+<CodeListing
+  packageName="@ethdebug/pointers"
+  sourcePath="src/evaluate.ts"
+  extract={
+    sourceFile => sourceFile.getFunction("evaluateResize")
+  } />
+
 ## Evaluating keccak256 hashes
 
 Many data types in storage are addressed by way of keccak256 hashing. This
@@ -137,15 +151,6 @@ See Solidity's
 documentation for an example of how one high-level EVM language makes heavy
 use of hashing to allocate persistent data.
 
-:::warning
-This area of the schema is likely incomplete and could still use additional
-specification. Be warned that, while this implementation may match the schema
-itself, it may not be fully sufficient for expressing all kinds of data
-allocations.
-
-Please stay tuned as this work continues being refined.
-:::
-
 <CodeListing
   packageName="@ethdebug/pointers"
   sourcePath="src/evaluate.ts"

packages/web/spec/pointer/expression.mdx

@@ -33,7 +33,7 @@ assumed to be left-padded to the bytes width appropriate for the context.
   pointer="#/$defs/Literal"
   />
 
-## Scalar variables
+## Variables
 
 An expression can be a string value equal to the identifier for a known
 scalar variable introduced by some pointer representation.
@@ -43,7 +43,7 @@ For an example where scalar variables may appear, see the
 
 <SchemaViewer
   schema={{ id: "schema:ethdebug/format/pointer/expression" }}
-  pointer="#/$defs/Literal"
+  pointer="#/$defs/Variable"
   />
 
 ## Arithmetic operations
@@ -92,6 +92,18 @@ hash of the concatenation of bytes specified by the list.
   pointer="#/$defs/Keccak256"
   />
 
+## Resize operations
+
+In certain situations, e.g. keccak256 hashes, it's crucially important to be
+able to express the bytes width of particular expression values. This schema
+provides primitives to allow specifying an explicit bytes width for a
+particular sub-expression.
+
+<SchemaViewer
+  schema={{ id: "schema:ethdebug/format/pointer/expression" }}
+  pointer="#/$defs/Resize"
+  />
+
 ## Region references
 
 Regions can be referenced either by name (which **must** be a defined region),