feat: refactor and unit test getDataColumnSidecars (#7072)

* refactor: getDataColumnSidecars

* test: unit test getDataColumnSidecars with mocks from c-kzg library

* refactor: use fromHex util

* chore: update numbering on mocks

* chore: update c-kzg to latest version

* chore: fix type export syntax

* test: add verification for cells from sidecars

* test: add verification to DataColumnSidecars tests

* refactor: getDataColumnSidecars for PR comments

* feat: narrow type and remove unnecessary conditional

* fix: getDataColumnSidecars param type

* refactor: rename to computeDataColumnSidecars

packages/beacon-node/package.json

@@ -132,7 +132,7 @@
     "@lodestar/utils": "^1.21.0",
     "@lodestar/validator": "^1.21.0",
     "@multiformats/multiaddr": "^12.1.3",
-    "c-kzg": "matthewkeil/c-kzg-4844#13aa01464479aa7c1ccafa64d52cbc17699ffa07",
+    "c-kzg": "matthewkeil/c-kzg-4844#853b22fa416d4eac376678a36bfba0bccb59dd78",
     "datastore-core": "^9.1.1",
     "datastore-level": "^10.1.1",
     "deepmerge": "^4.3.1",

packages/beacon-node/src/util/blobs.ts

@@ -7,6 +7,7 @@ import {
   ForkName,
   ForkAll,
   NUMBER_OF_COLUMNS,
+  ForkBlobs,
 } from "@lodestar/params";
 import {deneb, ssz, BeaconBlockBody, SignedBeaconBlock, SSZTypesFor, peerdas} from "@lodestar/types";
 import {ChainForkConfig} from "@lodestar/config";
@@ -63,6 +64,13 @@ export function computeBlobSidecars(
   });
 }
 
+/**
+ * Turns a SignedBeaconBlock and an array of Blobs from a given slot into an array of
+ * DataColumnSidecars that are ready to be served by gossip and req/resp.
+ *
+ * Implementation of get_data_column_sidecars
+ * https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#get_data_column_sidecars
+ */
 export function computeDataColumnSidecars(
   config: ChainForkConfig,
   signedBlock: SignedBeaconBlock,
@@ -75,28 +83,24 @@ export function computeDataColumnSidecars(
   if (blobKzgCommitments.length === 0) {
     return [];
   }
-
+  const {blobs} = contents;
+  const fork = config.getForkName(signedBlock.message.slot);
   const signedBlockHeader = signedBlockToSignedHeader(config, signedBlock);
-  const fork = config.getForkName(signedBlockHeader.message.slot);
   const kzgCommitmentsInclusionProof =
     contents.kzgCommitmentsInclusionProof ?? computeKzgCommitmentsInclusionProof(fork, signedBlock.message.body);
-  const cellsAndProofs = contents.blobs.map((blob) => ckzg.computeCellsAndKzgProofs(blob));
-  const dataColumnSidecars = Array.from({length: NUMBER_OF_COLUMNS}, (_, j) => {
-    // j'th column
-    const column = Array.from({length: contents.blobs.length}, (_, i) => cellsAndProofs[i][0][j]);
-    const kzgProofs = Array.from({length: contents.blobs.length}, (_, i) => cellsAndProofs[i][1][j]);
+  const cellsAndProofs = blobs.map((blob) => ckzg.computeCellsAndKzgProofs(blob));
 
-    const dataColumnSidecar = {
-      index: j,
+  return Array.from({length: NUMBER_OF_COLUMNS}, (_, columnIndex) => {
+    // columnIndex'th column
+    const column = Array.from({length: blobs.length}, (_, rowNumber) => cellsAndProofs[rowNumber][0][columnIndex]);
+    const kzgProofs = Array.from({length: blobs.length}, (_, rowNumber) => cellsAndProofs[rowNumber][1][columnIndex]);
+    return {
+      index: columnIndex,
       column,
       kzgCommitments: blobKzgCommitments,
       kzgProofs,
       signedBlockHeader,
       kzgCommitmentsInclusionProof,
-    } as peerdas.DataColumnSidecar;
-
-    return dataColumnSidecar;
+    };
   });
-
-  return dataColumnSidecars;
 }

packages/beacon-node/src/util/kzg.ts

@@ -13,36 +13,31 @@ function ckzgNotLoaded(): never {
 }
 
 export let ckzg: {
-  freeTrustedSetup(): void;
   loadTrustedSetup(precompute: number, filePath: string): void;
   blobToKzgCommitment(blob: Uint8Array): Uint8Array;
+  computeKzgProof(blob: Uint8Array, zBytes: Uint8Array): [Uint8Array, Uint8Array];
   computeBlobKzgProof(blob: Uint8Array, commitment: Uint8Array): Uint8Array;
+  verifyKzgProof(commitmentBytes: Uint8Array, zBytes: Uint8Array, yBytes: Uint8Array, proofBytes: Uint8Array): boolean;
   verifyBlobKzgProof(blob: Uint8Array, commitment: Uint8Array, proof: Uint8Array): boolean;
   verifyBlobKzgProofBatch(blobs: Uint8Array[], expectedKzgCommitments: Uint8Array[], kzgProofs: Uint8Array[]): boolean;
-  computeCells(blob: Uint8Array): Uint8Array[];
   computeCellsAndKzgProofs(blob: Uint8Array): [Uint8Array[], Uint8Array[]];
-  cellsToBlob(cells: Uint8Array[]): Uint8Array;
-  recoverAllCells(cellIds: number[], cells: Uint8Array[]): Uint8Array[];
-  verifyCellKzgProof(commitmentBytes: Uint8Array, cellId: number, cell: Uint8Array, proofBytes: Uint8Array): boolean;
+  recoverCellsAndKzgProofs(cellIndices: number[], cells: Uint8Array[]): [Uint8Array[], Uint8Array[]];
   verifyCellKzgProofBatch(
     commitmentsBytes: Uint8Array[],
-    rowIndices: number[],
-    columnIndices: number[],
+    cellIndices: number[],
     cells: Uint8Array[],
     proofsBytes: Uint8Array[]
   ): boolean;
 } = {
-  freeTrustedSetup: ckzgNotLoaded,
   loadTrustedSetup: ckzgNotLoaded,
   blobToKzgCommitment: ckzgNotLoaded,
+  computeKzgProof: ckzgNotLoaded,
   computeBlobKzgProof: ckzgNotLoaded,
+  verifyKzgProof: ckzgNotLoaded,
   verifyBlobKzgProof: ckzgNotLoaded,
   verifyBlobKzgProofBatch: ckzgNotLoaded,
-  computeCells: ckzgNotLoaded,
   computeCellsAndKzgProofs: ckzgNotLoaded,
-  cellsToBlob: ckzgNotLoaded,
-  recoverAllCells: ckzgNotLoaded,
-  verifyCellKzgProof: ckzgNotLoaded,
+  recoverCellsAndKzgProofs: ckzgNotLoaded,
   verifyCellKzgProofBatch: ckzgNotLoaded,
 };
 

packages/beacon-node/test/unit/util/kzg.test.ts

@@ -2,10 +2,12 @@ import {describe, it, expect, afterEach, beforeAll} from "vitest";
 import {bellatrix, deneb, ssz} from "@lodestar/types";
 import {BYTES_PER_FIELD_ELEMENT, BLOB_TX_TYPE} from "@lodestar/params";
 import {createBeaconConfig, createChainForkConfig, defaultChainConfig} from "@lodestar/config";
+import {signedBlockToSignedHeader} from "@lodestar/state-transition";
 import {getMockedBeaconChain} from "../../mocks/mockedBeaconChain.js";
-import {computeBlobSidecars, kzgCommitmentToVersionedHash} from "../../../src/util/blobs.js";
+import {computeBlobSidecars, computeDataColumnSidecars, kzgCommitmentToVersionedHash} from "../../../src/util/blobs.js";
 import {loadEthereumTrustedSetup, initCKZG, ckzg, FIELD_ELEMENTS_PER_BLOB_MAINNET} from "../../../src/util/kzg.js";
 import {validateBlobSidecars, validateGossipBlobSidecar} from "../../../src/chain/validation/blobSidecar.js";
+import {getBlobCellAndProofs} from "../../utils/getBlobCellAndProofs.js";
 
 describe("C-KZG", () => {
   const afterEachCallbacks: (() => Promise<unknown> | void)[] = [];
@@ -73,6 +75,51 @@ describe("C-KZG", () => {
       }
     });
   });
+
+  it("DataColumnSidecars", () => {
+    const config = createChainForkConfig({
+      ...defaultChainConfig,
+      ALTAIR_FORK_EPOCH: 0,
+      BELLATRIX_FORK_EPOCH: 0,
+      DENEB_FORK_EPOCH: 0,
+      ELECTRA_FORK_EPOCH: 0,
+    });
+    const signedBeaconBlock = ssz.deneb.SignedBeaconBlock.defaultValue();
+    const mocks = getBlobCellAndProofs();
+    const blobs = mocks.map(({blob}) => blob);
+    const kzgCommitments = blobs.map(ckzg.blobToKzgCommitment);
+    for (const commitment of kzgCommitments) {
+      signedBeaconBlock.message.body.executionPayload.transactions.push(transactionForKzgCommitment(commitment));
+      signedBeaconBlock.message.body.blobKzgCommitments.push(commitment);
+    }
+
+    const sidecars = computeDataColumnSidecars(config, signedBeaconBlock, {blobs});
+    const signedBlockHeader = signedBlockToSignedHeader(config, signedBeaconBlock);
+
+    sidecars.forEach((sidecar, column) => {
+      expect(sidecar.index).toBe(column);
+      expect(sidecar.signedBlockHeader).toStrictEqual(signedBlockHeader);
+      expect(sidecar.kzgCommitments).toStrictEqual(kzgCommitments);
+      expect(sidecar.column.length).toBe(blobs.length);
+      expect(sidecar.kzgProofs.length).toBe(blobs.length);
+      sidecar.column.forEach((cell, row) => {
+        expect(Uint8Array.from(cell)).toStrictEqual(mocks[row].cells[column]);
+        const proof = sidecar.kzgProofs[row];
+        expect(Uint8Array.from(proof)).toStrictEqual(mocks[row].proofs[column]);
+        const commitment = sidecar.kzgCommitments[row];
+        const cellIndex = sidecar.index;
+        expect(ckzg.verifyCellKzgProofBatch([commitment], [cellIndex], [cell], [proof])).toBeTruthy();
+      });
+      expect(
+        ckzg.verifyCellKzgProofBatch(
+          sidecar.kzgCommitments,
+          Array.from({length: sidecar.column.length}, () => sidecar.index),
+          sidecar.column,
+          sidecar.kzgProofs
+        )
+      ).toBeTruthy();
+    });
+  });
 });
 
 function transactionForKzgCommitment(kzgCommitment: deneb.KZGCommitment): bellatrix.Transaction {

packages/beacon-node/test/utils/getBlobCellAndProofs.ts

@@ -0,0 +1,39 @@
+import fs from "node:fs";
+import path from "node:path";
+import yaml from "js-yaml";
+import {Blob, Cell, KZGProof} from "c-kzg";
+import {fromHex} from "@lodestar/utils";
+
+interface BlobCellAndProofMock {
+  blob: Blob;
+  cells: Cell[];
+  proofs: KZGProof[];
+}
+
+interface BlobCellAndProofYamlFormat {
+  input: {
+    blob: string;
+  };
+  //output: [Cell[], KZGProof[]]
+  output: [string[], string[]];
+}
+
+export function getBlobCellAndProofs(): BlobCellAndProofMock[] {
+  const mocks = [] as BlobCellAndProofMock[];
+  const mocksDir = path.resolve(__dirname, "..", "fixtures", "blobsAndCells");
+  for (const file of fs.readdirSync(mocksDir)) {
+    const filepath = path.resolve(mocksDir, file);
+    if (fs.statSync(filepath).isFile()) {
+      const {
+        input: {blob},
+        output: [cells, proofs],
+      } = yaml.load(fs.readFileSync(filepath, "utf-8")) as BlobCellAndProofYamlFormat;
+      mocks.push({
+        blob: fromHex(blob),
+        cells: cells.map(fromHex),
+        proofs: proofs.map(fromHex),
+      });
+    }
+  }
+  return mocks;
+}

packages/types/src/peerdas/index.ts

@@ -1,3 +1,4 @@
 export * from "./types.js";
-export * as ts from "./types.js";
-export * as ssz from "./sszTypes.js";
+import * as ts from "./types.js";
+import * as ssz from "./sszTypes.js";
+export {ts, ssz};

yarn.lock

@@ -4693,9 +4693,9 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/byte-size/-/byte-size-8.1.1.tgz#3424608c62d59de5bfda05d31e0313c6174842ae"
   integrity sha512-tUkzZWK0M/qdoLEqikxBWe4kumyuwjl3HO6zHTr4yEI23EojPtLYXdG1+AQY7MN0cGyNDvEaJ8wiYQm6P2bPxg==
 
-c-kzg@matthewkeil/c-kzg-4844#13aa01464479aa7c1ccafa64d52cbc17699ffa07:
+c-kzg@matthewkeil/c-kzg-4844#853b22fa416d4eac376678a36bfba0bccb59dd78:
   version "4.0.0-alpha.0"
-  resolved "https://codeload.github.com/matthewkeil/c-kzg-4844/tar.gz/13aa01464479aa7c1ccafa64d52cbc17699ffa07"
+  resolved "https://codeload.github.com/matthewkeil/c-kzg-4844/tar.gz/853b22fa416d4eac376678a36bfba0bccb59dd78"
 
 cac@^6.7.14:
   version "6.7.14"