feat: validate data column sidecars

packages/beacon-node/package.json

@@ -132,7 +132,7 @@
     "@lodestar/utils": "^1.21.0",
     "@lodestar/validator": "^1.21.0",
     "@multiformats/multiaddr": "^12.1.3",
-    "c-kzg": "matthewkeil/c-kzg-4844#13aa01464479aa7c1ccafa64d52cbc17699ffa07",
+    "c-kzg": "^4.0.1",
     "datastore-core": "^9.1.1",
     "datastore-level": "^10.1.1",
     "deepmerge": "^4.3.1",

packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts

@@ -119,7 +119,7 @@ async function maybeValidateBlobs(
         const {dataColumns} = blockData;
         const skipProofsCheck = opts.validBlobSidecars === BlobSidecarValidation.Individual;
         // might require numColumns, custodyColumns from blockData as input to below
-        validateDataColumnsSidecars(blockSlot, beaconBlockRoot, blobKzgCommitments, dataColumns, {skipProofsCheck});
+        validateDataColumnsSidecars(blockSlot, beaconBlockRoot, dataColumns, {skipProofsCheck});
       }
 
       const availableBlockInput = getBlockInput.availableData(

packages/beacon-node/src/chain/validation/dataColumnSidecar.ts

@@ -4,12 +4,14 @@ import {
   DATA_COLUMN_SIDECAR_SUBNET_COUNT,
   NUMBER_OF_COLUMNS,
 } from "@lodestar/params";
-import {ssz, deneb, electra, Slot, Root} from "@lodestar/types";
-import {verifyMerkleBranch} from "@lodestar/utils";
+import {ssz, electra, Slot, Root} from "@lodestar/types";
+import {toHex, verifyMerkleBranch} from "@lodestar/utils";
 
 import {DataColumnSidecarGossipError, DataColumnSidecarErrorCode} from "../errors/dataColumnSidecarError.js";
 import {GossipAction} from "../errors/gossipValidation.js";
 import {IBeaconChain} from "../interface.js";
+import {ckzg} from "../../util/kzg.js";
+import {byteArrayEquals} from "../../util/bytes.js";
 
 export async function validateGossipDataColumnSidecar(
   chain: IBeaconChain,
@@ -45,14 +47,60 @@ export async function validateGossipDataColumnSidecar(
 }
 
 export function validateDataColumnsSidecars(
-  _blockSlot: Slot,
-  _blockRoot: Root,
-  _expectedKzgCommitments: deneb.BlobKzgCommitments,
-  _dataColumnSidecars: electra.DataColumnSidecars,
-  _opts: {skipProofsCheck: boolean} = {skipProofsCheck: false}
+  blockSlot: Slot,
+  blockRoot: Root,
+  dataColumnSidecars: electra.DataColumnSidecars,
+  opts: {skipProofsCheck: boolean} = {skipProofsCheck: false}
 ): void {
-  // stubbed
-  return;
+  const commitmentBytes: Uint8Array[] = [];
+  const cellIndices: number[] = [];
+  const cells: Uint8Array[] = [];
+  const proofBytes: Uint8Array[] = [];
+  for (let sidecarsIndex = 0; sidecarsIndex < dataColumnSidecars.length; sidecarsIndex++) {
+    const columnSidecar = dataColumnSidecars[sidecarsIndex];
+    const columnIndex = columnSidecar.index;
+    const columnBlockHeader = columnSidecar.signedBlockHeader.message;
+    const columnBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnBlockHeader);
+    if (columnBlockHeader.slot !== blockSlot || !byteArrayEquals(columnBlockRoot, blockRoot)) {
+      throw new Error(
+        `Invalid columnSidecar with slot=${columnBlockHeader.slot} columnBlockRoot=${toHex(columnBlockRoot)} columnIndex=${columnIndex} for the block blockRoot=${toHex(blockRoot)} slot=${blockSlot} sidecarsIndex=${sidecarsIndex}`
+      );
+    }
+
+    if (columnIndex < NUMBER_OF_COLUMNS) {
+      throw new Error(
+        `Invalid data column columnIndex=${columnIndex} in slot=${blockSlot} blockRoot=${toHex(blockRoot)} sidecarsIndex=${sidecarsIndex}`
+      );
+    }
+
+    const {column, kzgCommitments, kzgProofs} = columnSidecar;
+    if (column.length !== kzgCommitments.length || column.length !== kzgProofs.length) {
+      throw new Error(
+        `Invalid data column lengths for columnIndex=${columnIndex} in slot=${blockSlot} blockRoot=${toHex(blockRoot)}`
+      );
+    }
+
+    commitmentBytes.push(...kzgCommitments);
+    cellIndices.push(...Array.from({length: column.length}, () => columnIndex));
+    cells.push(...column);
+    proofBytes.push(...kzgProofs);
+  }
+
+  if (opts.skipProofsCheck) {
+    return;
+  }
+
+  let valid: boolean;
+  try {
+    valid = ckzg.verifyCellKzgProofBatch(commitmentBytes, cellIndices, cells, proofBytes);
+  } catch (err) {
+    (err as Error).message = `Error in verifyCellKzgProofBatch for slot=${blockSlot} blockRoot=${toHex(blockRoot)}`;
+    throw err;
+  }
+
+  if (!valid) {
+    throw new Error(`Invalid data column sidecars in slot=${blockSlot} blockRoot=${toHex(blockRoot)}`);
+  }
 }
 
 function validateInclusionProof(dataColumnSidecar: electra.DataColumnSidecar): boolean {

packages/beacon-node/src/node/nodejs.ts

@@ -20,7 +20,7 @@ import {MonitoringService} from "../monitoring/index.js";
 import {getApi, BeaconRestApiServer} from "../api/index.js";
 import {initializeExecutionEngine, initializeExecutionBuilder} from "../execution/index.js";
 import {initializeEth1ForBlockProduction} from "../eth1/index.js";
-import {initCKZG, loadEthereumTrustedSetup, TrustedFileMode} from "../util/kzg.js";
+import {initCKZG, loadEthereumTrustedSetup} from "../util/kzg.js";
 import {HistoricalStateRegen} from "../chain/historicalState/index.js";
 import {NodeId} from "../network/subnets/interface.js";
 import {IBeaconNodeOptions} from "./options.js";
@@ -164,7 +164,7 @@ export class BeaconNode {
     // If deneb is configured, load the trusted setup
     if (config.DENEB_FORK_EPOCH < Infinity) {
       await initCKZG();
-      loadEthereumTrustedSetup(TrustedFileMode.Txt, opts.chain.trustedSetupPrecompute, opts.chain.trustedSetup);
+      loadEthereumTrustedSetup(opts.chain.trustedSetupPrecompute, opts.chain.trustedSetup);
     }
 
     // Prune hot db repos

packages/beacon-node/src/util/kzg.ts

@@ -1,10 +1,3 @@
-import path from "node:path";
-import fs from "node:fs";
-import {fileURLToPath} from "node:url";
-import {fromHex, toHex} from "@lodestar/utils";
-
-/* eslint-disable @typescript-eslint/naming-convention */
-
 // "c-kzg" has hardcoded the mainnet value, do not use params
 export const FIELD_ELEMENTS_PER_BLOB_MAINNET = 4096;
 
@@ -14,20 +7,16 @@ function ckzgNotLoaded(): never {
 
 export let ckzg: {
   freeTrustedSetup(): void;
-  loadTrustedSetup(precompute: number, filePath: string): void;
+  loadTrustedSetup(precompute: number, filePath?: string): void;
   blobToKzgCommitment(blob: Uint8Array): Uint8Array;
   computeBlobKzgProof(blob: Uint8Array, commitment: Uint8Array): Uint8Array;
   verifyBlobKzgProof(blob: Uint8Array, commitment: Uint8Array, proof: Uint8Array): boolean;
   verifyBlobKzgProofBatch(blobs: Uint8Array[], expectedKzgCommitments: Uint8Array[], kzgProofs: Uint8Array[]): boolean;
-  computeCells(blob: Uint8Array): Uint8Array[];
   computeCellsAndKzgProofs(blob: Uint8Array): [Uint8Array[], Uint8Array[]];
-  cellsToBlob(cells: Uint8Array[]): Uint8Array;
-  recoverAllCells(cellIds: number[], cells: Uint8Array[]): Uint8Array[];
-  verifyCellKzgProof(commitmentBytes: Uint8Array, cellId: number, cell: Uint8Array, proofBytes: Uint8Array): boolean;
+  recoverCellsAndKzgProofs(cellIndices: number[], cells: Uint8Array[]): [Uint8Array[], Uint8Array[]];
   verifyCellKzgProofBatch(
     commitmentsBytes: Uint8Array[],
-    rowIndices: number[],
-    columnIndices: number[],
+    cellIndices: number[],
     cells: Uint8Array[],
     proofsBytes: Uint8Array[]
   ): boolean;
@@ -38,148 +27,40 @@ export let ckzg: {
   computeBlobKzgProof: ckzgNotLoaded,
   verifyBlobKzgProof: ckzgNotLoaded,
   verifyBlobKzgProofBatch: ckzgNotLoaded,
-  computeCells: ckzgNotLoaded,
   computeCellsAndKzgProofs: ckzgNotLoaded,
-  cellsToBlob: ckzgNotLoaded,
-  recoverAllCells: ckzgNotLoaded,
-  verifyCellKzgProof: ckzgNotLoaded,
+  recoverCellsAndKzgProofs: ckzgNotLoaded,
   verifyCellKzgProofBatch: ckzgNotLoaded,
 };
 
-// Global variable __dirname no longer available in ES6 modules.
-// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-export const TRUSTED_SETUP_BIN_FILEPATH = path.join(__dirname, "../../trusted_setup.bin");
-const TRUSTED_SETUP_JSON_FILEPATH = path.join(__dirname, "../../trusted_setup.json");
-const TRUSTED_SETUP_TXT_FILEPATH = path.join(__dirname, "../../trusted_setup.txt");
-
-const POINT_COUNT_BYTES = 4;
-const G1POINT_BYTES = 48;
-const G2POINT_BYTES = 96;
-const G1POINT_COUNT = FIELD_ELEMENTS_PER_BLOB_MAINNET;
-const G2POINT_COUNT = 65;
-const TOTAL_SIZE = 2 * POINT_COUNT_BYTES + G1POINT_BYTES * G1POINT_COUNT + G2POINT_BYTES * G2POINT_COUNT;
-
 export async function initCKZG(): Promise<void> {
   /* eslint-disable @typescript-eslint/ban-ts-comment */
   // @ts-ignore
   ckzg = (await import("c-kzg")).default as typeof ckzg;
   /* eslint-enable @typescript-eslint/ban-ts-comment */
 }
 
-export enum TrustedFileMode {
-  Bin = "bin",
-  Txt = "txt",
-}
-
 /**
  * Load our KZG trusted setup into C-KZG for later use.
  * We persist the trusted setup as serialized bytes to save space over TXT or JSON formats.
  * However the current c-kzg API **requires** to read from a file with a specific .txt format
  */
 export function loadEthereumTrustedSetup(
-  mode: TrustedFileMode = TrustedFileMode.Txt,
   precompute = 0, // default to 0 for testing
   filePath?: string
 ): void {
   try {
-    let setupFilePath;
-    if (mode === TrustedFileMode.Bin) {
-      const binPath = filePath ?? TRUSTED_SETUP_BIN_FILEPATH;
-      const bytes = fs.readFileSync(binPath);
-      const json = trustedSetupBinToJson(bytes);
-      const txt = trustedSetupJsonToTxt(json);
-      fs.writeFileSync(TRUSTED_SETUP_TXT_FILEPATH, txt);
-      setupFilePath = TRUSTED_SETUP_TXT_FILEPATH;
-    } else {
-      setupFilePath = filePath ?? TRUSTED_SETUP_TXT_FILEPATH;
-    }
-
     try {
       // in unit tests, calling loadTrustedSetup() twice has error so we have to free and retry
-      ckzg.loadTrustedSetup(precompute, setupFilePath);
+      ckzg.loadTrustedSetup(precompute, filePath);
     } catch (e) {
       if ((e as Error).message !== "Error trusted setup is already loaded") {
         throw e;
       }
     }
   } catch (e) {
-    (e as Error).message = `Error loading trusted setup ${TRUSTED_SETUP_JSON_FILEPATH}: ${(e as Error).message}`;
+    (e as Error).message = filePath
+      ? `Error loading trusted setup ${filePath}: ${(e as Error).message}`
+      : `Error loading default trusted setup: ${(e as Error).message}`;
     throw e;
   }
 }
-
-/* eslint-disable @typescript-eslint/naming-convention */
-export interface TrustedSetupJSON {
-  setup_G1: string[];
-  setup_G2: string[];
-}
-
-type TrustedSetupBin = Uint8Array;
-type TrustedSetupTXT = string;
-
-/**
- * Custom format defined in https://github.com/ethereum/c-kzg-4844/issues/3
- */
-export function trustedSetupJsonToBin(data: TrustedSetupJSON): TrustedSetupBin {
-  const out = new Uint8Array(TOTAL_SIZE);
-  const dv = new DataView(out.buffer, out.byteOffset, out.byteLength);
-
-  dv.setUint32(0, G1POINT_COUNT);
-  dv.setUint32(POINT_COUNT_BYTES, G2POINT_BYTES);
-
-  for (let i = 0; i < G1POINT_COUNT; i++) {
-    const point = fromHex(data.setup_G1[i]);
-    if (point.length !== G1POINT_BYTES) throw Error(`g1 point size ${point.length} != ${G1POINT_BYTES}`);
-    out.set(point, 2 * POINT_COUNT_BYTES + i * G1POINT_BYTES);
-  }
-
-  for (let i = 0; i < G2POINT_COUNT; i++) {
-    const point = fromHex(data.setup_G2[i]);
-    if (point.length !== G2POINT_BYTES) throw Error(`g2 point size ${point.length} != ${G2POINT_BYTES}`);
-    out.set(point, 2 * POINT_COUNT_BYTES + G1POINT_COUNT * G1POINT_BYTES + i * G2POINT_BYTES);
-  }
-
-  return out;
-}
-
-export function trustedSetupBinToJson(bytes: TrustedSetupBin): TrustedSetupJSON {
-  const data: TrustedSetupJSON = {
-    setup_G1: [],
-    setup_G2: [],
-  };
-
-  if (bytes.length < TOTAL_SIZE) {
-    throw Error(`trusted_setup size ${bytes.length} < ${TOTAL_SIZE}`);
-  }
-
-  for (let i = 0; i < G1POINT_COUNT; i++) {
-    const start = 2 * POINT_COUNT_BYTES + i * G1POINT_BYTES;
-    data.setup_G1.push(toHex(bytes.slice(start, start + G1POINT_BYTES)));
-  }
-
-  for (let i = 0; i < G2POINT_COUNT; i++) {
-    const start = 2 * POINT_COUNT_BYTES + G1POINT_COUNT * G1POINT_BYTES + i * G2POINT_BYTES;
-    data.setup_G1.push(toHex(bytes.slice(start, start + G2POINT_BYTES)));
-  }
-
-  return data;
-}
-
-export function trustedSetupJsonToTxt(data: TrustedSetupJSON): TrustedSetupTXT {
-  return [
-    // ↵
-    G1POINT_COUNT,
-    G2POINT_COUNT,
-    ...data.setup_G1.map(strip0xPrefix),
-    ...data.setup_G2.map(strip0xPrefix),
-  ].join("\n");
-}
-
-function strip0xPrefix(hex: string): string {
-  if (hex.startsWith("0x")) {
-    return hex.slice(2);
-  } else {
-    return hex;
-  }
-}