Blob support

src/Nethermind/Nethermind.Optimism.Test/CL/BlobDecoderTests.cs

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections;
+using System.IO;
+using System.Linq;
+using Nethermind.Core.Extensions;
+using Nethermind.Optimism.CL;
+using NUnit.Framework;
+
+namespace Nethermind.Optimism.Test.CL;
+
+public class BlobDecoderTests
+{
+    private static byte[] StringToByteArray(string hex) {
+        return Enumerable.Range(0, hex.Length)
+            .Where(x => x % 2 == 0)
+            .Select(x => Convert.ToByte(hex.Substring(x, 2), 16))
+            .ToArray();
+    }
+
+    [TestCase(994040)]
+    [TestCase(2243081)]
+    [TestCase(35443070)]
+    [TestCase(1484649)]
+    [TestCase(76888454)]
+    [TestCase(69007144)] // isLast = false
+    public void Blob_decode_test(int index)
+    {
+        StreamReader inputReader = new StreamReader($"/home/deffrian/Documents/testvectors/input{index}");
+        StreamReader outputReader = new StreamReader($"/home/deffrian/Documents/testvectors/output{index}");
+        byte[] blob = StringToByteArray(inputReader.ReadToEnd());
+        byte[] decoded = StringToByteArray(outputReader.ReadToEnd());
+        inputReader.Close();
+        outputReader.Close();
+        byte[] result = BlobDecoder.DecodeBlob(new BlobSidecar { Blob = blob });
+        Assert.That(decoded, Is.EqualTo(result));
+
+        var frames = FrameDecoder.DecodeFrames(result);
+        Assert.That(frames.Length, Is.EqualTo(1));
+        Assert.That(frames[0].IsLast, Is.True);
+
+        var end = ChannelDecoder.DecodeChannel(frames[0]);
+    }
+
+    public static IEnumerable BlobTestCases
+    {
+        get
+        {
+            yield return new TestCaseData(
+                Bytes.FromHexString(""),
+                Bytes.FromHexString("")
+            )
+            {
+                TestName = "Sepolia blob 1"
+            };
+        }
+    }
+}

src/Nethermind/Nethermind.Optimism/CL/Decoders/BlobDecoder.cs

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+
+namespace Nethermind.Optimism.CL;
+
+public class BlobDecoder
+{
+    static public byte[] DecodeBlob(BlobSidecar blobSidecar)
+    {
+        int MaxBlobDataSize = (4 * 31 + 3) * 1024 - 4;
+        int BlobSize = 4096 * 32;
+        int length = ((int)blobSidecar.Blob[2] << 16) | ((int)blobSidecar.Blob[3] << 8) |
+                     ((int)blobSidecar.Blob[4]);
+        if (length > MaxBlobDataSize)
+        {
+            throw new Exception("Blob size is too big");
+        }
+
+        byte[] output = new byte[MaxBlobDataSize];
+        for (int i = 0; i < 27; ++i)
+        {
+            output[i] = blobSidecar.Blob[i + 5];
+        }
+
+        byte[] encodedByte = new byte[4];
+        int blobPos = 32;
+        int outputPos = 28;
+
+        encodedByte[0] = blobSidecar.Blob[0];
+        for (int i = 1; i < 4; ++i)
+        {
+            (encodedByte[i], outputPos, blobPos) = DecodeFieldElement(blobSidecar.Blob, outputPos, blobPos, output);
+        }
+
+        outputPos = ReassembleBytes(outputPos, encodedByte, output);
+
+        for (int i = 1; i < 1024 && outputPos < length; i++)
+        {
+            for (int j = 0; j < 4; j++)
+            {
+                (encodedByte[j], outputPos, blobPos) =
+                    DecodeFieldElement(blobSidecar.Blob, outputPos, blobPos, output);
+            }
+
+            outputPos = ReassembleBytes(outputPos, encodedByte, output);
+        }
+
+        for (int i = length; i < MaxBlobDataSize; i++)
+        {
+            if (output[i] != 0)
+            {
+                throw new Exception("Wrong output");
+            }
+        }
+
+        output = output[..length];
+        for (; blobPos < BlobSize; blobPos++)
+        {
+            if (blobSidecar.Blob[blobPos] != 0)
+            {
+                throw new Exception("Blob excess data");
+            }
+        }
+
+        return output;
+    }
+
+    static private (byte, int, int) DecodeFieldElement(byte[] blob, int outPos, int blobPos, byte[] output) {
+        // two highest order bits of the first byte of each field element should always be 0
+        if ((blob[blobPos] & 0b1100_0000) != 0) {
+            // TODO: remove exception
+            throw new Exception("Invalid field element");
+        }
+
+        for (int i = 0; i < 31; i++)
+        {
+            output[outPos + i] = blob[blobPos + i + 1];
+        }
+
+        return (blob[blobPos], outPos + 32, blobPos + 32);
+    }
+
+    static int ReassembleBytes(int outPos, byte[] encodedByte, byte[] output)
+    {
+        outPos--; // account for fact that we don't output a 128th byte
+        byte x = (byte)((encodedByte[0] & 0b0011_1111) | ((encodedByte[1] & 0b0011_0000) << 2));
+        byte y = (byte)((encodedByte[1] & 0b0000_1111) | ((encodedByte[3] & 0b0000_1111) << 4));
+        byte z = (byte)((encodedByte[2] & 0b0011_1111) | ((encodedByte[3] & 0b0011_0000) << 2));
+        // put the re-assembled bytes in their appropriate output locations
+        output[outPos - 32] = z;
+        output[outPos - 32 * 2] = y;
+        output[outPos - 32 * 3] = x;
+        return outPos;
+    }
+}

src/Nethermind/Nethermind.Optimism/CL/Decoders/ChannelDecoder.cs

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using Nethermind.Core;
+using Nethermind.Serialization.Rlp;
+
+namespace Nethermind.Optimism.CL;
+
+public class ChannelDecoder
+{
+    public static byte[] DecodeChannel(Frame frame)
+    {
+        if ((frame.FrameData[0] & 0x0F) == 8 || (frame.FrameData[0] & 0x0F) == 15)
+        {
+            // zlib
+            var deflateStream = new DeflateStream(new MemoryStream(frame.FrameData[2..]), CompressionMode.Decompress);
+            var memoryStream = new MemoryStream();
+            deflateStream.CopyTo(memoryStream);
+            return memoryStream.ToArray();
+        } else if (frame.FrameData[0] == 1)
+        {
+            // brotli
+            throw new NotImplementedException("Brotli is not supported");
+        }
+        else
+        {
+            throw new Exception($"Unsupported compression algorithm {frame.FrameData[0]}");
+        }
+    }
+}

src/Nethermind/Nethermind.Optimism/CL/Decoders/FrameDecoder.cs

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Nethermind.Network.Rlpx;
+
+namespace Nethermind.Optimism.CL;
+
+public class FrameDecoder
+{
+    private static (Frame, int) DecodeFrame(byte[] data)
+    {
+        byte[] channelId = data[..16];
+        UInt16 frameNumber = BitConverter.ToUInt16(data[16..18].Reverse().ToArray());
+        UInt32 frameDataLength = BitConverter.ToUInt32(data[18..22].Reverse().ToArray());
+        byte[] frameData = data[22..(22 + (int)frameDataLength)];
+        byte isLast = data[22 + (int)frameDataLength];
+        if (isLast != 0 && isLast != 1)
+        {
+            throw new Exception("Invalid isLast flag");
+        }
+        return (new Frame()
+        {
+            ChannelId = channelId,
+            FrameNumber = frameNumber,
+            FrameData = frameData,
+            IsLast = isLast == 0
+        }, 23 + (int)frameDataLength);
+    }
+
+    public static Frame[] DecodeFrames(byte[] data)
+    {
+        byte version = data[0];
+        if (version != 0)
+        {
+            throw new Exception($"Frame Decoder version {version} is not supported.");
+        }
+
+        List<Frame> frames = new List<Frame>();
+        int pos = 1;
+        while (pos < data.Length)
+        {
+            (Frame frame, int decoded) = DecodeFrame(data[pos..]);
+            pos += decoded;
+            frames.Add(frame);
+        }
+
+        if (pos != data.Length)
+        {
+            throw new Exception("Excess frame data");
+        }
+        return frames.ToArray();
+    }
+}
+
+public struct Frame
+{
+    public byte[] ChannelId;
+    public UInt16 FrameNumber;
+    public byte[] FrameData;
+    public bool IsLast;
+}

src/Nethermind/Nethermind.Optimism/CL/Driver.cs

@@ -1,41 +1,46 @@
 // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
 using System.Linq;
-using Nethermind.Blockchain.Find;
+using System.Threading;
 using Nethermind.Core;
-using Nethermind.Core.Crypto;
-using Nethermind.Facade.Eth;
-using Nethermind.JsonRpc.Modules.Eth;
+using Nethermind.Logging;
 
 namespace Nethermind.Optimism.CL;
 
 public class Driver
 {
     private readonly ICLConfig _config;
     private readonly IL1Bridge _l1Bridge;
+    private readonly ILogger _logger;
 
-    public Driver(IL1Bridge l1Bridge, ICLConfig config)
+    public Driver(IL1Bridge l1Bridge, ICLConfig config, ILogger logger)
     {
         _config = config;
         _l1Bridge = l1Bridge;
+        _logger = logger;
     }
 
-    private void Start()
+    public void Start()
     {
         _l1Bridge.OnNewL1Head += OnNewL1Head;
     }
 
-    private void OnNewL1Head(BlockForRpc block, ulong slotNumber)
+    private void OnNewL1Head(BeaconBlock block, ulong slotNumber)
     {
+        _logger.Error("INVOKED");
+        Address sepoliaBatcher = new("0x8F23BB38F531600e5d8FDDaAEC41F13FaB46E98c");
+        Address batcherInboxAddress = new("0xff00000000000000000000000000000011155420");
         // Filter batch submitter transaction
-        foreach (TransactionForRpc transaction in block.Transactions.Cast<TransactionForRpc>())
+        foreach (Transaction transaction in block.Transactions)
         {
-            if (_config.BatcherInboxAddress == transaction.To && _config.BatcherAddress == transaction.From)
+            // _logger.Error($"Tx To: {transaction.To}, From: {transaction.SenderAddress} end");
+            if (batcherInboxAddress == transaction.To && sepoliaBatcher == transaction.SenderAddress)
             {
                 if (transaction.Type == TxType.Blob)
                 {
-                    ProcessBlobBatcherTransaction(transaction);
+                    ProcessBlobBatcherTransaction(transaction, slotNumber);
                 }
                 else
                 {
@@ -45,16 +50,35 @@ private void OnNewL1Head(BlockForRpc block, ulong slotNumber)
         }
     }
 
-    private void ProcessBlobBatcherTransaction(TransactionForRpc transaction)
+    private async void ProcessBlobBatcherTransaction(Transaction transaction, ulong slotNumber)
     {
-        int numberOfBlobs = transaction.BlobVersionedHashes!.Length;
-        for (int i = 0; i < numberOfBlobs; ++i)
+        if (_logger.IsError)
         {
-
+            _logger.Error($"GOT BLOB TRANSACTION To: {transaction.To}, From: {transaction.SenderAddress}");
+        }
+        BlobSidecar[] blobSidecars = await _l1Bridge.GetBlobSidecars(slotNumber);
+        for (int i = 0; i < transaction.BlobVersionedHashes!.Length; i++)
+        {
+            for (int j = 0; j < blobSidecars.Length; ++j)
+            {
+                if (blobSidecars[j].BlobVersionedHash.SequenceEqual(transaction.BlobVersionedHashes[i]!))
+                {
+                    _logger.Error($"GOT BLOB VERSIONED HASH: {BitConverter.ToString(transaction.BlobVersionedHashes[i]!).Replace("-", "")}");
+                    _logger.Error($"BLOB: {BitConverter.ToString(blobSidecars[j].Blob[..32]).Replace("-", "")}");
+                    byte[] data = BlobDecoder.DecodeBlob(blobSidecars[j]);
+                    FrameDecoder.DecodeFrames(data);
+                    // _logger.Error($"DATA: {BitConverter.ToString(data).Replace("-", "")}");
+                }
+            }
         }
+
     }
 
-    private void ProcessCalldataBatcherTransaction(TransactionForRpc transaction)
+    private void ProcessCalldataBatcherTransaction(Transaction transaction)
     {
+        if (_logger.IsError)
+        {
+            _logger.Error($"GOT REGULAR TRANSACTION");
+        }
     }
 }