From c8748770fe88fecf3b0b36226296e5c0858da051 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 1 Dec 2024 17:56:10 +0000 Subject: [PATCH] Optimize EvmStack (#7843) --- .../Nethermind.Core/Extensions/Bytes.cs | 10 +- .../EvmStackBenchmarks.cs | 12 +- .../Tracing/DebugTracerTests.cs | 2 +- src/Nethermind/Nethermind.Evm/EvmStack.cs | 208 +++++++----------- .../Nethermind.Evm/VirtualMachine.cs | 2 +- .../Nethermind.State/StateReader.cs | 2 +- 6 files changed, 93 insertions(+), 143 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 21ed6e8e66a..39d8429cec8 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -27,8 +27,12 @@ public static unsafe partial class Bytes public static readonly IEqualityComparer EqualityComparer = new BytesEqualityComparer(); public static readonly IEqualityComparer NullableEqualityComparer = new NullableBytesEqualityComparer(); public static readonly BytesComparer Comparer = new(); - public static readonly ReadOnlyMemory ZeroByte = new byte[] { 0 }; - public static readonly ReadOnlyMemory OneByte = new byte[] { 1 }; + // The ReadOnlyMemory needs to be initialized = or it will be created each time. + public static ReadOnlyMemory ZeroByte = new byte[] { 0 }; + public static ReadOnlyMemory OneByte = new byte[] { 1 }; + // The Jit converts a ReadOnlySpan => new byte[] to a data section load, no allocation. + public static ReadOnlySpan ZeroByteSpan => new byte[] { 0 }; + public static ReadOnlySpan OneByteSpan => new byte[] { 1 }; public const string ZeroHexValue = "0x0"; public const string ZeroValue = "0"; @@ -210,7 +214,7 @@ public static ReadOnlySpan WithoutLeadingZeros(this Span bytes) public static ReadOnlySpan WithoutLeadingZeros(this ReadOnlySpan bytes) { - if (bytes.Length == 0) return ZeroByte.Span; + if (bytes.Length == 0) return ZeroByteSpan; int nonZeroIndex = bytes.IndexOfAnyExcept((byte)0); // Keep one or it will be interpreted as null diff --git a/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs b/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs index 1c3c0599562..4ea3983493b 100644 --- a/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs +++ b/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs @@ -30,7 +30,7 @@ public void GlobalSetup() [ArgumentsSource(nameof(ValueSource))] public UInt256 Uint256(UInt256 v) { - EvmStack stack = new(_stack.AsSpan(), 0, NullTxTracer.Instance); + EvmStack stack = new(0, NullTxTracer.Instance, _stack.AsSpan()); stack.PushUInt256(in v); stack.PopUInt256(out UInt256 value); @@ -50,7 +50,7 @@ public UInt256 Uint256(UInt256 v) [Benchmark(OperationsPerInvoke = 4)] public byte Byte() { - EvmStack stack = new(_stack.AsSpan(), 0, NullTxTracer.Instance); + EvmStack stack = new(0, NullTxTracer.Instance, _stack.AsSpan()); byte b = 1; @@ -72,7 +72,7 @@ public byte Byte() [Benchmark(OperationsPerInvoke = 4)] public void PushZero() { - EvmStack stack = new(_stack.AsSpan(), 0, NullTxTracer.Instance); + EvmStack stack = new(0, NullTxTracer.Instance, _stack.AsSpan()); stack.PushZero(); stack.PushZero(); @@ -83,7 +83,7 @@ public void PushZero() [Benchmark(OperationsPerInvoke = 4)] public void PushOne() { - EvmStack stack = new(_stack.AsSpan(), 0, NullTxTracer.Instance); + EvmStack stack = new(0, NullTxTracer.Instance, _stack.AsSpan()); stack.PushOne(); stack.PushOne(); @@ -94,7 +94,7 @@ public void PushOne() [Benchmark(OperationsPerInvoke = 4)] public void Swap() { - EvmStack stack = new(_stack.AsSpan(), 2, NullTxTracer.Instance); + EvmStack stack = new(0, NullTxTracer.Instance, _stack.AsSpan()); stack.Swap(2); stack.Swap(2); @@ -105,7 +105,7 @@ public void Swap() [Benchmark(OperationsPerInvoke = 4)] public void Dup() { - EvmStack stack = new(_stack.AsSpan(), 1, NullTxTracer.Instance); + EvmStack stack = new(1, NullTxTracer.Instance, _stack.AsSpan()); stack.Dup(1); stack.Dup(1); diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/DebugTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/DebugTracerTests.cs index b2fd2017894..6f4f5831642 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/DebugTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/DebugTracerTests.cs @@ -276,7 +276,7 @@ public void Debugger_Can_Alter_Data_Stack(string bytecodeHex) if (tracer.CanReadState) { // we pop the condition and overwrite it with a false to force breaking out of the loop - EvmStack stack = new(tracer.CurrentState.DataStack, tracer.CurrentState.DataStackHead, tracer); + EvmStack stack = new(tracer.CurrentState.DataStackHead, tracer, tracer.CurrentState.DataStack); stack.PopLimbo(); stack.PushByte(0x00); diff --git a/src/Nethermind/Nethermind.Evm/EvmStack.cs b/src/Nethermind/Nethermind.Evm/EvmStack.cs index ef6dc068bde..ebe1eabd04f 100644 --- a/src/Nethermind/Nethermind.Evm/EvmStack.cs +++ b/src/Nethermind/Nethermind.Evm/EvmStack.cs @@ -19,6 +19,7 @@ namespace Nethermind.Evm; using static VirtualMachine; using Word = Vector256; +[StructLayout(LayoutKind.Auto)] public ref struct EvmStack where TTracing : struct, IIsTracing { @@ -26,36 +27,44 @@ public ref struct EvmStack public const int WordSize = EvmStack.WordSize; public const int AddressSize = EvmStack.AddressSize; - public EvmStack(scoped in Span bytes, scoped in int head, ITxTracer txTracer) + public EvmStack(scoped in int head, ITxTracer txTracer, scoped in Span bytes) { - _bytes = bytes; Head = head; _tracer = txTracer; + _bytes = bytes; } + private readonly ITxTracer _tracer; + private readonly Span _bytes; public int Head; - private readonly Span _bytes; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref byte PushBytesRef() + { + // Workhorse method + int head = Head; + if ((Head = head + 1) >= MaxStackSize) + { + EvmStack.ThrowEvmStackOverflowException(); + } - private readonly ITxTracer _tracer; + return ref Unsafe.Add(ref MemoryMarshal.GetReference(_bytes), head * WordSize); + } public void PushBytes(scoped ReadOnlySpan value) { if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(value); + ref byte bytes = ref PushBytesRef(); if (value.Length != WordSize) { - ClearWordAtHead(); - value.CopyTo(_bytes.Slice(Head * WordSize + WordSize - value.Length, value.Length)); + // Not full entry, clear first + Unsafe.As(ref bytes) = default; + value.CopyTo(MemoryMarshal.CreateSpan(ref Unsafe.Add(ref bytes, WordSize - value.Length), value.Length)); } else { - Unsafe.WriteUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetReference(_bytes), Head * WordSize), Unsafe.As(ref MemoryMarshal.GetReference(value))); - } - - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); + Unsafe.As(ref bytes) = Unsafe.As(ref MemoryMarshal.GetReference(value)); } } @@ -63,94 +72,75 @@ public void PushBytes(scoped in ZeroPaddedSpan value) { if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(value); + ref byte bytes = ref PushBytesRef(); ReadOnlySpan valueSpan = value.Span; if (valueSpan.Length != WordSize) { - ClearWordAtHead(); - Span stack = _bytes.Slice(Head * WordSize, valueSpan.Length); - valueSpan.CopyTo(stack); + // Not full entry, clear first + Unsafe.As(ref bytes) = default; + valueSpan.CopyTo(MemoryMarshal.CreateSpan(ref bytes, value.Length)); } else { - Unsafe.WriteUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetReference(_bytes), Head * WordSize), Unsafe.As(ref MemoryMarshal.GetReference(valueSpan))); - } - - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); + Unsafe.As(ref bytes) = Unsafe.As(ref MemoryMarshal.GetReference(valueSpan)); } } - public ref byte PushBytesRef() + public void PushLeftPaddedBytes(ReadOnlySpan value, int paddingLength) { - ref byte bytes = ref _bytes[Head * WordSize]; + if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(value); - if (++Head >= MaxStackSize) + ref byte bytes = ref PushBytesRef(); + if (value.Length != WordSize) { - EvmStack.ThrowEvmStackOverflowException(); + // Not full entry, clear first + Unsafe.As(ref bytes) = default; + value.CopyTo(MemoryMarshal.CreateSpan(ref Unsafe.Add(ref bytes, WordSize - paddingLength), value.Length)); + } + else + { + Unsafe.As(ref bytes) = Unsafe.As(ref MemoryMarshal.GetReference(value)); } - - return ref bytes; } public void PushByte(byte value) { if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(value); - ClearWordAtHead(); - _bytes[Head * WordSize + WordSize - sizeof(byte)] = value; - - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); - } + ref byte bytes = ref PushBytesRef(); + // Not full entry, clear first + Unsafe.As(ref bytes) = default; + Unsafe.Add(ref bytes, WordSize - sizeof(byte)) = value; } - private static ReadOnlySpan OneStackItem() => Bytes.OneByte.Span; - public void PushOne() { - if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(OneStackItem()); - - ClearWordAtHead(); - _bytes[Head * WordSize + WordSize - sizeof(byte)] = 1; + if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(Bytes.OneByteSpan); - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); - } + ref byte bytes = ref PushBytesRef(); + // Not full entry, clear first + Unsafe.As(ref bytes) = default; + Unsafe.Add(ref bytes, WordSize - sizeof(byte)) = 1; } - private static ReadOnlySpan ZeroStackItem() => Bytes.ZeroByte.Span; - public void PushZero() { - if (typeof(TTracing) == typeof(IsTracing)) - { - _tracer.ReportStackPush(ZeroStackItem()); - } + if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(Bytes.ZeroByteSpan); - ClearWordAtHead(); - - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); - } + ref byte bytes = ref PushBytesRef(); + Unsafe.As(ref bytes) = default; } public void PushUInt32(in int value) { - ClearWordAtHead(); + ref byte bytes = ref PushBytesRef(); + // Not full entry, clear first + Unsafe.As(ref bytes) = default; - Span intPlace = _bytes.Slice(Head * WordSize + WordSize - sizeof(uint), sizeof(uint)); + Span intPlace = MemoryMarshal.CreateSpan(ref Unsafe.Add(ref bytes, WordSize - sizeof(uint)), sizeof(uint)); BinaryPrimitives.WriteInt32BigEndian(intPlace, value); if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(intPlace); - - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); - } } /// @@ -161,17 +151,14 @@ public void PushUInt32(in int value) /// public void PushUInt256(in UInt256 value) { - Span word = _bytes.Slice(Head * WordSize, WordSize); - ref byte bytes = ref MemoryMarshal.GetReference(word); - + ref byte bytes = ref PushBytesRef(); if (Avx2.IsSupported) { Word shuffle = Vector256.Create( - (byte) - 31, 30, 29, 28, 27, 26, 25, 24, - 23, 22, 21, 20, 19, 18, 17, 16, - 15, 14, 13, 12, 11, 10, 9, 8, - 7, 6, 5, 4, 3, 2, 1, 0); + 0x18191a1b1c1d1e1ful, + 0x1011121314151617ul, + 0x08090a0b0c0d0e0ful, + 0x0001020304050607ul).AsByte(); if (Avx512Vbmi.VL.IsSupported) { Word data = Unsafe.As(ref Unsafe.AsRef(in value)); @@ -205,18 +192,13 @@ public void PushUInt256(in UInt256 value) Unsafe.WriteUnaligned(ref bytes, Vector256.Create(u3, u2, u1, u0)); } - if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(word); - - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); - } + if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(MemoryMarshal.CreateReadOnlySpan(ref bytes, WordSize)); } public void PushSignedInt256(in Int256.Int256 value) { // tail call into UInt256 - PushUInt256(Unsafe.As(ref Unsafe.AsRef(in value))); + PushUInt256(in Unsafe.As(ref Unsafe.AsRef(in value))); } public void PopLimbo() @@ -245,11 +227,10 @@ public bool PopUInt256(out UInt256 result) { Word data = Unsafe.ReadUnaligned(ref bytes); Word shuffle = Vector256.Create( - (byte) - 31, 30, 29, 28, 27, 26, 25, 24, - 23, 22, 21, 20, 19, 18, 17, 16, - 15, 14, 13, 12, 11, 10, 9, 8, - 7, 6, 5, 4, 3, 2, 1, 0); + 0x18191a1b1c1d1e1ful, + 0x1011121314151617ul, + 0x08090a0b0c0d0e0ful, + 0x0001020304050607ul).AsByte(); if (Avx512Vbmi.VL.IsSupported) { Word convert = Avx512Vbmi.VL.PermuteVar32x8(data, shuffle); @@ -312,54 +293,33 @@ public readonly Span PeekWord256() public Address? PopAddress() => Head-- == 0 ? null : new Address(_bytes.Slice(Head * WordSize + WordSize - AddressSize, AddressSize).ToArray()); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public ref byte PopBytesByRef() { - if (Head-- == 0) + int head = Head; + if (head == 0) { return ref Unsafe.NullRef(); } - - return ref _bytes[Head * WordSize]; + head--; + Head = head; + return ref Unsafe.Add(ref MemoryMarshal.GetReference(_bytes), head * WordSize); } public Span PopWord256() { - if (Head-- == 0) - { - EvmStack.ThrowEvmStackUnderflowException(); - } + ref byte bytes = ref PopBytesByRef(); + if (Unsafe.IsNullRef(ref bytes)) EvmStack.ThrowEvmStackUnderflowException(); - return _bytes.Slice(Head * WordSize, WordSize); + return MemoryMarshal.CreateSpan(ref bytes, WordSize); } public byte PopByte() { - if (Head-- == 0) - { - EvmStack.ThrowEvmStackUnderflowException(); - } - - return _bytes[Head * WordSize + WordSize - sizeof(byte)]; - } - - public void PushLeftPaddedBytes(ReadOnlySpan value, int paddingLength) - { - if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(value); - - if (value.Length != WordSize) - { - ClearWordAtHead(); - value.CopyTo(_bytes.Slice(Head * WordSize + WordSize - paddingLength, value.Length)); - } - else - { - Unsafe.WriteUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetReference(_bytes), Head * WordSize), Unsafe.As(ref MemoryMarshal.GetReference(value))); - } + ref byte bytes = ref PopBytesByRef(); + if (Unsafe.IsNullRef(ref bytes)) EvmStack.ThrowEvmStackUnderflowException(); - if (++Head >= MaxStackSize) - { - EvmStack.ThrowEvmStackOverflowException(); - } + return Unsafe.Add(ref bytes, WordSize - sizeof(byte)); } public bool Dup(in int depth) @@ -387,14 +347,7 @@ public bool Dup(in int depth) } public readonly bool EnsureDepth(int depth) - { - if (Head < depth) - { - return false; - } - - return true; - } + => Head >= depth; public readonly bool Swap(int depth) { @@ -424,12 +377,6 @@ private readonly void Trace(int depth) _tracer.ReportStackPush(_bytes.Slice(Head * WordSize - i * WordSize, WordSize)); } } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly void ClearWordAtHead() - { - Unsafe.WriteUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetReference(_bytes), Head * WordSize), Word.Zero); - } } public static class EvmStack @@ -440,7 +387,6 @@ public static class EvmStack public const int WordSize = 32; public const int AddressSize = 20; - [StackTraceHidden] [DoesNotReturn] internal static void ThrowEvmStackUnderflowException() diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index 3512f53574c..2d12fccfef9 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -648,7 +648,7 @@ private CallResult ExecuteCall(EvmState vmState, ReadOnlyM } vmState.InitStacks(); - EvmStack stack = new(vmState.DataStack.AsSpan(), vmState.DataStackHead, _txTracer); + EvmStack stack = new(vmState.DataStackHead, _txTracer, vmState.DataStack.AsSpan()); long gasAvailable = vmState.GasAvailable; if (previousCallResult is not null) diff --git a/src/Nethermind/Nethermind.State/StateReader.cs b/src/Nethermind/Nethermind.State/StateReader.cs index 9b4e73229e5..269bd1cc12c 100644 --- a/src/Nethermind/Nethermind.State/StateReader.cs +++ b/src/Nethermind/Nethermind.State/StateReader.cs @@ -29,7 +29,7 @@ public ReadOnlySpan GetStorage(Hash256 stateRoot, Address address, in UInt ValueHash256 storageRoot = account.StorageRoot; if (storageRoot == Keccak.EmptyTreeHash) { - return Bytes.ZeroByte.Span; + return Bytes.ZeroByteSpan; } Metrics.StorageReaderReads++;