From 3f6271a816c6f75f4f140732e673b2a0b9719849 Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Thu, 20 Jun 2024 15:42:54 -0400 Subject: [PATCH 1/5] Markdown-ify trace.py --- src/ethereum/trace.py | 178 +++++++++++++++++++++++++++++++++++------- 1 file changed, 150 insertions(+), 28 deletions(-) diff --git a/src/ethereum/trace.py b/src/ethereum/trace.py index e2dd49a100..b4873f7fb1 100644 --- a/src/ethereum/trace.py +++ b/src/ethereum/trace.py @@ -1,87 +1,150 @@ """ -.. _trace: +Defines the functions required for creating EVM traces during execution. -EVM Trace -^^^^^^^^^ +A _trace_ is a log of operations that took place during an event or period of +time. In the case of an EVM trace, the log is built from a series of +[`TraceEvent`]s emitted during the execution of a transaction. -.. contents:: Table of Contents - :backlinks: none - :local: +Note that this module _does not_ contain a trace implementation. Instead, it +defines only the events that can be collected into a trace by some other +package. See [`EvmTracer`]. -Introduction ------------- +See [EIP-3155] for more details on EVM traces. -Defines the functions required for creating evm traces during execution. +[`EvmTracer`]: ref:ethereum.trace.EvmTracer +[`TraceEvent`]: ref:ethereum.trace.TraceEvent +[EIP-3155]: https://eips.ethereum.org/EIPS/eip-3155 """ import enum from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional, Protocol, Union @dataclass class TransactionStart: - """Trace event that is triggered at the start of a transaction.""" - - pass + """ + Trace event that is triggered at the start of a transaction. + """ @dataclass class TransactionEnd: - """Trace event that is triggered at the end of a transaction.""" + """ + Trace event that is triggered at the end of a transaction. + """ gas_used: int + """ + Total gas consumed by this transaction. + """ + output: bytes + """ + Return value or revert reason of the outermost frame of execution. + """ + error: Optional[Exception] + """ + The exception, if any, that caused the transaction to fail. + + See [`ethereum.exceptions`] as well as fork-specific modules like + [`ethereum.frontier.vm.exceptions`][vm] for details. + + [`ethereum.exceptions`]: ref:ethereum.exceptions + [vm]: ref:ethereum.frontier.vm.exceptions + """ @dataclass class PrecompileStart: - """Trace event that is triggered before executing a precompile.""" + """ + Trace event that is triggered before executing a precompile. + """ address: bytes + """ + Precompile that is about to be executed. + """ @dataclass class PrecompileEnd: - """Trace event that is triggered after executing a precompile.""" - - pass + """ + Trace event that is triggered after executing a precompile. + """ @dataclass class OpStart: - """Trace event that is triggered before executing an opcode.""" + """ + Trace event that is triggered before executing an opcode. + """ op: enum.Enum + """ + Opcode that is about to be executed. + + Will be an instance of a fork-specific type like, for example, + [`ethereum.frontier.vm.instructions.Ops`][ops]. + + [ops]: ref:ethereum.frontier.vm.instructions.Ops + """ @dataclass class OpEnd: - """Trace event that is triggered after executing an opcode.""" - - pass + """ + Trace event that is triggered after executing an opcode. + """ @dataclass class OpException: - """Trace event that is triggered when an opcode raises an exception.""" + """ + Trace event that is triggered when an opcode raises an exception. + """ error: Exception + """ + Exception that was raised. + + See [`ethereum.exceptions`] as well as fork-specific modules like + [`ethereum.frontier.vm.exceptions`][vm] for examples of exceptions that + might be raised. + + [`ethereum.exceptions`]: ref:ethereum.exceptions + [vm]: ref:ethereum.frontier.vm.exceptions + """ @dataclass class EvmStop: - """Trace event that is triggered when the EVM stops.""" + """ + Trace event that is triggered when the EVM stops. + """ op: enum.Enum + """ + Last opcode executed. + + Will be an instance of a fork-specific type like, for example, + [`ethereum.frontier.vm.instructions.Ops`][ops]. + + [ops]: ref:ethereum.frontier.vm.instructions.Ops + """ @dataclass class GasAndRefund: - """Trace event that is triggered when gas is deducted.""" + """ + Trace event that is triggered when gas is deducted. + """ gas_cost: int + """ + Amount of gas charged or refunded. + """ TraceEvent = Union[ @@ -95,9 +158,14 @@ class GasAndRefund: EvmStop, GasAndRefund, ] +""" +All possible types of events that an [`EvmTracer`] is expected to handle. +[`EvmTracer`]: ref:ethereum.trace.EvmTracer +""" -def evm_trace( + +def discard_evm_trace( evm: object, event: TraceEvent, trace_memory: bool = False, @@ -105,6 +173,60 @@ def evm_trace( trace_return_data: bool = False, ) -> None: """ - Create a trace of the event. + An [`EvmTracer`] that discards all events. + + [`EvmTracer`]: ref:ethereum.trace.EvmTracer """ - pass + + +class EvmTracer(Protocol): + """ + [`Protocol`] that describes tracer functions. + + See [`ethereum.trace`] for details about tracing in general, and + [`__call__`] for more on how to implement a tracer. + + [`Protocol`]: https://docs.python.org/3/library/typing.html#typing.Protocol + [`ethereum.trace`]: ref:ethereum.trace + [`__call__`]: ref:ethereum.trace.EvmTracer.__call__ + """ + + def __call__( + self, + evm: object, + event: TraceEvent, + /, + trace_memory: bool = False, + trace_stack: bool = True, + trace_return_data: bool = False, + ) -> None: + """ + Call `self` as a function, recording a trace event. + + `evm` is the live state of the EVM, and will be a fork-specific type + like [`ethereum.frontier.vm.Evm`][evm]. + + `event`, a [`TraceEvent`], is the reason why the tracer was triggered. + + `trace_memory` requests a full memory dump in the resulting trace. + + `trace_stack` requests the full stack in the resulting trace. + + `trace_return_data` requests that return data be included in the + resulting trace. + + See [`discard_evm_trace`] for an example function implementing this + protocol. + + [`discard_evm_trace`]: ref:ethereum.trace.discard_evm_trace + [evm]: ref:ethereum.frontier.vm.Evm + [`TraceEvent`]: ref:ethereum.trace.TraceEvent + """ + + +evm_trace: EvmTracer = discard_evm_trace +""" +Active [`EvmTracer`] that is used for generating traces. + +[`EvmTracer`]: ref:ethereum.trace.EvmTracer +""" From dffe5c23eaf36732e76e09595d9b0d9811d5f74f Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Thu, 20 Jun 2024 16:58:26 -0400 Subject: [PATCH 2/5] Rewrite RLP decoding --- setup.cfg | 2 +- src/ethereum/arrow_glacier/fork.py | 4 +- src/ethereum/arrow_glacier/trie.py | 14 +- src/ethereum/base_types.py | 23 +- src/ethereum/berlin/fork.py | 4 +- src/ethereum/berlin/trie.py | 14 +- src/ethereum/byzantium/fork.py | 4 +- src/ethereum/byzantium/trie.py | 14 +- src/ethereum/cancun/trie.py | 16 +- src/ethereum/constantinople/fork.py | 4 +- src/ethereum/constantinople/trie.py | 14 +- src/ethereum/dao_fork/fork.py | 4 +- src/ethereum/dao_fork/trie.py | 14 +- src/ethereum/frontier/fork.py | 4 +- src/ethereum/frontier/trie.py | 14 +- src/ethereum/gray_glacier/fork.py | 4 +- src/ethereum/gray_glacier/trie.py | 14 +- src/ethereum/homestead/fork.py | 4 +- src/ethereum/homestead/trie.py | 14 +- src/ethereum/istanbul/fork.py | 4 +- src/ethereum/istanbul/trie.py | 14 +- src/ethereum/london/fork.py | 4 +- src/ethereum/london/trie.py | 14 +- src/ethereum/muir_glacier/fork.py | 4 +- src/ethereum/muir_glacier/trie.py | 14 +- src/ethereum/paris/trie.py | 14 +- src/ethereum/rlp.py | 295 +++++++++++------- src/ethereum/shanghai/trie.py | 16 +- src/ethereum/spurious_dragon/fork.py | 4 +- src/ethereum/spurious_dragon/trie.py | 16 +- src/ethereum/tangerine_whistle/fork.py | 4 +- src/ethereum/tangerine_whistle/trie.py | 16 +- .../evm_tools/b11r/b11r_types.py | 5 +- src/ethereum_spec_tools/sync.py | 8 +- tests/berlin/test_rlp.py | 2 +- tests/byzantium/test_rlp.py | 2 +- tests/cancun/test_rlp.py | 2 +- tests/constantinople/test_rlp.py | 2 +- tests/frontier/test_rlp.py | 2 +- tests/helpers/load_state_tests.py | 2 +- tests/homestead/test_rlp.py | 2 +- tests/istanbul/test_rlp.py | 2 +- tests/london/test_rlp.py | 2 +- tests/paris/test_rlp.py | 2 +- tests/shanghai/test_rlp.py | 2 +- tests/spurious_dragon/test_rlp.py | 2 +- tests/tangerine_whistle/test_rlp.py | 2 +- tests/test_rlp.py | 16 +- 48 files changed, 362 insertions(+), 297 deletions(-) diff --git a/setup.cfg b/setup.cfg index 92f81c9ade..275eb48c0f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -162,7 +162,7 @@ test = lint = types-setuptools>=68.1.0.1,<69 isort==5.13.2 - mypy==1.5.1 + mypy==1.10.0 black==23.12.0 flake8==6.1.0 flake8-bugbear==23.12.2 diff --git a/src/ethereum/arrow_glacier/fork.py b/src/ethereum/arrow_glacier/fork.py index 57172b3e0f..818e48a838 100644 --- a/src/ethereum/arrow_glacier/fork.py +++ b/src/ethereum/arrow_glacier/fork.py @@ -335,7 +335,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -350,7 +350,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.timestamp, header.extra_data, header.base_fee_per_gas, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/arrow_glacier/trie.py b/src/ethereum/arrow_glacier/trie.py index a6f8f3aba8..62fc9776cf 100644 --- a/src/ethereum/arrow_glacier/trie.py +++ b/src/ethereum/arrow_glacier/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/base_types.py b/src/ethereum/base_types.py index 3e44fe8478..645a5e7ca1 100644 --- a/src/ethereum/base_types.py +++ b/src/ethereum/base_types.py @@ -652,6 +652,18 @@ def to_be_bytes(self) -> "Bytes": # TODO: Implement neg, pos, abs ... + @classmethod + def from_be_bytes(cls: Type[T], buffer: "Bytes") -> T: + """ + Converts a sequence of bytes into a fixed sized unsigned integer + from its big endian representation. + """ + max_length = (7 + cls.MAX_VALUE.bit_length()) // 8 + if len(buffer) > max_length: + raise ValueError() + + return cls(int.from_bytes(buffer, "big")) + class U256(FixedUint): """ @@ -665,17 +677,6 @@ class U256(FixedUint): __slots__ = () - @classmethod - def from_be_bytes(cls: Type, buffer: "Bytes") -> "U256": - """ - Converts a sequence of bytes into a fixed sized unsigned integer - from its big endian representation. - """ - if len(buffer) > 32: - raise ValueError() - - return cls(int.from_bytes(buffer, "big")) - @classmethod def from_signed(cls: Type, value: int) -> "U256": """ diff --git a/src/ethereum/berlin/fork.py b/src/ethereum/berlin/fork.py index aa5e385886..720f7e4c8e 100644 --- a/src/ethereum/berlin/fork.py +++ b/src/ethereum/berlin/fork.py @@ -259,7 +259,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -273,7 +273,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/berlin/trie.py b/src/ethereum/berlin/trie.py index f6972e828f..1497569384 100644 --- a/src/ethereum/berlin/trie.py +++ b/src/ethereum/berlin/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/byzantium/fork.py b/src/ethereum/byzantium/fork.py index 48cfb370d1..610ec867a0 100644 --- a/src/ethereum/byzantium/fork.py +++ b/src/ethereum/byzantium/fork.py @@ -253,7 +253,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -267,7 +267,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/byzantium/trie.py b/src/ethereum/byzantium/trie.py index a49094a51a..c523a96a65 100644 --- a/src/ethereum/byzantium/trie.py +++ b/src/ethereum/byzantium/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/cancun/trie.py b/src/ethereum/cancun/trie.py index 99d109ec96..a77475672b 100644 --- a/src/ethereum/cancun/trie.py +++ b/src/ethereum/cancun/trie.py @@ -81,7 +81,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -90,7 +90,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -98,14 +98,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,10 +121,10 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: Returns ------- - encoded : `rlp.RLP` + encoded : `rlp.Extended` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -159,7 +159,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, Withdrawal, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/constantinople/fork.py b/src/ethereum/constantinople/fork.py index 7dd466e467..dab9200777 100644 --- a/src/ethereum/constantinople/fork.py +++ b/src/ethereum/constantinople/fork.py @@ -253,7 +253,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -267,7 +267,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/constantinople/trie.py b/src/ethereum/constantinople/trie.py index 4cfe556ac1..c27e15edad 100644 --- a/src/ethereum/constantinople/trie.py +++ b/src/ethereum/constantinople/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/dao_fork/fork.py b/src/ethereum/dao_fork/fork.py index fca9533d3a..a475de7d4e 100644 --- a/src/ethereum/dao_fork/fork.py +++ b/src/ethereum/dao_fork/fork.py @@ -265,7 +265,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -279,7 +279,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/dao_fork/trie.py b/src/ethereum/dao_fork/trie.py index 549c026c9b..70adaeab56 100644 --- a/src/ethereum/dao_fork/trie.py +++ b/src/ethereum/dao_fork/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/frontier/fork.py b/src/ethereum/frontier/fork.py index 28345a3e9c..742bb64e01 100644 --- a/src/ethereum/frontier/fork.py +++ b/src/ethereum/frontier/fork.py @@ -245,7 +245,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -259,7 +259,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/frontier/trie.py b/src/ethereum/frontier/trie.py index 6103ecbbce..7b966394b1 100644 --- a/src/ethereum/frontier/trie.py +++ b/src/ethereum/frontier/trie.py @@ -77,7 +77,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -86,7 +86,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -94,14 +94,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -120,7 +120,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -155,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/gray_glacier/fork.py b/src/ethereum/gray_glacier/fork.py index 540624611d..4a80e96bdb 100644 --- a/src/ethereum/gray_glacier/fork.py +++ b/src/ethereum/gray_glacier/fork.py @@ -335,7 +335,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -350,7 +350,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.timestamp, header.extra_data, header.base_fee_per_gas, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/gray_glacier/trie.py b/src/ethereum/gray_glacier/trie.py index 050084684e..1e11c1ffb7 100644 --- a/src/ethereum/gray_glacier/trie.py +++ b/src/ethereum/gray_glacier/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/homestead/fork.py b/src/ethereum/homestead/fork.py index 27bf18e9ec..46173ff765 100644 --- a/src/ethereum/homestead/fork.py +++ b/src/ethereum/homestead/fork.py @@ -247,7 +247,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -261,7 +261,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/homestead/trie.py b/src/ethereum/homestead/trie.py index 63dbadaf86..903d8af408 100644 --- a/src/ethereum/homestead/trie.py +++ b/src/ethereum/homestead/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/istanbul/fork.py b/src/ethereum/istanbul/fork.py index 719dda888b..945dffa498 100644 --- a/src/ethereum/istanbul/fork.py +++ b/src/ethereum/istanbul/fork.py @@ -253,7 +253,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -267,7 +267,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/istanbul/trie.py b/src/ethereum/istanbul/trie.py index 065830b751..4fd91e9ed9 100644 --- a/src/ethereum/istanbul/trie.py +++ b/src/ethereum/istanbul/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/london/fork.py b/src/ethereum/london/fork.py index c60d7aa423..9aae34f0f0 100644 --- a/src/ethereum/london/fork.py +++ b/src/ethereum/london/fork.py @@ -343,7 +343,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -358,7 +358,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.timestamp, header.extra_data, header.base_fee_per_gas, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/london/trie.py b/src/ethereum/london/trie.py index 649426b8f7..b9956dae35 100644 --- a/src/ethereum/london/trie.py +++ b/src/ethereum/london/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/muir_glacier/fork.py b/src/ethereum/muir_glacier/fork.py index 1fa34f82cd..751c932139 100644 --- a/src/ethereum/muir_glacier/fork.py +++ b/src/ethereum/muir_glacier/fork.py @@ -253,7 +253,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -267,7 +267,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/muir_glacier/trie.py b/src/ethereum/muir_glacier/trie.py index c7a50e348b..e979370730 100644 --- a/src/ethereum/muir_glacier/trie.py +++ b/src/ethereum/muir_glacier/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/paris/trie.py b/src/ethereum/paris/trie.py index ea24c4be84..36ce79f33e 100644 --- a/src/ethereum/paris/trie.py +++ b/src/ethereum/paris/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,7 +121,7 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: encoded : `rlp.RLP` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/rlp.py b/src/ethereum/rlp.py index 84be5cfaaa..7fcc85335c 100644 --- a/src/ethereum/rlp.py +++ b/src/ethereum/rlp.py @@ -15,14 +15,45 @@ """ from dataclasses import astuple, fields, is_dataclass -from typing import Any, List, Sequence, Tuple, Type, TypeVar, Union, cast +from typing import ( + Any, + ClassVar, + Dict, + Protocol, + Sequence, + Tuple, + Type, + TypeAlias, + TypeVar, + Union, + cast, + get_args, + get_origin, + get_type_hints, + overload, +) from ethereum.crypto.hash import Hash32, keccak256 from ethereum.exceptions import RLPDecodingError, RLPEncodingError -from .base_types import Bytes, Bytes0, Bytes20, FixedBytes, FixedUint, Uint +from .base_types import Bytes, FixedBytes, FixedUint, Uint -RLP = Any + +class RLP(Protocol): + """ + [`Protocol`] that describes the requirements to be RLP-encodable. + + [`Protocol`]: https://docs.python.org/3/library/typing.html#typing.Protocol + """ + + __dataclass_fields__: ClassVar[Dict] + + +Simple: TypeAlias = Union[Sequence["Simple"], bytes] + +Extended: TypeAlias = Union[ + Sequence["Extended"], bytearray, bytes, Uint, FixedUint, str, bool, RLP +] # @@ -30,7 +61,7 @@ # -def encode(raw_data: RLP) -> Bytes: +def encode(raw_data: Extended) -> Bytes: """ Encodes `raw_data` into a sequence of bytes using RLP. @@ -45,19 +76,20 @@ def encode(raw_data: RLP) -> Bytes: encoded : `ethereum.base_types.Bytes` The RLP encoded bytes representing `raw_data`. """ - if isinstance(raw_data, (bytearray, bytes)): - return encode_bytes(raw_data) + if isinstance(raw_data, Sequence): + if isinstance(raw_data, (bytearray, bytes)): + return encode_bytes(raw_data) + elif isinstance(raw_data, str): + return encode_bytes(raw_data.encode()) + else: + return encode_sequence(raw_data) elif isinstance(raw_data, (Uint, FixedUint)): return encode(raw_data.to_be_bytes()) - elif isinstance(raw_data, str): - return encode_bytes(raw_data.encode()) elif isinstance(raw_data, bool): if raw_data: return encode_bytes(b"\x01") else: return encode_bytes(b"") - elif isinstance(raw_data, Sequence): - return encode_sequence(raw_data) elif is_dataclass(raw_data): return encode(astuple(raw_data)) else: @@ -96,7 +128,7 @@ def encode_bytes(raw_bytes: Bytes) -> Bytes: ) -def encode_sequence(raw_sequence: Sequence[RLP]) -> Bytes: +def encode_sequence(raw_sequence: Sequence[Extended]) -> Bytes: """ Encodes a list of RLP encodable objects (`raw_sequence`) using RLP. @@ -124,7 +156,7 @@ def encode_sequence(raw_sequence: Sequence[RLP]) -> Bytes: ) -def get_joined_encodings(raw_sequence: Sequence[RLP]) -> Bytes: +def get_joined_encodings(raw_sequence: Sequence[Extended]) -> Bytes: """ Obtain concatenation of rlp encoding for each item in the sequence raw_sequence. @@ -148,7 +180,7 @@ def get_joined_encodings(raw_sequence: Sequence[RLP]) -> Bytes: # -def decode(encoded_data: Bytes) -> RLP: +def decode(encoded_data: Bytes) -> Simple: """ Decodes an integer, byte sequence, or list of RLP encodable objects from the byte sequence `encoded_data`, using RLP. @@ -174,129 +206,158 @@ def decode(encoded_data: Bytes) -> RLP: return decode_to_sequence(encoded_data) -T = TypeVar("T") +U = TypeVar("U", bound=Extended) -def decode_to(cls: Type[T], encoded_data: Bytes) -> T: +def decode_to(cls: Type[U], encoded_data: Bytes) -> U: """ Decode the bytes in `encoded_data` to an object of type `cls`. `cls` can be a `Bytes` subclass, a dataclass, `Uint`, `U256` or `Tuple[cls]`. Parameters ---------- - cls: `Type[T]` + cls: `Type[U]` The type to decode to. encoded_data : A sequence of bytes, in RLP form. Returns ------- - decoded_data : `T` + decoded_data : `U` Object decoded from `encoded_data`. """ - return _decode_to(cls, decode(encoded_data)) + decoded = decode(encoded_data) + return _deserialize_to(cls, decoded) -def _decode_to(cls: Type[T], raw_rlp: RLP) -> T: - """ - Decode the rlp structure in `encoded_data` to an object of type `cls`. - `cls` can be a `Bytes` subclass, a dataclass, `Uint`, `U256`, - `Tuple[cls, ...]`, `Tuple[cls1, cls2]` or `Union[Bytes, cls]`. +@overload +def _deserialize_to(class_: Type[U], value: Simple) -> U: + pass - Parameters - ---------- - cls: `Type[T]` - The type to decode to. - raw_rlp : - A decoded rlp structure. - Returns - ------- - decoded_data : `T` - Object decoded from `encoded_data`. - """ - if isinstance(cls, type(Tuple[Uint, ...])) and cls._name == "Tuple": # type: ignore # noqa: E501 - if not isinstance(raw_rlp, list): - raise RLPDecodingError - if cls.__args__[1] == ...: # type: ignore - args = [] - for raw_item in raw_rlp: - args.append(_decode_to(cls.__args__[0], raw_item)) # type: ignore # noqa: E501 - return tuple(args) # type: ignore - else: - args = [] - if len(raw_rlp) != len(cls.__args__): # type: ignore - raise RLPDecodingError - for t, raw_item in zip(cls.__args__, raw_rlp): # type: ignore - args.append(_decode_to(t, raw_item)) - return tuple(args) # type: ignore - elif cls == Union[Bytes0, Bytes20]: - if not isinstance(raw_rlp, Bytes): - raise RLPDecodingError - if len(raw_rlp) == 0: - return Bytes0() # type: ignore - elif len(raw_rlp) == 20: - return Bytes20(raw_rlp) # type: ignore - else: - raise RLPDecodingError( - "Bytes has length {}, expected 0 or 20".format(len(raw_rlp)) - ) - elif isinstance(cls, type(List[Bytes])) and cls._name == "List": # type: ignore # noqa: E501 - if not isinstance(raw_rlp, list): - raise RLPDecodingError - items = [] - for raw_item in raw_rlp: - items.append(_decode_to(cls.__args__[0], raw_item)) # type: ignore - return items # type: ignore - elif isinstance(cls, type(Union[Bytes, List[Bytes]])) and cls.__origin__ == Union: # type: ignore # noqa: E501 - if len(cls.__args__) != 2 or Bytes not in cls.__args__: # type: ignore - raise RLPDecodingError( - "RLP Decoding to type {} is not supported".format(cls) - ) - if isinstance(raw_rlp, Bytes): - return raw_rlp # type: ignore - elif cls.__args__[0] == Bytes: # type: ignore - return _decode_to(cls.__args__[1], raw_rlp) # type: ignore - else: - return _decode_to(cls.__args__[0], raw_rlp) # type: ignore - elif issubclass(cls, bool): - if raw_rlp == b"\x01": - return cls(True) # type: ignore - elif raw_rlp == b"": - return cls(False) # type: ignore - else: - raise TypeError("Cannot decode {} as {}".format(raw_rlp, cls)) - elif issubclass(cls, FixedBytes): - if not isinstance(raw_rlp, Bytes): - raise RLPDecodingError - if len(raw_rlp) != cls.LENGTH: - raise RLPDecodingError - return cls(raw_rlp) # type: ignore - elif issubclass(cls, Bytes): - if not isinstance(raw_rlp, Bytes): - raise RLPDecodingError - return raw_rlp # type: ignore - elif issubclass(cls, (Uint, FixedUint)): - if not isinstance(raw_rlp, Bytes): - raise RLPDecodingError +@overload +def _deserialize_to(class_: object, value: Simple) -> Extended: + pass + + +def _deserialize_to(class_: object, value: Simple) -> Extended: + if not isinstance(class_, type): + return _deserialize_to_annotation(class_, value) + elif is_dataclass(class_): + return _deserialize_to_dataclass(class_, value) + elif issubclass(class_, (Uint, FixedUint)): + return _deserialize_to_uint(class_, value) + elif issubclass(class_, (Bytes, FixedBytes)): + return _deserialize_to_bytes(class_, value) + elif class_ is bool: + return _deserialize_to_bool(value) + else: + raise NotImplementedError(class_) + + +def _deserialize_to_dataclass(cls: Type[U], decoded: Simple) -> U: + assert is_dataclass(cls) + hints = get_type_hints(cls) + target_fields = fields(cls) + + if isinstance(decoded, bytes): + raise RLPDecodingError + + if len(target_fields) != len(decoded): + raise RLPDecodingError + + values: Dict[str, Any] = {} + + for value, target_field in zip(decoded, target_fields): + resolved_type = hints[target_field.name] + values[target_field.name] = _deserialize_to(resolved_type, value) + + result = cls(**values) + assert isinstance(result, cls) + return cast(U, result) + + +def _deserialize_to_bool(value: Simple) -> bool: + if value == b"": + return False + elif value == b"\x01": + return True + else: + raise RLPDecodingError + + +def _deserialize_to_bytes( + class_: Union[Type[Bytes], Type[FixedBytes]], value: Simple +) -> Union[Bytes, FixedBytes]: + if not isinstance(value, bytes): + raise RLPDecodingError + try: + return class_(value) + except ValueError as e: + raise RLPDecodingError from e + + +def _deserialize_to_uint( + class_: Union[Type[Uint], Type[FixedUint]], decoded: Simple +) -> Union[Uint, FixedUint]: + if not isinstance(decoded, bytes): + raise RLPDecodingError + try: + return class_.from_be_bytes(decoded) + except ValueError as e: + raise RLPDecodingError from e + + +def _deserialize_to_annotation(annotation: object, value: Simple) -> Extended: + origin = get_origin(annotation) + if origin is Union: + return _deserialize_to_union(annotation, value) + elif origin in (Tuple, tuple): + return _deserialize_to_tuple(annotation, value) + elif origin is None: + raise Exception(annotation) + else: + raise NotImplementedError(f"RLP non-type {origin!r}") + + +def _deserialize_to_union(annotation: object, value: Simple) -> Extended: + arguments = get_args(annotation) + successes = [] + failures = [] + for argument in arguments: try: - return cls.from_be_bytes(raw_rlp) # type: ignore - except ValueError: - raise RLPDecodingError - elif is_dataclass(cls): - if not isinstance(raw_rlp, list): - raise RLPDecodingError - assert isinstance(raw_rlp, list) - args = [] - if len(fields(cls)) != len(raw_rlp): - raise RLPDecodingError - for field, rlp_item in zip(fields(cls), raw_rlp): - args.append(_decode_to(field.type, rlp_item)) - return cast(T, cls(*args)) + success = _deserialize_to(argument, value) + except Exception as e: + failures.append(e) + continue + + successes.append(success) + + if len(successes) == 1: + return successes[0] + elif not successes: + raise RLPDecodingError(f"no matching union variant\n{failures!r}") else: - raise RLPDecodingError( - "RLP Decoding to type {} is not supported".format(cls) - ) + raise RLPDecodingError("multiple matching union variants") + + +def _deserialize_to_tuple( + annotation: object, values: Simple +) -> Sequence[Extended]: + if isinstance(values, bytes): + raise RLPDecodingError + arguments = list(get_args(annotation)) + + if arguments[-1] is Ellipsis: + arguments.pop() + fill_count = len(values) - len(arguments) + arguments = list(arguments) + [arguments[-1]] * fill_count + + decoded = [] + for argument, value in zip(arguments, values): + decoded.append(_deserialize_to(argument, value)) + + return tuple(decoded) def decode_to_bytes(encoded_bytes: Bytes) -> Bytes: @@ -343,7 +404,7 @@ def decode_to_bytes(encoded_bytes: Bytes) -> Bytes: return encoded_bytes[decoded_data_start_idx:decoded_data_end_idx] -def decode_to_sequence(encoded_sequence: Bytes) -> List[RLP]: +def decode_to_sequence(encoded_sequence: Bytes) -> Sequence[Simple]: """ Decodes a rlp encoded byte stream assuming that the decoded data should be of type `Sequence` of objects. @@ -386,7 +447,7 @@ def decode_to_sequence(encoded_sequence: Bytes) -> List[RLP]: return decode_joined_encodings(joined_encodings) -def decode_joined_encodings(joined_encodings: Bytes) -> List[RLP]: +def decode_joined_encodings(joined_encodings: Bytes) -> Sequence[Simple]: """ Decodes `joined_encodings`, which is a concatenation of RLP encoded objects. @@ -489,7 +550,7 @@ def decode_item_length(encoded_data: Bytes) -> int: return 1 + length_length + decoded_data_length -def rlp_hash(data: RLP) -> Hash32: +def rlp_hash(data: Extended) -> Hash32: """ Obtain the keccak-256 hash of the rlp encoding of the passed in data. diff --git a/src/ethereum/shanghai/trie.py b/src/ethereum/shanghai/trie.py index 35b87cf2c6..c1e6f1d9f9 100644 --- a/src/ethereum/shanghai/trie.py +++ b/src/ethereum/shanghai/trie.py @@ -81,7 +81,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -90,7 +90,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -98,14 +98,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -121,10 +121,10 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: Returns ------- - encoded : `rlp.RLP` + encoded : `rlp.Extended` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -159,7 +159,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, Withdrawal, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/spurious_dragon/fork.py b/src/ethereum/spurious_dragon/fork.py index c04ad36155..8dd4f63755 100644 --- a/src/ethereum/spurious_dragon/fork.py +++ b/src/ethereum/spurious_dragon/fork.py @@ -249,7 +249,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -263,7 +263,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/spurious_dragon/trie.py b/src/ethereum/spurious_dragon/trie.py index 606211e8dd..44e3bf7550 100644 --- a/src/ethereum/spurious_dragon/trie.py +++ b/src/ethereum/spurious_dragon/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -118,10 +118,10 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: Returns ------- - encoded : `rlp.RLP` + encoded : `rlp.Extended` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/tangerine_whistle/fork.py b/src/ethereum/tangerine_whistle/fork.py index 27bf18e9ec..46173ff765 100644 --- a/src/ethereum/tangerine_whistle/fork.py +++ b/src/ethereum/tangerine_whistle/fork.py @@ -247,7 +247,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: hash : `Hash32` The PoW valid rlp hash of the passed in header. """ - header_data_without_pow_artefacts = [ + header_data_without_pow_artefacts = ( header.parent_hash, header.ommers_hash, header.coinbase, @@ -261,7 +261,7 @@ def generate_header_hash_for_pow(header: Header) -> Hash32: header.gas_used, header.timestamp, header.extra_data, - ] + ) return rlp.rlp_hash(header_data_without_pow_artefacts) diff --git a/src/ethereum/tangerine_whistle/trie.py b/src/ethereum/tangerine_whistle/trie.py index 338540e6f9..f175385b34 100644 --- a/src/ethereum/tangerine_whistle/trie.py +++ b/src/ethereum/tangerine_whistle/trie.py @@ -78,7 +78,7 @@ class LeafNode: """Leaf node in the Merkle Trie""" rest_of_key: Bytes - value: rlp.RLP + value: rlp.Extended @slotted_freezable @@ -87,7 +87,7 @@ class ExtensionNode: """Extension node in the Merkle Trie""" key_segment: Bytes - subnode: rlp.RLP + subnode: rlp.Extended @slotted_freezable @@ -95,14 +95,14 @@ class ExtensionNode: class BranchNode: """Branch node in the Merkle Trie""" - subnodes: List[rlp.RLP] - value: rlp.RLP + subnodes: List[rlp.Extended] + value: rlp.Extended InternalNode = Union[LeafNode, ExtensionNode, BranchNode] -def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: +def encode_internal_node(node: Optional[InternalNode]) -> rlp.Extended: """ Encodes a Merkle Trie node into its RLP form. The RLP will then be serialized into a `Bytes` and hashed unless it is less that 32 bytes @@ -118,10 +118,10 @@ def encode_internal_node(node: Optional[InternalNode]) -> rlp.RLP: Returns ------- - encoded : `rlp.RLP` + encoded : `rlp.Extended` The node encoded as RLP. """ - unencoded: rlp.RLP + unencoded: rlp.Extended if node is None: unencoded = b"" elif isinstance(node, LeafNode): @@ -156,7 +156,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.RLP, node)) + return rlp.encode(cast(rlp.Extended, node)) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum_spec_tools/evm_tools/b11r/b11r_types.py b/src/ethereum_spec_tools/evm_tools/b11r/b11r_types.py index 19eb2d9b62..2726dfa4d2 100644 --- a/src/ethereum_spec_tools/evm_tools/b11r/b11r_types.py +++ b/src/ethereum_spec_tools/evm_tools/b11r/b11r_types.py @@ -31,8 +31,8 @@ class Body: A class representing a block body. """ - transactions: rlp.RLP - ommers: rlp.RLP + transactions: rlp.Extended + ommers: rlp.Extended withdrawals: Optional[List[Tuple[U64, U64, Bytes20, Uint]]] def __init__(self, options: Any, stdin: Any = None): @@ -65,6 +65,7 @@ def __init__(self, options: Any, stdin: Any = None): self.ommers = [] for ommer in ommers_data: decoded_ommer = rlp.decode(hex_to_bytes(ommer)) + assert not isinstance(decoded_ommer, bytes) self.ommers.append(decoded_ommer[0]) # ommer[0] is the header # Parse withdrawals diff --git a/src/ethereum_spec_tools/sync.py b/src/ethereum_spec_tools/sync.py index d302f041d6..9e41dcecbb 100644 --- a/src/ethereum_spec_tools/sync.py +++ b/src/ethereum_spec_tools/sync.py @@ -260,9 +260,11 @@ def fetch_blocks_debug( blocks.append(block_rlp) else: # Unfortunately we have to decode the RLP twice. - timestamp = rlp.decode_to( - U64, rlp.decode(block_rlp)[0][11] - ) + decoded_block = rlp.decode(block_rlp) + assert not isinstance(decoded_block, bytes) + assert not isinstance(decoded_block[0], bytes) + assert isinstance(decoded_block[0][11], bytes) + timestamp = rlp.decode_to(U64, decoded_block[0][11]) self.advance_block(timestamp) blocks.append( rlp.decode_to(self.module("blocks").Block, block_rlp) diff --git a/tests/berlin/test_rlp.py b/tests/berlin/test_rlp.py index fb6efabe09..b1d7e4e4b7 100644 --- a/tests/berlin/test_rlp.py +++ b/tests/berlin/test_rlp.py @@ -123,7 +123,7 @@ receipt, ], ) -def test_berlin_rlp(rlp_object: rlp.RLP) -> None: +def test_berlin_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/byzantium/test_rlp.py b/tests/byzantium/test_rlp.py index c84150b00a..bbce187093 100644 --- a/tests/byzantium/test_rlp.py +++ b/tests/byzantium/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_byzantium_rlp(rlp_object: rlp.RLP) -> None: +def test_byzantium_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/cancun/test_rlp.py b/tests/cancun/test_rlp.py index 5631234ac3..747eeb7e48 100644 --- a/tests/cancun/test_rlp.py +++ b/tests/cancun/test_rlp.py @@ -151,7 +151,7 @@ withdrawal, ], ) -def test_cancun_rlp(rlp_object: rlp.RLP) -> None: +def test_cancun_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/constantinople/test_rlp.py b/tests/constantinople/test_rlp.py index 58acd7ffa7..198d8c1dad 100644 --- a/tests/constantinople/test_rlp.py +++ b/tests/constantinople/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_constantinople_rlp(rlp_object: rlp.RLP) -> None: +def test_constantinople_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/frontier/test_rlp.py b/tests/frontier/test_rlp.py index 5ebc51cbb1..432d3c03ad 100644 --- a/tests/frontier/test_rlp.py +++ b/tests/frontier/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_frontier_rlp(rlp_object: rlp.RLP) -> None: +def test_frontier_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/helpers/load_state_tests.py b/tests/helpers/load_state_tests.py index 63c87e2868..d2e0c1619e 100644 --- a/tests/helpers/load_state_tests.py +++ b/tests/helpers/load_state_tests.py @@ -99,7 +99,7 @@ def add_block_to_chain( ) = load.json_to_block(json_block) assert rlp.rlp_hash(block.header) == block_header_hash - assert rlp.encode(cast(rlp.RLP, block)) == block_rlp + assert rlp.encode(cast(rlp.Extended, block)) == block_rlp if not mock_pow: load.fork.state_transition(chain, block) diff --git a/tests/homestead/test_rlp.py b/tests/homestead/test_rlp.py index c1221a9fb9..e73ccec886 100644 --- a/tests/homestead/test_rlp.py +++ b/tests/homestead/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_homestead_rlp(rlp_object: rlp.RLP) -> None: +def test_homestead_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/istanbul/test_rlp.py b/tests/istanbul/test_rlp.py index 28fd68d122..992c7762f0 100644 --- a/tests/istanbul/test_rlp.py +++ b/tests/istanbul/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_istanbul_rlp(rlp_object: rlp.RLP) -> None: +def test_istanbul_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/london/test_rlp.py b/tests/london/test_rlp.py index afee4d591a..56f6a4f81c 100644 --- a/tests/london/test_rlp.py +++ b/tests/london/test_rlp.py @@ -143,7 +143,7 @@ receipt, ], ) -def test_london_rlp(rlp_object: rlp.RLP) -> None: +def test_london_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/paris/test_rlp.py b/tests/paris/test_rlp.py index ab5bc8b736..695f385bd4 100644 --- a/tests/paris/test_rlp.py +++ b/tests/paris/test_rlp.py @@ -143,7 +143,7 @@ receipt, ], ) -def test_paris_rlp(rlp_object: rlp.RLP) -> None: +def test_paris_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/shanghai/test_rlp.py b/tests/shanghai/test_rlp.py index b26dd5da07..2b073783c5 100644 --- a/tests/shanghai/test_rlp.py +++ b/tests/shanghai/test_rlp.py @@ -148,7 +148,7 @@ withdrawal, ], ) -def test_shanghai_rlp(rlp_object: rlp.RLP) -> None: +def test_shanghai_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/spurious_dragon/test_rlp.py b/tests/spurious_dragon/test_rlp.py index 1c3e3d4317..d57e640292 100644 --- a/tests/spurious_dragon/test_rlp.py +++ b/tests/spurious_dragon/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_spurious_dragon_rlp(rlp_object: rlp.RLP) -> None: +def test_spurious_dragon_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/tangerine_whistle/test_rlp.py b/tests/tangerine_whistle/test_rlp.py index a6f9603b0a..1d0414f7d9 100644 --- a/tests/tangerine_whistle/test_rlp.py +++ b/tests/tangerine_whistle/test_rlp.py @@ -104,6 +104,6 @@ "rlp_object", [transaction1, transaction2, header, block, log1, log2, receipt], ) -def test_tangerine_whistle_rlp(rlp_object: rlp.RLP) -> None: +def test_tangerine_whistle_rlp(rlp_object: rlp.Extended) -> None: encoded = rlp.encode(rlp_object) assert rlp.decode_to(type(rlp_object), encoded) == rlp_object diff --git a/tests/test_rlp.py b/tests/test_rlp.py index 8b9dadefaa..c2f58032ef 100644 --- a/tests/test_rlp.py +++ b/tests/test_rlp.py @@ -7,7 +7,7 @@ from ethereum import rlp from ethereum.exceptions import RLPDecodingError, RLPEncodingError from ethereum.frontier.fork_types import U256, Bytes, Uint -from ethereum.rlp import RLP +from ethereum.rlp import Extended from ethereum.utils.hexadecimal import hex_to_bytes from tests.helpers import TEST_FIXTURES @@ -140,7 +140,7 @@ def test_rlp_encode_20_elem_byte_uint_combo() -> None: def test_rlp_encode_nested_sequence() -> None: - nested_sequence: Sequence["RLP"] = [ + nested_sequence: Sequence["Extended"] = [ b"hello", Uint(255), [b"how", [b"are", b"you", [b"doing"]]], @@ -171,7 +171,7 @@ def test_rlp_encode_successfully() -> None: ), ] for raw_data, expected_encoding in test_cases: - assert rlp.encode(cast(RLP, raw_data)) == expected_encoding + assert rlp.encode(cast(Extended, raw_data)) == expected_encoding def test_rlp_encode_fails() -> None: @@ -181,7 +181,7 @@ def test_rlp_encode_fails() -> None: ] for raw_data in test_cases: with pytest.raises(RLPEncodingError): - rlp.encode(cast(RLP, raw_data)) + rlp.encode(cast(Extended, raw_data)) # @@ -338,7 +338,7 @@ def test_roundtrip_encoding_and_decoding() -> None: [[b"hello", b"world"], [b"how", b"are"], [b"you", b"doing"]], ] for raw_data in test_cases: - assert rlp.decode(rlp.encode(cast(RLP, raw_data))) == raw_data + assert rlp.decode(rlp.encode(cast(Extended, raw_data))) == raw_data # @@ -348,7 +348,7 @@ def test_roundtrip_encoding_and_decoding() -> None: def convert_to_rlp_native( obj: Union[str, int, Sequence[Union[str, int]]] -) -> RLP: +) -> Extended: if isinstance(obj, str): return bytes(obj, "utf-8") elif isinstance(obj, int): @@ -360,7 +360,7 @@ def convert_to_rlp_native( def ethtest_fixtures_as_pytest_fixtures( *test_files: str, -) -> List[Tuple[RLP, Bytes]]: +) -> List[Tuple[Extended, Bytes]]: base_path = f"{ETHEREUM_TESTS_PATH}/RLPTests/" test_data = dict() @@ -390,7 +390,7 @@ def ethtest_fixtures_as_pytest_fixtures( ethtest_fixtures_as_pytest_fixtures("rlptest.json"), ) def test_ethtest_fixtures_for_rlp_encoding( - raw_data: RLP, expected_encoded_data: Bytes + raw_data: Extended, expected_encoded_data: Bytes ) -> None: assert rlp.encode(raw_data) == expected_encoded_data From f5f8060c040b3dae7a82802013a77c45cdd8580f Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Fri, 21 Jun 2024 12:51:52 -0400 Subject: [PATCH 3/5] Re-enable assertion (ethereum/execution-spec-tests#64) --- tests/helpers/load_state_tests.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/helpers/load_state_tests.py b/tests/helpers/load_state_tests.py index d2e0c1619e..8c39b7162e 100644 --- a/tests/helpers/load_state_tests.py +++ b/tests/helpers/load_state_tests.py @@ -48,13 +48,8 @@ def run_blockchain_st_test(test_case: Dict, load: Load) -> None: genesis_header_hash = hex_to_bytes(json_data["genesisBlockHeader"]["hash"]) assert rlp.rlp_hash(genesis_header) == genesis_header_hash - # FIXME: Re-enable this assertion once the genesis block RLP is - # correctly encoded for Shanghai. - # See https://github.com/ethereum/execution-spec-tests/issues/64 - # assert ( - # rlp.encode(cast(rlp.RLP, genesis_block)) - # == test_data["genesis_block_rlp"] - # ) + genesis_rlp = hex_to_bytes(json_data["genesisRLP"]) + assert rlp.encode(genesis_block) == genesis_rlp chain = load.fork.BlockChain( blocks=[genesis_block], From 789312eccc5604d5e7c7b587538a6b824f9f1fdf Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Fri, 21 Jun 2024 12:52:08 -0400 Subject: [PATCH 4/5] Remove unnecessary casts --- src/ethereum/arrow_glacier/trie.py | 3 +-- src/ethereum/berlin/trie.py | 3 +-- src/ethereum/byzantium/trie.py | 3 +-- src/ethereum/cancun/trie.py | 3 +-- src/ethereum/constantinople/trie.py | 3 +-- src/ethereum/dao_fork/trie.py | 3 +-- src/ethereum/frontier/trie.py | 3 +-- src/ethereum/gray_glacier/trie.py | 3 +-- src/ethereum/homestead/trie.py | 3 +-- src/ethereum/istanbul/trie.py | 3 +-- src/ethereum/london/trie.py | 3 +-- src/ethereum/muir_glacier/trie.py | 3 +-- src/ethereum/paris/trie.py | 3 +-- src/ethereum/shanghai/trie.py | 3 +-- src/ethereum/spurious_dragon/trie.py | 3 +-- src/ethereum/tangerine_whistle/trie.py | 3 +-- tests/helpers/load_state_tests.py | 4 ++-- tests/test_rlp.py | 8 ++++---- 18 files changed, 22 insertions(+), 38 deletions(-) diff --git a/src/ethereum/arrow_glacier/trie.py b/src/ethereum/arrow_glacier/trie.py index 62fc9776cf..ab2eda6f0f 100644 --- a/src/ethereum/arrow_glacier/trie.py +++ b/src/ethereum/arrow_glacier/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/berlin/trie.py b/src/ethereum/berlin/trie.py index 1497569384..e268dc2fd5 100644 --- a/src/ethereum/berlin/trie.py +++ b/src/ethereum/berlin/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/byzantium/trie.py b/src/ethereum/byzantium/trie.py index c523a96a65..5b5be2092f 100644 --- a/src/ethereum/byzantium/trie.py +++ b/src/ethereum/byzantium/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/cancun/trie.py b/src/ethereum/cancun/trie.py index a77475672b..66b40e19c2 100644 --- a/src/ethereum/cancun/trie.py +++ b/src/ethereum/cancun/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -159,7 +158,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, Withdrawal, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/constantinople/trie.py b/src/ethereum/constantinople/trie.py index c27e15edad..7e3537197b 100644 --- a/src/ethereum/constantinople/trie.py +++ b/src/ethereum/constantinople/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.byzantium import trie as previous_trie @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/dao_fork/trie.py b/src/ethereum/dao_fork/trie.py index 70adaeab56..4bb4e7bd14 100644 --- a/src/ethereum/dao_fork/trie.py +++ b/src/ethereum/dao_fork/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/frontier/trie.py b/src/ethereum/frontier/trie.py index 7b966394b1..2e250c64f9 100644 --- a/src/ethereum/frontier/trie.py +++ b/src/ethereum/frontier/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -155,7 +154,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/gray_glacier/trie.py b/src/ethereum/gray_glacier/trie.py index 1e11c1ffb7..afebc7a302 100644 --- a/src/ethereum/gray_glacier/trie.py +++ b/src/ethereum/gray_glacier/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.arrow_glacier import trie as previous_trie @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/homestead/trie.py b/src/ethereum/homestead/trie.py index 903d8af408..f261d33c9a 100644 --- a/src/ethereum/homestead/trie.py +++ b/src/ethereum/homestead/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/istanbul/trie.py b/src/ethereum/istanbul/trie.py index 4fd91e9ed9..636949f1e1 100644 --- a/src/ethereum/istanbul/trie.py +++ b/src/ethereum/istanbul/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.constantinople import trie as previous_trie @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/london/trie.py b/src/ethereum/london/trie.py index b9956dae35..68d1271950 100644 --- a/src/ethereum/london/trie.py +++ b/src/ethereum/london/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.berlin import trie as previous_trie @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/muir_glacier/trie.py b/src/ethereum/muir_glacier/trie.py index e979370730..b3fde0f4c7 100644 --- a/src/ethereum/muir_glacier/trie.py +++ b/src/ethereum/muir_glacier/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/paris/trie.py b/src/ethereum/paris/trie.py index 36ce79f33e..24ccd3ac10 100644 --- a/src/ethereum/paris/trie.py +++ b/src/ethereum/paris/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/shanghai/trie.py b/src/ethereum/shanghai/trie.py index c1e6f1d9f9..25244e03fc 100644 --- a/src/ethereum/shanghai/trie.py +++ b/src/ethereum/shanghai/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -159,7 +158,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (LegacyTransaction, Receipt, Withdrawal, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/spurious_dragon/trie.py b/src/ethereum/spurious_dragon/trie.py index 44e3bf7550..aa7923cfce 100644 --- a/src/ethereum/spurious_dragon/trie.py +++ b/src/ethereum/spurious_dragon/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/src/ethereum/tangerine_whistle/trie.py b/src/ethereum/tangerine_whistle/trie.py index f175385b34..06fdb23f96 100644 --- a/src/ethereum/tangerine_whistle/trie.py +++ b/src/ethereum/tangerine_whistle/trie.py @@ -26,7 +26,6 @@ Sequence, TypeVar, Union, - cast, ) from ethereum.crypto.hash import keccak256 @@ -156,7 +155,7 @@ def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: assert storage_root is not None return encode_account(node, storage_root) elif isinstance(node, (Transaction, Receipt, U256)): - return rlp.encode(cast(rlp.Extended, node)) + return rlp.encode(node) elif isinstance(node, Bytes): return node else: diff --git a/tests/helpers/load_state_tests.py b/tests/helpers/load_state_tests.py index 8c39b7162e..b4cb9e9cd4 100644 --- a/tests/helpers/load_state_tests.py +++ b/tests/helpers/load_state_tests.py @@ -3,7 +3,7 @@ import os.path import re from glob import glob -from typing import Any, Dict, Generator, Tuple, Union, cast +from typing import Any, Dict, Generator, Tuple, Union from unittest.mock import call, patch import pytest @@ -94,7 +94,7 @@ def add_block_to_chain( ) = load.json_to_block(json_block) assert rlp.rlp_hash(block.header) == block_header_hash - assert rlp.encode(cast(rlp.Extended, block)) == block_rlp + assert rlp.encode(block) == block_rlp if not mock_pow: load.fork.state_transition(chain, block) diff --git a/tests/test_rlp.py b/tests/test_rlp.py index c2f58032ef..2c1a0f2d19 100644 --- a/tests/test_rlp.py +++ b/tests/test_rlp.py @@ -152,7 +152,7 @@ def test_rlp_encode_nested_sequence() -> None: def test_rlp_encode_successfully() -> None: - test_cases = [ + test_cases: List[Tuple[rlp.Extended, Union[bytes, bytearray]]] = [ (b"", bytearray([0x80])), (b"\x83" * 55, bytearray([0xB7]) + bytearray(b"\x83" * 55)), (Uint(0), b"\x80"), @@ -171,7 +171,7 @@ def test_rlp_encode_successfully() -> None: ), ] for raw_data, expected_encoding in test_cases: - assert rlp.encode(cast(Extended, raw_data)) == expected_encoding + assert rlp.encode(raw_data) == expected_encoding def test_rlp_encode_fails() -> None: @@ -324,7 +324,7 @@ def test_rlp_decode_failure_empty_bytes() -> None: def test_roundtrip_encoding_and_decoding() -> None: - test_cases = [ + test_cases: List[Extended] = [ b"", b"h", b"hello how are you doing today?", @@ -338,7 +338,7 @@ def test_roundtrip_encoding_and_decoding() -> None: [[b"hello", b"world"], [b"how", b"are"], [b"you", b"doing"]], ] for raw_data in test_cases: - assert rlp.decode(rlp.encode(cast(Extended, raw_data))) == raw_data + assert rlp.decode(rlp.encode(raw_data)) == raw_data # From 153e6f4e07bdaf32d6acf1ef420c5a3acb26cd5f Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Fri, 5 Jul 2024 20:20:36 -0400 Subject: [PATCH 5/5] Better RLP error message for mismatched element count --- src/ethereum/rlp.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ethereum/rlp.py b/src/ethereum/rlp.py index 7fcc85335c..3ada0f4fac 100644 --- a/src/ethereum/rlp.py +++ b/src/ethereum/rlp.py @@ -261,10 +261,15 @@ def _deserialize_to_dataclass(cls: Type[U], decoded: Simple) -> U: target_fields = fields(cls) if isinstance(decoded, bytes): - raise RLPDecodingError + raise RLPDecodingError(f"got `bytes` while decoding `{cls.__name__}`") if len(target_fields) != len(decoded): - raise RLPDecodingError + name = cls.__name__ + actual = len(decoded) + expected = len(target_fields) + raise RLPDecodingError( + f"`{name}` needs {expected} field(s), but got {actual} instead" + ) values: Dict[str, Any] = {}