diff --git a/docs/executing_tests/index.md b/docs/executing_tests/index.md new file mode 100644 index 0000000000..b36e1bcf7b --- /dev/null +++ b/docs/executing_tests/index.md @@ -0,0 +1,117 @@ +# Executing Tests on Local Networks or Hive + +@ethereum/execution-spec-tests is capable of running tests on local networks or on Hive with a few considerations. This page describes how to do so. + +## The `execute` command and `pytest` plugin + +The `execute` command is capable of parse and execute all tests in the `tests` directory, collect the transactions it requires, send them to a client connected to a network, wait for the network to include them in a block and, finally, check the resulting state of the involved smart-contracts against the expected state to validate the behavior of the clients. + +It will not check for the state of the network itself, only the state of the smart-contracts, accounts and transactions involved in the tests, so it is possible that the network becomes unstable or forks during the execution of the tests, but this will not be detected by the command. + +The way this is achieved is by using a pytest plugin that will collect all the tests the same way as the fill plugin does, but instead of compiling the transactions and sending them as a batch to the transition tool, they are prepared and sent to the client one by one. + +Before sending the actual test transactions to the client, the plugin uses a special pre-allocation object that collects the contracts and EOAs that are used by the tests and, instead of pre-allocating them in a dictionary as the fill plugin does, it sends transactions to deploy contracts or fund the accounts for them to be available in the network. + +The pre-allocation object requires a seed account with funds available in the network to be able to deploy contracts and fund accounts. In the case of a live remote network, the seed account needs to be provided via a command-line parameter, but in the case of a local hive network, the seed account is automatically created and funded by the plugin via the genesis file. + +At the end of each test, the plugin will also check the remaining balance of all accounts and will attempt to automatically recover the funds back to the seed account in order to execute the following tests. + +## Differences between the `fill` and `execute` plugins + +The test execution with the `execute` plugin is different from the `fill` plugin in a few ways: + +### EOA and Contract Addresses + +The `fill` plugin will pre-allocate all the accounts and contracts that are used in the tests, so the addresses of the accounts and contracts will be known before the tests are executed, Further more, the test contracts will start from the same address on different tests, so there are collisions on the account addresses used across different tests. This is not the case with the `execute` plugin, as the accounts and contracts are deployed on the fly, from sender keys that are randomly generated and therefore are different in each execution. + +Reasoning behind the random generation of the sender keys is that one can execute the same test multiple times in the same network and the plugin will not fail because the accounts and contracts are already deployed. + +### Transactions Gas Price + +The `fill` plugin will use a fixed and minimum gas price for all the transactions it uses for testing, but this is not possible with the `execute` plugin, as the gas price is determined by the current state of the network. + +At the moment, the `execute` plugin does not query the client for the current gas price, but instead uses a fixed increment to the gas price in order to avoid the transactions to be stuck in the mempool. + +## Running Tests on a Hive Single-Client Local Network + +Tests can be executed on a local hive-controlled single-client network by running the `execute hive` command. + +This command requires hive to be running in `--dev` mode: + +```bash +./hive --dev --client go-ethereum +``` + +This will start hive in dev mode with the single go-ethereum client available for launching tests. + +By default, the hive server will be listening on `http://127.0.0.1:3000`, but this can be changed by setting the `--dev.addr` flag: + +```bash +./hive --dev --client go-ethereum --dev.addr http://127.0.0.1:5000 +``` + +The `execute hive` can now be executed to connect to the hive server, but the environment variable `HIVE_SIMULATOR` needs to be set to the address of the hive server: + +```bash +export HIVE_SIMULATOR=http://127.0.0.1:3000 +``` + +And the tests can be executed with: + +```bash +uv run execute hive --fork=Cancun +``` + +This will execute all available tests in the `tests` directory on the `Cancun` fork by connecting to the hive server running on `http://127.0.0.1:3000` and launching a single client with the appropriate genesis file. + +The genesis file is passed to the client with the appropriate configuration for the fork schedule, system contracts and pre-allocated seed account. + +All tests will be executed in the same network, in the same client, and serially, but when the `-n auto` parameter is passed to the command, the tests can also be executed in parallel. + +One important feature of the `execute hive` command is that, since there is no consensus client running in the network, the command drives the chain by the use of the Engine API to prompt the execution client to generate new blocks and include the transactions in them. + +## Running Test on a Live Remote Network + +Tests can be executed on a live remote network by running the `execute remote` command. + +The command also accepts the `--fork` flag which should match the fork that is currently active in the network (fork transition tests are not supported yet). + +The `execute remote` command requires to be pointed to an RPC endpoint of a client that is connected to the network, which can be specified by using the `--rpc-endpoint` flag: + +```bash +uv run execute remote --rpc-endpoint=https://rpc.endpoint.io +``` + +Another requirement is that the command is provided with a seed account that has funds available in the network to deploy contracts and fund accounts. This can be done by setting the `--rpc-seed-key` flag: + +```bash +uv run execute remote --rpc-endpoint=https://rpc.endpoint.io --rpc-seed-key 0x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f +``` + +The value needs to be a private key that is used to sign the transactions that deploy the contracts and fund the accounts. + +One last requirement is that the `--rpc-chain-id` flag is set to the chain id of the network that is being tested: + +```bash +uv run execute remote --rpc-endpoint=https://rpc.endpoint.io --rpc-seed-key 0x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f --rpc-chain-id 12345 +``` + +## `execute` Command Test Execution + +After executing wither `execute hive` or `execute remote`, the command will first create a random sender account from which all required test accounts will be deployed and funded, and this account is funded by sweeping (by default) the seed account. + +The sweep amount can be configured by setting the `--seed-account-sweep-amount` flag: + +```bash +--seed-account-sweep-amount "1000 ether" +``` + +Once the sender account is funded, the command will start executing tests one by one by sending the transactions from this account to the network. + +Test transactions are not sent from the main sender account though, they are sent from a different unique account that is created for each test (accounts returned by `pre.fund_eoa`). + +If the command is run using the `-n` flag, the tests will be executed in parallel, and each process will have its own separate sender account, so the amount that is swept from the seed account is divided by the number of processes, so this has to be taken into account when setting the sweep amount and also when funding the seed account. + +After finishing each test the command will check the remaining balance of all accounts and will attempt to recover the funds back to the sender account, and at the end of all tests, the remaining balance of the sender account will be swept back to the seed account. + +There are instances where it will be impossible to recover the funds back from a test, for example, funds that are sent to a contract that has no built-in way to send them back, the funds will be stuck in the contract and they will not be recoverable. diff --git a/docs/navigation.md b/docs/navigation.md index 5148ce6a01..eeb08a83c3 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -27,6 +27,7 @@ * [EOF Tests](consuming_tests/eof_test.md) * [Common Types](consuming_tests/common_types.md) * [Exceptions](consuming_tests/exceptions.md) + * [Executing Tests](executing_tests/index.md) * [Getting Help](getting_help/index.md) * [Developer Doc](dev/index.md) * [Managing Configurations](dev/configurations.md) diff --git a/docs/writing_tests/test_markers.md b/docs/writing_tests/test_markers.md index ce43edab4d..c8af8b3a7e 100644 --- a/docs/writing_tests/test_markers.md +++ b/docs/writing_tests/test_markers.md @@ -271,6 +271,48 @@ def test_something_with_all_tx_types_but_skip_type_1(state_test_only, tx_type): In this example, the test will be skipped if `tx_type` is equal to 1 by returning a `pytest.mark.skip` marker, and return `None` otherwise. +## Fill/Execute Markers + +These markers are used to apply different markers to a test depending on whether it is being filled or executed. + +### `@pytest.mark.fill` + +This marker is used to apply markers to a test when it is being filled. + +```python +import pytest + +from ethereum_test_tools import Alloc, StateTestFiller + +@pytest.mark.fill(pytest.mark.skip(reason="Only for execution")) +def test_something( + state_test: StateTestFiller, + pre: Alloc +): + pass +``` + +In this example, the test will be skipped when it is being filled. + +### `@pytest.mark.execute` + +This marker is used to apply markers to a test when it is being executed. + +```python +import pytest + +from ethereum_test_tools import Alloc, StateTestFiller + +@pytest.mark.execute(pytest.mark.xfail(reason="Depends on block context")) +def test_something( + state_test: StateTestFiller, + pre: Alloc +): + pass +``` + +In this example, the test will be marked as expected to fail when it is being executed, which is particularly useful so that the test is still executed but does not fail the test run. + ## Other Markers ### `@pytest.mark.slow` diff --git a/pyproject.toml b/pyproject.toml index 0bf639deef..557a282a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "ethereum-types>=0.2.1,<0.3", "pyyaml>=6.0.2", "types-pyyaml>=6.0.12.20240917", + "pytest-json-report>=1.5.0,<2", ] [project.urls] @@ -85,6 +86,7 @@ docs = [ [project.scripts] fill = "cli.pytest_commands.fill:fill" phil = "cli.pytest_commands.fill:phil" +execute = "cli.pytest_commands.execute:execute" tf = "cli.pytest_commands.fill:tf" checkfixtures = "cli.check_fixtures:check_fixtures" consume = "cli.pytest_commands.consume:consume" diff --git a/pytest-execute-hive.ini b/pytest-execute-hive.ini new file mode 100644 index 0000000000..05f156d99d --- /dev/null +++ b/pytest-execute-hive.ini @@ -0,0 +1,24 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = *.py +testpaths = tests/ +markers = + slow + pre_alloc_modify +addopts = + -p pytest_plugins.concurrency + -p pytest_plugins.execute.sender + -p pytest_plugins.execute.pre_alloc + -p pytest_plugins.solc.solc + -p pytest_plugins.execute.rpc.hive + -p pytest_plugins.execute.execute + -p pytest_plugins.shared.execute_fill + -p pytest_plugins.forks.forks + -p pytest_plugins.spec_version_checker.spec_version_checker + -p pytest_plugins.pytest_hive.pytest_hive + -p pytest_plugins.help.help + -m "not eip_version_check" + --tb short + --dist loadscope + --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/ diff --git a/pytest-execute-recover.ini b/pytest-execute-recover.ini new file mode 100644 index 0000000000..72f8b7393e --- /dev/null +++ b/pytest-execute-recover.ini @@ -0,0 +1,15 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = *.py +testpaths = src/pytest_plugins/execute/test_recover.py +markers = + slow + pre_alloc_modify +addopts = + -p pytest_plugins.execute.rpc.remote + -p pytest_plugins.execute.recover + -p pytest_plugins.help.help + -m "not eip_version_check" + --tb short + --dist loadscope diff --git a/pytest-execute.ini b/pytest-execute.ini new file mode 100644 index 0000000000..984f836f1b --- /dev/null +++ b/pytest-execute.ini @@ -0,0 +1,24 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = *.py +testpaths = tests/ +markers = + slow + pre_alloc_modify +addopts = + -p pytest_plugins.concurrency + -p pytest_plugins.execute.sender + -p pytest_plugins.execute.pre_alloc + -p pytest_plugins.solc.solc + -p pytest_plugins.execute.execute + -p pytest_plugins.shared.execute_fill + -p pytest_plugins.execute.rpc.remote_seed_sender + -p pytest_plugins.execute.rpc.remote + -p pytest_plugins.forks.forks + -p pytest_plugins.spec_version_checker.spec_version_checker + -p pytest_plugins.help.help + -m "not eip_version_check" + --tb short + --dist loadscope + --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/ diff --git a/pytest-framework.ini b/pytest-framework.ini index b3295bffe9..a9fcf19738 100644 --- a/pytest-framework.ini +++ b/pytest-framework.ini @@ -14,3 +14,4 @@ addopts = --ignore=src/pytest_plugins/consume/direct/test_via_direct.py --ignore=src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py --ignore=src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py + --ignore=src/pytest_plugins/execute/test_recover.py diff --git a/pytest.ini b/pytest.ini index ede8b05674..73b0b441cb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,6 +11,7 @@ addopts = -p pytest_plugins.filler.pre_alloc -p pytest_plugins.solc.solc -p pytest_plugins.filler.filler + -p pytest_plugins.shared.execute_fill -p pytest_plugins.forks.forks -p pytest_plugins.spec_version_checker.spec_version_checker -p pytest_plugins.eels_resolver diff --git a/src/cli/pytest_commands/common.py b/src/cli/pytest_commands/common.py index 018e303bfe..114d658021 100644 --- a/src/cli/pytest_commands/common.py +++ b/src/cli/pytest_commands/common.py @@ -2,7 +2,7 @@ Common functions for CLI pytest-based entry points. """ -from typing import Any, Callable, List +from typing import Any, Callable, Dict, List import click @@ -38,6 +38,31 @@ def common_click_options(func: Callable[..., Any]) -> Decorator: return click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED)(func) +REQUIRED_FLAGS: Dict[str, List] = { + "fill": [], + "consume": [], + "execute": [ + "--rpc-endpoint", + "x", + "--rpc-seed-key", + "x", + "--rpc-chain-id", + "1", + ], + "execute-hive": [], + "execute-recover": [ + "--rpc-endpoint", + "x", + "--rpc-chain-id", + "1", + "--start-eoa-index", + "1", + "--destination", + "0x1234567890123456789012345678901234567890", + ], +} + + def handle_help_flags(pytest_args: List[str], pytest_type: str) -> List[str]: """ Modifies the help arguments passed to the click CLI command before forwarding to @@ -49,7 +74,11 @@ def handle_help_flags(pytest_args: List[str], pytest_type: str) -> List[str]: ctx = click.get_current_context() if ctx.params.get("help_flag"): - return [f"--{pytest_type}-help"] if pytest_type in {"consume", "fill"} else pytest_args + return ( + [f"--{pytest_type}-help", *REQUIRED_FLAGS[pytest_type]] + if pytest_type in {"consume", "fill", "execute", "execute-hive", "execute-recover"} + else pytest_args + ) elif ctx.params.get("pytest_help_flag"): return ["--help"] diff --git a/src/cli/pytest_commands/execute.py b/src/cli/pytest_commands/execute.py new file mode 100644 index 0000000000..a949577baa --- /dev/null +++ b/src/cli/pytest_commands/execute.py @@ -0,0 +1,71 @@ +""" +CLI entry point for the `execute` pytest-based command. +""" + +import sys +from typing import Tuple + +import click +import pytest + +from .common import common_click_options, handle_help_flags + + +@click.group(context_settings=dict(help_option_names=["-h", "--help"])) +def execute() -> None: + """ + Execute command to run tests in hive or live networks. + """ + pass + + +@execute.command(context_settings=dict(ignore_unknown_options=True)) +@common_click_options +def hive( + pytest_args: Tuple[str, ...], + **kwargs, +) -> None: + """ + Execute tests using hive in dev-mode as backend, requires hive to be running + (using command: `./hive --dev`). + """ + pytest_type = "execute-hive" + args = handle_help_flags(list(pytest_args), pytest_type=pytest_type) + ini_file = "pytest-execute-hive.ini" + args = ["-c", ini_file] + args + result = pytest.main(args) + sys.exit(result) + + +@execute.command(context_settings=dict(ignore_unknown_options=True)) +@common_click_options +def remote( + pytest_args: Tuple[str, ...], + **kwargs, +) -> None: + """ + Execute tests using a remote RPC endpoint. + """ + pytest_type = "execute" + args = handle_help_flags(list(pytest_args), pytest_type=pytest_type) + ini_file = "pytest-execute.ini" + args = ["-c", ini_file] + args + result = pytest.main(args) + sys.exit(result) + + +@execute.command(context_settings=dict(ignore_unknown_options=True)) +@common_click_options +def recover( + pytest_args: Tuple[str, ...], + **kwargs, +) -> None: + """ + Recover funds from a failed test execution using a remote RPC endpoint. + """ + pytest_type = "execute-recover" + args = handle_help_flags(list(pytest_args), pytest_type=pytest_type) + ini_file = "pytest-execute-recover.ini" + args = ["-c", ini_file] + args + result = pytest.main(args) + sys.exit(result) diff --git a/src/ethereum_test_execution/__init__.py b/src/ethereum_test_execution/__init__.py new file mode 100644 index 0000000000..b6a2fffc66 --- /dev/null +++ b/src/ethereum_test_execution/__init__.py @@ -0,0 +1,20 @@ +""" +Ethereum test execution package. +""" +from typing import Dict + +from .base import BaseExecute, ExecuteFormat +from .transaction_post import TransactionPost + +EXECUTE_FORMATS: Dict[str, ExecuteFormat] = { + f.execute_format_name: f # type: ignore + for f in [ + TransactionPost, + ] +} +__all__ = [ + "BaseExecute", + "ExecuteFormat", + "TransactionPost", + "EXECUTE_FORMATS", +] diff --git a/src/ethereum_test_execution/base.py b/src/ethereum_test_execution/base.py new file mode 100644 index 0000000000..dd5df826c2 --- /dev/null +++ b/src/ethereum_test_execution/base.py @@ -0,0 +1,29 @@ +""" +Ethereum test execution base types. +""" +from abc import abstractmethod +from typing import ClassVar, Type + +from ethereum_test_base_types import CamelModel +from ethereum_test_rpc import EthRPC + + +class BaseExecute(CamelModel): + """ + Represents a base execution format. + """ + + # Execute format properties + execute_format_name: ClassVar[str] = "unset" + description: ClassVar[str] = "Unknown execute format; it has not been set." + + @abstractmethod + def execute(self, eth_rpc: EthRPC): + """ + Execute the format. + """ + pass + + +# Type alias for a base execute class +ExecuteFormat = Type[BaseExecute] diff --git a/src/ethereum_test_execution/transaction_post.py b/src/ethereum_test_execution/transaction_post.py new file mode 100644 index 0000000000..5c71be5f43 --- /dev/null +++ b/src/ethereum_test_execution/transaction_post.py @@ -0,0 +1,75 @@ +""" +Simple transaction-send then post-check execution format. +""" + +from typing import ClassVar, List + +import pytest + +from ethereum_test_base_types import Alloc, Hash +from ethereum_test_rpc import EthRPC, SendTransactionException +from ethereum_test_types import Transaction + +from .base import BaseExecute + + +class TransactionPost(BaseExecute): + """ + Represents a simple transaction-send then post-check execution format. + """ + + transactions: List[Transaction] + post: Alloc + + execute_format_name: ClassVar[str] = "transaction_post" + description: ClassVar[ + str + ] = "Simple transaction sending, then post-check after all transactions are included" + + def execute(self, eth_rpc: EthRPC): + """ + Execute the format. + """ + assert not any( + tx.ty == 3 for tx in self.transactions + ), "Transaction type 3 is not supported in execute mode." + if any(tx.error is not None for tx in self.transactions): + for transaction in self.transactions: + if transaction.error is None: + eth_rpc.send_wait_transaction(transaction.with_signature_and_sender()) + else: + with pytest.raises(SendTransactionException): + eth_rpc.send_transaction(transaction.with_signature_and_sender()) + else: + eth_rpc.send_wait_transactions( + [tx.with_signature_and_sender() for tx in self.transactions] + ) + + for address, account in self.post.root.items(): + balance = eth_rpc.get_balance(address) + code = eth_rpc.get_code(address) + nonce = eth_rpc.get_transaction_count(address) + if account is None: + assert balance == 0, f"Balance of {address} is {balance}, expected 0." + assert code == b"", f"Code of {address} is {code}, expected 0x." + assert nonce == 0, f"Nonce of {address} is {nonce}, expected 0." + else: + if "balance" in account.model_fields_set: + assert ( + balance == account.balance + ), f"Balance of {address} is {balance}, expected {account.balance}." + if "code" in account.model_fields_set: + assert ( + code == account.code + ), f"Code of {address} is {code}, expected {account.code}." + if "nonce" in account.model_fields_set: + assert ( + nonce == account.nonce + ), f"Nonce of {address} is {nonce}, expected {account.nonce}." + if "storage" in account.model_fields_set: + for key, value in account.storage.items(): + storage_value = eth_rpc.get_storage_at(address, Hash(key)) + assert storage_value == value, ( + f"Storage value at {key} of {address} is {storage_value}," + f"expected {value}." + ) diff --git a/src/ethereum_test_fixtures/base.py b/src/ethereum_test_fixtures/base.py index 05e83ff62e..2344698ff1 100644 --- a/src/ethereum_test_fixtures/base.py +++ b/src/ethereum_test_fixtures/base.py @@ -59,7 +59,7 @@ def json_dict_with_info(self, hash_only: bool = False) -> Dict[str, Any]: def fill_info( self, t8n_version: str, - fixture_description: str, + test_case_description: str, fixture_source_url: str, ref_spec: ReferenceSpec | None, ): @@ -69,7 +69,7 @@ def fill_info( if "comment" not in self.info: self.info["comment"] = "`execution-spec-tests` generated test" self.info["filling-transition-tool"] = t8n_version - self.info["description"] = fixture_description + self.info["description"] = test_case_description self.info["url"] = fixture_source_url if ref_spec is not None: ref_spec.write_info(self.info) diff --git a/src/ethereum_test_rpc/rpc.py b/src/ethereum_test_rpc/rpc.py index 7aa0f31744..2b144be7db 100644 --- a/src/ethereum_test_rpc/rpc.py +++ b/src/ethereum_test_rpc/rpc.py @@ -182,9 +182,7 @@ def send_transaction(self, transaction: Transaction) -> Hash: `eth_sendRawTransaction`: Send a transaction to the client. """ try: - result_hash = Hash( - self.post_request("sendRawTransaction", f"0x{transaction.rlp.hex()}") - ) + result_hash = Hash(self.post_request("sendRawTransaction", f"{transaction.rlp.hex()}")) assert result_hash == transaction.hash assert result_hash is not None return transaction.hash diff --git a/src/ethereum_test_specs/base.py b/src/ethereum_test_specs/base.py index 3dc4e24dc3..f8484268a8 100644 --- a/src/ethereum_test_specs/base.py +++ b/src/ethereum_test_specs/base.py @@ -14,6 +14,7 @@ from ethereum_clis import Result, TransitionTool from ethereum_test_base_types import to_hex +from ethereum_test_execution import BaseExecute, ExecuteFormat from ethereum_test_fixtures import BaseFixture, FixtureFormat from ethereum_test_forks import Fork from ethereum_test_types import Environment, Withdrawal @@ -53,6 +54,7 @@ class BaseTest(BaseModel): _t8n_call_counter: Iterator[int] = count(0) supported_fixture_formats: ClassVar[List[FixtureFormat]] = [] + supported_execute_formats: ClassVar[List[ExecuteFormat]] = [] @abstractmethod def generate( @@ -69,6 +71,18 @@ def generate( """ pass + def execute( + self, + *, + fork: Fork, + execute_format: ExecuteFormat, + eips: Optional[List[int]] = None, + ) -> BaseExecute: + """ + Generate the list of test fixtures. + """ + raise Exception(f"Unsupported execute format: {execute_format}") + @classmethod def pytest_parameter_name(cls) -> str: """ diff --git a/src/ethereum_test_specs/blockchain.py b/src/ethereum_test_specs/blockchain.py index 7d58979d94..171c57e04a 100644 --- a/src/ethereum_test_specs/blockchain.py +++ b/src/ethereum_test_specs/blockchain.py @@ -22,6 +22,7 @@ Number, ) from ethereum_test_exceptions import BlockException, EngineAPIError, TransactionException +from ethereum_test_execution import BaseExecute, ExecuteFormat, TransactionPost from ethereum_test_fixtures import ( BaseFixture, BlockchainEngineFixture, @@ -329,6 +330,9 @@ class BlockchainTest(BaseTest): BlockchainFixture, BlockchainEngineFixture, ] + supported_execute_formats: ClassVar[List[ExecuteFormat]] = [ + TransactionPost, + ] def make_genesis( self, @@ -751,6 +755,26 @@ def generate( raise Exception(f"Unknown fixture format: {fixture_format}") + def execute( + self, + *, + fork: Fork, + execute_format: ExecuteFormat, + eips: Optional[List[int]] = None, + ) -> BaseExecute: + """ + Generate the list of test fixtures. + """ + if execute_format == TransactionPost: + txs: List[Transaction] = [] + for block in self.blocks: + txs += block.txs + return TransactionPost( + transactions=txs, + post=self.post, + ) + raise Exception(f"Unsupported execute format: {execute_format}") + BlockchainTestSpec = Callable[[str], Generator[BlockchainTest, None, None]] BlockchainTestFiller = Type[BlockchainTest] diff --git a/src/ethereum_test_specs/eof.py b/src/ethereum_test_specs/eof.py index 3096dfff69..dcaf9ae6f3 100644 --- a/src/ethereum_test_specs/eof.py +++ b/src/ethereum_test_specs/eof.py @@ -15,6 +15,7 @@ from ethereum_clis import EvmoneExceptionMapper, TransitionTool from ethereum_test_base_types import Account, Bytes from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList, to_pipe_str +from ethereum_test_execution import BaseExecute, ExecuteFormat, TransactionPost from ethereum_test_fixtures import ( BaseFixture, BlockchainEngineFixture, @@ -289,6 +290,8 @@ def generate( raise Exception(f"Unknown fixture format: {fixture_format}") + # TODO: Implement execute method for EOF tests + EOFTestSpec = Callable[[str], Generator[EOFTest, None, None]] EOFTestFiller = Type[EOFTest] @@ -401,6 +404,22 @@ def generate( raise Exception(f"Unknown fixture format: {fixture_format}") + def execute( + self, + *, + fork: Fork, + execute_format: ExecuteFormat, + eips: Optional[List[int]] = None, + ) -> BaseExecute: + """ + Generate the list of test fixtures. + """ + if execute_format == TransactionPost: + return self.generate_state_test().execute( + fork=fork, execute_format=execute_format, eips=eips + ) + raise Exception(f"Unsupported execute format: {execute_format}") + EOFStateTestSpec = Callable[[str], Generator[EOFStateTest, None, None]] EOFStateTestFiller = Type[EOFStateTest] diff --git a/src/ethereum_test_specs/state.py b/src/ethereum_test_specs/state.py index 80c7f5333e..a39add940e 100644 --- a/src/ethereum_test_specs/state.py +++ b/src/ethereum_test_specs/state.py @@ -9,6 +9,7 @@ from ethereum_clis import TransitionTool from ethereum_test_exceptions import EngineAPIError +from ethereum_test_execution import BaseExecute, ExecuteFormat, TransactionPost from ethereum_test_fixtures import ( BaseFixture, BlockchainEngineFixture, @@ -52,6 +53,9 @@ class StateTest(BaseTest): BlockchainEngineFixture, StateFixture, ] + supported_execute_formats: ClassVar[List[ExecuteFormat]] = [ + TransactionPost, + ] def _generate_blockchain_genesis_environment(self) -> Environment: """ @@ -197,6 +201,23 @@ def generate( raise Exception(f"Unknown fixture format: {fixture_format}") + def execute( + self, + *, + fork: Fork, + execute_format: ExecuteFormat, + eips: Optional[List[int]] = None, + ) -> BaseExecute: + """ + Generate the list of test fixtures. + """ + if execute_format == TransactionPost: + return TransactionPost( + transactions=[self.tx], + post=self.post, + ) + raise Exception(f"Unsupported execute format: {execute_format}") + class StateTestOnly(StateTest): """ diff --git a/src/pytest_plugins/consume/hive_simulators/conftest.py b/src/pytest_plugins/consume/hive_simulators/conftest.py index 58a0249796..56f561824f 100644 --- a/src/pytest_plugins/consume/hive_simulators/conftest.py +++ b/src/pytest_plugins/consume/hive_simulators/conftest.py @@ -79,7 +79,7 @@ def eest_consume_commands( @pytest.fixture(scope="function") -def fixture_description( +def test_case_description( blockchain_fixture: BlockchainFixtureCommon, test_case: TestCaseIndexFile | TestCaseStream, hive_consume_command: str, diff --git a/src/pytest_plugins/execute/__init__.py b/src/pytest_plugins/execute/__init__.py new file mode 100644 index 0000000000..80ae1fab28 --- /dev/null +++ b/src/pytest_plugins/execute/__init__.py @@ -0,0 +1,3 @@ +""" +A pytest plugin that provides fixtures that exectute tests in live devnets/testnets. +""" diff --git a/src/pytest_plugins/execute/execute.py b/src/pytest_plugins/execute/execute.py new file mode 100644 index 0000000000..0d28baadd2 --- /dev/null +++ b/src/pytest_plugins/execute/execute.py @@ -0,0 +1,395 @@ +""" +Test execution plugin for pytest, to run Ethereum tests using in live networks. +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Generator, List, Type + +import pytest +from pytest_metadata.plugin import metadata_key # type: ignore + +from ethereum_test_base_types import Number +from ethereum_test_execution import EXECUTE_FORMATS, BaseExecute +from ethereum_test_forks import Fork +from ethereum_test_rpc import EthRPC +from ethereum_test_tools import SPEC_TYPES, BaseTest, TestInfo, Transaction +from ethereum_test_types import TransactionDefaults +from pytest_plugins.spec_version_checker.spec_version_checker import EIPSpecTestItem + +from .pre_alloc import Alloc + + +def default_html_report_file_path() -> str: + """ + The default file to store the generated HTML test report. Defined as a + function to allow for easier testing. + """ + return "./execution_results/report_execute.html" + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + execute_group = parser.getgroup("execute", "Arguments defining test execution behavior") + execute_group.addoption( + "--default-gas-price", + action="store", + dest="default_gas_price", + type=int, + default=10**9, + help=("Default gas price used for transactions, unless overridden by the test."), + ) + execute_group.addoption( + "--default-max-fee-per-gas", + action="store", + dest="default_max_fee_per_gas", + type=int, + default=10**9, + help=("Default max fee per gas used for transactions, unless overridden by the test."), + ) + execute_group.addoption( + "--default-max-priority-fee-per-gas", + action="store", + dest="default_max_priority_fee_per_gas", + type=int, + default=10**9, + help=( + "Default max priority fee per gas used for transactions, " + "unless overridden by the test." + ), + ) + + report_group = parser.getgroup("tests", "Arguments defining html report behavior") + report_group.addoption( + "--no-html", + action="store_true", + dest="disable_html", + default=False, + help=( + "Don't generate an HTML test report. " + "The --html flag can be used to specify a different path." + ), + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + """ + Pytest hook called after command line options have been parsed and before + test collection begins. + + Couple of notes: + 1. Register the plugin's custom markers and process command-line options. + + Custom marker registration: + https://docs.pytest.org/en/7.1.x/how-to/writing_plugins.html#registering-custom-markers + + 2. `@pytest.hookimpl(tryfirst=True)` is applied to ensure that this hook is + called before the pytest-html plugin's pytest_configure to ensure that + it uses the modified `htmlpath` option. + """ + if config.option.collectonly: + return + if config.getoption("disable_html") and config.getoption("htmlpath") is None: + # generate an html report by default, unless explicitly disabled + config.option.htmlpath = Path(default_html_report_file_path()) + + command_line_args = "fill " + " ".join(config.invocation_params.args) + config.stash[metadata_key]["Command-line args"] = f"{command_line_args}" + + if len(config.fork_set) != 1: + pytest.exit( + f""" + Expected exactly one fork to be specified, got {len(config.fork_set)}. + Make sure to specify exactly one fork using the --fork command line argument. + """, + returncode=pytest.ExitCode.USAGE_ERROR, + ) + + +def pytest_metadata(metadata): + """ + Add or remove metadata to/from the pytest report. + """ + metadata.pop("JAVA_HOME", None) + + +def pytest_html_results_table_header(cells): + """ + Customize the table headers of the HTML report table. + """ + cells.insert(3, 'Sender') + cells.insert(4, 'Funded Accounts') + cells.insert( + 5, 'Deployed Contracts' + ) + del cells[-1] # Remove the "Links" column + + +def pytest_html_results_table_row(report, cells): + """ + Customize the table rows of the HTML report table. + """ + if hasattr(report, "user_properties"): + user_props = dict(report.user_properties) + if "sender_address" in user_props and user_props["sender_address"] is not None: + sender_address = user_props["sender_address"] + cells.insert(3, f"{sender_address}") + else: + cells.insert(3, "Not available") + + if "funded_accounts" in user_props and user_props["funded_accounts"] is not None: + funded_accounts = user_props["funded_accounts"] + cells.insert(4, f"{funded_accounts}") + else: + cells.insert(4, "Not available") + + del cells[-1] # Remove the "Links" column + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + This hook is called when each test is run and a report is being made. + + Make each test's fixture json path available to the test report via + user_properties. + """ + outcome = yield + report = outcome.get_result() + + if call.when == "call": + for property_name in ["sender_address", "funded_accounts"]: + if hasattr(item.config, property_name): + report.user_properties.append((property_name, getattr(item.config, property_name))) + + +def pytest_html_report_title(report): + """ + Set the HTML report title (pytest-html plugin). + """ + report.title = "Execute Test Report" + + +@pytest.fixture(scope="session") +def default_gas_price(request) -> int: + """ + Returns the default gas price used for transactions. + """ + return request.config.getoption("default_gas_price") + + +@pytest.fixture(scope="session") +def default_max_fee_per_gas(request) -> int: + """ + Returns the default max fee per gas used for transactions. + """ + return request.config.getoption("default_max_fee_per_gas") + + +@pytest.fixture(scope="session") +def default_max_priority_fee_per_gas(request) -> int: + """ + Returns the default max priority fee per gas used for transactions. + """ + return request.config.getoption("default_max_priority_fee_per_gas") + + +@pytest.fixture(autouse=True, scope="session") +def modify_transaction_defaults( + default_gas_price: int, default_max_fee_per_gas: int, default_max_priority_fee_per_gas: int +): + """ + Modify transaction defaults to values better suited for live networks. + """ + TransactionDefaults.gas_price = default_gas_price + TransactionDefaults.max_fee_per_gas = default_max_fee_per_gas + TransactionDefaults.max_priority_fee_per_gas = default_max_priority_fee_per_gas + + +@dataclass(kw_only=True) +class Collector: + """ + A class that collects transactions and post-allocations for every test case. + """ + + eth_rpc: EthRPC + collected_tests: Dict[str, BaseExecute] = field(default_factory=dict) + + def collect(self, test_name: str, execute_format: BaseExecute): + """ + Collects the transactions and post-allocations for the test case. + """ + self.collected_tests[test_name] = execute_format + + +@pytest.fixture(scope="session") +def collector( + request, + eth_rpc: EthRPC, +) -> Generator[Collector, None, None]: + """ + Returns the configured fixture collector instance used for all tests + in one test module. + """ + collector = Collector(eth_rpc=eth_rpc) + yield collector + + +def node_to_test_info(node) -> TestInfo: + """ + Returns the test info of the current node item. + """ + return TestInfo( + name=node.name, + id=node.nodeid, + original_name=node.originalname, + path=Path(node.path), + ) + + +def base_test_parametrizer(cls: Type[BaseTest]): + """ + Generates a pytest.fixture for a given BaseTest subclass. + + Implementation detail: All spec fixtures must be scoped on test function level to avoid + leakage between tests. + """ + + @pytest.fixture( + scope="function", + name=cls.pytest_parameter_name(), + ) + def base_test_parametrizer_func( + request: Any, + fork: Fork, + pre: Alloc, + eips: List[int], + eth_rpc: EthRPC, + collector: Collector, + default_gas_price: int, + ): + """ + Fixture used to instantiate an auto-fillable BaseTest object from within + a test function. + + Every test that defines a test filler must explicitly specify its parameter name + (see `pytest_parameter_name` in each implementation of BaseTest) in its function + arguments. + + When parametrize, indirect must be used along with the fixture format as value. + """ + execute_format = request.param + assert execute_format in EXECUTE_FORMATS.values() + + class BaseTestWrapper(cls): # type: ignore + def __init__(self, *args, **kwargs): + kwargs["t8n_dump_dir"] = None + if "pre" not in kwargs: + kwargs["pre"] = pre + elif kwargs["pre"] != pre: + raise ValueError("The pre-alloc object was modified by the test.") + + request.node.config.sender_address = str(pre._sender) + + super(BaseTestWrapper, self).__init__(*args, **kwargs) + + # wait for pre-requisite transactions to be included in blocks + pre.wait_for_transactions() + for deployed_contract, deployed_code in pre._deployed_contracts: + + if eth_rpc.get_code(deployed_contract) == deployed_code: + pass + else: + raise Exception( + f"Deployed test contract didn't match expected code at address " + f"{deployed_contract} (not enough gas_limit?)." + ) + request.node.config.funded_accounts = ", ".join( + [str(eoa) for eoa in pre._funded_eoa] + ) + + execute = self.execute(fork=fork, execute_format=execute_format, eips=eips) + execute.execute(eth_rpc) + collector.collect(request.node.nodeid, execute) + + sender_start_balance = eth_rpc.get_balance(pre._sender) + + yield BaseTestWrapper + + # Refund all EOAs (regardless of whether the test passed or failed) + refund_txs = [] + for eoa in pre._funded_eoa: + remaining_balance = eth_rpc.get_balance(eoa) + eoa.nonce = Number(eth_rpc.get_transaction_count(eoa)) + refund_gas_limit = 21_000 + tx_cost = refund_gas_limit * default_gas_price + if remaining_balance < tx_cost: + continue + refund_txs.append( + Transaction( + sender=eoa, + to=pre._sender, + gas_limit=21_000, + gas_price=default_gas_price, + value=remaining_balance - tx_cost, + ).with_signature_and_sender() + ) + eth_rpc.send_wait_transactions(refund_txs) + + sender_end_balance = eth_rpc.get_balance(pre._sender) + used_balance = sender_start_balance - sender_end_balance + print(f"Used balance={used_balance / 10**18:.18f}") + + return base_test_parametrizer_func + + +# Dynamically generate a pytest fixture for each test spec type. +for cls in SPEC_TYPES: + # Fixture needs to be defined in the global scope so pytest can detect it. + globals()[cls.pytest_parameter_name()] = base_test_parametrizer(cls) + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + """ + Pytest hook used to dynamically generate test cases for each fixture format a given + test spec supports. + """ + for test_type in SPEC_TYPES: + if test_type.pytest_parameter_name() in metafunc.fixturenames: + metafunc.parametrize( + [test_type.pytest_parameter_name()], + [ + pytest.param( + execute_format, + id=execute_format.execute_format_name.lower(), + marks=[getattr(pytest.mark, execute_format.execute_format_name.lower())], + ) + for execute_format in test_type.supported_execute_formats + ], + scope="function", + indirect=True, + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item]): + """ + Remove pre-Paris tests parametrized to generate hive type fixtures; these + can't be used in the Hive Pyspec Simulator. + + This can't be handled in this plugins pytest_generate_tests() as the fork + parametrization occurs in the forks plugin. + """ + for item in items[:]: # use a copy of the list, as we'll be modifying it + if isinstance(item, EIPSpecTestItem): + continue + for marker in item.iter_markers(): + if marker.name == "execute": + for mark in marker.args: + item.add_marker(mark) + elif marker.name == "valid_at_transition_to": + item.add_marker(pytest.mark.skip(reason="transition tests not executable")) + if "yul" in item.fixturenames: # type: ignore + item.add_marker(pytest.mark.yul_test) diff --git a/src/pytest_plugins/execute/pre_alloc.py b/src/pytest_plugins/execute/pre_alloc.py new file mode 100644 index 0000000000..c8ffec4f9a --- /dev/null +++ b/src/pytest_plugins/execute/pre_alloc.py @@ -0,0 +1,395 @@ +""" +Pre-allocation fixtures using for test filling. +""" + +from itertools import count +from random import randint +from typing import Iterator, List, Literal, Tuple + +import pytest +from pydantic import PrivateAttr + +from ethereum_test_base_types import Number, StorageRootType, ZeroPaddedHexNumber +from ethereum_test_base_types.conversions import ( + BytesConvertible, + FixedSizeBytesConvertible, + NumberConvertible, +) +from ethereum_test_rpc import EthRPC +from ethereum_test_rpc.types import TransactionByHashResponse +from ethereum_test_tools import EOA, Account, Address +from ethereum_test_tools import Alloc as BaseAlloc +from ethereum_test_tools import AuthorizationTuple, Initcode +from ethereum_test_tools import Opcodes as Op +from ethereum_test_tools import ( + Storage, + Transaction, + cost_memory_bytes, + eip_2028_transaction_data_cost, +) +from ethereum_test_types.eof.v1 import Container +from ethereum_test_vm import Bytecode, EVMCodeType, Opcodes + +MAX_BYTECODE_SIZE = 24576 + +MAX_INITCODE_SIZE = MAX_BYTECODE_SIZE * 2 + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + pre_alloc_group = parser.getgroup( + "pre_alloc", "Arguments defining pre-allocation behavior during test execution" + ) + pre_alloc_group.addoption( + "--eoa-start", + action="store", + dest="eoa_iterator_start", + default=randint(0, 2**256), + type=int, + help="The start private key from which tests will deploy EOAs.", + ) + pre_alloc_group.addoption( + "--evm-code-type", + action="store", + dest="evm_code_type", + default=None, + type=EVMCodeType, + choices=list(EVMCodeType), + help="Type of EVM code to deploy in each test by default.", + ) + pre_alloc_group.addoption( + "--eoa-fund-amount-default", + action="store", + dest="eoa_fund_amount_default", + default=10**18, + type=int, + help="The default amount of wei to fund each EOA in each test with.", + ) + + +@pytest.hookimpl(trylast=True) +def pytest_report_header(config): + """A pytest hook called to obtain the report header.""" + bold = "\033[1m" + reset = "\033[39;49m" + eoa_start = config.getoption("eoa_iterator_start") + header = [ + (bold + f"Start seed for EOA: {hex(eoa_start)} " + reset), + ] + return header + + +@pytest.fixture(scope="session") +def eoa_iterator(request) -> Iterator[EOA]: + """ + Returns an iterator that generates EOAs. + """ + eoa_start = request.config.getoption("eoa_iterator_start") + print(f"Starting EOA index: {hex(eoa_start)}") + return iter(EOA(key=i, nonce=0) for i in count(start=eoa_start)) + + +class Alloc(BaseAlloc): + """ + A custom class that inherits from the original Alloc class. + """ + + _sender: EOA = PrivateAttr(...) + _eth_rpc: EthRPC = PrivateAttr(...) + _txs: List[Transaction] = PrivateAttr(default_factory=list) + _deployed_contracts: List[Tuple[Address, bytes]] = PrivateAttr(default_factory=list) + _funded_eoa: List[EOA] = PrivateAttr(default_factory=list) + _evm_code_type: EVMCodeType | None = PrivateAttr(None) + _chain_id: int = PrivateAttr(...) + + def __init__( + self, + *args, + sender: EOA, + eth_rpc: EthRPC, + eoa_iterator: Iterator[EOA], + chain_id: int, + eoa_fund_amount_default: int, + evm_code_type: EVMCodeType | None = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + self._sender = sender + self._eth_rpc = eth_rpc + self._eoa_iterator = eoa_iterator + self._evm_code_type = evm_code_type + self._chain_id = chain_id + self._eoa_fund_amount_default = eoa_fund_amount_default + + def __setitem__(self, address: Address | FixedSizeBytesConvertible, account: Account | None): + """ + Sets the account associated with an address. + """ + raise ValueError("Tests are not allowed to set pre-alloc items in execute mode") + + def code_pre_processor( + self, code: Bytecode | Container, *, evm_code_type: EVMCodeType | None + ) -> Bytecode | Container: + """ + Pre-processes the code before setting it. + """ + if evm_code_type is None: + evm_code_type = self._evm_code_type + if evm_code_type == EVMCodeType.EOF_V1: + if not isinstance(code, Container): + if isinstance(code, Bytecode) and not code.terminating: + return Container.Code(code + Opcodes.STOP) + return Container.Code(code) + return code + + def deploy_contract( + self, + code: BytesConvertible, + *, + storage: Storage | StorageRootType = {}, + balance: NumberConvertible = 0, + nonce: NumberConvertible = 1, + address: Address | None = None, + evm_code_type: EVMCodeType | None = None, + label: str | None = None, + ) -> Address: + """ + Deploy a contract to the allocation. + """ + assert address is None, "address parameter is not supported" + + if not isinstance(storage, Storage): + storage = Storage(storage) # type: ignore + + initcode_prefix = Bytecode() + + deploy_gas_limit = 21_000 + 32_000 + + if len(storage.root) > 0: + initcode_prefix += sum(Op.SSTORE(key, value) for key, value in storage.root.items()) + deploy_gas_limit += len(storage.root) * 22_600 + + assert isinstance(code, Bytecode) or isinstance( + code, Container + ), f"incompatible code type: {type(code)}" + code = self.code_pre_processor(code, evm_code_type=evm_code_type) + + assert len(code) <= MAX_BYTECODE_SIZE, f"code too large: {len(code)} > {MAX_BYTECODE_SIZE}" + + deploy_gas_limit += len(bytes(code)) * 200 + + initcode: Bytecode | Container + + if evm_code_type == EVMCodeType.EOF_V1: + assert isinstance(code, Container) + initcode = Container.Init(deploy_container=code, initcode_prefix=initcode_prefix) + else: + initcode = Initcode(deploy_code=code, initcode_prefix=initcode_prefix) + deploy_gas_limit += cost_memory_bytes(len(bytes(initcode)), 0) + + assert ( + len(initcode) <= MAX_INITCODE_SIZE + ), f"initcode too large {len(initcode)} > {MAX_INITCODE_SIZE}" + + deploy_gas_limit += eip_2028_transaction_data_cost(bytes(initcode)) + + # Limit the gas limit + deploy_gas_limit = min(deploy_gas_limit * 2, 30_000_000) + print(f"Deploying contract with gas limit: {deploy_gas_limit}") + + deploy_tx = Transaction( + sender=self._sender, + to=None, + data=initcode, + value=balance, + gas_limit=deploy_gas_limit, + ).with_signature_and_sender() + self._eth_rpc.send_transaction(deploy_tx) + self._txs.append(deploy_tx) + + contract_address = deploy_tx.created_contract + self._deployed_contracts.append((contract_address, bytes(code))) + + assert Number(nonce) >= 1, "impossible to deploy contract with nonce lower than one" + + super().__setitem__( + contract_address, + Account( + nonce=nonce, + balance=balance, + code=code, + storage=storage, + ), + ) + + contract_address.label = label + return contract_address + + def fund_eoa( + self, + amount: NumberConvertible | None = None, + label: str | None = None, + storage: Storage | None = None, + delegation: Address | Literal["Self"] | None = None, + nonce: NumberConvertible | None = None, + ) -> EOA: + """ + Add a previously unused EOA to the pre-alloc with the balance specified by `amount`. + """ + assert nonce is None, "nonce parameter is not supported for execute" + eoa = next(self._eoa_iterator) + # Send a transaction to fund the EOA + if amount is None: + amount = self._eoa_fund_amount_default + + if delegation is not None or storage is not None: + if storage is not None: + sstore_address = self.deploy_contract( + code=( + sum(Op.SSTORE(key, value) for key, value in storage.root.items()) + Op.STOP + ) + ) + set_storage_tx = Transaction( + sender=self._sender, + to=eoa, + authorization_list=[ + AuthorizationTuple( + chain_id=self._chain_id, + address=sstore_address, + nonce=eoa.nonce, + signer=eoa, + ), + ], + gas_limit=100_000, + ).with_signature_and_sender() + eoa.nonce = Number(eoa.nonce + 1) + self._eth_rpc.send_transaction(set_storage_tx) + self._txs.append(set_storage_tx) + + if delegation is not None: + if not isinstance(delegation, Address) and delegation == "Self": + delegation = eoa + # TODO: This tx has side-effects on the EOA state because of the delegation + fund_tx = Transaction( + sender=self._sender, + to=eoa, + value=amount, + authorization_list=[ + AuthorizationTuple( + chain_id=self._chain_id, + address=delegation, + nonce=eoa.nonce, + signer=eoa, + ), + ], + gas_limit=100_000, + ).with_signature_and_sender() + eoa.nonce = Number(eoa.nonce + 1) + else: + fund_tx = Transaction( + sender=self._sender, + to=eoa, + value=amount, + authorization_list=[ + AuthorizationTuple( + chain_id=self._chain_id, + address=0, # Reset delegation to an address without code + nonce=eoa.nonce, + signer=eoa, + ), + ], + gas_limit=100_000, + ).with_signature_and_sender() + eoa.nonce = Number(eoa.nonce + 1) + + else: + fund_tx = Transaction( + sender=self._sender, + to=eoa, + value=amount, + ).with_signature_and_sender() + + self._eth_rpc.send_transaction(fund_tx) + self._txs.append(fund_tx) + super().__setitem__( + eoa, + Account( + nonce=eoa.nonce, + balance=amount, + ), + ) + self._funded_eoa.append(eoa) + return eoa + + def fund_address(self, address: Address, amount: NumberConvertible): + """ + Fund an address with a given amount. + + If the address is already present in the pre-alloc the amount will be + added to its existing balance. + """ + fund_tx = Transaction( + sender=self._sender, + to=address, + value=amount, + ).with_signature_and_sender() + self._eth_rpc.send_transaction(fund_tx) + self._txs.append(fund_tx) + if address in self: + account = self[address] + if account is not None: + current_balance = account.balance or 0 + account.balance = ZeroPaddedHexNumber(current_balance + Number(amount)) + return + + super().__setitem__(address, Account(balance=amount)) + + def wait_for_transactions(self) -> List[TransactionByHashResponse]: + """ + Wait for all transactions to be included in blocks. + """ + return self._eth_rpc.wait_for_transactions(self._txs) + + +@pytest.fixture(autouse=True) +def evm_code_type(request: pytest.FixtureRequest) -> EVMCodeType: + """ + Returns the default EVM code type for all tests (LEGACY). + """ + parameter_evm_code_type = request.config.getoption("evm_code_type") + if parameter_evm_code_type is not None: + assert type(parameter_evm_code_type) is EVMCodeType, "Invalid EVM code type" + return parameter_evm_code_type + return EVMCodeType.LEGACY + + +@pytest.fixture(scope="session") +def eoa_fund_amount_default(request: pytest.FixtureRequest) -> int: + """ + Get the gas price for the funding transactions. + """ + return request.config.option.eoa_fund_amount_default + + +@pytest.fixture(autouse=True, scope="function") +def pre( + sender_key: EOA, + eoa_iterator: Iterator[EOA], + eth_rpc: EthRPC, + evm_code_type: EVMCodeType, + chain_id: int, + eoa_fund_amount_default: int, +) -> Alloc: + """ + Returns the default pre allocation for all tests (Empty alloc). + """ + return Alloc( + sender=sender_key, + eth_rpc=eth_rpc, + eoa_iterator=eoa_iterator, + evm_code_type=evm_code_type, + chain_id=chain_id, + eoa_fund_amount_default=eoa_fund_amount_default, + ) diff --git a/src/pytest_plugins/execute/recover.py b/src/pytest_plugins/execute/recover.py new file mode 100644 index 0000000000..2b169b59f6 --- /dev/null +++ b/src/pytest_plugins/execute/recover.py @@ -0,0 +1,66 @@ +""" +Pytest plugin to recover funds from a failed remote execution. +""" +import pytest + +from ethereum_test_base_types import Address, HexNumber +from ethereum_test_types import EOA + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + recover_group = parser.getgroup("execute", "Arguments defining fund recovery behavior.") + recover_group.addoption( + "--start-eoa-index", + action="store", + dest="start_eoa_index", + type=HexNumber, + required=True, + default=None, + help=("Starting private key index to use for EOA generation."), + ) + recover_group.addoption( + "--destination", + action="store", + dest="destination", + type=Address, + required=True, + default=None, + help=("Address to send the recovered funds to."), + ) + recover_group.addoption( + "--max-index", + action="store", + dest="max_index", + type=int, + default=100, + help=("Maximum private key index to use for EOA generation."), + ) + + +@pytest.fixture(scope="session") +def destination(request: pytest.FixtureRequest) -> Address: + """ + Get the destination address. + """ + return request.config.option.destination + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + """ + Pytest hook used to dynamically generate test cases. + """ + max_index = metafunc.config.option.max_index + start_eoa_index = metafunc.config.option.start_eoa_index + + print(f"Generating {max_index} test cases starting from index {start_eoa_index}") + + indexes_keys = [(index, EOA(key=start_eoa_index + index)) for index in range(max_index)] + + metafunc.parametrize( + ["index", "eoa"], + indexes_keys, + ids=[f"{index}-{eoa}" for index, eoa in indexes_keys], + ) diff --git a/src/pytest_plugins/execute/rpc/__init__.py b/src/pytest_plugins/execute/rpc/__init__.py new file mode 100644 index 0000000000..530d0bba81 --- /dev/null +++ b/src/pytest_plugins/execute/rpc/__init__.py @@ -0,0 +1,3 @@ +""" +RPC plugins to execute tests in different environments. +""" diff --git a/src/pytest_plugins/execute/rpc/hive.py b/src/pytest_plugins/execute/rpc/hive.py new file mode 100644 index 0000000000..57f2f9cc3d --- /dev/null +++ b/src/pytest_plugins/execute/rpc/hive.py @@ -0,0 +1,871 @@ +""" +Pytest plugin to run the test-execute in hive-mode. +""" + +import io +import json +import os +import time +from dataclasses import asdict, replace +from pathlib import Path +from random import randint +from typing import Any, Dict, Generator, List, Mapping, Tuple, cast + +import pytest +from ethereum.crypto.hash import keccak256 +from filelock import FileLock +from hive.client import Client, ClientType +from hive.simulation import Simulation +from hive.testing import HiveTest, HiveTestResult, HiveTestSuite +from pydantic import RootModel + +from ethereum_test_base_types import EmptyOmmersRoot, EmptyTrieRoot, HexNumber, to_json +from ethereum_test_fixtures.blockchain import FixtureHeader +from ethereum_test_forks import Fork, get_forks +from ethereum_test_rpc import EngineRPC +from ethereum_test_rpc import EthRPC as BaseEthRPC +from ethereum_test_rpc.types import ( + ForkchoiceState, + PayloadAttributes, + PayloadStatusEnum, + TransactionByHashResponse, +) +from ethereum_test_tools import ( + EOA, + Account, + Address, + Alloc, + Environment, + Hash, + Transaction, + Withdrawal, +) +from ethereum_test_types import Requests +from pytest_plugins.consume.hive_simulators.ruleset import ruleset + + +class HashList(RootModel[List[Hash]]): + """Hash list class""" + + root: List[Hash] + + def append(self, item: Hash): + """Append an item to the list""" + self.root.append(item) + + def clear(self): + """Clear the list""" + self.root.clear() + + def remove(self, item: Hash): + """Remove an item from the list""" + self.root.remove(item) + + def __contains__(self, item: Hash): + """Check if an item is in the list""" + return item in self.root + + def __len__(self): + """Get the length of the list""" + return len(self.root) + + def __iter__(self): + """Iterate over the list""" + return iter(self.root) + + +class AddressList(RootModel[List[Address]]): + """Address list class""" + + root: List[Address] + + def append(self, item: Address): + """Append an item to the list""" + self.root.append(item) + + def clear(self): + """Clear the list""" + self.root.clear() + + def remove(self, item: Address): + """Remove an item from the list""" + self.root.remove(item) + + def __contains__(self, item: Address): + """Check if an item is in the list""" + return item in self.root + + def __len__(self): + """Get the length of the list""" + return len(self.root) + + def __iter__(self): + """Iterate over the list""" + return iter(self.root) + + +def get_fork_option(request, option_name: str) -> Fork | None: + """Post-process get option to allow for external fork conditions.""" + option = request.config.getoption(option_name) + if option := request.config.getoption(option_name): + if option == "Merge": + option = "Paris" + for fork in get_forks(): + if option == fork.name(): + return fork + return None + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + hive_rpc_group = parser.getgroup( + "hive_rpc", "Arguments defining the hive RPC client properties for the test." + ) + hive_rpc_group.addoption( + "--transactions-per-block", + action="store", + dest="transactions_per_block", + type=int, + default=None, + help=("Number of transactions to send before producing the next block."), + ) + hive_rpc_group.addoption( + "--get-payload-wait-time", + action="store", + dest="get_payload_wait_time", + type=float, + default=0.3, + help=("Time to wait after sending a forkchoice_updated before getting the payload."), + ) + hive_rpc_group.addoption( + "--sender-key-initial-balance", + action="store", + dest="sender_key_initial_balance", + type=int, + default=10**26, + help=( + "Initial balance of each sender key. There is one sender key per worker process " + "(`-n` option)." + ), + ) + hive_rpc_group.addoption( + "--tx-wait-timeout", + action="store", + dest="tx_wait_timeout", + type=int, + default=10, # Lowered from Remote RPC because of the consistent block production + help="Maximum time in seconds to wait for a transaction to be included in a block", + ) + + +def pytest_configure(config): # noqa: D103 + config.test_suite_scope = "session" + + +@pytest.fixture(scope="session") +def base_fork(request) -> Fork: + """ + Get the base fork for all tests. + """ + fork = get_fork_option(request, "single_fork") + assert fork is not None, "invalid fork requested" + return fork + + +@pytest.fixture(scope="session") +def seed_sender(session_temp_folder: Path) -> EOA: + """ + Determine the seed sender account for the client's genesis. + """ + base_name = "seed_sender" + base_file = session_temp_folder / base_name + base_lock_file = session_temp_folder / f"{base_name}.lock" + + with FileLock(base_lock_file): + if base_file.exists(): + with base_file.open("r") as f: + seed_sender_key = Hash(f.read()) + seed_sender = EOA(key=seed_sender_key) + else: + seed_sender = EOA(key=randint(0, 2**256)) + with base_file.open("w") as f: + f.write(str(seed_sender.key)) + return seed_sender + + +@pytest.fixture(scope="session") +def base_pre(request, seed_sender: EOA, worker_count: int) -> Alloc: + """ + Base pre-allocation for the client's genesis. + """ + sender_key_initial_balance = request.config.getoption("sender_key_initial_balance") + return Alloc( + {seed_sender: Account(balance=(worker_count * sender_key_initial_balance) + 10**18)} + ) + + +@pytest.fixture(scope="session") +def base_pre_genesis( + base_fork: Fork, + base_pre: Alloc, +) -> Tuple[Alloc, FixtureHeader]: + """ + Create a genesis block from the blockchain test definition. + """ + env = Environment().set_fork_requirements(base_fork) + assert ( + env.withdrawals is None or len(env.withdrawals) == 0 + ), "withdrawals must be empty at genesis" + assert env.parent_beacon_block_root is None or env.parent_beacon_block_root == Hash( + 0 + ), "parent_beacon_block_root must be empty at genesis" + + pre_alloc = Alloc.merge( + Alloc.model_validate(base_fork.pre_allocation_blockchain()), + base_pre, + ) + if empty_accounts := pre_alloc.empty_accounts(): + raise Exception(f"Empty accounts in pre state: {empty_accounts}") + state_root = pre_alloc.state_root() + genesis = FixtureHeader( + parent_hash=0, + ommers_hash=EmptyOmmersRoot, + fee_recipient=0, + state_root=state_root, + transactions_trie=EmptyTrieRoot, + receipts_root=EmptyTrieRoot, + logs_bloom=0, + difficulty=0x20000 if env.difficulty is None else env.difficulty, + number=0, + gas_limit=env.gas_limit, + gas_used=0, + timestamp=1, + extra_data=b"\x00", + prev_randao=0, + nonce=0, + base_fee_per_gas=env.base_fee_per_gas, + blob_gas_used=env.blob_gas_used, + excess_blob_gas=env.excess_blob_gas, + withdrawals_root=Withdrawal.list_root(env.withdrawals) + if env.withdrawals is not None + else None, + parent_beacon_block_root=env.parent_beacon_block_root, + requests_root=Requests(root=[]).trie_root + if base_fork.header_requests_required(0, 0) + else None, + ) + + return (pre_alloc, genesis) + + +@pytest.fixture(scope="session") +def base_genesis_header(base_pre_genesis: Tuple[Alloc, FixtureHeader]) -> FixtureHeader: + """ + Return the genesis header for the current test fixture. + """ + return base_pre_genesis[1] + + +@pytest.fixture(scope="session") +def client_genesis(base_pre_genesis: Tuple[Alloc, FixtureHeader]) -> dict: + """ + Convert the fixture's genesis block header and pre-state to a client genesis state. + """ + genesis = to_json(base_pre_genesis[1]) # NOTE: to_json() excludes None values + alloc = to_json(base_pre_genesis[0]) + # NOTE: nethermind requires account keys without '0x' prefix + genesis["alloc"] = {k.replace("0x", ""): v for k, v in alloc.items()} + return genesis + + +@pytest.fixture(scope="session") +def buffered_genesis(client_genesis: dict) -> io.BufferedReader: + """ + Create a buffered reader for the genesis block header of the current test + fixture. + """ + genesis_json = json.dumps(client_genesis) + genesis_bytes = genesis_json.encode("utf-8") + return io.BufferedReader(cast(io.RawIOBase, io.BytesIO(genesis_bytes))) + + +@pytest.fixture(scope="session") +def client_files( + buffered_genesis: io.BufferedReader, +) -> Mapping[str, io.BufferedReader]: + """ + Define the files that hive will start the client with. + + For this type of test, only the genesis is passed + """ + files = {} + files["/genesis.json"] = buffered_genesis + return files + + +@pytest.fixture(scope="session") +def environment(base_fork: Fork) -> dict: + """ + Define the environment that hive will start the client with using the fork + rules specific for the simulator. + """ + assert base_fork.name() in ruleset, f"fork '{base_fork.name()}' missing in hive ruleset" + return { + "HIVE_CHAIN_ID": "1", + "HIVE_FORK_DAO_VOTE": "1", + "HIVE_NODETYPE": "full", + **{k: f"{v:d}" for k, v in ruleset[base_fork.name()].items()}, + } + + +@pytest.fixture(scope="session") +def test_suite_name() -> str: + """ + The name of the hive test suite used in this simulator. + """ + return "EEST Execute Test, Hive Mode" + + +@pytest.fixture(scope="session") +def test_suite_description() -> str: + """ + The description of the hive test suite used in this simulator. + """ + return "Execute EEST tests using hive endpoint." + + +@pytest.fixture(autouse=True, scope="session") +def base_hive_test( + request: pytest.FixtureRequest, test_suite: HiveTestSuite, session_temp_folder: Path +) -> Generator[HiveTest, None, None]: + """ + Base test used to deploy the main client to be used throughout all tests. + """ + base_name = "base_hive_test" + base_file = session_temp_folder / base_name + base_lock_file = session_temp_folder / f"{base_name}.lock" + with FileLock(base_lock_file): + if base_file.exists(): + with open(base_file, "r") as f: + test = HiveTest(**json.load(f)) + else: + test = test_suite.start_test( + name="Base Hive Test", + description=( + "Base test used to deploy the main client to be used throughout all tests." + ), + ) + with open(base_file, "w") as f: + json.dump(asdict(test), f) + + users_file_name = f"{base_name}_users" + users_file = session_temp_folder / users_file_name + users_lock_file = session_temp_folder / f"{users_file_name}.lock" + with FileLock(users_lock_file): + if users_file.exists(): + with open(users_file, "r") as f: + users = json.load(f) + else: + users = 0 + users += 1 + with open(users_file, "w") as f: + json.dump(users, f) + + yield test + + test_pass = True + test_details = "All tests have completed" + if request.session.testsfailed > 0: # noqa: SC200 + test_pass = False + test_details = "One or more tests have failed" + + with FileLock(users_lock_file): + with open(users_file, "r") as f: + users = json.load(f) + users -= 1 + with open(users_file, "w") as f: + json.dump(users, f) + if users == 0: + test.end(result=HiveTestResult(test_pass=test_pass, details=test_details)) + base_file.unlink() + users_file.unlink() + + +@pytest.fixture(scope="session") +def client_type(simulator: Simulation) -> ClientType: + """ + The type of client to be used in the test. + """ + return simulator.client_types()[0] + + +@pytest.fixture(autouse=True, scope="session") +def client( + base_hive_test: HiveTest, + client_files: dict, + environment: dict, + client_type: ClientType, + session_temp_folder: Path, +) -> Generator[Client, None, None]: + """ + Initialize the client with the appropriate files and environment variables. + """ + base_name = "hive_client" + base_file = session_temp_folder / base_name + base_error_file = session_temp_folder / f"{base_name}.err" + base_lock_file = session_temp_folder / f"{base_name}.lock" + client: Client | None = None + with FileLock(base_lock_file): + if not base_error_file.exists(): + if base_file.exists(): + with open(base_file, "r") as f: + client = Client(**json.load(f)) + else: + base_error_file.touch() # Assume error + client = base_hive_test.start_client( + client_type=client_type, environment=environment, files=client_files + ) + if client is not None: + base_error_file.unlink() # Success + with open(base_file, "w") as f: + json.dump( + asdict(replace(client, config=None)), # type: ignore + f, + ) + + error_message = ( + f"Unable to connect to the client container ({client_type.name}) via Hive during test " + "setup. Check the client or Hive server logs for more information." + ) + assert client is not None, error_message + + users_file_name = f"{base_name}_users" + users_file = session_temp_folder / users_file_name + users_lock_file = session_temp_folder / f"{users_file_name}.lock" + with FileLock(users_lock_file): + if users_file.exists(): + with open(users_file, "r") as f: + users = json.load(f) + else: + users = 0 + users += 1 + with open(users_file, "w") as f: + json.dump(users, f) + + yield client + + with FileLock(users_lock_file): + with open(users_file, "r") as f: + users = json.load(f) + users -= 1 + with open(users_file, "w") as f: + json.dump(users, f) + if users == 0: + client.stop() + base_file.unlink() + users_file.unlink() + + +class PendingTxHashes: + """ + A class to manage the pending transaction hashes in a multi-process environment. + + It uses a lock file to ensure that only one process can access the pending hashes file at a + time. + """ + + pending_hashes_file: Path + pending_hashes_lock: Path + pending_tx_hashes: HashList | None + lock: FileLock | None + + def __init__(self, temp_folder: Path): + self.pending_hashes_file = temp_folder / "pending_tx_hashes" + self.pending_hashes_lock = temp_folder / "pending_tx_hashes.lock" + self.pending_tx_hashes = None + self.lock = None + + def __enter__(self): + """ + Lock the pending hashes file and load it. + """ + assert self.lock is None, "Lock already acquired" + self.lock = FileLock(self.pending_hashes_lock, timeout=-1) + self.lock.acquire() + assert self.pending_tx_hashes is None, "Pending transaction hashes already loaded" + if self.pending_hashes_file.exists(): + with open(self.pending_hashes_file, "r") as f: + self.pending_tx_hashes = HashList.model_validate_json(f.read()) + else: + self.pending_tx_hashes = HashList([]) + return self + + def __exit__(self, exc_type, exc_value, traceback): + """ + Flush the pending hashes to the file and release the lock. + """ + assert self.lock is not None, "Lock not acquired" + assert self.pending_tx_hashes is not None, "Pending transaction hashes not loaded" + with open(self.pending_hashes_file, "w") as f: + f.write(self.pending_tx_hashes.model_dump_json()) + self.lock.release() + self.lock = None + self.pending_tx_hashes = None + + def append(self, tx_hash: Hash): + """ + Add a transaction hash to the pending list. + """ + assert self.lock is not None, "Lock not acquired" + assert self.pending_tx_hashes is not None, "Pending transaction hashes not loaded" + self.pending_tx_hashes.append(tx_hash) + + def clear(self): + """ + Remove a transaction hash from the pending list. + """ + assert self.lock is not None, "Lock not acquired" + self.pending_tx_hashes.clear() + + def remove(self, tx_hash: Hash): + """ + Remove a transaction hash from the pending list. + """ + assert self.lock is not None, "Lock not acquired" + assert self.pending_tx_hashes is not None, "Pending transaction hashes not loaded" + self.pending_tx_hashes.remove(tx_hash) + + def __contains__(self, tx_hash: Hash): + """ + Check if a transaction hash is in the pending list. + """ + assert self.lock is not None, "Lock not acquired" + assert self.pending_tx_hashes is not None, "Pending transaction hashes not loaded" + return tx_hash in self.pending_tx_hashes + + def __len__(self): + """ + Get the number of pending transaction hashes. + """ + assert self.lock is not None, "Lock not acquired" + assert self.pending_tx_hashes is not None, "Pending transaction hashes not loaded" + return len(self.pending_tx_hashes) + + def __iter__(self): + """ + Iterate over the pending transaction hashes. + """ + assert self.lock is not None, "Lock not acquired" + assert self.pending_tx_hashes is not None, "Pending transaction hashes not loaded" + return iter(self.pending_tx_hashes) + + +class EthRPC(BaseEthRPC): + """ + Ethereum RPC client for the hive simulator which automatically sends Engine API requests to + generate blocks after a certain number of transactions have been sent. + """ + + fork: Fork + engine_rpc: EngineRPC + transactions_per_block: int + get_payload_wait_time: float + pending_tx_hashes: PendingTxHashes + + def __init__( + self, + *, + client: Client, + fork: Fork, + base_genesis_header: FixtureHeader, + transactions_per_block: int, + session_temp_folder: Path, + get_payload_wait_time: float, + initial_forkchoice_update_retries: int = 5, + transaction_wait_timeout: int = 60, + ): + """ + Initialize the Ethereum RPC client for the hive simulator. + """ + super().__init__( + f"http://{client.ip}:8545", + transaction_wait_timeout=transaction_wait_timeout, + ) + self.fork = fork + self.engine_rpc = EngineRPC(f"http://{client.ip}:8551") + self.transactions_per_block = transactions_per_block + self.pending_tx_hashes = PendingTxHashes(session_temp_folder) + self.get_payload_wait_time = get_payload_wait_time + + # Send initial forkchoice updated only if we are the first worker + base_name = "eth_rpc_forkchoice_updated" + base_file = session_temp_folder / base_name + base_error_file = session_temp_folder / f"{base_name}.err" + base_lock_file = session_temp_folder / f"{base_name}.lock" + + with FileLock(base_lock_file): + if base_error_file.exists(): + raise Exception("Error occurred during initial forkchoice_updated") + if not base_file.exists(): + base_error_file.touch() # Assume error + # Send initial forkchoice updated + forkchoice_state = ForkchoiceState( + head_block_hash=base_genesis_header.block_hash, + ) + forkchoice_version = self.fork.engine_forkchoice_updated_version() + assert ( + forkchoice_version is not None + ), "Fork does not support engine forkchoice_updated" + for _ in range(initial_forkchoice_update_retries): + response = self.engine_rpc.forkchoice_updated( + forkchoice_state, + None, + version=forkchoice_version, + ) + if response.payload_status.status == PayloadStatusEnum.VALID: + break + time.sleep(0.5) + else: + raise Exception("Initial forkchoice_updated was invalid") + base_error_file.unlink() # Success + base_file.touch() + + def generate_block(self: "EthRPC"): + """ + Generate a block using the Engine API. + """ + # Get the head block hash + head_block = self.get_block_by_number("latest") + + forkchoice_state = ForkchoiceState( + head_block_hash=head_block["hash"], + ) + parent_beacon_block_root = Hash(0) if self.fork.header_beacon_root_required(0, 0) else None + payload_attributes = PayloadAttributes( + timestamp=HexNumber(head_block["timestamp"]) + 1, + prev_randao=Hash(0), + suggested_fee_recipient=Address(0), + withdrawals=[] if self.fork.header_withdrawals_required() else None, + parent_beacon_block_root=parent_beacon_block_root, + ) + forkchoice_updated_version = self.fork.engine_forkchoice_updated_version() + assert ( + forkchoice_updated_version is not None + ), "Fork does not support engine forkchoice_updated" + response = self.engine_rpc.forkchoice_updated( + forkchoice_state, + payload_attributes, + version=forkchoice_updated_version, + ) + assert response.payload_status.status == PayloadStatusEnum.VALID, "Payload was invalid" + assert response.payload_id is not None, "payload_id was not returned by the client" + time.sleep(self.get_payload_wait_time) + get_payload_version = self.fork.engine_get_payload_version() + assert get_payload_version is not None, "Fork does not support engine get_payload" + new_payload = self.engine_rpc.get_payload( + response.payload_id, + version=get_payload_version, + ) + new_payload_args: List[Any] = [new_payload.execution_payload] + if new_payload.blobs_bundle is not None: + new_payload_args.append(new_payload.blobs_bundle.blob_versioned_hashes()) + if parent_beacon_block_root is not None: + new_payload_args.append(parent_beacon_block_root) + new_payload_version = self.fork.engine_new_payload_version() + assert new_payload_version is not None, "Fork does not support engine new_payload" + new_payload_response = self.engine_rpc.new_payload( + *new_payload_args, version=new_payload_version + ) + assert new_payload_response.status == PayloadStatusEnum.VALID, "Payload was invalid" + + new_forkchoice_state = ForkchoiceState( + head_block_hash=new_payload.execution_payload.block_hash, + ) + response = self.engine_rpc.forkchoice_updated( + new_forkchoice_state, + None, + version=forkchoice_updated_version, + ) + assert response.payload_status.status == PayloadStatusEnum.VALID, "Payload was invalid" + for tx in new_payload.execution_payload.transactions: + tx_hash = Hash(keccak256(tx)) + if tx_hash in self.pending_tx_hashes: + self.pending_tx_hashes.remove(tx_hash) + + def send_transaction(self, transaction: Transaction) -> Hash: + """ + `eth_sendRawTransaction`: Send a transaction to the client. + """ + returned_hash = super().send_transaction(transaction) + with self.pending_tx_hashes: + self.pending_tx_hashes.append(transaction.hash) + if len(self.pending_tx_hashes) >= self.transactions_per_block: + self.generate_block() + return returned_hash + + def wait_for_transaction(self, transaction: Transaction) -> TransactionByHashResponse: + """ + Wait for a specific transaction to be included in a block. + + Waits for a specific transaction to be included in a block by polling + `eth_getTransactionByHash` until it is confirmed or a timeout occurs. + + Args: + transaction: The transaction to track. + + Returns: + The transaction details after it is included in a block. + """ + return self.wait_for_transactions([transaction])[0] + + def wait_for_transactions( + self, transactions: List[Transaction] + ) -> List[TransactionByHashResponse]: + """ + Wait for all transactions in the provided list to be included in a block. + + Waits for all transactions in the provided list to be included in a block + by polling `eth_getTransactionByHash` until they are confirmed or a + timeout occurs. + + Args: + transactions: A list of transactions to track. + + Returns: + A list of transaction details after they are included in a block. + + Raises: + Exception: If one or more transactions are not included in a block + within the timeout period. + """ + tx_hashes = [tx.hash for tx in transactions] + responses: List[TransactionByHashResponse] = [] + pending_responses: Dict[Hash, TransactionByHashResponse] = {} + + start_time = time.time() + pending_transactions_handler = PendingTransactionHandler(self) + while True: + tx_id = 0 + pending_responses = {} + while tx_id < len(tx_hashes): + tx_hash = tx_hashes[tx_id] + tx = self.get_transaction_by_hash(tx_hash) + if tx.block_number is not None: + responses.append(tx) + tx_hashes.pop(tx_id) + else: + pending_responses[tx_hash] = tx + tx_id += 1 + + if not tx_hashes: + return responses + + pending_transactions_handler.handle() + + if (time.time() - start_time) > self.transaction_wait_timeout: + break + time.sleep(0.1) + + missing_txs_strings = [ + f"{tx.hash} ({tx.model_dump_json()})" for tx in transactions if tx.hash in tx_hashes + ] + + pending_tx_responses_string = "\n".join( + [f"{tx_hash}: {tx.model_dump_json()}" for tx_hash, tx in pending_responses.items()] + ) + raise Exception( + f"Transactions {', '.join(missing_txs_strings)} were not included in a block " + f"within {self.transaction_wait_timeout} seconds:\n" + f"{pending_tx_responses_string}" + ) + + +class PendingTransactionHandler: + """Manages block generation based on the number of pending transactions or a block generation + interval. + + Attributes: + block_generation_interval: The number of iterations after which a block + is generated if no new transactions are added (default: 10). + """ + + eth_rpc: EthRPC + block_generation_interval: int + last_pending_tx_hashes_count: int | None = None + i: int = 0 + + def __init__(self, eth_rpc: EthRPC, block_generation_interval: int = 10): + """Initialize the pending transaction handler.""" + self.eth_rpc = eth_rpc + self.block_generation_interval = block_generation_interval + + def handle(self): + """Handle pending transactions and generate blocks if necessary. + + If the number of pending transactions reaches the limit, a block is generated. + + If no new transactions have been added to the pending list and the block + generation interval has been reached, a block is generated to avoid potential + deadlock. + """ + with self.eth_rpc.pending_tx_hashes: + if len(self.eth_rpc.pending_tx_hashes) >= self.eth_rpc.transactions_per_block: + self.eth_rpc.generate_block() + else: + if ( + self.last_pending_tx_hashes_count is not None + and len(self.eth_rpc.pending_tx_hashes) == self.last_pending_tx_hashes_count + and self.i % self.block_generation_interval == 0 + ): + # If no new transactions have been added to the pending list, + # generate a block to avoid potential deadlock. + self.eth_rpc.generate_block() + self.last_pending_tx_hashes_count = len(self.eth_rpc.pending_tx_hashes) + self.i += 1 + + +@pytest.fixture(scope="session") +def transactions_per_block(request) -> int: # noqa: D103 + if transactions_per_block := request.config.getoption("transactions_per_block"): + return transactions_per_block + + # Get the number of workers for the test + worker_count_env = os.environ.get("PYTEST_XDIST_WORKER_COUNT") + if not worker_count_env: + return 1 + return max(int(worker_count_env), 1) + + +@pytest.fixture(scope="session") +def chain_id() -> int: + """ + Returns the chain id where the tests will be executed. + """ + return 1 + + +@pytest.fixture(autouse=True, scope="session") +def eth_rpc( + request: pytest.FixtureRequest, + client: Client, + base_genesis_header: FixtureHeader, + base_fork: Fork, + transactions_per_block: int, + session_temp_folder: Path, +) -> EthRPC: + """ + Initialize ethereum RPC client for the execution client under test. + """ + get_payload_wait_time = request.config.getoption("get_payload_wait_time") + tx_wait_timeout = request.config.getoption("tx_wait_timeout") + return EthRPC( + client=client, + fork=base_fork, + base_genesis_header=base_genesis_header, + transactions_per_block=transactions_per_block, + session_temp_folder=session_temp_folder, + get_payload_wait_time=get_payload_wait_time, + transaction_wait_timeout=tx_wait_timeout, + ) diff --git a/src/pytest_plugins/execute/rpc/remote.py b/src/pytest_plugins/execute/rpc/remote.py new file mode 100644 index 0000000000..6d7dc965d6 --- /dev/null +++ b/src/pytest_plugins/execute/rpc/remote.py @@ -0,0 +1,67 @@ +""" +Pytest plugin to run the execute in remote-rpc-mode. +""" + +import pytest + +from ethereum_test_rpc import EthRPC +from ethereum_test_types import TransactionDefaults + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + remote_rpc_group = parser.getgroup("remote_rpc", "Arguments defining remote RPC configuration") + remote_rpc_group.addoption( + "--rpc-endpoint", + required=True, + action="store", + dest="rpc_endpoint", + help="RPC endpoint to an execution client", + ) + remote_rpc_group.addoption( + "--rpc-chain-id", + action="store", + dest="rpc_chain_id", + required=True, + type=int, + default=None, + help="ID of the chain where the tests will be executed.", + ) + remote_rpc_group.addoption( + "--tx-wait-timeout", + action="store", + dest="tx_wait_timeout", + type=int, + default=60, + help="Maximum time in seconds to wait for a transaction to be included in a block", + ) + + +@pytest.fixture(autouse=True, scope="session") +def rpc_endpoint(request) -> str: + """ + Returns the remote RPC endpoint to be used to make requests to the execution client. + """ + return request.config.getoption("rpc_endpoint") + + +@pytest.fixture(autouse=True, scope="session") +def chain_id(request) -> int: + """ + Returns the chain id where the tests will be executed. + """ + chain_id = request.config.getoption("rpc_chain_id") + if chain_id is not None: + TransactionDefaults.chain_id = chain_id + return chain_id + + +@pytest.fixture(autouse=True, scope="session") +def eth_rpc(request, rpc_endpoint: str) -> EthRPC: + """ + Initialize ethereum RPC client for the execution client under test. + """ + tx_wait_timeout = request.config.getoption("tx_wait_timeout") + return EthRPC(rpc_endpoint, transaction_wait_timeout=tx_wait_timeout) diff --git a/src/pytest_plugins/execute/rpc/remote_seed_sender.py b/src/pytest_plugins/execute/rpc/remote_seed_sender.py new file mode 100644 index 0000000000..99265364c6 --- /dev/null +++ b/src/pytest_plugins/execute/rpc/remote_seed_sender.py @@ -0,0 +1,43 @@ +""" +Seed sender on a remote execution client. +""" +import pytest + +from ethereum_test_base_types import Hash, Number +from ethereum_test_rpc import EthRPC +from ethereum_test_types import EOA + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + remote_seed_sender_group = parser.getgroup( + "remote_seed_sender", + "Arguments for the remote seed sender", + ) + + remote_seed_sender_group.addoption( + "--rpc-seed-key", + action="store", + required=True, + dest="rpc_seed_key", + help=( + "Seed key used to fund all sender keys. This account must have a balance of at least " + "`sender_key_initial_balance` * `workers` + gas fees. It should also be " + "exclusively used by this command because the nonce is only checked once and if " + "it's externally increased, the seed transactions might fail." + ), + ) + + +@pytest.fixture(scope="session") +def seed_sender(request, eth_rpc: EthRPC) -> EOA: + """ + Setup the seed sender account by checking its balance and nonce. + """ + rpc_seed_key = Hash(request.config.getoption("rpc_seed_key")) + # check the nonce through the rpc client + seed_sender = EOA(key=rpc_seed_key) + seed_sender.nonce = Number(eth_rpc.get_transaction_count(seed_sender)) + return seed_sender diff --git a/src/pytest_plugins/execute/sender.py b/src/pytest_plugins/execute/sender.py new file mode 100644 index 0000000000..5700b63a0d --- /dev/null +++ b/src/pytest_plugins/execute/sender.py @@ -0,0 +1,201 @@ +""" +Sender mutex class that allows sending transactions one at a time. +""" +from pathlib import Path +from typing import Generator, Iterator + +import pytest +from filelock import FileLock +from pytest_metadata.plugin import metadata_key # type: ignore + +from ethereum_test_base_types import Number, Wei +from ethereum_test_rpc import EthRPC +from ethereum_test_tools import EOA, Transaction + + +def pytest_addoption(parser): + """ + Adds command-line options to pytest. + """ + sender_group = parser.getgroup( + "sender", + "Arguments for the sender key fixtures", + ) + + sender_group.addoption( + "--seed-account-sweep-amount", + action="store", + dest="seed_account_sweep_amount", + type=Wei, + default=None, + help="Amount of wei to sweep from the seed account to the sender account. " + "Default=None (Entire balance)", + ) + + sender_group.addoption( + "--sender-funding-txs-gas-price", + action="store", + dest="sender_funding_transactions_gas_price", + type=Wei, + default=10**9, + help=("Gas price set for the funding transactions of each worker's sender key."), + ) + + sender_group.addoption( + "--sender-fund-refund-gas-limit", + action="store", + dest="sender_fund_refund_gas_limit", + type=Wei, + default=21_000, + help=("Gas limit set for the funding transactions of each worker's sender key."), + ) + + +@pytest.fixture(scope="session") +def sender_funding_transactions_gas_price(request: pytest.FixtureRequest) -> int: + """ + Get the gas price for the funding transactions. + """ + return request.config.option.sender_funding_transactions_gas_price + + +@pytest.fixture(scope="session") +def sender_fund_refund_gas_limit(request: pytest.FixtureRequest) -> int: + """ + Get the gas limit of the funding transactions. + """ + return request.config.option.sender_fund_refund_gas_limit + + +@pytest.fixture(scope="session") +def seed_account_sweep_amount(request: pytest.FixtureRequest) -> int | None: + """ + Get the seed account sweep amount. + """ + return request.config.option.seed_account_sweep_amount + + +@pytest.fixture(scope="session") +def sender_key_initial_balance( + seed_sender: EOA, + eth_rpc: EthRPC, + session_temp_folder: Path, + worker_count: int, + sender_funding_transactions_gas_price: int, + sender_fund_refund_gas_limit: int, + seed_account_sweep_amount: int | None, +) -> int: + """ + Calculate the initial balance of each sender key. + + The way to do this is to fetch the seed sender balance and divide it by the number of + workers. This way we can ensure that each sender key has the same initial balance. + + We also only do this once per session, because if we try to fetch the balance again, it + could be that another worker has already sent a transaction and the balance is different. + + It's not really possible to calculate the transaction costs of each test that each worker + is going to run, so we can't really calculate the initial balance of each sender key + based on that. + """ + base_name = "sender_key_initial_balance" + base_file = session_temp_folder / base_name + base_lock_file = session_temp_folder / f"{base_name}.lock" + + with FileLock(base_lock_file): + if base_file.exists(): + with base_file.open("r") as f: + sender_key_initial_balance = int(f.read()) + else: + if seed_account_sweep_amount is None: + seed_account_sweep_amount = eth_rpc.get_balance(seed_sender) + seed_sender_balance_per_worker = seed_account_sweep_amount // worker_count + assert seed_sender_balance_per_worker > 100, "Seed sender balance too low" + # Subtract the cost of the transaction that is going to be sent to the seed sender + sender_key_initial_balance = seed_sender_balance_per_worker - ( + sender_fund_refund_gas_limit * sender_funding_transactions_gas_price + ) + + with base_file.open("w") as f: + f.write(str(sender_key_initial_balance)) + return sender_key_initial_balance + + +@pytest.fixture(scope="session") +def sender_key( + request: pytest.FixtureRequest, + seed_sender: EOA, + sender_key_initial_balance: int, + eoa_iterator: Iterator[EOA], + eth_rpc: EthRPC, + session_temp_folder: Path, + sender_funding_transactions_gas_price: int, + sender_fund_refund_gas_limit: int, +) -> Generator[EOA, None, None]: + """ + Get the sender keys for all tests. + + The seed sender is going to be shared among different processes, so we need to lock it + before we produce each funding transaction. + """ + # For the seed sender we do need to keep track of the nonce because it is shared among + # different processes, and there might not be a new block produced between the transactions. + seed_sender_nonce_file_name = "seed_sender_nonce" + seed_sender_lock_file_name = f"{seed_sender_nonce_file_name}.lock" + seed_sender_nonce_file = session_temp_folder / seed_sender_nonce_file_name + seed_sender_lock_file = session_temp_folder / seed_sender_lock_file_name + + sender = next(eoa_iterator) + + # prepare funding transaction + with FileLock(seed_sender_lock_file): + if seed_sender_nonce_file.exists(): + with seed_sender_nonce_file.open("r") as f: + seed_sender.nonce = Number(f.read()) + fund_tx = Transaction( + sender=seed_sender, + to=sender, + gas_limit=sender_fund_refund_gas_limit, + gas_price=sender_funding_transactions_gas_price, + value=sender_key_initial_balance, + ).with_signature_and_sender() + eth_rpc.send_transaction(fund_tx) + with seed_sender_nonce_file.open("w") as f: + f.write(str(seed_sender.nonce)) + eth_rpc.wait_for_transaction(fund_tx) + + yield sender + + # refund seed sender + remaining_balance = eth_rpc.get_balance(sender) + used_balance = sender_key_initial_balance - remaining_balance + request.config.stash[metadata_key]["Senders"][ + str(sender) + ] = f"Used balance={used_balance / 10**18:.18f}" + + refund_gas_limit = sender_fund_refund_gas_limit + refund_gas_price = sender_funding_transactions_gas_price + tx_cost = refund_gas_limit * refund_gas_price + + if (remaining_balance - 1) < tx_cost: + return + + # Update the nonce of the sender in case one of the pre-alloc transactions failed + sender.nonce = Number(eth_rpc.get_transaction_count(sender)) + + refund_tx = Transaction( + sender=sender, + to=seed_sender, + gas_limit=refund_gas_limit, + gas_price=refund_gas_price, + value=remaining_balance - tx_cost - 1, + ).with_signature_and_sender() + + eth_rpc.send_wait_transaction(refund_tx) + + +def pytest_sessionstart(session): # noqa: SC200 + """ + Reset the sender info before the session starts. + """ + session.config.stash[metadata_key]["Senders"] = {} diff --git a/src/pytest_plugins/execute/test_recover.py b/src/pytest_plugins/execute/test_recover.py new file mode 100644 index 0000000000..9733c9bfa3 --- /dev/null +++ b/src/pytest_plugins/execute/test_recover.py @@ -0,0 +1,45 @@ +""" +Pytest test to recover funds from a failed remote execution. +""" + +import pytest + +from ethereum_test_base_types import Address +from ethereum_test_rpc import EthRPC +from ethereum_test_types import EOA, Transaction + + +@pytest.fixture(scope="session") +def gas_price(eth_rpc: EthRPC) -> int: + """ + Get the gas price for the funding transactions. + """ + return eth_rpc.gas_price() + + +def test_recover_funds( + destination: Address, + index: int, + eoa: EOA, + gas_price: int, + eth_rpc: EthRPC, +) -> None: + """ + Recover funds from a failed remote execution. + """ + remaining_balance = eth_rpc.get_balance(eoa) + refund_gas_limit = 21_000 + tx_cost = refund_gas_limit * gas_price + if remaining_balance < tx_cost: + pytest.skip(f"Balance {remaining_balance} is less than the transaction cost {tx_cost}") + + refund_tx = Transaction( + sender=eoa, + to=destination, + gas_limit=refund_gas_limit, + gas_price=gas_price, + value=remaining_balance - tx_cost, + ).with_signature_and_sender() + + eth_rpc.send_wait_transaction(refund_tx) + print(f"Recovered {remaining_balance} from {eoa} to {destination}") diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index d4874f2152..d3033b65e0 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -22,14 +22,9 @@ from cli.gen_index import generate_fixtures_index from ethereum_clis import TransitionTool from ethereum_test_base_types import Alloc, ReferenceSpec -from ethereum_test_fixtures import FIXTURE_FORMATS, BaseFixture, FixtureCollector, TestInfo -from ethereum_test_forks import ( - Fork, - get_closest_fork_with_solc_support, - get_forks_with_solc_support, -) +from ethereum_test_fixtures import BaseFixture, FixtureCollector, TestInfo +from ethereum_test_forks import Fork from ethereum_test_specs import SPEC_TYPES, BaseTest -from ethereum_test_tools import Yul from ethereum_test_tools.utility.versioning import ( generate_github_url, get_current_commit_hash_or_tag, @@ -208,19 +203,6 @@ def pytest_configure(config): called before the pytest-html plugin's pytest_configure to ensure that it uses the modified `htmlpath` option. """ - for fixture_format in FIXTURE_FORMATS.values(): - config.addinivalue_line( - "markers", - (f"{fixture_format.fixture_format_name.lower()}: {fixture_format.description}"), - ) - config.addinivalue_line( - "markers", - "yul_test: a test case that compiles Yul code.", - ) - config.addinivalue_line( - "markers", - "compile_yul_with(fork): Always compile Yul source using the corresponding evm version.", - ) if config.option.collectonly: return if not config.getoption("disable_html") and config.getoption("htmlpath") is None: @@ -265,8 +247,7 @@ def pytest_report_header(config: pytest.Config): if config.option.collectonly: return t8n_version = config.stash[metadata_key]["Tools"]["t8n"] - solc_version = config.stash[metadata_key]["Tools"]["solc"] - return [(f"{t8n_version}, {solc_version}")] + return [(f"{t8n_version}")] def pytest_report_teststatus(report, config: pytest.Config): @@ -689,62 +670,6 @@ def filler_path(request: pytest.FixtureRequest) -> Path: return request.config.getoption("filler_path") -@pytest.fixture(autouse=True) -def eips(): - """ - A fixture specifying that, by default, no EIPs should be activated for - tests. - - This fixture (function) may be redefined in test filler modules in order - to overwrite this default and return a list of integers specifying which - EIPs should be activated for the tests in scope. - """ - return [] - - -@pytest.fixture -def yul(fork: Fork, request): - """ - A fixture that allows contract code to be defined with Yul code. - - This fixture defines a class that wraps the ::ethereum_test_tools.Yul - class so that upon instantiation within the test case, it provides the - test case's current fork parameter. The forks is then available for use - in solc's arguments for the Yul code compilation. - - Test cases can override the default value by specifying a fixed version - with the @pytest.mark.compile_yul_with(FORK) marker. - """ - solc_target_fork: Fork | None - marker = request.node.get_closest_marker("compile_yul_with") - if marker: - if not marker.args[0]: - pytest.fail( - f"{request.node.name}: Expected one argument in 'compile_yul_with' marker." - ) - for fork in request.config.forks: - if fork.name() == marker.args[0]: - solc_target_fork = fork - break - else: - pytest.fail(f"{request.node.name}: Fork {marker.args[0]} not found in forks list.") - assert solc_target_fork in get_forks_with_solc_support(request.config.solc_version) - else: - solc_target_fork = get_closest_fork_with_solc_support(fork, request.config.solc_version) - assert solc_target_fork is not None, "No fork supports provided solc version." - if solc_target_fork != fork and request.config.getoption("verbose") >= 1: - warnings.warn(f"Compiling Yul for {solc_target_fork.name()}, not {fork.name()}.") - - class YulWrapper(Yul): - def __new__(cls, *args, **kwargs): - return super(YulWrapper, cls).__new__(cls, *args, **kwargs, fork=solc_target_fork) - - return YulWrapper - - -SPEC_TYPES_PARAMETERS: List[str] = [s.pytest_parameter_name() for s in SPEC_TYPES] - - def node_to_test_info(node: pytest.Item) -> TestInfo: """ Returns the test info of the current node item. @@ -771,24 +696,6 @@ def fixture_source_url(request: pytest.FixtureRequest) -> str: return github_url -@pytest.fixture(scope="function") -def fixture_description(request: pytest.FixtureRequest) -> str: - """Fixture to extract and combine docstrings from the test class and the test function.""" - description_unavailable = ( - "No description available - add a docstring to the python test class or function." - ) - test_class_doc = f"Test class documentation:\n{request.cls.__doc__}" if request.cls else "" - test_function_doc = ( - f"Test function documentation:\n{request.function.__doc__}" - if request.function.__doc__ - else "" - ) - if not test_class_doc and not test_function_doc: - return description_unavailable - combined_docstring = f"{test_class_doc}\n\n{test_function_doc}".strip() - return combined_docstring - - def base_test_parametrizer(cls: Type[BaseTest]): """ Generates a pytest.fixture for a given BaseTest subclass. @@ -811,7 +718,7 @@ def base_test_parametrizer_func( output_dir: Path, dump_dir_parameter_level: Path | None, fixture_collector: FixtureCollector, - fixture_description: str, + test_case_description: str, fixture_source_url: str, ): """ @@ -842,7 +749,7 @@ def __init__(self, *args, **kwargs): ) fixture.fill_info( t8n.version(), - fixture_description, + test_case_description, fixture_source_url=fixture_source_url, ref_spec=reference_spec, ) @@ -912,38 +819,9 @@ def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item if spec_name in params and not params[spec_name].supports_fork(fork): items.remove(item) break + for marker in item.iter_markers(): + if marker.name == "fill": + for mark in marker.args: + item.add_marker(mark) if "yul" in item.fixturenames: # type: ignore item.add_marker(pytest.mark.yul_test) - - -def pytest_make_parametrize_id(config, val, argname): - """ - Pytest hook called when generating test ids. We use this to generate - more readable test ids for the generated tests. - """ - return f"{argname}_{val}" - - -def pytest_runtest_call(item): - """ - Pytest hook called in the context of test execution. - """ - if isinstance(item, EIPSpecTestItem): - return - - class InvalidFiller(Exception): - def __init__(self, message): - super().__init__(message) - - if "state_test" in item.fixturenames and "blockchain_test" in item.fixturenames: - raise InvalidFiller( - "A filler should only implement either a state test or " "a blockchain test; not both." - ) - - # Check that the test defines either test type as parameter. - if not any([i for i in item.funcargs if i in SPEC_TYPES_PARAMETERS]): - pytest.fail( - "Test must define either one of the following parameters to " - + "properly generate a test: " - + ", ".join(SPEC_TYPES_PARAMETERS) - ) diff --git a/src/pytest_plugins/forks/forks.py b/src/pytest_plugins/forks/forks.py index bf54356f58..c0eb18952a 100644 --- a/src/pytest_plugins/forks/forks.py +++ b/src/pytest_plugins/forks/forks.py @@ -422,11 +422,12 @@ def get_fork_option(config, option_name: str, parameter_name: str) -> Set[Fork]: if config.option.collectonly: return - evm_bin = config.getoption("evm_bin") - t8n = TransitionTool.from_binary_path(binary_path=evm_bin) - config.unsupported_forks = frozenset( # type: ignore - filter(lambda fork: not t8n.is_fork_supported(fork), fork_set) - ) + evm_bin = config.getoption("evm_bin", None) + if evm_bin is not None: + t8n = TransitionTool.from_binary_path(binary_path=evm_bin) + config.unsupported_forks = frozenset( # type: ignore + filter(lambda fork: not t8n.is_fork_supported(fork), fork_set) + ) @pytest.hookimpl(trylast=True) diff --git a/src/pytest_plugins/help/help.py b/src/pytest_plugins/help/help.py index f034255923..d47ed8c8db 100644 --- a/src/pytest_plugins/help/help.py +++ b/src/pytest_plugins/help/help.py @@ -28,6 +28,27 @@ def pytest_addoption(parser): default=False, help="Show help options specific to the consume command and exit.", ) + help_group.addoption( + "--execute-help", + action="store_true", + dest="show_execute_help", + default=False, + help="Show help options specific to the execute's command remote and exit.", + ) + help_group.addoption( + "--execute-hive-help", + action="store_true", + dest="show_execute_hive_help", + default=False, + help="Show help options specific to the execute's command hive and exit.", + ) + help_group.addoption( + "--execute-recover-help", + action="store_true", + dest="show_execute_recover_help", + default=False, + help="Show help options specific to the execute's command recover and exit.", + ) @pytest.hookimpl(tryfirst=True) @@ -45,11 +66,51 @@ def pytest_configure(config): "fork range", "filler location", "defining debug", - "pre-allocation behavior", + "pre-allocation behavior during test filling", ], ) elif config.getoption("show_consume_help"): - show_specific_help(config, "pytest-consume.ini", ["consuming"]) + show_specific_help( + config, + "pytest-consume.ini", + [ + "consuming", + ], + ) + elif config.getoption("show_execute_help"): + show_specific_help( + config, + "pytest-execute.ini", + [ + "execute", + "remote RPC configuration", + "pre-allocation behavior during test execution", + "sender key fixtures", + "remote seed sender", + ], + ) + elif config.getoption("show_execute_hive_help"): + show_specific_help( + config, + "pytest-execute-hive.ini", + [ + "execute", + "hive RPC client", + "pre-allocation behavior during test execution", + "sender key fixtures", + "remote seed sender", + ], + ) + elif config.getoption("show_execute_recover_help"): + show_specific_help( + config, + "pytest-execute-recover.ini", + [ + "fund recovery", + "remote RPC configuration", + "remote seed sender", + ], + ) def show_specific_help(config, expected_ini, substrings): @@ -58,7 +119,9 @@ def show_specific_help(config, expected_ini, substrings): """ pytest_ini = Path(config.inifile) if pytest_ini.name != expected_ini: - raise ValueError(f"Unexpected {expected_ini} file option generating help.") + raise ValueError( + f"Unexpected {expected_ini}!={pytest_ini.name} file option generating help." + ) test_parser = argparse.ArgumentParser() for group in config._parser.optparser._action_groups: diff --git a/src/pytest_plugins/pytest_hive/pytest_hive.py b/src/pytest_plugins/pytest_hive/pytest_hive.py index 056e39894c..f11f77c477 100644 --- a/src/pytest_plugins/pytest_hive/pytest_hive.py +++ b/src/pytest_plugins/pytest_hive/pytest_hive.py @@ -100,7 +100,18 @@ def simulator(request): # noqa: D103 return request.config.hive_simulator -@pytest.fixture(scope="module") +def get_test_suite_scope(fixture_name, config: pytest.Config): + """ + Return the appropriate scope of the test suite. + + See: https://docs.pytest.org/en/stable/how-to/fixtures.html#dynamic-scope + """ + if hasattr(config, "test_suite_scope"): + return config.test_suite_scope + return "module" + + +@pytest.fixture(scope=get_test_suite_scope) def test_suite( simulator: Simulation, session_temp_folder: Path, @@ -158,16 +169,16 @@ def hive_test(request, test_suite: HiveTestSuite): Propagate the pytest test case and its result to the hive server. """ try: - fixture_description = request.getfixturevalue("fixture_description") + test_case_description = request.getfixturevalue("test_case_description") except pytest.FixtureLookupError: pytest.exit( - "Error: The 'fixture_description' fixture has not been defined by the simulator " + "Error: The 'test_case_description' fixture has not been defined by the simulator " "or pytest plugin using this plugin!" ) test_parameter_string = request.node.name # consume pytest test id test: HiveTest = test_suite.start_test( name=test_parameter_string, - description=fixture_description, + description=test_case_description, ) yield test try: diff --git a/src/pytest_plugins/shared/execute_fill.py b/src/pytest_plugins/shared/execute_fill.py new file mode 100644 index 0000000000..6db25af2bc --- /dev/null +++ b/src/pytest_plugins/shared/execute_fill.py @@ -0,0 +1,178 @@ +""" +Shared pytest fixtures and hooks for EEST generation modes (fill and execute). +""" + +import warnings +from typing import List, cast + +import pytest + +from ethereum_test_execution import EXECUTE_FORMATS +from ethereum_test_fixtures import FIXTURE_FORMATS +from ethereum_test_forks import ( + Fork, + get_closest_fork_with_solc_support, + get_forks_with_solc_support, +) +from ethereum_test_specs import SPEC_TYPES +from ethereum_test_tools import Yul +from pytest_plugins.spec_version_checker.spec_version_checker import EIPSpecTestItem + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config): + """ + Pytest hook called after command line options have been parsed and before + test collection begins. + + Couple of notes: + 1. Register the plugin's custom markers and process command-line options. + + Custom marker registration: + https://docs.pytest.org/en/7.1.x/how-to/writing_plugins.html#registering-custom-markers + + 2. `@pytest.hookimpl(tryfirst=True)` is applied to ensure that this hook is + called before the pytest-html plugin's pytest_configure to ensure that + it uses the modified `htmlpath` option. + """ + if config.pluginmanager.has_plugin("pytest_plugins.filler.filler"): + for fixture_format in FIXTURE_FORMATS.values(): + config.addinivalue_line( + "markers", + (f"{fixture_format.fixture_format_name.lower()}: {fixture_format.description}"), + ) + elif config.pluginmanager.has_plugin("pytest_plugins.execute.execute"): + for execute_format in EXECUTE_FORMATS.values(): + config.addinivalue_line( + "markers", + (f"{execute_format.execute_format_name.lower()}: {execute_format.description}"), + ) + else: + raise Exception("Neither the filler nor the execute plugin is loaded.") + + config.addinivalue_line( + "markers", + "yul_test: a test case that compiles Yul code.", + ) + config.addinivalue_line( + "markers", + "compile_yul_with(fork): Always compile Yul source using the corresponding evm version.", + ) + config.addinivalue_line( + "markers", + "fill: Markers to be added in fill mode only.", + ) + config.addinivalue_line( + "markers", + "execute: Markers to be added in execute mode only.", + ) + + +@pytest.fixture(autouse=True) +def eips(): + """ + A fixture specifying that, by default, no EIPs should be activated for + tests. + + This fixture (function) may be redefined in test filler modules in order + to overwrite this default and return a list of integers specifying which + EIPs should be activated for the tests in scope. + """ + return [] + + +@pytest.fixture +def yul(fork: Fork, request: pytest.FixtureRequest): + """ + A fixture that allows contract code to be defined with Yul code. + + This fixture defines a class that wraps the ::ethereum_test_tools.Yul + class so that upon instantiation within the test case, it provides the + test case's current fork parameter. The forks is then available for use + in solc's arguments for the Yul code compilation. + + Test cases can override the default value by specifying a fixed version + with the @pytest.mark.compile_yul_with(FORK) marker. + """ + solc_target_fork: Fork | None + marker = request.node.get_closest_marker("compile_yul_with") + assert hasattr(request.config, "solc_version"), "solc_version not set in pytest config." + if marker: + if not marker.args[0]: + pytest.fail( + f"{request.node.name}: Expected one argument in 'compile_yul_with' marker." + ) + for fork in request.config.forks: # type: ignore + if fork.name() == marker.args[0]: + solc_target_fork = fork + break + else: + pytest.fail(f"{request.node.name}: Fork {marker.args[0]} not found in forks list.") + assert solc_target_fork in get_forks_with_solc_support(request.config.solc_version) + else: + solc_target_fork = get_closest_fork_with_solc_support(fork, request.config.solc_version) + assert solc_target_fork is not None, "No fork supports provided solc version." + if solc_target_fork != fork and request.config.getoption("verbose") >= 1: + warnings.warn(f"Compiling Yul for {solc_target_fork.name()}, not {fork.name()}.") + + class YulWrapper(Yul): + def __new__(cls, *args, **kwargs): + return super(YulWrapper, cls).__new__(cls, *args, **kwargs, fork=solc_target_fork) + + return YulWrapper + + +@pytest.fixture(scope="function") +def test_case_description(request: pytest.FixtureRequest) -> str: + """Fixture to extract and combine docstrings from the test class and the test function.""" + description_unavailable = ( + "No description available - add a docstring to the python test class or function." + ) + test_class_doc = f"Test class documentation:\n{request.cls.__doc__}" if request.cls else "" + test_function_doc = ( + f"Test function documentation:\n{request.function.__doc__}" + if request.function.__doc__ + else "" + ) + if not test_class_doc and not test_function_doc: + return description_unavailable + combined_docstring = f"{test_class_doc}\n\n{test_function_doc}".strip() + return combined_docstring + + +def pytest_make_parametrize_id(config: pytest.Config, val: str, argname: str): + """ + Pytest hook called when generating test ids. We use this to generate + more readable test ids for the generated tests. + """ + return f"{argname}_{val}" + + +SPEC_TYPES_PARAMETERS: List[str] = [s.pytest_parameter_name() for s in SPEC_TYPES] + + +def pytest_runtest_call(item: pytest.Item): + """ + Pytest hook called in the context of test execution. + """ + if isinstance(item, EIPSpecTestItem): + return + + class InvalidFiller(Exception): + def __init__(self, message): + super().__init__(message) + + item = cast(pytest.Function, item) # help mypy infer type + + if "state_test" in item.fixturenames and "blockchain_test" in item.fixturenames: + raise InvalidFiller( + "A filler should only implement either a state test or " "a blockchain test; not both." + ) + + # Check that the test defines either test type as parameter. + if not any([i for i in item.funcargs if i in SPEC_TYPES_PARAMETERS]): + pytest.fail( + "Test must define either one of the following parameters to " + + "properly generate a test: " + + ", ".join(SPEC_TYPES_PARAMETERS) + ) diff --git a/src/pytest_plugins/solc/solc.py b/src/pytest_plugins/solc/solc.py index 01b1d1a565..a666a74f62 100644 --- a/src/pytest_plugins/solc/solc.py +++ b/src/pytest_plugins/solc/solc.py @@ -101,3 +101,12 @@ def solc_bin(request: pytest.FixtureRequest): Returns the configured solc binary path. """ return request.config.getoption("solc_bin") + + +@pytest.hookimpl(trylast=True) +def pytest_report_header(config, start_path): + """Add lines to pytest's console output header""" + if config.option.collectonly: + return + solc_version = config.stash[metadata_key]["Tools"]["solc"] + return [(f"solc: {solc_version}")] diff --git a/uv.lock b/uv.lock index bbfb7d8f5d..5ea4ef75fb 100644 --- a/uv.lock +++ b/uv.lock @@ -527,6 +527,7 @@ dependencies = [ { name = "pytest" }, { name = "pytest-custom-report" }, { name = "pytest-html" }, + { name = "pytest-json-report" }, { name = "pytest-metadata" }, { name = "pytest-xdist" }, { name = "pyyaml" }, @@ -612,6 +613,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.1.0,<5" }, { name = "pytest-custom-report", specifier = ">=1.0.1,<2" }, { name = "pytest-html", specifier = ">=4.1.0,<5" }, + { name = "pytest-json-report", specifier = ">=1.5.0,<2" }, { name = "pytest-metadata", specifier = ">=3,<4" }, { name = "pytest-xdist", specifier = ">=3.3.1,<4" }, { name = "pyyaml", specifier = ">=6.0.2" }, @@ -1594,6 +1596,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c7/c160021cbecd956cc1a6f79e5fe155f7868b2e5b848f1320dad0b3e3122f/pytest_html-4.1.1-py3-none-any.whl", hash = "sha256:c8152cea03bd4e9bee6d525573b67bbc6622967b72b9628dda0ea3e2a0b5dd71", size = 23491 }, ] +[[package]] +name = "pytest-json-report" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "pytest-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/d3/765dae9712fcd68d820338908c1337e077d5fdadccd5cacf95b9b0bea278/pytest-json-report-1.5.0.tar.gz", hash = "sha256:2dde3c647851a19b5f3700729e8310a6e66efb2077d674f27ddea3d34dc615de", size = 21241 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/35/d07400c715bf8a88aa0c1ee9c9eb6050ca7fe5b39981f0eea773feeb0681/pytest_json_report-1.5.0-py3-none-any.whl", hash = "sha256:9897b68c910b12a2e48dd849f9a284b2c79a732a8a9cb398452ddd23d3c8c325", size = 13222 }, +] + [[package]] name = "pytest-metadata" version = "3.1.1" diff --git a/whitelist.txt b/whitelist.txt index 52216aeb82..dcde317783 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -135,6 +135,7 @@ enum env envvar EOA +EOAs eof EOF1 EOFException @@ -267,6 +268,7 @@ marioevz markdownlint md mem +mempool metaclass mixhash mkdocs @@ -407,6 +409,7 @@ TestAddress testscollected TestContractCreationGasUsage TestMultipleWithdrawalsSameAddress +testrun textwrap tf ThreeHrSleep @@ -431,6 +434,7 @@ typehints u256 ubuntu ukiyo +uid uncomment undersize unlink