From 4e09ca0dea52e80c0f2b222903e9d82b773ed7e2 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 22 Jun 2023 17:21:35 +0200 Subject: [PATCH 1/2] chore: add a few helper comments to doc script --- docs/gen_test_case_reference.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/gen_test_case_reference.py b/docs/gen_test_case_reference.py index 7d69518417..8816b52830 100644 --- a/docs/gen_test_case_reference.py +++ b/docs/gen_test_case_reference.py @@ -235,10 +235,14 @@ def non_recursive_os_walk(top_dir): all_directories = [directory for directory in fork_directories if os.path.exists(directory)] all_directories.insert(0, source_directory) +# Loop over directories here instead of walking tests/ to ensure we +# get a reverse chronological listing of forks in the nav bar. for directory in all_directories: if directory is source_directory: + # Process files within tests/ but don't walk it recursively. walk_directory_output = non_recursive_os_walk(directory) else: + # Walk each tests/fork/ directory recursively. # sorted() is a bit of a hack to order nav content for each fork walk_directory_output = sorted(os.walk(directory)) for root, _, files in walk_directory_output: From bc2ec3630631f366fb03df21b172096a2e3d6611 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 22 Jun 2023 17:22:55 +0200 Subject: [PATCH 2/2] docs: fix paths due to directory structure, forgotten in #156 --- README.md | 6 +-- .../executing_tests_command_line.md | 48 ++++++++++++++----- docs/getting_started/quick_start.md | 38 +++++++-------- docs/writing_tests/adding_a_new_test.md | 31 +++++++----- 4 files changed, 76 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 25183dd201..6abee82e73 100644 --- a/README.md +++ b/README.md @@ -97,10 +97,10 @@ This guide installs stable versions of the required external `evm` and `solc` ex Expected console output: ![Screenshot of pytest test collection console output](docs/getting_started/img/pytest_collect_only.png) - 2. Execute the test cases (verbosely) in the `./tests/example/test_acl_example.py` module: + 2. Execute the test cases (verbosely) in the `./tests/berlin/eip2930_access_list/test_acl.py` module: ```console - fill -v tests/example/test_acl_example.py + fill -v tests/berlin/eip2930_access_list/test_acl.py ``` Expected console output: @@ -111,7 +111,7 @@ This guide installs stable versions of the required external `evm` and `solc` ex 2. The corresponding fixture file has been generated: ```console - head fixtures/example/acl_example/test_access_list.json + head fixtures/berlin/eip2930_access_list/acl/access_list.json ``` ## Coverage diff --git a/docs/getting_started/executing_tests_command_line.md b/docs/getting_started/executing_tests_command_line.md index 4c8b675c6c..a8ba057d1e 100644 --- a/docs/getting_started/executing_tests_command_line.md +++ b/docs/getting_started/executing_tests_command_line.md @@ -1,47 +1,68 @@ # Executing Tests at a Prompt +The execution-spec-tests repo uses [pytest](https://docs.pytest.org/en/latest/) as its test framework. The `fill` command is essentially an alias for `pytest`. + ## Collection - Test Exploration The test cases implemented in the `./tests` sub-directory can be listed in the console using: + ```console fill --collect-only ``` + and can be filtered (by test path, function and parameter substring): + ```console fill --collect-only -k warm_coinbase ``` + Docstrings are additionally displayed when ran verbosely: + ```console fill --collect-only -k warm_coinbase -vv ``` ## Execution -To generate all the test fixtures defined in the `./tests` sub-directory and write them to the `./fixtures` directory, run `fill` in the top-level directory as: +By default, all test cases are executed for all forks deployed to mainnet, but not for forks still under active development, i.e., + +```console +fill +``` + +will generate fixtures for test cases from Frontier to Shanghai (as of time of writing, Q2 2023). + +To generate all the test fixtures defined in the `./tests/shanghai` sub-directory and write them to the `./fixtures-shanghai` directory, run `fill` in the top-level directory as: + ```console -fill --output="fixtures" +fill ./tests/shanghai --output="fixtures-shanghai" ``` !!! note "Test case verification" - Note, that the test `post` conditions are tested against the output of the `evm t8n` command for transition tests, respectively `evm b11r` command for blockchain tests, during test generation. + Note, that the (limited set of) test `post` conditions are tested against the output of the `evm t8n` command for transition tests, respectively `evm b11r` command for blockchain tests, during test generation. + +To generate all the test fixtures in the `tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py` module, for example, run: -To generate all the test fixtures in the `./tests/eips/` sub-directory (category), for example, run: ```console -fill tests/eips +fill tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py ``` -To generate all the test fixtures in the `./tests/eips/test_eip3651.py` module, for example, run: +To generate specific test fixtures from a specific test function or even test function and parameter set, obtain the corresponding test ID using: + ```console -fill ./tests/eips/test_eip3651.py +fill --collect-only -q -k test_warm_coinbase ``` -To generate specific test fixtures, such as those from the test function `test_warm_coinbase_call_out_of_gas()`, for example, run: +This filters the tests by `test_warm_coinbase`. Then find the relevant test ID in the console output and provide it to fill, for example, for a test function: + ```console -fill -k "test_warm_coinbase_call_out_of_gas" +fill tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py::test_warm_coinbase_gas_usage ``` -or, additionally, only for the for Shanghai fork: + +or, for a test function and specific parameter combination: + ```console -fill -k "test_warm_coinbase_call_out_of_gas and shanghai" +fill tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py::test_warm_coinbase_gas_usage[fork=Merge-DELEGATECALL] ``` ## Execution for Development Forks @@ -51,7 +72,7 @@ fill -k "test_warm_coinbase_call_out_of_gas and shanghai" In order to execute test cases for an upcoming fork, ensure that the `evm` tool used supports that fork and features under test and use the `--until` or `--fork` flag. - For example, as of May 2023, the current fork under active development is `Cancun`: + For example, as of Q2 2023, the current fork under active development is `Cancun`: ```console fill --until Cancun ``` @@ -61,15 +82,16 @@ fill -k "test_warm_coinbase_call_out_of_gas and shanghai" ## Useful pytest/fill Command-Line Options Custom `fill` command-line options: + ```console fill --traces # Collect traces of the execution information from the transition tool fill --evm=EVM_BIN # Specify the evm executable to generate fixtures with ``` Pytest command-line options: + ```console fill -vv # More verbose output fill -x # Exit instantly on first error or failed test case fill --pdb # drop into the debugger upon error in a test case ``` - diff --git a/docs/getting_started/quick_start.md b/docs/getting_started/quick_start.md index a4292418a5..7e732f2064 100644 --- a/docs/getting_started/quick_start.md +++ b/docs/getting_started/quick_start.md @@ -59,31 +59,31 @@ The following requires a Python 3.10 installation. ``` 3. Verify installation: 1. Explore test cases: + 2. ```console + fill --collect-only + ``` + + Expected console output: +
+ ![Screenshot of pytest test collection console output](./img/pytest_collect_only.png){align=center} +
+ 3. Execute the test cases (verbosely) in the `./tests/berlin/eip2930_access_list/test_acl.py` module: ```console - fill --collect-only + fill -v tests/berlin/eip2930_access_list/test_acl.py ``` Expected console output: +
- ![Screenshot of pytest test collection console output](./img/pytest_collect_only.png){align=center} + ![Screenshot of pytest test collection console output](./img/pytest_run_example.png){align=center}
- - 2. Execute the test cases (verbosely) in the `./tests/example/test_acl_example.py` module: - ```console - fill -v tests/example/test_acl_example.py - ``` - Expected console output: -
- ![Screenshot of pytest test collection console output](./img/pytest_run_example.png){align=center} -
- Check: - - 1. The versions of the `evm` and `solc` tools are as expected (your versions may differ from those in the highlighted box). - 2. The corresponding fixture file has been generated: - - ```console - head fixtures/example/acl_example/test_access_list.json - ``` + Check: + 1. The versions of the `evm` and `solc` tools are as expected (your versions may differ from those in the highlighted box). + 2. The corresponding fixture file has been generated: + + ``` + head fixtures/berlin/eip2930_access_list/acl/access_list.json + ``` ## Next Steps diff --git a/docs/writing_tests/adding_a_new_test.md b/docs/writing_tests/adding_a_new_test.md index 1e7607707f..f15b3bcdbe 100644 --- a/docs/writing_tests/adding_a_new_test.md +++ b/docs/writing_tests/adding_a_new_test.md @@ -1,20 +1,27 @@ # Adding a New Test -All test cases are located in the `tests` directory, which is composed of many subdirectories, each one represents a different test category. The sub-directories may contain sub-categories, if necessary. +All test cases are located underneath the `tests` directory, which are then organized by fork. Each fork contains sub-directories containing test sub-categories. ``` 📁 execution-test-specs/ -├─╴📁 tests/ # test cases -│ ├── 📁 eips/ -│ | ├── 📁 eip4844/ -| | | ├── 📄 test_blobhash_opcode.py -| | | └── 📄 test_excess_data_gas.py -| | ├── 📄 test_eip3855.py -| | └── 📄 test_eip3860.py -│ ├── 📁 example/ -│ ├── 📁 security/ -│ ├── 📁 vm/ -│ ├── 📁 withdrawals/ +├─╴📁 tests/ +| ├── 📄 __init__.py +│ ├── 📁 cancun/ +| | ├── 📄 __init__.py +│ | └── 📁 eip4844_blobs/ +| | ├── 📄 __init__.py +| | ├── 📄 test_blobhash_opcode.py +| | ├── 📄 test_excess_data_gas.py +| | └── 📄 ... +| ├── 📁 shanghai +| | ├── 📁 eip3651_warm_coinbase +| | | ├── 📄 __init__.py +| | | └── 📄 test_warm_coinbase.py +| | ├── 📁 eip3855_push0 +| | | ├── 📄 __init__.py +| | | └── 📄 test_push0.py +| | ├── 📁... +| | ... │ └── 📁 ... ```