diff --git a/.gitignore b/.gitignore index 3769738d037..9cc42fe1a04 100644 --- a/.gitignore +++ b/.gitignore @@ -89,4 +89,7 @@ logs/ env.yaml +# Client binary config (user-specific paths) +validate.toml + site/ diff --git a/docs/running_tests/consume/exceptions.md b/docs/running_tests/consume/exceptions.md index a5a82c499ab..b9f8f1c3b81 100644 --- a/docs/running_tests/consume/exceptions.md +++ b/docs/running_tests/consume/exceptions.md @@ -32,7 +32,56 @@ GETH_EXCEPTIONS = { } ``` -## Updating Client Exception Messages +## External Client Exception Mappers + +Client repositories can extend EEST's built-in exception mappers with a +client-owned YAML file. External mappings are additive: they do not remove or +replace mappings maintained in EEST. + +```yaml +version: 1 +name: geth-ci +substring: + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: + - "insufficient funds for gas * price + value" +regex: + BlockException.INVALID_GASLIMIT: + - 'child gas_limit \d+ .*' +``` + +Rules: + +- `version` must be `1`. +- Exception keys must be exact EEST names such as + `TransactionException.INSUFFICIENT_ACCOUNT_FUNDS` or + `BlockException.INVALID_GASLIMIT`. +- `substring` and `regex` values can be a string or a list of strings. +- Empty patterns, unknown exception names, unknown top-level sections, and + invalid regular expressions are rejected when the file is loaded. + +For `validate`, add the mapper path to the relevant client section in +`validate.toml`. Relative paths are resolved from the directory containing +`validate.toml`. + +```toml +[geth] +bin = "../go-ethereum/build/bin/evm" +exception-mapper = "../go-ethereum/eest-exceptions.yaml" +``` + +For Hive-backed `consume` commands, pass one or more mapper files on the +command line: + +```bash +uv run consume engine \ + --input ./fixtures \ + --exception-mapper geth=../go-ethereum/eest-exceptions.yaml +``` + +The `CLIENT` key is matched against Hive client names. For example, `geth` +extends the mapper selected for `go-ethereum`. + +## Updating Built-In Client Exception Messages When clients change their error messages or you encounter unmapped exceptions: diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml index 40fee87eb1d..5ef05a4a4ea 100644 --- a/packages/testing/pyproject.toml +++ b/packages/testing/pyproject.toml @@ -84,6 +84,7 @@ checkfixtures = "execution_testing.cli.check_fixtures:check_fixtures" check_eip_versions = "execution_testing.cli.pytest_commands.check_eip_versions:check_eip_versions" consume = "execution_testing.cli.pytest_commands.consume:consume" protec = "execution_testing.cli.pytest_commands.consume:consume" +validate = "execution_testing.cli.pytest_commands.validate:validate" checklist = "execution_testing.cli.pytest_commands.checklist:checklist" generate_checklist_stubs = "execution_testing.cli.generate_checklist_stubs:generate_checklist_stubs" genindex = "execution_testing.cli.gen_index:generate_fixtures_index_cli" @@ -134,6 +135,7 @@ addopts = [ "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/test_cache.py", "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/direct/", "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/simulators/", + "--ignore=src/execution_testing/cli/pytest_commands/plugins/validate/", ] markers = [ "pre_alloc_group: Custom marker for pre-allocation grouping", diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/consume.py index 3059f463c05..b43ec1e08e1 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/consume.py @@ -55,10 +55,6 @@ def get_command_logic_test_paths(command_name: str) -> List[Path]: command_logic_test_paths = [ base_path / "simulators" / "simulator_logic" / "test_via_sync.py" ] - elif command_name == "direct": - command_logic_test_paths = [ - base_path / "direct" / "test_via_direct.py" - ] else: raise ValueError(f"Unexpected command: {command_name}.") return command_logic_test_paths @@ -70,6 +66,8 @@ def consume() -> None: pass + + def consume_command( is_hive: bool = False, ) -> Callable[[Callable[..., Any]], click.Command]: @@ -102,12 +100,6 @@ def command(pytest_args: List[str], **kwargs: Any) -> None: return decorator -@consume_command(is_hive=False) -def direct() -> None: - """Clients consume directly via the `blocktest` interface.""" - pass - - @consume_command(is_hive=True) def rlp() -> None: """Client consumes RLP-encoded blocks on startup.""" @@ -144,3 +136,5 @@ def cache(pytest_args: List[str], **kwargs: Any) -> None: command_logic_test_paths=[], is_hive=False ) cache_cmd.execute(list(pytest_args)) + + diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/exceptions.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/exceptions.py index 9201cd24036..d9f3441428f 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/exceptions.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/exceptions.py @@ -1,11 +1,17 @@ """Pytest plugin that defines options and fixtures for client exceptions.""" +from pathlib import Path from typing import Dict, List import pytest from hive.client import ClientType -from execution_testing.exceptions import ExceptionMapper +from execution_testing.exceptions import ( + ExceptionMapper, + ExternalExceptionMapper, + extend_exception_mapper, + load_external_exception_mapper, +) from execution_testing.fixtures import ( BlockchainFixtureCommon, ) @@ -28,6 +34,77 @@ def pytest_addoption(parser: pytest.Parser) -> None: "NOT use strict exception matching." ), ) + consume_group.addoption( + "--exception-mapper", + action="append", + dest="external_exception_mappers", + default=[], + metavar="CLIENT=PATH", + help=( + "Extend a client's built-in exception mapper with a YAML mapper " + "file. Can be repeated, e.g. " + "--exception-mapper geth=../go-ethereum/eest-exceptions.yaml." + ), + ) + + +def _parse_external_exception_mapper_options( + options: List[str], +) -> Dict[str, ExternalExceptionMapper]: + """Load external exception mapper files from CLIENT=PATH options.""" + external_mappers: Dict[str, ExternalExceptionMapper] = {} + for option in options: + if "=" not in option: + raise ValueError( + "--exception-mapper must use CLIENT=PATH syntax" + ) + client, path_string = option.split("=", 1) + client = client.strip().lower() + path_string = path_string.strip() + if not client or not path_string: + raise ValueError( + "--exception-mapper must use non-empty CLIENT=PATH values" + ) + external_mappers[client] = load_external_exception_mapper( + Path(path_string).expanduser() + ) + return external_mappers + + +def _client_key_matches( + configured_key: str, + hive_client_name: str, + built_in_key: str, +) -> bool: + """Return whether an external mapper key applies to a Hive client.""" + configured_key = configured_key.lower() + hive_client_name = hive_client_name.lower() + built_in_key = built_in_key.lower() + return ( + configured_key in hive_client_name + or configured_key == built_in_key + or (configured_key == "geth" and built_in_key == "go-ethereum") + ) + + +def get_configured_exception_mapper( + client_name: str, + external_exception_mappers: Dict[str, ExternalExceptionMapper], +) -> ExceptionMapper | None: + """Return the built-in mapper extended with a matching external mapper.""" + normalized_client_name = client_name.lower() + for client_key, built_in_mapper in EXCEPTION_MAPPERS.items(): + if client_key not in normalized_client_name: + continue + external_mapper = None + for configured_key, mapper in external_exception_mappers.items(): + if _client_key_matches( + configured_key, client_name, client_key + ): + external_mapper = mapper + break + return extend_exception_mapper(built_in_mapper, external_mapper) + return None @pytest.fixture(scope="session") @@ -36,21 +113,30 @@ def client_exception_mapper_cache() -> Dict[str, ExceptionMapper | None]: return {} +@pytest.fixture(scope="session") +def external_exception_mappers( + request: pytest.FixtureRequest, +) -> Dict[str, ExternalExceptionMapper]: + """Load external exception mapper files requested on the command line.""" + return _parse_external_exception_mapper_options( + request.config.getoption("external_exception_mappers") + ) + + @pytest.fixture(scope="function") def client_exception_mapper( client_type: ClientType, client_exception_mapper_cache: Dict[str, ExceptionMapper | None], + external_exception_mappers: Dict[str, ExternalExceptionMapper], ) -> ExceptionMapper | None: """Return the exception mapper for the client type, with caching.""" if client_type.name not in client_exception_mapper_cache: - for client in EXCEPTION_MAPPERS: - if client in client_type.name: - client_exception_mapper_cache[client_type.name] = ( - EXCEPTION_MAPPERS[client] - ) - break - else: - client_exception_mapper_cache[client_type.name] = None + client_exception_mapper_cache[client_type.name] = ( + get_configured_exception_mapper( + client_type.name, + external_exception_mappers, + ) + ) return client_exception_mapper_cache[client_type.name] diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py index 94ddfb14dba..4015bb2fb65 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py @@ -15,7 +15,10 @@ from hive.testing import HiveTest from execution_testing.base_types import to_json -from execution_testing.exceptions import ExceptionMapper +from execution_testing.exceptions import ( + ExceptionMapper, + ExternalExceptionMapper, +) from execution_testing.fixtures import BlockchainEngineSyncFixture from execution_testing.rpc import AdminRPC, EngineRPC, EthRPC, NetRPC @@ -251,19 +254,18 @@ def sync_client( def sync_client_exception_mapper( sync_client_type: ClientType, client_exception_mapper_cache: Dict[str, ExceptionMapper | None], + external_exception_mappers: Dict[str, ExternalExceptionMapper], ) -> ExceptionMapper | None: """Return the exception mapper for the sync client type, with caching.""" if sync_client_type.name not in client_exception_mapper_cache: - from ..exceptions import EXCEPTION_MAPPERS + from ..exceptions import get_configured_exception_mapper - for client in EXCEPTION_MAPPERS: - if client in sync_client_type.name: - client_exception_mapper_cache[sync_client_type.name] = ( - EXCEPTION_MAPPERS[client] - ) - break - else: - client_exception_mapper_cache[sync_client_type.name] = None + client_exception_mapper_cache[sync_client_type.name] = ( + get_configured_exception_mapper( + sync_client_type.name, + external_exception_mappers, + ) + ) return client_exception_mapper_cache[sync_client_type.name] diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_exception_mappers.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_exception_mappers.py new file mode 100644 index 00000000000..023af022ad0 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_exception_mappers.py @@ -0,0 +1,84 @@ +"""Tests for consume exception mapper selection.""" + +from pathlib import Path + +from execution_testing.cli.pytest_commands.plugins.consume.simulators import ( + exceptions, +) +from execution_testing.exceptions import ( + BlockException, + TransactionException, +) + + +def test_parse_exception_mapper_option(tmp_path: Path) -> None: + """Parse repeatable CLIENT=PATH mapper options.""" + mapper_path = tmp_path / "mapper.yaml" + mapper_path.write_text( + """ +version: 1 +substring: + BlockException.INVALID_GASLIMIT: custom gas +""" + ) + + mappers = exceptions._parse_external_exception_mapper_options( + [f"geth={mapper_path}"] + ) + + assert list(mappers) == ["geth"] + assert mappers["geth"].message_to_exception("custom gas") == [ + BlockException.INVALID_GASLIMIT + ] + + +def test_configured_exception_mapper_extends_matching_client( + tmp_path: Path, +) -> None: + """Extend the selected built-in mapper for matching Hive client names.""" + mapper_path = tmp_path / "mapper.yaml" + mapper_path.write_text( + """ +version: 1 +substring: + BlockException.INVALID_GASLIMIT: custom gas +""" + ) + external = exceptions._parse_external_exception_mapper_options( + [f"geth={mapper_path}"] + ) + + mapper = exceptions.get_configured_exception_mapper( + "go-ethereum", external + ) + + assert mapper is not None + assert mapper.message_to_exception("custom gas") == [ + BlockException.INVALID_GASLIMIT + ] + + +def test_unmatched_exception_mapper_option_is_ignored( + tmp_path: Path, +) -> None: + """Do not apply external mappings to non-matching clients.""" + mapper_path = tmp_path / "mapper.yaml" + mapper_path.write_text( + """ +version: 1 +substring: + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: custom funds +""" + ) + external = exceptions._parse_external_exception_mapper_options( + [f"besu={mapper_path}"] + ) + + mapper = exceptions.get_configured_exception_mapper( + "go-ethereum", external + ) + + assert mapper is not None + assert mapper.message_to_exception("custom funds") != [ + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS + ] diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/help/help.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/help/help.py index 9d299b1bc9d..fc91571228c 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/help/help.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/help/help.py @@ -57,6 +57,16 @@ def pytest_addoption(parser: pytest.Parser) -> None: "Show help options specific to the execute hive command and exit." ), ) + help_group.addoption( + "--validate-help", + action="store_true", + dest="show_validate_help", + default=False, + help=( + "Show help options specific to the validate command " + "and exit." + ), + ) help_group.addoption( "--execute-recover-help", action="store_true", @@ -116,6 +126,16 @@ def pytest_configure(config: pytest.Config) -> None: "consuming", ], ) + elif config.getoption("show_validate_help"): + show_specific_help( + config, + "pytest-validate.ini", + [ + "validating fixtures", + "fixture input", + "debug behavior", + ], + ) elif config.getoption("show_execute_help"): show_specific_help( config, @@ -199,7 +219,10 @@ def show_specific_help( kwargs["type"] = action.type if action.nargs: kwargs["nargs"] = action.nargs - new_group.add_argument(*action.option_strings, **kwargs) + try: + new_group.add_argument(*action.option_strings, **kwargs) + except argparse.ArgumentError: + pass # skip conflicting options (e.g. -h/--help) print(test_parser.format_help()) pytest.exit("After displaying help.", returncode=pytest.ExitCode.OK) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/__init__.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/__init__.py new file mode 100644 index 00000000000..610a3c8f7a0 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/__init__.py @@ -0,0 +1 @@ +"""Pytest plugins for validate commands.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/conftest.py new file mode 100644 index 00000000000..332058ebeaf --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/conftest.py @@ -0,0 +1,870 @@ +""" +A pytest plugin that configures the validate command to run test fixtures +against client "direct" consumer interfaces. + +Reads client configuration from validate.toml (falling back to +consume-direct.toml) and creates fixture consumer instances based on the +--validate-type and --validate-clients options injected by the validate +click subcommands. +""" + +import argparse +import json +import sys +import tempfile +import warnings +from pathlib import Path +from typing import Any, Generator, List + +import pytest +import rich + +from execution_testing.base_types import to_json +from execution_testing.cli.gen_index import ( + generate_fixtures_index, +) +from execution_testing.cli.pytest_commands.plugins.consume.consume import ( + CACHED_DOWNLOADS_DIRECTORY, + FixturesSource, + SimLimitBehavior, + default_html_report_file_path, + default_input, +) +from execution_testing.client_clis.ethereum_cli import EthereumCLI +from execution_testing.client_clis.fixture_consumer_tool import ( + FixtureConsumerTool, +) +from execution_testing.exceptions import ( + extend_exception_mapper, + load_external_exception_mapper, +) +from execution_testing.fixtures import ( + BaseFixture, + BlockchainEngineFixture, + BlockchainEngineXFixture, + BlockchainFixture, + FixtureFormat, + StateFixture, +) +from execution_testing.fixtures.consume import ( + IndexFile, + TestCaseIndexFile, + TestCases, + TestCaseStream, +) +from execution_testing.fixtures.file import Fixtures +from execution_testing.forks import ( + get_forks, + get_relative_fork_markers, + get_transition_forks, +) +from execution_testing.tools.utility.versioning import ( + get_current_commit_hash_or_tag, +) + + +class CollectOnlyCLI(EthereumCLI): + """A dummy CLI for use with `--collect-only`.""" + + def __init__(self) -> None: # noqa: D107 + pass + + +class CollectOnlyFixtureConsumer( + FixtureConsumerTool, + CollectOnlyCLI, + fixture_formats=list(BaseFixture.formats.values()), +): + """A dummy fixture consumer for use with `--collect-only`.""" + + def consume_fixture( + self, *args: Any, **kwargs: Any + ) -> None: + """Consume a fixture without executing a client.""" + pass + + +NAME_MAP = { + "GethFixtureConsumer": "geth", + "ErigonFixtureConsumer": "erigon", + "BesuFixtureConsumer": "besu", + "NethtestFixtureConsumer": "nethermind", + "RethFixtureConsumer": "reth", + "EthrexFixtureConsumer": "ethrex", + "NimbusFixtureConsumer": "nimbus", +} + +RECOMMENDED_WORKERS: dict[str, int] = { + "GethFixtureConsumer": 8, + "ErigonFixtureConsumer": 8, + "BesuFixtureConsumer": 8, + "NethtestFixtureConsumer": 4, + "EthrexFixtureConsumer": 8, +} + +RECOMMENDED_N: dict[str, int] = { + "RethFixtureConsumer": 2, +} + + +def _build_client_class_map() -> dict[str, type]: + """Build mapping from client name to consumer class.""" + from execution_testing.client_clis.clis.besu import ( + BesuFixtureConsumer, + ) + from execution_testing.client_clis.clis.erigon import ( + ErigonFixtureConsumer, + ) + from execution_testing.client_clis.clis.ethrex import ( + EthrexFixtureConsumer, + ) + from execution_testing.client_clis.clis.geth import ( + GethFixtureConsumer, + ) + from execution_testing.client_clis.clis.nethermind import ( + NethtestFixtureConsumer, + ) + from execution_testing.client_clis.clis.nimbus import ( + NimbusFixtureConsumer, + ) + from execution_testing.client_clis.clis.reth import ( + RethFixtureConsumer, + ) + + return { + "geth": GethFixtureConsumer, + "go-ethereum": GethFixtureConsumer, + "erigon": ErigonFixtureConsumer, + "besu": BesuFixtureConsumer, + "nethermind": NethtestFixtureConsumer, + "reth": RethFixtureConsumer, + "ethrex": EthrexFixtureConsumer, + "nimbus": NimbusFixtureConsumer, + } + + +CLIENT_CLASS_MAP: dict[str, type] = _build_client_class_map() + + +def _resolve_config_path() -> Path | None: + """Find validate.toml or fall back to consume-direct.toml.""" + for name in ("validate.toml", "consume-direct.toml"): + path = Path.cwd() / name + if path.exists(): + return path + return None + + +def _resolve_config_relative_path( + path_string: str, + config_path: Path | None, +) -> Path: + """Resolve a configured path relative to its TOML file.""" + path = Path(path_string).expanduser() + if path.is_absolute() or config_path is None: + return path + return config_path.parent / path + + +def pytest_addoption(parser: pytest.Parser) -> None: # noqa: D103 + fixtures_group = parser.getgroup( + "fixtures", + "Arguments related to fixture input", + ) + fixtures_group.addoption( + "--input", + action="store", + dest="fixtures_source", + default=None, + help=( + "Specify the JSON test fixtures source. Can be a " + "local directory, a URL pointing to a " + "fixtures.tar.gz archive, a release name and " + "version in the form of `NAME@v1.2.3` (`stable` " + "and `develop` are valid release names, and " + "`latest` is a valid version), or the special " + "keyword 'stdin'. Defaults to the following local " + f"directory: '{default_input()}'." + ), + ) + fixtures_group.addoption( + "--cache-folder", + action="store", + dest="fixture_cache_folder", + default=CACHED_DOWNLOADS_DIRECTORY, + help=( + "Specify the path where the downloaded fixtures " + "are cached. Defaults to the following directory: " + f"'{CACHED_DOWNLOADS_DIRECTORY}'." + ), + ) + fixtures_group.addoption( + "--no-html", + action="store_true", + dest="disable_html", + default=False, + help=( + "Don't generate an HTML test report (in the " + "output directory). The --html flag can be used " + "to specify a different path." + ), + ) + fixtures_group.addoption( + "--sim.limit", + action="store", + dest="sim_limit", + type=SimLimitBehavior.from_string, + default=SimLimitBehavior(".*"), + help=( + "Filter tests by either a regex pattern or a " + "literal test case ID. To match a test case by " + "its exact ID, prefix the ID with `id:`. " + "Without the `id:` prefix, the argument is " + "interpreted as a Python regex pattern. To see " + "which test cases are matched without executing " + "them, prefix with `collectonly:`." + ), + ) + validate_group = parser.getgroup( + "validate", + "Arguments related to validating fixtures against a client", + ) + validate_group.addoption( + "--client", + action="append", + dest="_client_help_only", + default=[], + help="Client name (e.g. geth, besu). Required. Can be repeated.", + ) + validate_group.addoption( + "--validate-type", + action="store", + dest="validate_type", + type=str, + default=None, + choices=["state", "block", "engine"], + help=argparse.SUPPRESS, + ) + validate_group.addoption( + "--validate-clients", + action="store", + dest="validate_clients", + type=str, + default="", + help=argparse.SUPPRESS, + ) + validate_group.addoption( + "--no-exception-check", + action="store_true", + dest="no_exception_check", + default=False, + help=( + "Skip exception matching (useful for clients that " + "don't report errors)." + ), + ) + validate_group.addoption( + "--bin-workers", + action="store", + dest="num_workers", + type=int, + default=1, + help=( + "Number of parallel workers passed to the client " + "binary's --workers flag." + ), + ) + validate_group.addoption( + "--traces", + action="store_true", + dest="consumer_collect_traces", + default=False, + help=( + "Collect traces of the execution information from " + "the fixture consumer tool." + ), + ) + debug_group = parser.getgroup( + "debug", "Arguments defining debug behavior" + ) + debug_group.addoption( + "--dump-dir", + action="store", + dest="base_dump_dir", + type=Path, + default=None, + help="Path to dump the fixture consumer tool debug output.", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: # noqa: D103 + validate_type = config.getoption("validate_type", None) + + # Health subcommand does not need client/fixture setup + if validate_type is None: + return + + # --- Fixture source loading --- + fixtures_source: FixturesSource + if config.option.fixtures_source is None: + fixtures_source = FixturesSource( + input_option=default_input(), + path=Path(default_input()), + ) + else: + fixtures_source = FixturesSource.from_input( + config.option.fixtures_source, + Path(config.option.fixture_cache_folder), + ) + config.fixtures_source = fixtures_source # type: ignore[attr-defined] + config.fixture_source_flags = [ # type: ignore[attr-defined] + "--input", + fixtures_source.input_option, + ] + + if fixtures_source.is_stdin: + config.test_cases = TestCases.from_stream( # type: ignore[attr-defined] + sys.stdin + ) + else: + index_file = ( + fixtures_source.path / ".meta" / "index.json" + ) + index_file.parent.mkdir(parents=True, exist_ok=True) + if not index_file.exists(): + rich.print( + f"Generating index file " + f"[bold cyan]{index_file}[/]..." + ) + generate_fixtures_index( + fixtures_source.path, + quiet_mode=False, + force_flag=False, + ) + index = IndexFile.model_validate_json( + index_file.read_text() + ) + config.test_cases = index.test_cases # type: ignore[attr-defined] + + for fixture_format in BaseFixture.formats.values(): + config.addinivalue_line( + "markers", + f"{fixture_format.format_name}: " + f"Tests in `{fixture_format.format_name}` " + f"format ", + ) + all_forks = { + fork + for fork in set(get_forks()) | get_transition_forks() + if not fork.ignore() + } + all_forks.update(getattr(index, "forks", [])) + for fork in all_forks: + config.addinivalue_line( + "markers", + f"{fork}: Tests for the {fork} fork", + ) + + if config.option.sim_limit: + if config.option.dest_regex != ".*": + pytest.exit( + "Both the --sim.limit (via env var?) and the " + "--regex flags are set. Please only set one " + "of them." + ) + config.option.dest_regex = ( + config.option.sim_limit.pattern + ) + if config.option.sim_limit.collectonly: + config.option.collectonly = True + config.option.verbose = -1 + + if config.option.collectonly or config.option.markers: + return + if ( + not config.getoption("disable_html") + and config.getoption("htmlpath") is None + ): + config.option.htmlpath = ( + config.fixtures_source.path # type: ignore[attr-defined] + / default_html_report_file_path() + ) + + # Resolve config file + config_path = _resolve_config_path() + if not config_path and not config.option.collectonly: + warnings.warn( + "No validate.toml or consume-direct.toml found. " + "Run `validate health` to check client binaries.", + stacklevel=1, + ) + + # Parse client names from the hidden option + clients_str = config.getoption("validate_clients", "") + client_aliases = {"go-ethereum": "geth"} + client_names = [ + client_aliases.get(c.strip(), c.strip()) + for c in clients_str.split(",") if c.strip() + ] + + # Help passthrough — skip client setup + if "__help__" in client_names: + return + + if not client_names and not config.option.collectonly: + pytest.exit( + "No clients specified. Use --client to " + "specify a fixture consumer." + ) + + # Load TOML config + try: + import tomllib + except ModuleNotFoundError: + import tomli as tomllib # type: ignore[no-redef] + + toml_config: dict[str, dict[str, str]] = {} + if config_path: + with open(config_path, "rb") as f: + toml_config = tomllib.load(f) + + # Resolve client names to binary paths and configs + bin_paths: list[Path] = [] + client_configs: dict[str, dict[str, str]] = {} + + for name in client_names: + entry = toml_config.get(name, {}) + bin_str = entry.get("bin", "") + if not bin_str: + config_name = ( + config_path.name if config_path else "validate.toml" + ) + pytest.exit( + f"Client '{name}' not configured in {config_name}." + ) + resolved = Path(bin_str).expanduser() + bin_paths.append(resolved) + client_configs[str(resolved)] = { + **entry, + "_client_name": name, + } + + # Set supported fixture formats based on validate type + type_to_formats = { + "state": [StateFixture], + "block": [BlockchainFixture], + "engine": [BlockchainEngineFixture], + } + config.supported_fixture_formats = type_to_formats[validate_type] # type: ignore[attr-defined] + + num_workers = config.getoption("num_workers", 1) + + # Create fixture consumers + fixture_consumers = [] + for bin_path in bin_paths: + entry = client_configs.get(str(bin_path), {}) + + # Resolve per-type binary overrides from TOML config + extra_kwargs: dict[str, Any] = {} + state_bin_str = entry.get("state-bin", "") + if state_bin_str: + extra_kwargs["state_binary"] = ( + Path(state_bin_str).expanduser() + ) + block_bin_str = entry.get("block-bin", "") + if block_bin_str: + extra_kwargs["block_binary"] = ( + Path(block_bin_str).expanduser() + ) + engine_bin_str = entry.get("engine-bin", "") + if engine_bin_str: + extra_kwargs["engine_binary"] = ( + Path(engine_bin_str).expanduser() + ) + + client_name = entry.get("_client_name", "") + consumer = None + + # Directly instantiate the right class to avoid ambiguous + # auto-detection (e.g. erigon and geth both use `evm`). + if client_name and client_name in CLIENT_CLASS_MAP: + cls = CLIENT_CLASS_MAP[client_name] + try: + consumer = cls( + binary=bin_path, + trace=config.getoption( + "consumer_collect_traces" + ), + **extra_kwargs, + ) + except Exception: + pass + + if consumer is None: + try: + consumer = FixtureConsumerTool.from_binary_path( + binary_path=bin_path, + trace=config.getoption( + "consumer_collect_traces" + ), + **extra_kwargs, + ) + except Exception: + from execution_testing.client_clis.clis.nethermind import ( + NethtestFixtureConsumer, + ) + + try: + consumer = ( + NethtestFixtureConsumer.from_binary_path( + binary_path=bin_path, + trace=config.getoption( + "consumer_collect_traces" + ), + ) + ) + except Exception as exception: + raise Exception( + f"Unknown CLI binary: {bin_path}. " + f"Could not detect as native binary " + f"or dotnet project." + ) from exception + + # Check that consumer supports the requested format(s) + supported = set(getattr(consumer, "fixture_formats", [])) + requested = set( + config.supported_fixture_formats # type: ignore[attr-defined] + ) + unsupported = requested - supported + if unsupported: + friendly = NAME_MAP.get( + type(consumer).__name__, type(consumer).__name__ + ) + unsupported_names = ", ".join( + f.format_name for f in unsupported + ) + supported_names = ", ".join( + f.format_name for f in supported + ) + hints = [] + if ( + BlockchainEngineFixture in unsupported + and BlockchainEngineXFixture in supported + ): + hints.append( + f"Use --type enginex instead of --type engine " + f"({friendly} needs enginex as engine is " + f"too slow)." + ) + if ( + BlockchainEngineXFixture in unsupported + and BlockchainEngineFixture in supported + ): + hints.append( + "Use --type engine instead of --type enginex." + ) + hint_str = ( + " " + " ".join(hints) if hints else "" + ) + pytest.exit( + f"{friendly} does not support: " + f"{unsupported_names}. " + f"Supported: {supported_names}.{hint_str}" + ) + + mapper_path_str = entry.get("exception-mapper", "") + if mapper_path_str: + mapper_path = _resolve_config_relative_path( + mapper_path_str, + config_path, + ) + external_mapper = load_external_exception_mapper(mapper_path) + consumer.exception_mapper = extend_exception_mapper( # type: ignore[attr-defined] + getattr(consumer, "exception_mapper", None), + external_mapper, + ) + fixture_consumers.append(consumer) + + if config.option.markers: + return + elif not fixture_consumers and config.option.collectonly: + warnings.warn( + ( + "No fixture consumer binaries provided; using a " + "dummy consumer for collect-only; all possible " + "fixture formats will be collected." + ), + stacklevel=1, + ) + fixture_consumers = [CollectOnlyFixtureConsumer()] + elif not fixture_consumers: + pytest.exit( + "No fixture consumer binaries provided; use " + "--client to specify a fixture consumer." + ) + + # Block -n for clients with top-level caching + no_xdist_reasons: dict[str, tuple[str, str]] = { + "BesuFixtureConsumer": ( + "JVM startup is expensive per xdist worker", + "Use --bin-workers instead. " + "Recommended: --bin-workers 8.", + ), + "NethtestFixtureConsumer": ( + "dotnet startup is expensive per xdist worker", + "Use --bin-workers instead. " + "Recommended: --bin-workers 4.", + ), + } + n_workers = config.getoption("numprocesses", None) + if n_workers and n_workers > 0: + for consumer in fixture_consumers: + cls_name = type(consumer).__name__ + xdist_entry = no_xdist_reasons.get(cls_name) + if xdist_entry: + friendly = NAME_MAP.get(cls_name, cls_name) + reason, suggestion = xdist_entry + pytest.exit( + f"{friendly} does not support -n (xdist): " + f"{reason}. {suggestion}" + ) + + # Reject --bin-workers for clients that don't support it + no_workers_clients: dict[str, tuple[str, str]] = { + "RethFixtureConsumer": ( + "parallelism is handled internally by rayon", + "Use -n instead. Recommended: -n 2.", + ), + "NimbusFixtureConsumer": ( + "--workers is not yet implemented", + "-n doesn't add much. Recommended: keep default (-n 1).", + ), + } + user_set_workers = "--bin-workers" in sys.argv + if user_set_workers and num_workers != 1: + for consumer in fixture_consumers: + cls_name = type(consumer).__name__ + entry = no_workers_clients.get(cls_name) + if entry: + reason, suggestion = entry + friendly = NAME_MAP.get(cls_name, cls_name) + pytest.exit( + f"{friendly} does not support " + f"--bin-workers ({reason}). " + f"{suggestion}" + ) + + # Auto-set -n for clients that use xdist + user_set_n = "-n" in sys.argv + if not user_set_n: + for consumer in fixture_consumers: + rec_n = RECOMMENDED_N.get(type(consumer).__name__) + if rec_n: + config.option.numprocesses = rec_n + break + + # Auto-set recommended --bin-workers + if not user_set_workers and num_workers == 1: + for consumer in fixture_consumers: + rec = RECOMMENDED_WORKERS.get( + type(consumer).__name__ + ) + if rec: + num_workers = rec + break + + no_exception_check = config.getoption( + "no_exception_check", False + ) + for consumer in fixture_consumers: + consumer.workers = num_workers # type: ignore[attr-defined] + consumer.exception_check = not no_exception_check # type: ignore[attr-defined] + + config.fixture_consumers = fixture_consumers # type: ignore[attr-defined] + + +def pytest_html_report_title(report: Any) -> None: + """Set the HTML report title (pytest-html plugin).""" + report.title = "Validate Test Report" + + +def pytest_report_header( + config: pytest.Config, +) -> List[str]: + """Add fixtures source and client info to report header.""" + lines: List[str] = [ + f"validate ref: {get_current_commit_hash_or_tag()}", + ] + source = getattr(config, "fixtures_source", None) + if source is not None: + lines.append(f"fixtures: {source.path}") + if not source.is_local and not source.is_stdin: + lines.append(f"fixtures url: {source.url}") + if source.release_page: + lines.append( + f"fixtures release: {source.release_page}" + ) + + consumers = getattr(config, "fixture_consumers", []) + if not consumers: + return lines + + cli_workers = config.getoption("num_workers", 1) + tw = config.get_terminal_writer() + + for consumer in consumers: + cls_name = type(consumer).__name__ + friendly = NAME_MAP.get(cls_name, cls_name) + actual_workers = getattr(consumer, "workers", cli_workers) + auto = ( + " (auto)" + if cli_workers == 1 and actual_workers != 1 + else "" + ) + tw.write( + f"client: {friendly} " + f"(bin-workers: {actual_workers}{auto})\n", + yellow=True, + ) + + n_workers = config.getoption("numprocesses", None) + user_set_n = "-n" in sys.argv + if n_workers: + auto_n = " (auto)" if not user_set_n else "" + tw.write( + f"xdist workers: {n_workers}{auto_n}\n", + yellow=True, + ) + + tw.write( + "Note: initial binary startup may take a moment " + "(especially JVM/dotnet clients)\n", + yellow=True, + ) + return lines + + +@pytest.fixture(scope="function") +def test_dump_dir( + request: pytest.FixtureRequest, + fixture_path: Path, + fixture_name: str, +) -> Path | None: + """The directory to write evm debug output to.""" + base_dump_dir = request.config.getoption("base_dump_dir") + if not base_dump_dir: + return None + if len(fixture_name) > 142: + fixture_name = ( + fixture_name[:70] + "..." + fixture_name[-70:] + ) + return ( + base_dump_dir + / fixture_path.stem + / fixture_name.replace("/", "-") + ) + + +@pytest.fixture +def fixture_path( + test_case: TestCaseIndexFile | TestCaseStream, + fixtures_source: FixturesSource, +) -> Generator[Path, None, None]: + """ + Path to the current JSON fixture file. + + If the fixture source is stdin, the fixture is written to a + temporary json file. + """ + if fixtures_source.is_stdin: + assert isinstance(test_case, TestCaseStream) + temp_dir = tempfile.TemporaryDirectory() + fixture_path = ( + Path(temp_dir.name) + / f"{test_case.id.replace('/', '_')}.json" + ) + fixtures = Fixtures({test_case.id: test_case.fixture}) + with open(fixture_path, "w") as f: + json.dump(to_json(fixtures), f, indent=4) + yield fixture_path + temp_dir.cleanup() + else: + assert isinstance(test_case, TestCaseIndexFile) + yield fixtures_source.path / test_case.json_path + + +@pytest.fixture(scope="function") +def fixture_name( + test_case: TestCaseIndexFile | TestCaseStream, +) -> str: + """Name of the current fixture.""" + return test_case.id + + +@pytest.fixture(scope="session") +def fixtures_source( + request: pytest.FixtureRequest, +) -> FixturesSource: + """Return the resolved fixtures source.""" + return request.config.fixtures_source # type: ignore[attr-defined] + + +@pytest.fixture(scope="session") +def fixture_source_flags( + request: pytest.FixtureRequest, +) -> List[str]: + """Return the input flags used to specify the fixture source.""" + return request.config.fixture_source_flags # type: ignore[attr-defined] + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Parametrize test cases and fixture consumers.""" + # Parametrize test_case from loaded fixtures index + if "test_case" in metafunc.fixturenames: + if not hasattr(metafunc.config, "test_cases"): + return + test_cases = metafunc.config.test_cases # type: ignore[attr-defined] + supported: List[FixtureFormat] = getattr( + metafunc.config, "supported_fixture_formats", [] + ) + param_list = [] + for test_case in test_cases: + if test_case.format not in supported: + continue + fork_markers = get_relative_fork_markers( + test_case.fork, strict_mode=False + ) + marks = [ + getattr(pytest.mark, m) for m in fork_markers + ] + [ + getattr( + pytest.mark, test_case.format.format_name + ) + ] + param_list.append( + pytest.param( + test_case, id=test_case.id, marks=marks + ) + ) + metafunc.parametrize("test_case", param_list) + + # Parametrize fixture_consumer from configured clients + if "fixture_consumer" in metafunc.fixturenames: + consumers = getattr( + metafunc.config, "fixture_consumers", None + ) + if consumers is None: + return + metafunc.parametrize( + "fixture_consumer", + ( + pytest.param( + fixture_consumer, + id=str( + fixture_consumer.__class__.__name__ + ), + ) + for fixture_consumer in consumers + ), + ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/health/__init__.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/health/__init__.py new file mode 100644 index 00000000000..a0e21b8b9bd --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/health/__init__.py @@ -0,0 +1 @@ +"""Health check tests for validate.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/health/test_health.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/health/test_health.py new file mode 100644 index 00000000000..56829b0abac --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/health/test_health.py @@ -0,0 +1,360 @@ +"""Health check tests for validate client binaries.""" + +import json +import subprocess +from pathlib import Path +from typing import Any, Dict, Optional + +import pytest + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[no-redef] + +from execution_testing.client_clis.fixture_consumer_tool import ( + FixtureConsumerTool, +) +from execution_testing.fixtures import ( + BlockchainEngineFixture, + BlockchainFixture, + StateFixture, +) + +CONFIG_FILES = ("validate.toml", "consume-direct.toml") + +SANITY_DIR = Path(__file__).parent.parent / "sanity_fixtures" + +FORMAT_MAP = { + "state": StateFixture, + "block": BlockchainFixture, + "engine": BlockchainEngineFixture, +} + +TYPE_TO_SUBDIR = { + "state": "state_tests", + "block": "blockchain_tests", + "engine": "blockchain_tests_engine", +} + + +def _find_config_file() -> str: + """Find the first existing config file.""" + for name in CONFIG_FILES: + if (Path.cwd() / name).exists(): + return name + return CONFIG_FILES[0] + + +def load_config() -> Dict[str, Dict[str, str]]: + """Load client configuration from validate.toml or consume-direct.toml.""" + config_file = _find_config_file() + config_path = Path.cwd() / config_file + if not config_path.exists(): + pytest.skip( + f"No config file found ({', '.join(CONFIG_FILES)}). " + f"Create one to configure client binaries." + ) + with open(config_path, "rb") as f: + return tomllib.load(f) + + +def detect_version(bin_path: Path) -> Optional[str]: + """Detect version from a binary or dotnet project.""" + is_dotnet = ( + bin_path.is_dir() + or bin_path.suffix in (".csproj", ".dll") + ) + for flag in ["--version", "-v", "version"]: + try: + if is_dotnet: + project = bin_path + if project.is_dir(): + csproj = list(project.glob("*.csproj")) + if csproj: + project = csproj[0] + cmd = [ + "dotnet", "run", "--no-build", "-c", "Release", + "--project", str(project), "--", flag, + ] + else: + cmd = [str(bin_path), flag] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode == 0 and result.stdout.strip(): + lines = [ + l.strip() + for l in result.stdout.strip().splitlines() + if l.strip() + ] + return lines[-1] if lines else None + except Exception: + continue + return None + + +def get_consumer(client: str, bin_path: Path) -> Any: + """Create consumer for a client, passing extra config like state-bin.""" + from execution_testing.client_clis.clis.besu import BesuFixtureConsumer + from execution_testing.client_clis.clis.ethrex import EthrexFixtureConsumer + from execution_testing.client_clis.clis.geth import GethFixtureConsumer + from execution_testing.client_clis.clis.nethermind import NethtestFixtureConsumer + from execution_testing.client_clis.clis.nimbus import NimbusFixtureConsumer + from execution_testing.client_clis.clis.reth import RethFixtureConsumer + + class_map: Dict[str, type] = { + "geth": GethFixtureConsumer, + "besu": BesuFixtureConsumer, + "nethermind": NethtestFixtureConsumer, + "reth": RethFixtureConsumer, + "ethrex": EthrexFixtureConsumer, + "nimbus": NimbusFixtureConsumer, + } + + config = load_config() + client_config = config.get(client, {}) + kwargs: Dict[str, Any] = {"binary": bin_path, "trace": False} + + state_bin_str = client_config.get("state-bin", "") + if state_bin_str: + kwargs["state_binary"] = Path(state_bin_str).expanduser().resolve() + + block_bin_str = client_config.get("block-bin", "") + if block_bin_str: + kwargs["block_binary"] = Path(block_bin_str).expanduser().resolve() + + engine_bin_str = client_config.get("engine-bin", "") + if engine_bin_str: + kwargs["engine_binary"] = Path(engine_bin_str).expanduser().resolve() + + cls = class_map.get(client) + if cls: + return cls(**kwargs) + + return FixtureConsumerTool.from_binary_path( + binary_path=bin_path, trace=False + ) + + +def get_bin_path(client: str, test_type: str | None = None) -> Path: + """Resolve binary path for a client from config. + + Support per-type binary overrides (e.g. state-bin for reth/revm). + Skip if not configured. + """ + config = load_config() + client_config = config.get(client, {}) + config_file = _find_config_file() + # Check for type-specific binary first + bin_str = "" + if test_type: + bin_str = client_config.get(f"{test_type}-bin", "") + if not bin_str: + bin_str = client_config.get("bin", "") + if not bin_str: + pytest.skip(f"{client}: not configured in {config_file}") + bin_path = Path(bin_str).expanduser().resolve() + assert bin_path.exists(), f"binary not found: {bin_str}" + return bin_path + + +def run_version(client: str) -> None: + """Check binary exists and version is detectable.""" + bin_path = get_bin_path(client) + version = detect_version(bin_path) + if version is None: + import warnings + warnings.warn( + f"{client}: could not detect version from {bin_path}" + ) + + +def run_health(client: str, test_type: str) -> None: + """Run a sanity fixture for a client + test type.""" + # Always detect consumer from main binary, not type-specific override + main_bin = get_bin_path(client) + consumer = get_consumer(client, main_bin) + # Verify type-specific binary exists if configured + type_bin = get_bin_path(client, test_type) + _ = type_bin # just check it exists (get_bin_path asserts) + + fixture_format = FORMAT_MAP[test_type] + assert fixture_format in consumer.fixture_formats, ( + f"{client}: {test_type} not supported" + ) + + subdir = TYPE_TO_SUBDIR[test_type] + fixture_file = next((SANITY_DIR / subdir).glob("*.json"), None) + assert fixture_file is not None, ( + f"sanity fixture missing for {test_type}" + ) + + data = json.loads(fixture_file.read_text()) + first_name = next(iter(data.keys())) + + consumer.consume_fixture( + fixture_format=fixture_format, + fixture_path=fixture_file, + fixture_name=first_name, + ) + + +# --- geth --- + + +def test_geth_version() -> None: + """Geth version detection.""" + run_version("geth") + + +def test_geth_state() -> None: + """Geth state test sanity check.""" + run_health("geth", "state") + + +def test_geth_block() -> None: + """Geth block test sanity check.""" + run_health("geth", "block") + + +def test_geth_engine() -> None: + """Geth engine test sanity check.""" + run_health("geth", "engine") + + +# --- besu --- + + +def test_besu_version() -> None: + """Besu version detection.""" + run_version("besu") + + +def test_besu_state() -> None: + """Besu state test sanity check.""" + run_health("besu", "state") + + +def test_besu_block() -> None: + """Besu block test sanity check.""" + run_health("besu", "block") + + +def test_besu_engine() -> None: + """Besu engine test sanity check.""" + run_health("besu", "engine") + + +# --- nethermind --- + + +def test_nethermind_version() -> None: + """Nethermind version detection.""" + run_version("nethermind") + + +def test_nethermind_state() -> None: + """Nethermind state test sanity check.""" + run_health("nethermind", "state") + + +def test_nethermind_block() -> None: + """Nethermind block test sanity check.""" + run_health("nethermind", "block") + + +def test_nethermind_engine() -> None: + """Nethermind engine test sanity check.""" + run_health("nethermind", "engine") + + +# --- erigon --- + + +def test_erigon_version() -> None: + """Erigon version detection.""" + run_version("erigon") + + +def test_erigon_state() -> None: + """Erigon state test sanity check.""" + run_health("erigon", "state") + + +def test_erigon_block() -> None: + """Erigon block test sanity check.""" + run_health("erigon", "block") + + +def test_erigon_engine() -> None: + """Erigon engine test sanity check.""" + run_health("erigon", "engine") + + +# --- reth --- + + +def test_reth_version() -> None: + """Reth version detection.""" + run_version("reth") + + +def test_reth_state() -> None: + """Reth state test sanity check.""" + run_health("reth", "state") + + +def test_reth_engine() -> None: + """Reth engine test sanity check.""" + run_health("reth", "engine") + + +# --- ethrex --- + + +def test_ethrex_version() -> None: + """Ethrex version detection.""" + run_version("ethrex") + + +def test_ethrex_state() -> None: + """Ethrex state test sanity check.""" + run_health("ethrex", "state") + + +def test_ethrex_block() -> None: + """Ethrex block test sanity check.""" + run_health("ethrex", "block") + + +def test_ethrex_engine() -> None: + """Ethrex engine test sanity check.""" + run_health("ethrex", "engine") + + +# --- nimbus --- + + +def test_nimbus_version() -> None: + """Nimbus version detection.""" + run_version("nimbus") + + +def test_nimbus_state() -> None: + """Nimbus state test sanity check.""" + run_health("nimbus", "state") + + +def test_nimbus_block() -> None: + """Nimbus block test sanity check.""" + run_health("nimbus", "block") + + +def test_nimbus_engine() -> None: + """Nimbus engine test sanity check.""" + run_health("nimbus", "engine") diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/blockchain_tests/sanity.json b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/blockchain_tests/sanity.json new file mode 100644 index 00000000000..f0a29c31e70 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/blockchain_tests/sanity.json @@ -0,0 +1,138 @@ +{ + "tests/frontier/opcodes/test_all_opcodes.py::test_cover_revert[fork_Cancun-blockchain_test_from_state_test]": { + "network": "Cancun", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0xb5dd68136df8950b7c99d24e768837d396cea55192f2da330e53db9aacd760fd", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x00", + "gasLimit": "0x055d4a80", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0xf7240ae46bb2d8f8aa09a7522623f5981536cdc884aa596fc2d73f3062ce8707" + }, + "pre": { + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "nonce": "0x01", + "balance": "0x00", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "storage": {} + }, + "0x0a16f360235334164b5f50ca2fa0d37e420cedb8": { + "nonce": "0x00", + "balance": "0x3635c9adc5dea00000", + "code": "0x", + "storage": {} + } + }, + "postState": { + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "nonce": "0x01", + "balance": "0x00", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "storage": { + "0x03e8": "0x03e8" + } + }, + "0x0a16f360235334164b5f50ca2fa0d37e420cedb8": { + "nonce": "0x01", + "balance": "0x3635c9adc5de94848c", + "code": "0x", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "nonce": "0x00", + "balance": "0x0371d6", + "code": "0x", + "storage": {} + } + }, + "lastblockhash": "0x3e732f38e89bdf0045ee30d1659bd10b84b0b4f38b90be85a961a1d41cdf5dc5", + "config": { + "network": "Cancun", + "chainid": "0x01", + "blobSchedule": { + "Cancun": { + "target": "0x03", + "max": "0x06", + "baseFeeUpdateFraction": "0x32f0ed" + } + } + }, + "genesisRLP": "0xf9023cf90236a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347940000000000000000000000000000000000000000a0b5dd68136df8950b7c99d24e768837d396cea55192f2da330e53db9aacd760fda056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808084055d4a80808000a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b4218080a00000000000000000000000000000000000000000000000000000000000000000c0c0c0", + "blocks": [ + { + "blockHeader": { + "parentHash": "0xf7240ae46bb2d8f8aa09a7522623f5981536cdc884aa596fc2d73f3062ce8707", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0xaa3819f9a42be7d90070fa4f27936ac749e2aebf38a3478ddd23d1e040813813", + "transactionsTrie": "0xc1b6cb0b90a73cdbdac1becdef3032d4fb12c05d6148455d7b89e9d645eb51dd", + "receiptTrie": "0xd9de6e9898ff9600756178bd3f830c936cda7fe19aec53ff07a9b789449e20ef", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x01", + "gasLimit": "0x055d4a80", + "gasUsed": "0x0125f2", + "timestamp": "0x03e8", + "extraData": "0x00", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x3e732f38e89bdf0045ee30d1659bd10b84b0b4f38b90be85a961a1d41cdf5dc5" + }, + "transactions": [ + { + "type": "0x00", + "chainId": "0x01", + "nonce": "0x00", + "gasPrice": "0x0a", + "gasLimit": "0x0f4240", + "value": "0x00", + "data": "0x600160015560006000fd", + "v": "0x1b", + "r": "0xaeb31a2cc02627d0117ed31cc20f490a9a0fd3f313eaf459cc45092b9467c67d", + "s": "0x7af3ebeaa748580d1fe8c12e92039fa199a8b299230f62577478ff86f405f3e9", + "sender": "0x0a16f360235334164b5f50ca2fa0d37e420cedb8", + "to": "" + } + ], + "uncleHeaders": [], + "withdrawals": [], + "rlp": "0xf9029af9023ba0f7240ae46bb2d8f8aa09a7522623f5981536cdc884aa596fc2d73f3062ce8707a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa0aa3819f9a42be7d90070fa4f27936ac749e2aebf38a3478ddd23d1e040813813a0c1b6cb0b90a73cdbdac1becdef3032d4fb12c05d6148455d7b89e9d645eb51dda0d9de6e9898ff9600756178bd3f830c936cda7fe19aec53ff07a9b789449e20efb9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800184055d4a80830125f28203e800a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b4218080a00000000000000000000000000000000000000000000000000000000000000000f858f856800a830f424080808a600160015560006000fd1ba0aeb31a2cc02627d0117ed31cc20f490a9a0fd3f313eaf459cc45092b9467c67da07af3ebeaa748580d1fe8c12e92039fa199a8b299230f62577478ff86f405f3e9c0c0", + "blocknumber": "1" + } + ], + "sealEngine": "NoProof", + "_info": { + "hash": "0xc31f9082a802c728c8cb8b44f3a4c52d52112315a1c7deda83a97fb3bd256935", + "comment": "`execution-spec-tests` generated test", + "filling-transition-tool": "ethereum-spec-evm-resolver 0.0.5", + "description": "Cover state revert from original tests for the coverage script.", + "url": "https://github.com/ethereum/execution-spec-tests/tree/v5.3.0/tests/frontier/opcodes/test_all_opcodes.py#L112", + "fixture-format": "blockchain_test", + "eels-resolution": { + "git-url": "https://github.com/ethereum/execution-specs.git", + "branch": "forks/bpos", + "commit": "7aacb2c54111391af5e98c505d5010b5698a770f" + } + } + } +} \ No newline at end of file diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/blockchain_tests_engine/sanity.json b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/blockchain_tests_engine/sanity.json new file mode 100644 index 00000000000..1c3080ae28b --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/blockchain_tests_engine/sanity.json @@ -0,0 +1,120 @@ +{ + "tests/frontier/opcodes/test_all_opcodes.py::test_cover_revert[fork_Cancun-blockchain_test_engine_from_state_test]": { + "network": "Cancun", + "lastblockhash": "0x3e732f38e89bdf0045ee30d1659bd10b84b0b4f38b90be85a961a1d41cdf5dc5", + "config": { + "network": "Cancun", + "chainid": "0x01", + "blobSchedule": { + "Cancun": { + "target": "0x03", + "max": "0x06", + "baseFeeUpdateFraction": "0x32f0ed" + } + } + }, + "pre": { + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "nonce": "0x01", + "balance": "0x00", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "storage": {} + }, + "0x0a16f360235334164b5f50ca2fa0d37e420cedb8": { + "nonce": "0x00", + "balance": "0x3635c9adc5dea00000", + "code": "0x", + "storage": {} + } + }, + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0xb5dd68136df8950b7c99d24e768837d396cea55192f2da330e53db9aacd760fd", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x00", + "gasLimit": "0x055d4a80", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0xf7240ae46bb2d8f8aa09a7522623f5981536cdc884aa596fc2d73f3062ce8707" + }, + "postState": { + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "nonce": "0x01", + "balance": "0x00", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "storage": { + "0x03e8": "0x03e8" + } + }, + "0x0a16f360235334164b5f50ca2fa0d37e420cedb8": { + "nonce": "0x01", + "balance": "0x3635c9adc5de94848c", + "code": "0x", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "nonce": "0x00", + "balance": "0x0371d6", + "code": "0x", + "storage": {} + } + }, + "engineNewPayloads": [ + { + "params": [ + { + "parentHash": "0xf7240ae46bb2d8f8aa09a7522623f5981536cdc884aa596fc2d73f3062ce8707", + "feeRecipient": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0xaa3819f9a42be7d90070fa4f27936ac749e2aebf38a3478ddd23d1e040813813", + "receiptsRoot": "0xd9de6e9898ff9600756178bd3f830c936cda7fe19aec53ff07a9b789449e20ef", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", + "gasLimit": "0x55d4a80", + "gasUsed": "0x125f2", + "timestamp": "0x3e8", + "extraData": "0x00", + "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas": "0x7", + "blobGasUsed": "0x0", + "excessBlobGas": "0x0", + "blockHash": "0x3e732f38e89bdf0045ee30d1659bd10b84b0b4f38b90be85a961a1d41cdf5dc5", + "transactions": [ + "0xf856800a830f424080808a600160015560006000fd1ba0aeb31a2cc02627d0117ed31cc20f490a9a0fd3f313eaf459cc45092b9467c67da07af3ebeaa748580d1fe8c12e92039fa199a8b299230f62577478ff86f405f3e9" + ], + "withdrawals": [] + }, + [], + "0x0000000000000000000000000000000000000000000000000000000000000000" + ], + "newPayloadVersion": "3", + "forkchoiceUpdatedVersion": "3" + } + ], + "_info": { + "hash": "0x3f492409853312965e745c49f708b039c9cd91baec99c014c7547540812bd2b9", + "comment": "`execution-spec-tests` generated test", + "filling-transition-tool": "ethereum-spec-evm-resolver 0.0.5", + "description": "Cover state revert from original tests for the coverage script.", + "url": "https://github.com/ethereum/execution-spec-tests/tree/v5.3.0/tests/frontier/opcodes/test_all_opcodes.py#L112", + "fixture-format": "blockchain_test_engine", + "eels-resolution": { + "git-url": "https://github.com/ethereum/execution-specs.git", + "branch": "forks/bpos", + "commit": "7aacb2c54111391af5e98c505d5010b5698a770f" + } + } + } +} \ No newline at end of file diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/state_tests/sanity.json b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/state_tests/sanity.json new file mode 100644 index 00000000000..e9867b320e0 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/sanity_fixtures/state_tests/sanity.json @@ -0,0 +1,89 @@ +{ + "tests/frontier/opcodes/test_all_opcodes.py::test_cover_revert[fork_Cancun-state_test]": { + "env": { + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "0x055d4a80", + "currentNumber": "0x01", + "currentTimestamp": "0x03e8", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000000000", + "currentDifficulty": "0x00", + "currentBaseFee": "0x07", + "currentExcessBlobGas": "0x00" + }, + "pre": { + "0x0a16f360235334164b5f50ca2fa0d37e420cedb8": { + "nonce": "0x00", + "balance": "0x3635c9adc5dea00000", + "code": "0x", + "storage": {} + } + }, + "transaction": { + "nonce": "0x00", + "gasPrice": "0x0a", + "gasLimit": [ + "0x0f4240" + ], + "value": [ + "0x00" + ], + "data": [ + "0x600160015560006000fd" + ], + "sender": "0x0a16f360235334164b5f50ca2fa0d37e420cedb8", + "secretKey": "0xf5f0650b2e65e494cf980df8220fc1635987686c02e1ab1184a2a2de8bf696f7", + "to": "" + }, + "post": { + "Cancun": [ + { + "hash": "0xdfe3e551e5a9ad38d8e078915698609e11a4fdd5fd4c0db4213a4532ab34ccee", + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "txbytes": "0xf856800a830f424080808a600160015560006000fd1ba0aeb31a2cc02627d0117ed31cc20f490a9a0fd3f313eaf459cc45092b9467c67da07af3ebeaa748580d1fe8c12e92039fa199a8b299230f62577478ff86f405f3e9", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "state": { + "0x0a16f360235334164b5f50ca2fa0d37e420cedb8": { + "nonce": "0x01", + "balance": "0x3635c9adc5de94848c", + "code": "0x", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "nonce": "0x00", + "balance": "0x0371d6", + "code": "0x", + "storage": {} + } + } + } + ] + }, + "config": { + "blobSchedule": { + "Cancun": { + "target": "0x03", + "max": "0x06", + "baseFeeUpdateFraction": "0x32f0ed" + } + }, + "chainid": "0x01" + }, + "_info": { + "hash": "0x6745d6e1d1a363b7e40e89713a9cac5e8c8b8ce4115656559496782c317ca5a6", + "comment": "`execution-spec-tests` generated test", + "filling-transition-tool": "ethereum-spec-evm-resolver 0.0.5", + "description": "Cover state revert from original tests for the coverage script.", + "url": "https://github.com/ethereum/execution-spec-tests/tree/v5.3.0/tests/frontier/opcodes/test_all_opcodes.py#L112", + "fixture-format": "state_test", + "eels-resolution": { + "git-url": "https://github.com/ethereum/execution-specs.git", + "branch": "forks/bpos", + "commit": "7aacb2c54111391af5e98c505d5010b5698a770f" + } + } + } +} \ No newline at end of file diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/test_validate.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/test_validate.py new file mode 100644 index 00000000000..bdd976c61e1 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/test_validate.py @@ -0,0 +1,30 @@ +""" +Execute a JSON test fixture directly against a client using a dedicated client +interface similar to geth's EVM 'blocktest' command. +""" + +from pathlib import Path + +from execution_testing.fixtures import FixtureConsumer +from execution_testing.fixtures.consume import ( + TestCaseIndexFile, + TestCaseStream, +) + + +def test_fixture( + test_case: TestCaseIndexFile | TestCaseStream, + fixture_consumer: FixtureConsumer, + fixture_path: Path, + test_dump_dir: Path | None, +) -> None: + """ + Generic test function used to call the fixture consumer with a given + fixture file path and a fixture name (for a single test run). + """ + fixture_consumer.consume_fixture( + test_case.format, + fixture_path, + fixture_name=test_case.id, + debug_output_path=test_dump_dir, + ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/tests/test_exception_mappers.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/tests/test_exception_mappers.py new file mode 100644 index 00000000000..1ba6c48a335 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/validate/tests/test_exception_mappers.py @@ -0,0 +1,22 @@ +"""Tests for validate exception mapper config helpers.""" + +from pathlib import Path + +from execution_testing.cli.pytest_commands.plugins.validate.conftest import ( + _resolve_config_relative_path, +) + + +def test_exception_mapper_path_resolves_relative_to_validate_toml( + tmp_path: Path, +) -> None: + """Resolve configured mapper paths relative to the TOML file.""" + config_path = tmp_path / "configs" / "validate.toml" + mapper_path = _resolve_config_relative_path( + "../client/eest-exceptions.yaml", + config_path, + ) + + assert mapper_path == ( + tmp_path / "configs" / "../client/eest-exceptions.yaml" + ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-validate.ini b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-validate.ini new file mode 100644 index 00000000000..47d7a28542b --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-validate.ini @@ -0,0 +1,11 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = test_* +addopts = + -rxXs + --tb short + -p execution_testing.cli.pytest_commands.plugins.concurrency + -p no:logging + -p execution_testing.cli.pytest_commands.plugins.custom_logging.plugin_logging + -p execution_testing.cli.pytest_commands.plugins.help.help diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/validate.py b/packages/testing/src/execution_testing/cli/pytest_commands/validate.py new file mode 100644 index 00000000000..f231e56a909 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/validate.py @@ -0,0 +1,186 @@ +"""CLI entry point for the `validate` pytest-based command.""" + +from pathlib import Path +from typing import Any, List + +import click + +from .base import ArgumentProcessor, PytestCommand, common_pytest_options +from .processors import ConsumeCommandProcessor, HelpFlagsProcessor + + +def get_validate_test_paths(command_name: str) -> List[Path]: + """Determine test paths based on the validate subcommand name.""" + base_path = Path("cli/pytest_commands/plugins/validate") + if command_name in ("state", "block", "engine"): + return [base_path / "test_validate.py"] + elif command_name == "health": + return [base_path / "health" / "test_health.py"] + else: + raise ValueError(f"Unexpected validate command: {command_name}.") + + +def create_validate_command( + *, + command_logic_test_paths: List[Path], +) -> PytestCommand: + """Initialize validate command with paths and processors.""" + processors: List[ArgumentProcessor] = [ + HelpFlagsProcessor("validate"), + ConsumeCommandProcessor(is_hive=False), + ] + return PytestCommand( + config_file="pytest-validate.ini", + argument_processors=processors, + command_logic_test_paths=command_logic_test_paths, + ) + + +class SectionedGroup(click.Group): + """Click group that displays commands in sections.""" + + sections = { + "Test Types": ["state", "block", "engine"], + "Utilities": ["health"], + } + + def format_commands( + self, ctx: click.Context, formatter: click.HelpFormatter + ) -> None: + """Write commands in sections.""" + for section, names in self.sections.items(): + cmds = [] + for name in names: + cmd = self.get_command(ctx, name) + if cmd is None: + continue + help_text = cmd.get_short_help_str(limit=150) + cmds.append((name, help_text)) + if cmds: + with formatter.section(section): + formatter.write_dl(cmds) + + +@click.group( + cls=SectionedGroup, + context_settings={"help_option_names": ["-h", "--help"]}, + invoke_without_command=True, +) +@click.pass_context +def validate(ctx: click.Context) -> None: + """Validate client EVM implementations against test fixtures.""" + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +def _run_validate( + test_type: str, + clients: tuple[str, ...], + pytest_args: List[str], + help_flag: bool = False, +) -> None: + """Common logic for state/block/engine subcommands.""" + if not clients: + click.echo( + f"Note: --client is required. " + f"Example: validate {test_type} --client geth\n" + ) + help_flag = True + if help_flag: + # Show filtered help via --validate-help + args = ["--validate-help"] + cmd = create_validate_command( + command_logic_test_paths=get_validate_test_paths(test_type), + ) + cmd.execute(args) + return + args = list(pytest_args) + args.extend(["--validate-type", test_type]) + args.extend(["--validate-clients", ",".join(clients)]) + cmd = create_validate_command( + command_logic_test_paths=get_validate_test_paths(test_type), + ) + cmd.execute(args) + + +@validate.command( + name="state", + context_settings={"ignore_unknown_options": True, "help_option_names": ["-h", "--help"]}, +) +@click.option( + "--client", + "clients", + multiple=True, + help="Client name (e.g. geth, besu). Can be used multiple times.", +) +@common_pytest_options +def state( + clients: tuple[str, ...], pytest_args: List[str], **kwargs: Any +) -> None: + """Validate client state test implementations.""" + _run_validate("state", clients, list(pytest_args), kwargs.get("help_flag", False)) + + +@validate.command( + name="block", + context_settings={"ignore_unknown_options": True, "help_option_names": ["-h", "--help"]}, +) +@click.option( + "--client", + "clients", + multiple=True, + help="Client name (e.g. geth, besu). Can be used multiple times.", +) +@common_pytest_options +def block( + clients: tuple[str, ...], pytest_args: List[str], **kwargs: Any +) -> None: + """Validate client block test implementations.""" + _run_validate("block", clients, list(pytest_args), kwargs.get("help_flag", False)) + + +@validate.command( + name="engine", + context_settings={"ignore_unknown_options": True, "help_option_names": ["-h", "--help"]}, +) +@click.option( + "--client", + "clients", + multiple=True, + help="Client name (e.g. geth, besu). Can be used multiple times.", +) +@common_pytest_options +def engine( + clients: tuple[str, ...], pytest_args: List[str], **kwargs: Any +) -> None: + """Validate client engine test implementations.""" + _run_validate("engine", clients, list(pytest_args), kwargs.get("help_flag", False)) + + +@validate.command( + name="health", + context_settings={"ignore_unknown_options": True, "help_option_names": ["-h", "--help"]}, +) +@click.option( + "--client", + "clients", + multiple=True, + default=(), + help="Filter health checks to specific client(s).", +) +@common_pytest_options +def health( + clients: tuple[str, ...], pytest_args: List[str], **kwargs: Any +) -> None: + """Run health checks for configured client binaries.""" + del kwargs + args = list(pytest_args) + if "-v" not in args and "--verbose" not in args: + args.insert(0, "-v") + if clients: + k_expr = " or ".join(clients) + args.extend(["-k", k_expr]) + cmd = create_validate_command( + command_logic_test_paths=get_validate_test_paths("health"), + ) + cmd.execute(args) diff --git a/packages/testing/src/execution_testing/client_clis/__init__.py b/packages/testing/src/execution_testing/client_clis/__init__.py index e44839fe578..ce540f91905 100644 --- a/packages/testing/src/execution_testing/client_clis/__init__.py +++ b/packages/testing/src/execution_testing/client_clis/__init__.py @@ -12,7 +12,12 @@ TransitionToolOutput, ) from .clis.besu import BesuFixtureConsumer, BesuTransitionTool +# Erigon must be imported before geth: both use the `evm` binary, +# and erigon's more-specific detect pattern (version >= 2.x) must +# be checked before geth's broader pattern. +from .clis.erigon import ErigonFixtureConsumer from .clis.ethereumjs import EthereumJSTransitionTool +from .clis.ethrex import EthrexFixtureConsumer from .clis.evmone import ( EvmOneBlockchainFixtureConsumer, EvmoneExceptionMapper, @@ -22,7 +27,8 @@ from .clis.execution_specs import ExecutionSpecsTransitionTool from .clis.geth import GethFixtureConsumer, GethTransitionTool from .clis.nethermind import Nethtest, NethtestFixtureConsumer -from .clis.nimbus import NimbusTransitionTool +from .clis.nimbus import NimbusFixtureConsumer, NimbusTransitionTool +from .clis.reth import RethFixtureConsumer from .ethereum_cli import CLINotFoundInPathError, UnknownCLIError from .fixture_consumer_tool import FixtureConsumerTool from .transition_tool import TransitionTool @@ -35,7 +41,9 @@ "BesuTransitionTool", "BlockExceptionWithMessage", "CLINotFoundInPathError", + "ErigonFixtureConsumer", "EthereumJSTransitionTool", + "EthrexFixtureConsumer", "EvmoneExceptionMapper", "EvmOneTransitionTool", "EvmOneStateFixtureConsumer", @@ -47,7 +55,9 @@ "LazyAlloc", "Nethtest", "NethtestFixtureConsumer", + "NimbusFixtureConsumer", "NimbusTransitionTool", + "RethFixtureConsumer", "Result", "Traces", "TransactionExceptionWithMessage", diff --git a/packages/testing/src/execution_testing/client_clis/cli_types.py b/packages/testing/src/execution_testing/client_clis/cli_types.py index 07dff0d72ae..e9ab38a5d25 100644 --- a/packages/testing/src/execution_testing/client_clis/cli_types.py +++ b/packages/testing/src/execution_testing/client_clis/cli_types.py @@ -39,6 +39,43 @@ logger = get_logger(__name__) +class FixtureTestResult(CamelModel): + """Base result fields shared by all fixture test types.""" + + name: str + """Full fixture key as it appears in the JSON file.""" + + pass_field: bool = Field(..., alias="pass") + """Whether the test passed.""" + + fork: str + """Fork name (e.g. 'Prague', 'CancunToPragueAtTime15k').""" + + error: str = "" + """Error or validation error string. Populated even on pass for invalid tests.""" + + +class StateTestResult(FixtureTestResult): + """Result from a client's statetest runner.""" + + state_root: str = Field(default="", alias="stateRoot") + """Hex-encoded final state root.""" + + +class BlockTestResult(FixtureTestResult): + """Result from a client's blocktest runner.""" + + last_block_hash: str = Field(default="", alias="lastBlockHash") + """Hex-encoded hash of the last block in the chain.""" + + +class EngineTestResult(BlockTestResult): + """Result from a client's enginetest runner.""" + + last_payload_status: str = Field(default="", alias="lastPayloadStatus") + """Payload status of the last newPayload call (VALID/INVALID/SYNCING).""" + + class TransactionExceptionWithMessage( ExceptionWithMessage[TransactionException] ): diff --git a/packages/testing/src/execution_testing/client_clis/clis/besu.py b/packages/testing/src/execution_testing/client_clis/clis/besu.py index a48ec82f50e..46384a77a66 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/besu.py +++ b/packages/testing/src/execution_testing/client_clis/clis/besu.py @@ -21,13 +21,21 @@ TransactionException, ) from execution_testing.fixtures import ( + BlockchainEngineFixture, BlockchainFixture, FixtureFormat, StateFixture, ) from execution_testing.forks import Fork -from ..cli_types import TransitionToolOutput +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, + TransitionToolOutput, +) +from ..validate_helpers import validate_test_result from ..ethereum_cli import EthereumCLI from ..fixture_consumer_tool import FixtureConsumerTool from ..transition_tool import ( @@ -57,7 +65,7 @@ def __init__( self.binary = binary if binary else self.default_binary self.trace = trace - def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: + def run_command(self, command: List[str]) -> subprocess.CompletedProcess: """Run a command and return the result.""" try: return subprocess.run( @@ -71,7 +79,7 @@ def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: except Exception as e: raise Exception("Unexpected exception calling evmtool.") from e - def _consume_debug_dump( + def validate_debug_dump( self, command: List[str], result: subprocess.CompletedProcess, @@ -87,22 +95,22 @@ def _consume_debug_dump( debug_fixture_path = str(debug_output_path / "fixtures.json") command[-1] = debug_fixture_path - consume_direct_call = " ".join(shlex.quote(arg) for arg in command) + validate_call = " ".join(shlex.quote(arg) for arg in command) - consume_direct_script = textwrap.dedent( + validate_script = textwrap.dedent( f"""\ #!/bin/bash - {consume_direct_call} + {validate_call} """ ) dump_files_to_directory( debug_output_path, { - "consume_direct_args.py": command, - "consume_direct_returncode.txt": result.returncode, - "consume_direct_stdout.txt": result.stdout, - "consume_direct_stderr.txt": result.stderr, - "consume_direct.sh+x": consume_direct_script, + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, }, ) shutil.copyfile(fixture_path, debug_fixture_path) @@ -424,12 +432,12 @@ class BesuExceptionMapper(ExceptionMapper): r"exceeds transaction sender account balance 0x[0-9a-f]+" ), TransactionException.INTRINSIC_GAS_TOO_LOW: ( - r"transaction invalid intrinsic gas cost \d+" + r"(?:transaction invalid )?intrinsic gas cost \d+" r"(?: \(regular \d+ \+ state \d+\))? " r"exceeds gas limit \d+" ), TransactionException.INTRINSIC_GAS_BELOW_FLOOR_GAS_COST: ( - r"transaction invalid intrinsic gas cost \d+" + r"(?:transaction invalid )?intrinsic gas cost \d+" r"(?: \(regular \d+ \+ state \d+\))? " r"exceeds gas limit \d+" ), @@ -490,155 +498,129 @@ class BesuExceptionMapper(ExceptionMapper): class BesuFixtureConsumer( BesuEvmTool, FixtureConsumerTool, - fixture_formats=[StateFixture, BlockchainFixture], + fixture_formats=[StateFixture, BlockchainFixture, BlockchainEngineFixture], ): """Besu's implementation of the fixture consumer.""" - def consume_blockchain_test( - self, - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ) -> None: - """ - Consume a single blockchain test. - - Besu's ``evmtool block-test`` accepts ``--test-name`` to - select a specific fixture from the file. - """ - subcommand = "block-test" - subcommand_options: List[str] = [] - if debug_output_path: - subcommand_options += ["--json"] - - if fixture_name: - subcommand_options += [ - "--test-name", - fixture_name, - ] - - command = ( - [str(self.binary)] - + [subcommand] - + subcommand_options - + [str(fixture_path)] - ) - - result = self._run_command(command) - - if debug_output_path: - self._consume_debug_dump( - command, result, fixture_path, debug_output_path - ) - - if result.returncode != 0: - raise Exception( - f"Unexpected exit code:\n{' '.join(command)}\n\n" - f"Error:\n{result.stderr}" - ) + subcommands: Dict[type, str] = { + StateFixture: "state-test", + BlockchainFixture: "block-test", + BlockchainEngineFixture: "engine-test", + } - # Parse text output for failures - stdout = result.stdout - if "Failed:" in stdout: - failed_match = re.search(r"Failed:\s+(\d+)", stdout) - if failed_match and int(failed_match.group(1)) > 0: - raise Exception(f"Blockchain test failed:\n{stdout}") + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + fixture_cache: Dict[str, Dict[str, Any]] = {} + exception_mapper: ExceptionMapper = BesuExceptionMapper() - @cache # noqa - def consume_state_test_file( + def get_dir_results( self, + fixture_format: FixtureFormat, fixture_path: Path, debug_output_path: Optional[Path] = None, - ) -> List[Dict[str, Any]]: - """ - Consume an entire state test file. + ) -> Dict[str, Dict[str, Any]]: + """Run evmtool once per type directory and cache results. - Besu's ``evmtool state-test`` outputs one JSON object per - line (NDJSON) with a ``test`` field instead of ``name``. - This method normalizes the output to match the expected - format. + Uses the top-level type directory (e.g. blockchain_tests_engine/) + to avoid repeated JVM startup per subdirectory. """ - subcommand = "state-test" - subcommand_options: List[str] = [] - if debug_output_path: - subcommand_options += ["--json"] + fmt = type(fixture_format) if not isinstance(fixture_format, type) else fixture_format + subcommand = self.subcommands[fmt] + type_dirs = {"state_tests", "blockchain_tests", "blockchain_tests_engine"} + dir_path = fixture_path if fixture_path.is_dir() else fixture_path.parent + while dir_path.name not in type_dirs and dir_path.parent != dir_path: + dir_path = dir_path.parent + cache_key = f"{subcommand}:{dir_path}" + + if cache_key not in self.dir_cache: + workers = getattr(self, "workers", 1) + command = [ + str(self.binary), subcommand, + "--json-array", "--workers", str(workers), + str(dir_path), + ] + if debug_output_path: + command.insert(-1, "--json") - command = ( - [str(self.binary)] - + [subcommand] - + subcommand_options - + [str(fixture_path)] - ) - result = self._run_command(command) + result = self.run_command(command) - if debug_output_path: - self._consume_debug_dump( - command, result, fixture_path, debug_output_path - ) + if debug_output_path: + self.validate_debug_dump( + command, result, fixture_path, debug_output_path + ) - if result.returncode != 0: - raise Exception( - f"Unexpected exit code:\n{' '.join(command)}\n\n" - f"Error:\n{result.stderr}" - ) + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n" + f"Error:\n{result.stderr}" + ) - # Parse NDJSON output, normalize "test" -> "name" - results: List[Dict[str, Any]] = [] - for line in result.stdout.strip().splitlines(): - line = line.strip() - if not line: - continue - try: - entry = json.loads(line) - if "test" in entry and "name" not in entry: - entry["name"] = entry["test"] - results.append(entry) - except json.JSONDecodeError as e: + # Besu mixes text output with JSON — find JSON array via '[{' + stdout = result.stdout + json_start = stdout.rfind("[{") + if json_start < 0: + json_start = stdout.rfind("[") + if json_start < 0: raise Exception( - f"Failed to parse Besu state-test output as JSON.\n" - f"Offending line:\n{line}\n\n" - f"Error: {e}" - ) from e - return results + f"No JSON array in evmtool {subcommand} output:\n" + f"{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + + result_model: type[FixtureTestResult] = { + StateFixture: StateTestResult, + BlockchainFixture: BlockTestResult, + BlockchainEngineFixture: EngineTestResult, + }.get(fmt, FixtureTestResult) + + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + validated = result_model.model_validate(r).model_dump( + by_alias=True + ) + indexed[validated["name"]] = validated + + self.dir_cache[cache_key] = indexed - def consume_state_test( + return self.dir_cache[cache_key] + + def validate_test( self, + fixture_format: FixtureFormat, + label: str, fixture_path: Path, fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """ - Consume a single state test. - - Uses the cached result from ``consume_state_test_file`` - and selects the requested fixture by name. - """ - file_results = self.consume_state_test_file( + """Generic consume method with directory cache and exception matching.""" + dir_results = self.get_dir_results( + fixture_format=fixture_format, fixture_path=fixture_path, debug_output_path=debug_output_path, ) if fixture_name: - test_result = [ - r for r in file_results if r["name"] == fixture_name - ] - assert len(test_result) < 2, ( - f"Multiple test results for {fixture_name}" - ) - assert len(test_result) == 1, ( - f"Test result for {fixture_name} missing" - ) - assert test_result[0]["pass"], ( - f"State test failed: " - f"{test_result[0].get('error', 'unknown error')}" + if fixture_name not in dir_results: + raise Exception( + f"{label} test result missing: {fixture_name} " + f"(client may have skipped or crashed on this test)" + ) + validate_test_result( + self.fixture_cache, self.exception_mapper, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=fixture_format == BlockchainEngineFixture, + is_block=fixture_format == BlockchainFixture, + is_state=fixture_format == StateFixture, + exception_check=getattr(self, "exception_check", True), ) else: - if any(not r["pass"] for r in file_results): - exception_text = "State test failed: \n" + "\n".join( - f"{r['name']}: " + r.get("error", "unknown error") - for r in file_results - if not r["pass"] + failures = [r for r in dir_results.values() if not r["pass"]] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join( + f"{r['name']}: {r['error']}" for r in failures + ) ) - raise Exception(exception_text) def consume_fixture( self, @@ -647,24 +629,14 @@ def consume_fixture( fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """ - Execute the appropriate Besu fixture consumer for the - fixture at ``fixture_path``. - """ - if fixture_format == BlockchainFixture: - self.consume_blockchain_test( - fixture_path=fixture_path, - fixture_name=fixture_name, - debug_output_path=debug_output_path, - ) - elif fixture_format == StateFixture: - self.consume_state_test( - fixture_path=fixture_path, - fixture_name=fixture_name, - debug_output_path=debug_output_path, - ) - else: - raise Exception( - f"Fixture format {fixture_format.format_name} " - f"not supported by {self.binary}" - ) + """Execute the appropriate Besu fixture consumer.""" + labels = { + StateFixture: "State", + BlockchainFixture: "Blockchain", + BlockchainEngineFixture: "Engine", + } + label = labels.get(fixture_format, "Unknown") + self.validate_test( + fixture_format, label, fixture_path, fixture_name, + debug_output_path, + ) diff --git a/packages/testing/src/execution_testing/client_clis/clis/erigon.py b/packages/testing/src/execution_testing/client_clis/clis/erigon.py index 699fd4f5744..804189870c9 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/erigon.py +++ b/packages/testing/src/execution_testing/client_clis/clis/erigon.py @@ -1,16 +1,44 @@ -"""Erigon execution client transition tool.""" +"""Erigon execution client fixture consumer interface.""" + +import json +import re +import shlex +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any, ClassVar, Dict, List, Optional from execution_testing.exceptions import ( BlockException, + ExceptionBase, ExceptionMapper, TransactionException, ) +from ..validate_helpers import validate_test_result +from execution_testing.fixtures import ( + BlockchainEngineFixture, + BlockchainEngineXFixture, + BlockchainFixture, + FixtureFormat, + StateFixture, +) + +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, +) +from ..ethereum_cli import EthereumCLI +from ..file_utils import dump_files_to_directory +from ..fixture_consumer_tool import FixtureConsumerTool class ErigonExceptionMapper(ExceptionMapper): - """Erigon exception mapper.""" + """Translate between EEST exceptions and error strings returned by Erigon.""" - mapping_substring = { + mapping_substring: ClassVar[Dict[ExceptionBase, str]] = { TransactionException.SENDER_NOT_EOA: "sender not an eoa", TransactionException.INITCODE_SIZE_EXCEEDED: ( "max initcode size exceeded" @@ -79,7 +107,9 @@ class ErigonExceptionMapper(ExceptionMapper): "invalid requests root hash in header" ), BlockException.INVALID_BLOCK_HASH: "invalid block hash", - BlockException.RLP_BLOCK_LIMIT_EXCEEDED: "block exceeds max rlp size", + BlockException.RLP_BLOCK_LIMIT_EXCEEDED: ( + "block exceeds max rlp size" + ), BlockException.INVALID_BASEFEE_PER_GAS: ( "invalid block: invalid baseFee" ), @@ -90,18 +120,26 @@ class ErigonExceptionMapper(ExceptionMapper): BlockException.EXTRA_DATA_TOO_BIG: ( "invalid block: extra-data longer than 32 bytes" ), - BlockException.INVALID_GASLIMIT: "invalid block: invalid gas limit", - BlockException.INVALID_STATE_ROOT: "invalid block: wrong trie root", + BlockException.INVALID_GASLIMIT: ( + "invalid block: invalid gas limit" + ), + BlockException.INVALID_STATE_ROOT: ( + "invalid block: wrong trie root" + ), BlockException.INVALID_RECEIPTS_ROOT: "receiptHash mismatch", BlockException.INVALID_LOG_BLOOM: "invalid bloom", BlockException.INVALID_BAL_MISSING_ACCOUNT: ( "block access list mismatch" ), - BlockException.INCORRECT_BLOCK_FORMAT: "invalid block access list", - BlockException.INVALID_BAL_EXTRA_ACCOUNT: "invalid block access list", + BlockException.INCORRECT_BLOCK_FORMAT: ( + "invalid block access list" + ), + BlockException.INVALID_BAL_EXTRA_ACCOUNT: ( + "invalid block access list" + ), BlockException.GAS_USED_OVERFLOW: "block gas used overflow", } - mapping_regex = { + mapping_regex: ClassVar[Dict[ExceptionBase, str]] = { BlockException.INVALID_BLOCK_ACCESS_LIST: ( r"invalid block access list|block access list mismatch" ), @@ -111,7 +149,9 @@ class ErigonExceptionMapper(ExceptionMapper): BlockException.INVALID_BAL_EXTRA_ACCOUNT: ( r"invalid block access list" ), - BlockException.INCORRECT_BLOCK_FORMAT: (r"invalid block access list"), + BlockException.INCORRECT_BLOCK_FORMAT: ( + r"invalid block access list" + ), BlockException.BLOCK_ACCESS_LIST_GAS_LIMIT_EXCEEDED: ( r"block access list too large" ), @@ -131,3 +171,334 @@ class ErigonExceptionMapper(ExceptionMapper): r"invalid gasUsed: have \d+, gasLimit \d+" ), } + + +class ErigonEvm(EthereumCLI): + """Erigon `evm` base class.""" + + default_binary = Path("evm") + detect_binary_pattern = re.compile(r"^evm(.exe)? version\b") + cached_version: Optional[str] = None + trace: bool + + def __init__( + self, + binary: Optional[Path] = None, + trace: bool = False, + ): + """Initialize the ErigonEvm class.""" + self.binary = binary if binary else self.default_binary + self.trace = trace + + def run_command( + self, command: List[str] + ) -> subprocess.CompletedProcess: + """Run a command and return the result.""" + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as e: + raise Exception( + "Command failed with non-zero status." + ) from e + except Exception as e: + raise Exception( + "Unexpected exception calling erigon evm tool." + ) from e + + def validate_debug_dump( + self, + command: List[str], + result: subprocess.CompletedProcess, + fixture_path: Path, + debug_output_path: Path, + ) -> None: + """Dump debug output for a consume command.""" + assert all(isinstance(x, str) for x in command), ( + f"Not all elements of 'command' list are strings: " + f"{command}" + ) + assert len(command) > 0 + + debug_fixture_path = str( + debug_output_path / "fixtures.json" + ) + command[-1] = debug_fixture_path + + validate_call = " ".join( + shlex.quote(arg) for arg in command + ) + + validate_script = textwrap.dedent( + f"""\ + #!/bin/bash + {validate_call} + """ + ) + dump_files_to_directory( + debug_output_path, + { + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, + }, + ) + shutil.copyfile(fixture_path, debug_fixture_path) + + +class ErigonFixtureConsumer( + ErigonEvm, + FixtureConsumerTool, + fixture_formats=[ + StateFixture, + BlockchainFixture, + BlockchainEngineFixture, + BlockchainEngineXFixture, + ], +): + """Erigon's implementation of the fixture consumer. + + Uses the erigon `evm` binary with statetest, blocktest, + enginetest, and enginextest subcommands. Requires `--jsonout` + for JSON output. Supports both --type engine (use with -n8 + --bin-workers 8) and --type enginex (pre-alloc cached, faster + with less parallelism). + """ + + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + fixture_cache: Dict[str, Dict[str, Any]] = {} + exception_mapper: ExceptionMapper = ErigonExceptionMapper() + + def get_dir_results( + self, + subcommand: str, + fixture_path: Path, + debug_output_path: Optional[Path] = None, + ) -> Dict[str, Dict[str, Any]]: + """ + Run a subcommand once per fixture directory and cache all + results indexed by test name. + """ + dir_path = ( + fixture_path + if fixture_path.is_dir() + else fixture_path.parent + ) + cache_key = f"{subcommand}:{dir_path}" + + if cache_key not in self.dir_cache: + workers = getattr(self, "workers", 1) + global_options: List[str] = [] + subcommand_options: List[str] = [ + "--jsonout", + "--workers", + str(workers), + ] + if debug_output_path: + global_options += ["--verbosity", "100"] + + command = ( + [str(self.binary)] + + global_options + + [subcommand] + + subcommand_options + + [str(dir_path)] + ) + result = self.run_command(command) + + if debug_output_path: + self.validate_debug_dump( + command, result, fixture_path, debug_output_path + ) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n" + f"{' '.join(command)}\n\n" + f"Error:\n{result.stderr}" + ) + + stdout = result.stdout + json_start = stdout.find("[") + if json_start < 0: + raise Exception( + f"No JSON array in evm {subcommand} output:\n" + f"{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + if not isinstance(result_json, list): + raise Exception( + f"Unexpected result from evm {subcommand}: " + f"{result_json}" + ) + + result_model: type[FixtureTestResult] = { + "statetest": StateTestResult, + "blocktest": BlockTestResult, + "enginetest": EngineTestResult, + "enginextest": EngineTestResult, + }.get(subcommand, FixtureTestResult) + + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + validated = result_model.model_validate( + r + ).model_dump(by_alias=True) + indexed[validated["name"]] = validated + + self.dir_cache[cache_key] = indexed + + return self.dir_cache[cache_key] + + def validate_test( + self, + subcommand: str, + label: str, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Generic consume method using directory-level cache.""" + dir_results = self.get_dir_results( + subcommand=subcommand, + fixture_path=fixture_path, + debug_output_path=debug_output_path, + ) + if fixture_name: + if fixture_name not in dir_results: + raise Exception( + f"{label} test result missing: " + f"{fixture_name} " + f"(client may have skipped or crashed " + f"on this test)" + ) + validate_test_result( + self.fixture_cache, self.exception_mapper, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=subcommand in ( + "enginetest", "enginextest", + ), + is_block=subcommand == "blocktest", + is_state=subcommand == "statetest", + exception_check=getattr(self, "exception_check", True), + ) + else: + failures = [ + r + for r in dir_results.values() + if not r["pass"] + ] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join( + f"{r['name']}: {r['error']}" + for r in failures + ) + ) + + def consume_state_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single state test.""" + self.validate_test( + "statetest", + "State", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_blockchain_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single blockchain test.""" + self.validate_test( + "blocktest", + "Blockchain", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_engine_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single engine test.""" + self.validate_test( + "enginetest", + "Engine", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_enginex_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single engine test via enginextest (pre-alloc cached).""" + self.validate_test( + "enginextest", + "EngineX", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Execute the appropriate erigon fixture consumer.""" + if fixture_format == BlockchainFixture: + self.consume_blockchain_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == BlockchainEngineFixture: + self.consume_engine_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == BlockchainEngineXFixture: + self.consume_enginex_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == StateFixture: + self.consume_state_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + else: + raise Exception( + f"Fixture format " + f"{fixture_format.format_name} " + f"not supported by {self.binary}" + ) diff --git a/packages/testing/src/execution_testing/client_clis/clis/ethrex.py b/packages/testing/src/execution_testing/client_clis/clis/ethrex.py index 8a7d03f26f8..dfc8f4dbfb2 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/ethrex.py +++ b/packages/testing/src/execution_testing/client_clis/clis/ethrex.py @@ -1,16 +1,43 @@ -"""Ethrex execution client transition tool.""" +"""Ethrex execution client fixture consumer interface.""" + +import json +import re +import shlex +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any, ClassVar, Dict, List, Optional from execution_testing.exceptions import ( BlockException, + ExceptionBase, ExceptionMapper, TransactionException, ) +from ..validate_helpers import validate_test_result +from execution_testing.fixtures import ( + BlockchainEngineFixture, + BlockchainFixture, + FixtureFormat, + StateFixture, +) + +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, +) +from ..ethereum_cli import EthereumCLI +from ..file_utils import dump_files_to_directory +from ..fixture_consumer_tool import FixtureConsumerTool class EthrexExceptionMapper(ExceptionMapper): - """Ethrex exception mapper.""" + """Translate between EEST exceptions and error strings returned by Ethrex.""" - mapping_substring = { + mapping_substring: ClassVar[Dict[ExceptionBase, str]] = { BlockException.INVALID_GASLIMIT: ( "Gas limit changed more than allowed from the parent" ), @@ -65,7 +92,7 @@ class EthrexExceptionMapper(ExceptionMapper): "Base fee per gas is incorrect" ), } - mapping_regex = { + mapping_regex: ClassVar[Dict[ExceptionBase, str]] = { TransactionException.PRIORITY_GREATER_THAN_MAX_FEE_PER_GAS: ( r"(?i)priority fee.* is greater than max fee.*" ), @@ -79,37 +106,46 @@ class EthrexExceptionMapper(ExceptionMapper): TransactionException.NONCE_MISMATCH_TOO_LOW: ( r"nonce \d+ too low, expected \d+|Nonce mismatch.*" ), - TransactionException.NONCE_MISMATCH_TOO_HIGH: r"Nonce mismatch.*", + TransactionException.NONCE_MISMATCH_TOO_HIGH: ( + r"Nonce mismatch.*" + ), TransactionException.TYPE_3_TX_MAX_BLOB_GAS_ALLOWANCE_EXCEEDED: ( r"blob gas used \d+ exceeds maximum allowance \d+" ), TransactionException.TYPE_3_TX_ZERO_BLOBS: ( - r"blob transactions present in pre-cancun payload|empty blobs|" + r"blob transactions present in pre-cancun payload|" + r"empty blobs|" r"Type 3 transaction without blobs" ), TransactionException.TYPE_3_TX_INVALID_BLOB_VERSIONED_HASH: ( - r"blob version not supported|Invalid blob versioned hash" + r"blob version not supported|" + r"Invalid blob versioned hash" ), TransactionException.TYPE_2_TX_PRE_FORK: ( - r"Type 2 transactions are not supported before the London fork" + r"Type 2 transactions are not supported " + r"before the London fork" ), TransactionException.TYPE_3_TX_PRE_FORK: ( r"blob versioned hashes not supported|" - r"Type 3 transactions are not supported before the Cancun fork" + r"Type 3 transactions are not supported " + r"before the Cancun fork" ), TransactionException.TYPE_4_TX_CONTRACT_CREATION: ( - r"unexpected length|Contract creation in type 4 transaction|" - r"Error decoding field 'to' of type primitive_types::H160: " - r"InvalidLength" + r"unexpected length|" + r"Contract creation in type 4 transaction|" + r"Error decoding field 'to' of type " + r"primitive_types::H160: InvalidLength" ), TransactionException.TYPE_3_TX_CONTRACT_CREATION: ( - r"unexpected length|Contract creation in type 3 transaction|" - r"Error decoding field 'to' of type primitive_types::H160: " - r"InvalidLength" + r"unexpected length|" + r"Contract creation in type 3 transaction|" + r"Error decoding field 'to' of type " + r"primitive_types::H160: InvalidLength" ), TransactionException.TYPE_4_TX_PRE_FORK: ( r"eip 7702 transactions present in pre-prague payload|" - r"Type 4 transactions are not supported before the Prague fork" + r"Type 4 transactions are not supported " + r"before the Prague fork" ), TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: ( r"lack of funds \(\d+\) for max fee \(\d+\)|" @@ -118,45 +154,50 @@ class EthrexExceptionMapper(ExceptionMapper): TransactionException.INTRINSIC_GAS_TOO_LOW: ( r"gas floor exceeds the gas limit|" r"call gas cost exceeds the gas limit|" - r"Transaction gas limit lower than the minimum gas cost " - r"to execute the transaction|" - r"Transaction gas limit lower than the gas cost floor " - r"for calldata tokens" + r"Transaction gas limit lower than the minimum " + r"gas cost to execute the transaction|" + r"Transaction gas limit lower than the gas cost " + r"floor for calldata tokens" ), TransactionException.INTRINSIC_GAS_BELOW_FLOOR_GAS_COST: ( - r"Transaction gas limit lower than the gas cost floor " - r"for calldata tokens" + r"Transaction gas limit lower than the gas cost " + r"floor for calldata tokens" ), TransactionException.INSUFFICIENT_MAX_FEE_PER_GAS: ( r"gas price is less than basefee|" r"Insufficient max fee per gas" ), TransactionException.INSUFFICIENT_MAX_FEE_PER_BLOB_GAS: ( - r"blob gas price is greater than max fee per blob gas|" + r"blob gas price is greater than " + r"max fee per blob gas|" r"Insufficient max fee per blob gas.*" ), TransactionException.INITCODE_SIZE_EXCEEDED: ( r"create initcode size limit|Initcode size exceeded.*" ), - TransactionException.NONCE_IS_MAX: (r"Nonce is max"), + TransactionException.NONCE_IS_MAX: r"Nonce is max", TransactionException.GAS_ALLOWANCE_EXCEEDED: ( r"Gas allowance exceeded.*" ), - BlockException.GAS_USED_OVERFLOW: (r"Block gas used overflow.*"), + BlockException.GAS_USED_OVERFLOW: r"Block gas used overflow.*", TransactionException.TYPE_3_TX_BLOB_COUNT_EXCEEDED: ( r"Blob count exceeded.*" ), TransactionException.GASLIMIT_PRICE_PRODUCT_OVERFLOW: ( - r"Invalid transaction: Gas limit price product overflow.*" + r"Invalid transaction: " + r"Gas limit price product overflow.*" ), TransactionException.GAS_LIMIT_EXCEEDS_MAXIMUM: ( r"Invalid transaction: " r"Transaction gas limit exceeds maximum.*" ), BlockException.INVALID_DEPOSIT_EVENT_LAYOUT: ( - r"Invalid deposit request layout|BAL validation failed.*" + r"Invalid deposit request layout|" + r"BAL validation failed.*" + ), + BlockException.SYSTEM_CONTRACT_CALL_FAILED: ( + r"System call failed.*" ), - BlockException.SYSTEM_CONTRACT_CALL_FAILED: (r"System call failed.*"), BlockException.SYSTEM_CONTRACT_EMPTY: ( r"System contract:.* has no code after deployment" ), @@ -176,16 +217,22 @@ class EthrexExceptionMapper(ExceptionMapper): r"Maximum block size exceeded.*" ), BlockException.INVALID_BAL_EXTRA_ACCOUNT: ( - r"Block access list accounts not in strictly ascending order.*|" - r"BAL validation failed: account .* was never accessed.*" + r"Block access list accounts not in strictly " + r"ascending order.*|" + r"BAL validation failed: account .* " + r"was never accessed.*" + ), + BlockException.INVALID_BAL_MISSING_ACCOUNT: ( + r"absent from BAL" ), - BlockException.INVALID_BAL_MISSING_ACCOUNT: (r"absent from BAL"), BlockException.INVALID_BLOCK_ACCESS_LIST: ( r"Block access list contains index \d+ " r"exceeding max valid index \d+|" r"Failed to RLP decode BAL|" - r"Block access list .+ not in strictly ascending order.*|" - r"BAL validation failed for (tx \d+|system_tx|withdrawal): .*|" + r"Block access list .+ not in strictly " + r"ascending order.*|" + r"BAL validation failed for " + r"(tx \d+|system_tx|withdrawal): .*|" r"BAL validation failed: .*|" r"Block access list slot .+ is in both " r"storage_changes and storage_reads.*" @@ -196,6 +243,329 @@ class EthrexExceptionMapper(ExceptionMapper): r"Block access list contains index \d+ " r"exceeding max valid index \d+|" r"Failed to RLP decode BAL|" - r"Block access list accounts not in strictly ascending order.*" + r"Block access list accounts not in strictly " + r"ascending order.*" ), } + + +class EthrexCLI(EthereumCLI): + """Ethrex base class for the ef_tests-* binaries. + + Uses a base binary path (e.g. `ef_tests`) and derives per-type + binaries by appending `-statetest`, `-blocktest`, `-enginetest`. + """ + + default_binary = Path("ef_tests-statetest") + detect_binary_pattern = re.compile(r"^ef_tests-statetest\b") + version_flag: str = "--version" + cached_version: Optional[str] = None + trace: bool + + def __init__( + self, + binary: Optional[Path] = None, + trace: bool = False, + ): + """Initialize the EthrexCLI class.""" + self.binary = binary if binary else self.default_binary + self.trace = trace + + def binary_for(self, test_type: str) -> Path: + """Derive the binary path for a given test type. + + If binary is `ef_tests-statetest`, derives siblings like + `ef_tests-blocktest`. If binary is a base like `ef_tests`, + appends `-statetest` etc. + """ + bin_str = str(self.binary) + # Strip any existing suffix to get the base + for suffix in ( + "-statetest", + "-blocktest", + "-enginetest", + ): + if bin_str.endswith(suffix): + base = bin_str[: -len(suffix)] + return Path(f"{base}-{test_type}") + # Binary is the base itself + return Path(f"{bin_str}-{test_type}") + + def run_command( + self, command: List[str] + ) -> subprocess.CompletedProcess: + """Run a command and return the result.""" + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as e: + raise Exception( + "Command failed with non-zero status." + ) from e + except Exception as e: + raise Exception( + "Unexpected exception calling ethrex tool." + ) from e + + def validate_debug_dump( + self, + command: List[str], + result: subprocess.CompletedProcess, + fixture_path: Path, + debug_output_path: Path, + ) -> None: + """Dump debug output for a consume command.""" + assert all(isinstance(x, str) for x in command), ( + f"Not all elements of 'command' list are strings: " + f"{command}" + ) + assert len(command) > 0 + + debug_fixture_path = str( + debug_output_path / "fixtures.json" + ) + + validate_call = " ".join( + shlex.quote(arg) for arg in command + ) + + validate_script = textwrap.dedent( + f"""\ + #!/bin/bash + {validate_call} + """ + ) + dump_files_to_directory( + debug_output_path, + { + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, + }, + ) + shutil.copyfile(fixture_path, debug_fixture_path) + + +class EthrexFixtureConsumer( + EthrexCLI, + FixtureConsumerTool, + fixture_formats=[ + StateFixture, + BlockchainFixture, + BlockchainEngineFixture, + ], +): + """Ethrex's implementation of the fixture consumer. + + Uses separate binaries per test type: + ef_tests-statetest, ef_tests-blocktest, ef_tests-enginetest. + All use --path --json --workers N. + """ + + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + fixture_cache: Dict[str, Dict[str, Any]] = {} + exception_mapper: ExceptionMapper = EthrexExceptionMapper() + + def get_dir_results( + self, + test_type: str, + fixture_path: Path, + debug_output_path: Optional[Path] = None, + ) -> Dict[str, Dict[str, Any]]: + """ + Run a binary once per fixture directory and cache all + results indexed by test name. + """ + dir_path = ( + fixture_path + if fixture_path.is_dir() + else fixture_path.parent + ) + cache_key = f"{test_type}:{dir_path}" + + if cache_key not in self.dir_cache: + workers = getattr(self, "workers", 1) + binary = self.binary_for(test_type) + + command = [ + str(binary), + "--path", + str(dir_path), + "--json", + "--workers", + str(workers), + ] + result = self.run_command(command) + + if debug_output_path: + self.validate_debug_dump( + command, + result, + fixture_path, + debug_output_path, + ) + + # Ethrex exits non-zero when any test fails, but still + # outputs JSON results. Parse JSON first, only fail if missing. + stdout = result.stdout + json_start = stdout.find("[") + if json_start < 0: + raise Exception( + f"No JSON array in {test_type} output:\n" + f"{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + if not isinstance(result_json, list): + raise Exception( + f"Unexpected result from {test_type}: " + f"{result_json}" + ) + + result_model: type[FixtureTestResult] = { + "statetest": StateTestResult, + "blocktest": BlockTestResult, + "enginetest": EngineTestResult, + }.get(test_type, FixtureTestResult) + + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + # Ethrex blocktest/enginetest may omit fork + # and use null for error; normalize before + # Pydantic validation. + if "fork" not in r: + r["fork"] = "" + if r.get("error") is None: + r["error"] = "" + validated = result_model.model_validate( + r + ).model_dump(by_alias=True) + indexed[validated["name"]] = validated + + self.dir_cache[cache_key] = indexed + + return self.dir_cache[cache_key] + + def validate_test( + self, + test_type: str, + label: str, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Generic consume method using directory-level cache.""" + dir_results = self.get_dir_results( + test_type=test_type, + fixture_path=fixture_path, + debug_output_path=debug_output_path, + ) + if fixture_name: + if fixture_name not in dir_results: + return # silently pass — client skipped this test (unsupported fork) + validate_test_result( + self.fixture_cache, self.exception_mapper, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=test_type == "enginetest", + is_block=test_type == "blocktest", + is_state=test_type == "statetest", + exception_check=getattr(self, "exception_check", True), + ) + else: + failures = [ + r + for r in dir_results.values() + if not r["pass"] + ] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join( + f"{r['name']}: {r['error']}" + for r in failures + ) + ) + + def consume_state_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single state test.""" + self.validate_test( + "statetest", + "State", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_blockchain_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single blockchain test.""" + self.validate_test( + "blocktest", + "Blockchain", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_engine_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single engine test.""" + self.validate_test( + "enginetest", + "Engine", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Execute the appropriate ethrex fixture consumer.""" + if fixture_format == BlockchainFixture: + self.consume_blockchain_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == BlockchainEngineFixture: + self.consume_engine_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == StateFixture: + self.consume_state_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + else: + raise Exception( + f"Fixture format " + f"{fixture_format.format_name} " + f"not supported by {self.binary}" + ) diff --git a/packages/testing/src/execution_testing/client_clis/clis/geth.py b/packages/testing/src/execution_testing/client_clis/clis/geth.py index 7fdedd348d6..8f579868b40 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/geth.py +++ b/packages/testing/src/execution_testing/client_clis/clis/geth.py @@ -16,13 +16,21 @@ ExceptionMapper, TransactionException, ) +from ..validate_helpers import validate_test_result from execution_testing.fixtures import ( + BlockchainEngineFixture, BlockchainFixture, FixtureFormat, StateFixture, ) from execution_testing.forks import Fork +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, +) from ..ethereum_cli import EthereumCLI from ..fixture_consumer_tool import FixtureConsumerTool from ..transition_tool import TransitionTool, dump_files_to_directory @@ -198,9 +206,9 @@ def __init__( """Initialize the GethEvm class.""" self.binary = binary if binary else self.default_binary self.trace = trace - self._info_metadata: Optional[Dict[str, Any]] = {} + self.info_metadata: Optional[Dict[str, Any]] = {} - def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: + def run_command(self, command: List[str]) -> subprocess.CompletedProcess: try: return subprocess.run( command, @@ -213,7 +221,7 @@ def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: except Exception as e: raise Exception("Unexpected exception calling evm tool.") from e - def _consume_debug_dump( + def validate_debug_dump( self, command: List[str], result: subprocess.CompletedProcess, @@ -231,22 +239,22 @@ def _consume_debug_dump( command[-1] = debug_fixture_path # ensure that flags with spaces are wrapped in double-quotes - consume_direct_call = " ".join(shlex.quote(arg) for arg in command) + validate_call = " ".join(shlex.quote(arg) for arg in command) - consume_direct_script = textwrap.dedent( + validate_script = textwrap.dedent( f"""\ #!/bin/bash - {consume_direct_call} + {validate_call} """ ) dump_files_to_directory( debug_output_path, { - "consume_direct_args.py": command, - "consume_direct_returncode.txt": result.returncode, - "consume_direct_stdout.txt": result.stdout, - "consume_direct_stderr.txt": result.stderr, - "consume_direct.sh+x": consume_direct_script, + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, }, ) shutil.copyfile(fixture_path, debug_fixture_path) @@ -258,7 +266,7 @@ def help(self, subcommand: str | None = None) -> str: if subcommand: help_command.append(subcommand) help_command.append("--help") - return self._run_command(help_command).stdout + return self.run_command(help_command).stdout class GethTransitionTool(GethEvm, TransitionTool): @@ -284,7 +292,7 @@ def __init__( self, binary=binary, exception_mapper=exception_mapper, trace=trace ) help_command = [str(self.binary), str(self.subcommand), "--help"] - result = self._run_command(help_command) + result = self.run_command(help_command) self.help_string = result.stdout def is_fork_supported(self, fork: Fork) -> bool: @@ -300,154 +308,150 @@ def is_fork_supported(self, fork: Fork) -> bool: class GethFixtureConsumer( GethEvm, FixtureConsumerTool, - fixture_formats=[StateFixture, BlockchainFixture], + fixture_formats=[StateFixture, BlockchainFixture, BlockchainEngineFixture], ): """Geth's implementation of the fixture consumer.""" - def consume_blockchain_test( + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + exception_mapper: ExceptionMapper = GethExceptionMapper() + + def get_dir_results( self, + subcommand: str, fixture_path: Path, - fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, - ) -> None: + ) -> Dict[str, Dict[str, Any]]: """ - Consume a single blockchain test. - - The `evm blocktest` command takes the `--run` argument which can be - used to select a specific fixture from the fixture file when executing. + Run a subcommand once per fixture directory and cache all results + indexed by test name. Subsequent calls for the same directory + return from cache instantly. """ - subcommand = "blocktest" - global_options = [] - subcommand_options = [] - if debug_output_path: - global_options += ["--verbosity", "100"] - subcommand_options += ["--trace"] - - if fixture_name: - subcommand_options += ["--run", re.escape(fixture_name)] - - command = ( - [str(self.binary)] - + global_options - + [subcommand] - + subcommand_options - + [str(fixture_path)] - ) - - result = self._run_command(command) - - if debug_output_path: - self._consume_debug_dump( - command, result, fixture_path, debug_output_path + dir_path = fixture_path if fixture_path.is_dir() else fixture_path.parent + cache_key = f"{subcommand}:{dir_path}" + + if cache_key not in self.dir_cache: + workers = getattr(self, "workers", 1) + global_options: List[str] = [] + subcommand_options: List[str] = ["--workers", str(workers)] + if debug_output_path: + global_options += ["--verbosity", "100"] + subcommand_options += ["--trace"] + + command = ( + [str(self.binary)] + + global_options + + [subcommand] + + subcommand_options + + [str(dir_path)] ) + result = self.run_command(command) - if result.returncode != 0: - raise Exception( - f"Unexpected exit code:\n{' '.join(command)}\n\n" - f"Error:\n{result.stderr}" - ) + if debug_output_path: + self.validate_debug_dump( + command, result, fixture_path, debug_output_path + ) - result_json = json.loads(result.stdout) - if not isinstance(result_json, list): - raise Exception( - f"Unexpected result from evm blocktest: {result_json}" - ) + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n" + f"Error:\n{result.stderr}" + ) - if any(not test_result["pass"] for test_result in result_json): - exception_text = "Blockchain test failed: \n" + "\n".join( - f"{test_result['name']}: " + test_result["error"] - for test_result in result_json - if not test_result["pass"] - ) - raise Exception(exception_text) + # Find JSON array start (geth may output debug info before it) + stdout = result.stdout + json_start = stdout.find("[") + if json_start < 0: + raise Exception( + f"No JSON array in evm {subcommand} output:\n{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + if not isinstance(result_json, list): + raise Exception( + f"Unexpected result from evm {subcommand}: {result_json}" + ) - @cache # noqa - def consume_state_test_file( - self, - fixture_path: Path, - debug_output_path: Optional[Path] = None, - ) -> List[Dict[str, Any]]: - """ - Consume an entire state test file. + result_model: type[FixtureTestResult] = { + "statetest": StateTestResult, + "blocktest": BlockTestResult, + "enginetest": EngineTestResult, + }.get(subcommand, FixtureTestResult) - The `evm statetest` will always execute all the tests contained in a - file without the possibility of selecting a single test, so this - function is cached in order to only call the command once and - `consume_state_test` can simply select the result that was requested. - """ - subcommand = "statetest" - global_options: List[str] = [] - subcommand_options: List[str] = [] - if debug_output_path: - global_options += ["--verbosity", "100"] - subcommand_options += ["--trace"] - - command = ( - [str(self.binary)] - + global_options - + [subcommand] - + subcommand_options - + [str(fixture_path)] - ) - result = self._run_command(command) + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + validated = result_model.model_validate(r).model_dump( + by_alias=True + ) + indexed[validated["name"]] = validated - if debug_output_path: - self._consume_debug_dump( - command, result, fixture_path, debug_output_path - ) + self.dir_cache[cache_key] = indexed - if result.returncode != 0: - raise Exception( - f"Unexpected exit code:\n{' '.join(command)}\n\n" - f"Error:\n{result.stderr}" - ) + return self.dir_cache[cache_key] - result_json = json.loads(result.stdout) - if not isinstance(result_json, list): - raise Exception( - f"Unexpected result from evm statetest: {result_json}" - ) - return result_json + fixture_cache: Dict[str, Dict[str, Any]] = {} - def consume_state_test( + def validate_test( self, + subcommand: str, + label: str, fixture_path: Path, fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """ - Consume a single state test. - - Uses the cached result from `consume_state_test_file` in order to not - call the command every time and select a single result from there. - """ - file_results = self.consume_state_test_file( + """Generic consume method using directory-level cache.""" + dir_results = self.get_dir_results( + subcommand=subcommand, fixture_path=fixture_path, debug_output_path=debug_output_path, ) if fixture_name: - test_result = [ - test_result - for test_result in file_results - if test_result["name"] == fixture_name - ] - assert len(test_result) < 2, ( - f"Multiple test results for {fixture_name}" - ) - assert len(test_result) == 1, ( - f"Test result for {fixture_name} missing" - ) - assert test_result[0]["pass"], ( - f"State test failed: {test_result[0]['error']}" + if fixture_name not in dir_results: + raise Exception( + f"{label} test result missing: {fixture_name} " + f"(client may have skipped or crashed on this test)" + ) + validate_test_result( + self.fixture_cache, self.exception_mapper, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=subcommand == "enginetest", + is_block=subcommand == "blocktest", + is_state=subcommand == "statetest", + exception_check=getattr(self, "exception_check", True), ) else: - if any(not test_result["pass"] for test_result in file_results): - exception_text = "State test failed: \n" + "\n".join( - f"{test_result['name']}: " + test_result["error"] - for test_result in file_results - if not test_result["pass"] + failures = [r for r in dir_results.values() if not r["pass"]] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join(f"{r['name']}: {r['error']}" for r in failures) ) - raise Exception(exception_text) + + def consume_state_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single state test.""" + self.validate_test("statetest", "State", fixture_path, fixture_name, debug_output_path) + + def consume_blockchain_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single blockchain test.""" + self.validate_test("blocktest", "Blockchain", fixture_path, fixture_name, debug_output_path) + + def consume_engine_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single engine test.""" + self.validate_test("enginetest", "Engine", fixture_path, fixture_name, debug_output_path) def consume_fixture( self, @@ -466,6 +470,12 @@ def consume_fixture( fixture_name=fixture_name, debug_output_path=debug_output_path, ) + elif fixture_format == BlockchainEngineFixture: + self.consume_engine_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) elif fixture_format == StateFixture: self.consume_state_test( fixture_path=fixture_path, diff --git a/packages/testing/src/execution_testing/client_clis/clis/nethermind.py b/packages/testing/src/execution_testing/client_clis/clis/nethermind.py index 90745b6ac6b..a0c571cb876 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/nethermind.py +++ b/packages/testing/src/execution_testing/client_clis/clis/nethermind.py @@ -11,15 +11,24 @@ from execution_testing.exceptions import ( BlockException, + ExceptionBase, ExceptionMapper, TransactionException, ) +from ..validate_helpers import validate_test_result from execution_testing.fixtures import ( + BlockchainEngineFixture, BlockchainFixture, FixtureFormat, StateFixture, ) +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, +) from ..ethereum_cli import EthereumCLI from ..file_utils import dump_files_to_directory from ..fixture_consumer_tool import FixtureConsumerTool @@ -36,6 +45,68 @@ class Nethtest(EthereumCLI): version_flag: str = "--version" cached_version: Optional[str] = None + @classmethod + def detect_binary(cls, binary_output: str) -> bool: + """ + Detect nethtest from version output, checking each line. + Handles the 'Using launch settings...' prefix from dotnet run. + """ + for line in binary_output.splitlines(): + line = line.strip() + if line and cls.detect_binary_pattern.match(line): + return True + return False + + @classmethod + def from_binary_path( + cls, + binary_path: Path, + **kwargs: Any, + ) -> "Nethtest": + """ + Create a Nethtest instance, handling .csproj/.dll paths that need + dotnet to run. + """ + binary = binary_path + suffix = binary.suffix.lower() + + # Try dotnet run for .csproj or .dll files + if suffix in (".csproj", ".dll") or ( + binary.is_dir() and list(binary.glob("*.csproj")) + ): + try: + if binary.is_dir(): + csproj = list(binary.glob("*.csproj"))[0] + else: + csproj = binary + result = subprocess.run( + ["dotnet", "run", "--no-build", "-c", "Release", + "--project", str(csproj), "--", "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + timeout=30, + ) + if result.returncode == 0 and cls.detect_binary(result.stdout): + instance = cls(binary=binary, **kwargs) + instance._needs_dotnet = True + return instance + except Exception: + pass + + # Fall back to direct execution + try: + result = subprocess.run( + [str(binary), "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + timeout=10, + ) + output = result.stdout.strip() + if cls.detect_binary(output): + return cls(binary=binary, **kwargs) + except Exception: + pass + + raise Exception(f"Could not detect nethtest at {binary}") + def __init__( self, binary: Path, @@ -45,10 +116,47 @@ def __init__( """Initialize the Nethtest class.""" self.binary = binary self.trace = trace - # TODO: Implement NethermindExceptionMapper self.exception_mapper = exception_mapper if exception_mapper else None - - def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: + # Detect if binary needs dotnet to run (.csproj, .dll, or directory with .csproj) + self.needs_dotnet = self.detect_dotnet(binary) + + @staticmethod + def detect_dotnet(binary: Path) -> bool: + """Check if the binary needs dotnet to run.""" + if binary.suffix in (".csproj", ".dll"): + return True + # Check if it's a directory containing a .csproj + if binary.is_dir() and list(binary.glob("*.csproj")): + return True + # Check if the binary fails to run directly (needs .NET runtime) + try: + result = subprocess.run( + [str(binary), "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + timeout=5, + ) + if "must install .NET" in result.stdout or "must install .NET" in result.stderr: + return True + except Exception: + pass + return False + + def build_base_command(self, args: List[str]) -> List[str]: + """Build a command, wrapping with dotnet if needed.""" + if self.needs_dotnet: + # Find the .csproj file + binary_path = self.binary + if binary_path.is_dir(): + csproj = list(binary_path.glob("*.csproj")) + if csproj: + binary_path = csproj[0] + return [ + "dotnet", "run", "--no-build", "-c", "Release", + "--project", str(binary_path), "--", + ] + args + return [str(self.binary)] + args + + def run_command(self, command: List[str]) -> subprocess.CompletedProcess: try: return subprocess.run( command, @@ -61,7 +169,7 @@ def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: except Exception as e: raise Exception("Unexpected exception calling evm tool.") from e - def _consume_debug_dump( + def validate_debug_dump( self, command: Tuple[str, ...], result: subprocess.CompletedProcess, @@ -73,23 +181,23 @@ def _consume_debug_dump( ) # ensure that flags with spaces are wrapped in double-quotes - consume_direct_call = " ".join(shlex.quote(arg) for arg in command) + validate_call = " ".join(shlex.quote(arg) for arg in command) - consume_direct_script = textwrap.dedent( + validate_script = textwrap.dedent( f"""\ #!/bin/bash - {consume_direct_call} + {validate_call} """ ) dump_files_to_directory( debug_output_path, { - "consume_direct_args.py": command, - "consume_direct_returncode.txt": result.returncode, - "consume_direct_stdout.txt": result.stdout, - "consume_direct_stderr.txt": result.stderr, - "consume_direct.sh+x": consume_direct_script, + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, }, ) @@ -100,167 +208,136 @@ def help(self, subcommand: str | None = None) -> str: if subcommand: help_command.append(subcommand) help_command.append("--help") - return self._run_command(help_command).stdout + return self.run_command(help_command).stdout class NethtestFixtureConsumer( Nethtest, FixtureConsumerTool, - fixture_formats=[StateFixture, BlockchainFixture], + fixture_formats=[StateFixture, BlockchainFixture, BlockchainEngineFixture], ): """Nethermind implementation of the fixture consumer.""" - def _build_command_with_options( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ) -> Tuple[str, ...]: - assert fixture_name, "Fixture name must be provided for nethtest." - command = [str(self.binary)] - if fixture_format is BlockchainFixture: - command += [ - "--blockTest", - "--filter", - f"{re.escape(fixture_name)}", - ] - elif fixture_format is StateFixture: - # TODO: consider using `--filter` here to readily access traces - # from the output - pass # no additional options needed - else: - raise Exception( - f"Fixture format {fixture_format.format_name} " - f"not supported by {self.binary}" - ) - command += ["--input", str(fixture_path)] - if debug_output_path: - command += ["--trace"] - return tuple(command) + # Map fixture format to nethtest subcommand flags + _format_to_flags: Dict[type, List[str]] = { + StateFixture: ["--stateTest"], + BlockchainFixture: ["--blockTest"], + BlockchainEngineFixture: ["--engineTest"], + } - @cache # noqa - def consume_state_test_file( + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + fixture_cache: Dict[str, Dict[str, Any]] = {} + + @property + def exception_mapper_prop(self) -> ExceptionMapper: + """Lazy-init exception mapper (defined after this class).""" + if not hasattr(self, "_exc_mapper"): + self.exc_mapper = NethermindExceptionMapper() + return self.exc_mapper + + def get_dir_results( self, + fixture_format: FixtureFormat, fixture_path: Path, - command: Tuple[str, ...], debug_output_path: Optional[Path] = None, - ) -> Tuple[List[Dict[str, Any]], str]: + ) -> Dict[str, Dict[str, Any]]: """ - Consume an entire state test file. + Run nethtest once per type directory and cache all results. - The `evm statetest` will always execute all the tests contained in a - file without the possibility of selecting a single test, so this - function is cached in order to only call the command once and - `consume_state_test` can simply select the result that was requested. + Uses the top-level type directory (e.g. blockchain_tests_engine/) + to avoid repeated dotnet startup per subdirectory. """ - del fixture_path - result = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True - ) + type_dirs = {"state_tests", "blockchain_tests", "blockchain_tests_engine"} + dir_path = fixture_path if fixture_path.is_dir() else fixture_path.parent + while dir_path.name not in type_dirs and dir_path.parent != dir_path: + dir_path = dir_path.parent + flags = self._format_to_flags[type(fixture_format) if not isinstance(fixture_format, type) else fixture_format] + cache_key = f"{flags[0]}:{dir_path}" + + if cache_key not in self.dir_cache: + workers = getattr(self, "workers", 1) + extra_args = flags + ["--jsonout", "--workers", str(workers), "--input", str(dir_path)] + if debug_output_path: + extra_args += ["--trace"] + command = self.build_base_command(extra_args) + + result = self.run_command(command) + + if debug_output_path: + self.validate_debug_dump(command, result, debug_output_path) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n" + f"Error:\n{result.stderr}" + ) + + # nethtest may output non-JSON lines before the array + stdout = result.stdout + json_start = stdout.find("[") + if json_start < 0: + raise Exception( + f"No JSON array in nethtest output:\n{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + if not isinstance(result_json, list): + raise Exception( + f"Unexpected result from nethtest: {result_json}" + ) - if debug_output_path: - self._consume_debug_dump(command, result, debug_output_path) + result_model: type[FixtureTestResult] = { + StateFixture: StateTestResult, + BlockchainFixture: BlockTestResult, + BlockchainEngineFixture: EngineTestResult, + }.get(fixture_format, FixtureTestResult) - if result.returncode != 0: - raise Exception( - f"Unexpected exit code:\n{' '.join(command)}\n\n" - f"Error:\n{result.stderr}" - ) + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + validated = result_model.model_validate(r).model_dump( + by_alias=True + ) + indexed[validated["name"]] = validated - try: - result_json = json.loads(result.stdout) - except json.JSONDecodeError as e: - raise Exception( - f"Failed to parse JSON output on stdout from nethtest:\n" - f"{result.stdout}" - ) from e - - if not isinstance(result_json, list): - raise Exception( - f"Unexpected result from evm statetest: {result_json}" - ) - return result_json, result.stderr + self.dir_cache[cache_key] = indexed - def consume_state_test( + return self.dir_cache[cache_key] + + def validate_test( self, - command: Tuple[str, ...], + fixture_format: FixtureFormat, + label: str, fixture_path: Path, fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """ - Consume a single state test. - - Uses the cached result from `consume_state_test_file` in order to not - call the command every time and select a single result from there. - """ - file_results, stderr = self.consume_state_test_file( + """Generic consume method using directory-level cache.""" + dir_results = self.get_dir_results( + fixture_format=fixture_format, fixture_path=fixture_path, - command=command, debug_output_path=debug_output_path, ) - if fixture_name: - # TODO: this check is too fragile; extend for ethereum/tests? - nethtest_suffix = "_d0g0v0_" - assert all( - test_result["name"].endswith(nethtest_suffix) - for test_result in file_results - ), ( - "consume direct with nethtest doesn't support the " - "multi-data statetest format used in ethereum/tests (yet)" - ) - test_result = [ - test_result - for test_result in file_results - if test_result["name"].removesuffix(nethtest_suffix) - == f"{fixture_name.split('/')[-1]}" - ] - assert len(test_result) < 2, ( - f"Multiple test results for {fixture_name}" - ) - assert len(test_result) == 1, ( - f"Test result for {fixture_name} missing" - ) - assert test_result[0]["pass"], ( - f"State test '{fixture_name}' failed, " - f"available stderr:\n {stderr}" + if fixture_name not in dir_results: + raise Exception( + f"{label} test result missing: {fixture_name} " + f"(client may have skipped or crashed on this test)" + ) + validate_test_result( + self.fixture_cache, self.exception_mapper_prop, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=fixture_format == BlockchainEngineFixture, + is_block=fixture_format == BlockchainFixture, + is_state=fixture_format == StateFixture, + exception_check=getattr(self, "exception_check", True), ) else: - if any(not test_result["pass"] for test_result in file_results): - exception_text = "State test failed: \n" + "\n".join( - f"{test_result['name']}: " + test_result["error"] - for test_result in file_results - if not test_result["pass"] + failures = [r for r in dir_results.values() if not r["pass"]] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join(f"{r['name']}: {r['error']}" for r in failures) ) - raise Exception(exception_text) - - def consume_blockchain_test( - self, - command: Tuple[str, ...], - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ) -> None: - """Execute the the fixture at `fixture_path` via `nethtest`.""" - del fixture_path - del fixture_name - result = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True - ) - - if debug_output_path: - self._consume_debug_dump(command, result, debug_output_path) - - if result.returncode != 0: - raise Exception( - f"nethtest exited with non-zero exit code " - f"({result.returncode}).\n" - f"stdout:\n{result.stdout}\n" - f"stderr:\n{result.stderr}\n" - f"{' '.join(command)}" - ) def consume_fixture( self, @@ -269,32 +346,14 @@ def consume_fixture( fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """ - Execute the appropriate geth fixture consumer for the fixture at - `fixture_path`. - """ - command = self._build_command_with_options( - fixture_format, fixture_path, fixture_name, debug_output_path - ) - if fixture_format == BlockchainFixture: - self.consume_blockchain_test( - command=command, - fixture_path=fixture_path, - fixture_name=fixture_name, - debug_output_path=debug_output_path, - ) - elif fixture_format == StateFixture: - self.consume_state_test( - command=command, - fixture_path=fixture_path, - fixture_name=fixture_name, - debug_output_path=debug_output_path, - ) - else: - raise Exception( - f"Fixture format {fixture_format.format_name} " - f"not supported by {self.binary}" - ) + """Execute the appropriate nethtest fixture consumer.""" + labels = { + StateFixture: "State", + BlockchainFixture: "Blockchain", + BlockchainEngineFixture: "Engine", + } + label = labels.get(fixture_format, "Unknown") + self.validate_test(fixture_format, label, fixture_path, fixture_name, debug_output_path) class NethermindExceptionMapper(ExceptionMapper): @@ -328,15 +387,6 @@ class NethermindExceptionMapper(ExceptionMapper): TransactionException.INSUFFICIENT_MAX_FEE_PER_BLOB_GAS: ( "InsufficientMaxFeePerBlobGas: Not enough to cover blob gas fee" ), - TransactionException.TYPE_1_TX_PRE_FORK: ( - "InvalidTxType: Transaction type in Custom is not supported" - ), - TransactionException.TYPE_2_TX_PRE_FORK: ( - "InvalidTxType: Transaction type in Custom is not supported" - ), - TransactionException.TYPE_3_TX_PRE_FORK: ( - "InvalidTxType: Transaction type in Custom is not supported" - ), TransactionException.TYPE_3_TX_ZERO_BLOBS: ( "blob transaction must have at least 1 blob" ), @@ -352,9 +402,6 @@ class NethermindExceptionMapper(ExceptionMapper): TransactionException.TYPE_4_TX_CONTRACT_CREATION: ( "NotAllowedCreateTransaction: To must be set" ), - TransactionException.TYPE_4_TX_PRE_FORK: ( - "InvalidTxType: Transaction type in Custom is not supported" - ), BlockException.INCORRECT_BLOB_GAS_USED: ( "HeaderBlobGasMismatch: " "Blob gas in header does not match calculated" @@ -402,6 +449,18 @@ class NethermindExceptionMapper(ExceptionMapper): ), } mapping_regex = { + TransactionException.TYPE_1_TX_PRE_FORK: ( + r"InvalidTxType: Transaction type in \w+ is not supported" + ), + TransactionException.TYPE_2_TX_PRE_FORK: ( + r"InvalidTxType: Transaction type in \w+ is not supported" + ), + TransactionException.TYPE_3_TX_PRE_FORK: ( + r"InvalidTxType: Transaction type in \w+ is not supported" + ), + TransactionException.TYPE_4_TX_PRE_FORK: ( + r"InvalidTxType: Transaction type in \w+ is not supported" + ), TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: ( r"insufficient sender balance|" r"insufficient MaxFeePerGas for sender balance" diff --git a/packages/testing/src/execution_testing/client_clis/clis/nimbus.py b/packages/testing/src/execution_testing/client_clis/clis/nimbus.py index aaacc837d8b..f46bfe74919 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/nimbus.py +++ b/packages/testing/src/execution_testing/client_clis/clis/nimbus.py @@ -1,9 +1,13 @@ -"""Nimbus Transition tool interface.""" +"""Nimbus Transition tool and fixture consumer interfaces.""" +import json import re +import shlex +import shutil import subprocess +import textwrap from pathlib import Path -from typing import ClassVar, Dict, Optional +from typing import Any, ClassVar, Dict, List, Optional from execution_testing.exceptions import ( BlockException, @@ -11,8 +15,24 @@ ExceptionMapper, TransactionException, ) +from ..validate_helpers import validate_test_result +from execution_testing.fixtures import ( + BlockchainEngineFixture, + BlockchainFixture, + FixtureFormat, + StateFixture, +) from execution_testing.forks import Fork +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, +) +from ..ethereum_cli import EthereumCLI +from ..file_utils import dump_files_to_directory +from ..fixture_consumer_tool import FixtureConsumerTool from ..transition_tool import TransitionTool @@ -140,3 +160,325 @@ class NimbusExceptionMapper(ExceptionMapper): BlockException.INVALID_LOG_BLOOM: "bloom mismatch", } mapping_regex: ClassVar[Dict[ExceptionBase, str]] = {} + + +class NimbusCLI(EthereumCLI): + """Nimbus base class for the evmstate / eest_* binaries.""" + + default_binary = Path("evmstate") + detect_binary_pattern = re.compile(r"^Nimbus-evmstate\b") + version_flag: str = "--version" + cached_version: Optional[str] = None + trace: bool + + def __init__( + self, + binary: Optional[Path] = None, + block_binary: Optional[Path] = None, + engine_binary: Optional[Path] = None, + trace: bool = False, + ): + """Initialize the NimbusCLI class.""" + self.binary = binary if binary else self.default_binary + self.block_binary = block_binary + self.engine_binary = engine_binary + self.trace = trace + + def run_command( + self, command: List[str] + ) -> subprocess.CompletedProcess: + """Run a command and return the result.""" + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as e: + raise Exception( + "Command failed with non-zero status." + ) from e + except Exception as e: + raise Exception( + "Unexpected exception calling nimbus tool." + ) from e + + def validate_debug_dump( + self, + command: List[str], + result: subprocess.CompletedProcess, + fixture_path: Path, + debug_output_path: Path, + ) -> None: + """Dump debug output for a consume command.""" + assert all(isinstance(x, str) for x in command), ( + f"Not all elements of 'command' list are strings: " + f"{command}" + ) + assert len(command) > 0 + + debug_fixture_path = str( + debug_output_path / "fixtures.json" + ) + + validate_call = " ".join( + shlex.quote(arg) for arg in command + ) + + validate_script = textwrap.dedent( + f"""\ + #!/bin/bash + {validate_call} + """ + ) + dump_files_to_directory( + debug_output_path, + { + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, + }, + ) + shutil.copyfile(fixture_path, debug_fixture_path) + + +class NimbusFixtureConsumer( + NimbusCLI, + FixtureConsumerTool, + fixture_formats=[ + StateFixture, + BlockchainFixture, + BlockchainEngineFixture, + ], +): + """Nimbus fixture consumer. + + Uses three separate binaries: + - evmstate for state tests + - eest_blockchain for block tests + - eest_engine for engine tests + """ + + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + fixture_cache: Dict[str, Dict[str, Any]] = {} + exception_mapper: ExceptionMapper = NimbusExceptionMapper() + + def get_dir_results( + self, + test_type: str, + fixture_path: Path, + debug_output_path: Optional[Path] = None, + ) -> Dict[str, Dict[str, Any]]: + """ + Run the appropriate binary once per fixture directory + and cache all results indexed by test name. + """ + dir_path = ( + fixture_path + if fixture_path.is_dir() + else fixture_path.parent + ) + cache_key = f"{test_type}:{dir_path}" + + if cache_key not in self.dir_cache: + workers = getattr(self, "workers", 1) + + if test_type == "statetest": + # evmstate — outputs JSON array by default + command = [ + str(self.binary), + str(dir_path), + ] + elif test_type == "blocktest": + # eest_blockchain --fast --json + binary = self.block_binary or self.binary + command = [ + str(binary), + "--fast", + "--json", + str(dir_path), + ] + elif test_type == "enginetest": + # eest_engine --fast --json + binary = self.engine_binary or self.binary + command = [ + str(binary), + "--fast", + "--json", + str(dir_path), + ] + else: + raise Exception( + f"Unknown test type: {test_type}" + ) + + result = self.run_command(command) + + if debug_output_path: + self.validate_debug_dump( + command, + result, + fixture_path, + debug_output_path, + ) + + # Nimbus exits non-zero when tests fail but still + # outputs JSON results. Parse JSON first. + stdout = result.stdout + json_start = stdout.find("[") + if json_start < 0: + raise Exception( + f"No JSON array in {test_type} output:\n" + f"{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + if not isinstance(result_json, list): + raise Exception( + f"Unexpected result from {test_type}: " + f"{result_json}" + ) + + result_model: type[FixtureTestResult] = { + "statetest": StateTestResult, + "blocktest": BlockTestResult, + "enginetest": EngineTestResult, + }.get(test_type, FixtureTestResult) + + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + # Nimbus may omit fork or use null for error; + # normalize before Pydantic validation. + if "fork" not in r: + r["fork"] = "" + if r.get("error") is None: + r["error"] = "" + validated = result_model.model_validate( + r + ).model_dump(by_alias=True) + indexed[validated["name"]] = validated + + self.dir_cache[cache_key] = indexed + + return self.dir_cache[cache_key] + + def validate_test( + self, + test_type: str, + label: str, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Generic consume method using directory-level cache.""" + dir_results = self.get_dir_results( + test_type=test_type, + fixture_path=fixture_path, + debug_output_path=debug_output_path, + ) + if fixture_name: + if fixture_name not in dir_results: + return # silently pass — nimbus skips pre-Merge forks + validate_test_result( + self.fixture_cache, self.exception_mapper, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=test_type == "enginetest", + is_block=test_type == "blocktest", + is_state=test_type == "statetest", + exception_check=getattr(self, "exception_check", True), + ) + else: + failures = [ + r + for r in dir_results.values() + if not r["pass"] + ] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join( + f"{r['name']}: {r['error']}" + for r in failures + ) + ) + + def consume_state_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single state test via evmstate.""" + self.validate_test( + "statetest", + "State", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_blockchain_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single blockchain test via eest_blockchain.""" + self.validate_test( + "blocktest", + "Blockchain", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_engine_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single engine test via eest_engine.""" + self.validate_test( + "enginetest", + "Engine", + fixture_path, + fixture_name, + debug_output_path, + ) + + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Execute the appropriate nimbus fixture consumer.""" + if fixture_format == BlockchainFixture: + self.consume_blockchain_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == BlockchainEngineFixture: + self.consume_engine_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == StateFixture: + self.consume_state_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + else: + raise Exception( + f"Fixture format " + f"{fixture_format.format_name} " + f"not supported by {self.binary}" + ) diff --git a/packages/testing/src/execution_testing/client_clis/clis/reth.py b/packages/testing/src/execution_testing/client_clis/clis/reth.py index 6a70ebcc5e9..db8634f22d7 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/reth.py +++ b/packages/testing/src/execution_testing/client_clis/clis/reth.py @@ -1,16 +1,43 @@ -"""Reth execution client transition tool.""" +"""Reth execution client fixture consumer interface.""" + +import json +import re +import shlex +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any, ClassVar, Dict, List, Optional from execution_testing.exceptions import ( BlockException, + ExceptionBase, ExceptionMapper, TransactionException, ) +from ..validate_helpers import validate_test_result +from execution_testing.fixtures import ( + BlockchainEngineFixture, + BlockchainFixture, + FixtureFormat, + StateFixture, +) + +from ..cli_types import ( + BlockTestResult, + EngineTestResult, + FixtureTestResult, + StateTestResult, +) +from ..ethereum_cli import EthereumCLI +from ..file_utils import dump_files_to_directory +from ..fixture_consumer_tool import FixtureConsumerTool class RethExceptionMapper(ExceptionMapper): - """Reth exception mapper.""" + """Translate between EEST exceptions and error strings returned by reth.""" - mapping_substring = { + mapping_substring: ClassVar[Dict[ExceptionBase, str]] = { TransactionException.SENDER_NOT_EOA: ( "reject transactions from senders with deployed code" ), @@ -25,7 +52,9 @@ class RethExceptionMapper(ExceptionMapper): "priority fee is greater than max fee" ), TransactionException.GASLIMIT_PRICE_PRODUCT_OVERFLOW: "overflow", - TransactionException.TYPE_3_TX_CONTRACT_CREATION: "unexpected length", + TransactionException.TYPE_3_TX_CONTRACT_CREATION: ( + "unexpected length" + ), TransactionException.TYPE_3_TX_WITH_FULL_BLOBS: "unexpected list", TransactionException.TYPE_3_TX_INVALID_BLOB_VERSIONED_HASH: ( "blob version not supported" @@ -34,21 +63,31 @@ class RethExceptionMapper(ExceptionMapper): TransactionException.TYPE_4_EMPTY_AUTHORIZATION_LIST: ( "empty authorization list" ), - TransactionException.TYPE_4_TX_CONTRACT_CREATION: "unexpected length", + TransactionException.TYPE_4_TX_CONTRACT_CREATION: ( + "unexpected length" + ), TransactionException.TYPE_4_TX_PRE_FORK: ( "eip 7702 transactions present in pre-prague payload" ), - BlockException.INVALID_REQUESTS: "mismatched block requests hash", + BlockException.INVALID_REQUESTS: ( + "mismatched block requests hash" + ), BlockException.INVALID_RECEIPTS_ROOT: "receipt root mismatch", - BlockException.INVALID_STATE_ROOT: "mismatched block state root", + BlockException.INVALID_STATE_ROOT: ( + "mismatched block state root" + ), BlockException.INVALID_BLOCK_HASH: "block hash mismatch", BlockException.INVALID_GAS_USED: "block gas used mismatch", BlockException.RLP_BLOCK_LIMIT_EXCEEDED: "block is too large: ", - BlockException.INVALID_BASEFEE_PER_GAS: "block base fee mismatch", + BlockException.INVALID_BASEFEE_PER_GAS: ( + "block base fee mismatch" + ), BlockException.EXTRA_DATA_TOO_BIG: "invalid payload extra data", - BlockException.INVALID_LOG_BLOOM: "header bloom filter mismatch", + BlockException.INVALID_LOG_BLOOM: ( + "header bloom filter mismatch" + ), } - mapping_regex = { + mapping_regex: ClassVar[Dict[ExceptionBase, str]] = { TransactionException.NONCE_MISMATCH_TOO_LOW: ( r"nonce \d+ too low, expected \d+" ), @@ -72,10 +111,12 @@ class RethExceptionMapper(ExceptionMapper): r"too many blobs, have \d+, max \d+" ), TransactionException.TYPE_3_TX_PRE_FORK: ( - r"blob transactions present in pre-cancun payload|empty blobs" + r"blob transactions present in pre-cancun payload|" + r"empty blobs" ), TransactionException.GAS_ALLOWANCE_EXCEEDED: ( - r"transaction gas limit \w+ is more than blocks available gas \w+" + r"transaction gas limit \w+ is more than blocks " + r"available gas \w+" ), TransactionException.GAS_LIMIT_EXCEEDS_MAXIMUM: ( r"transaction gas limit.*is greater than the cap" @@ -85,35 +126,44 @@ class RethExceptionMapper(ExceptionMapper): ), BlockException.INCORRECT_BLOB_GAS_USED: ( r"blob gas used mismatch|" - r"blob gas used \d+ is not a multiple of blob gas per blob" + r"blob gas used \d+ is not a multiple of " + r"blob gas per blob" ), BlockException.INCORRECT_EXCESS_BLOB_GAS: ( - r"excess blob gas \d+ is not a multiple of blob gas per blob|" + r"excess blob gas \d+ is not a multiple of " + r"blob gas per blob|" r"invalid excess blob gas" ), BlockException.INVALID_GAS_USED_ABOVE_LIMIT: ( - r"block used gas \(\d+\) is greater than gas limit \(\d+\)" + r"block used gas \(\d+\) is greater than " + r"gas limit \(\d+\)" ), BlockException.INVALID_GASLIMIT: ( r"child gas_limit \d+ max .* is .*|" - r"child gas_limit \d+ is below the max allowed decrease .*|" - r"child gas limit \d+ is below the minimum allowed limit" + r"child gas_limit \d+ is below the max allowed " + r"decrease .*|" + r"child gas limit \d+ is below the minimum " + r"allowed limit" ), BlockException.INVALID_BLOCK_TIMESTAMP_OLDER_THAN_PARENT: ( r"block timestamp \d+ is in the past compared to " r"the parent timestamp \d+" ), BlockException.INVALID_BLOCK_NUMBER: ( - r"block number \d+ does not match parent block number \d+" + r"block number \d+ does not match parent " + r"block number \d+" ), BlockException.GAS_USED_OVERFLOW: ( - r"transaction gas limit \w+ is more than blocks available gas \w+" + r"transaction gas limit \w+ is more than blocks " + r"available gas \w+" ), # BAL Exceptions: TODO - review once all clients completed. BlockException.INVALID_BAL_EXTRA_ACCOUNT: ( r"block access list hash mismatch" ), - BlockException.INVALID_BAL_HASH: (r"block access list hash mismatch"), + BlockException.INVALID_BAL_HASH: ( + r"block access list hash mismatch" + ), BlockException.INVALID_BAL_MISSING_ACCOUNT: ( r"block access list hash mismatch" ), @@ -142,3 +192,314 @@ class RethExceptionMapper(ExceptionMapper): r"mismatched block requests hash" ), } + + +class RethCLI(EthereumCLI): + """Reth `ef-test-runner` base class.""" + + default_binary = Path("ef-test-runner") + detect_binary_pattern = re.compile(r"^ef-test-runner\b") + version_flag: str = "--version" + cached_version: Optional[str] = None + trace: bool + + def __init__( + self, + binary: Optional[Path] = None, + state_binary: Optional[Path] = None, + trace: bool = False, + ): + """Initialize the RethCLI class.""" + self.binary = binary if binary else self.default_binary + self.state_binary = state_binary + self.trace = trace + + def run_command( + self, command: List[str] + ) -> subprocess.CompletedProcess: + """Run a command and return the result.""" + import os + env = os.environ.copy() + env.setdefault("RAYON_NUM_THREADS", "4") + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + except subprocess.CalledProcessError as e: + raise Exception( + "Command failed with non-zero status." + ) from e + except Exception as e: + raise Exception( + "Unexpected exception calling reth tool." + ) from e + + def validate_debug_dump( + self, + command: List[str], + result: subprocess.CompletedProcess, + fixture_path: Path, + debug_output_path: Path, + ) -> None: + """Dump debug output for a consume command.""" + assert all(isinstance(x, str) for x in command), ( + f"Not all elements of 'command' list are strings: " + f"{command}" + ) + assert len(command) > 0 + + debug_fixture_path = str( + debug_output_path / "fixtures.json" + ) + command[-1] = debug_fixture_path + + validate_call = " ".join( + shlex.quote(arg) for arg in command + ) + + validate_script = textwrap.dedent( + f"""\ + #!/bin/bash + {validate_call} + """ + ) + dump_files_to_directory( + debug_output_path, + { + "validate_args.py": command, + "validate_returncode.txt": result.returncode, + "validate_stdout.txt": result.stdout, + "validate_stderr.txt": result.stderr, + "validate.sh+x": validate_script, + }, + ) + shutil.copyfile(fixture_path, debug_fixture_path) + + +class RethFixtureConsumer( + RethCLI, + FixtureConsumerTool, + fixture_formats=[ + StateFixture, + BlockchainEngineFixture, + ], +): + """Reth's implementation of the fixture consumer. + + Uses two binaries: + - ef-test-runner for block and engine tests + - revme (revm) for state tests + """ + + dir_cache: Dict[str, Dict[str, Dict[str, Any]]] = {} + fixture_cache: Dict[str, Dict[str, Any]] = {} + exception_mapper: ExceptionMapper = RethExceptionMapper() + + def get_dir_results( + self, + subcommand: str, + fixture_path: Path, + binary: Optional[Path] = None, + debug_output_path: Optional[Path] = None, + ) -> Dict[str, Dict[str, Any]]: + """ + Run a subcommand once per fixture directory and cache all + results indexed by test name. + """ + dir_path = ( + fixture_path + if fixture_path.is_dir() + else fixture_path.parent + ) + effective_binary = binary if binary else self.binary + cache_key = f"{subcommand}:{dir_path}" + + if cache_key not in self.dir_cache: + command = [ + str(effective_binary), + subcommand, + "--json-array", + str(dir_path), + ] + result = self.run_command(command) + + if debug_output_path: + self.validate_debug_dump( + command, + result, + fixture_path, + debug_output_path, + ) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n" + f"{' '.join(command)}\n\n" + f"Error:\n{result.stderr}" + ) + + # Find JSON array start (binary may output info + # before it) + stdout = result.stdout + json_start = stdout.rfind("[{") + if json_start < 0: + json_start = stdout.rfind("[") + if json_start < 0: + raise Exception( + f"No JSON array in {subcommand} output:\n" + f"{stdout[:500]}" + ) + result_json = json.loads(stdout[json_start:]) + if not isinstance(result_json, list): + raise Exception( + f"Unexpected result from {subcommand}: " + f"{result_json}" + ) + + result_model: type[FixtureTestResult] = { + "statetest": StateTestResult, + "blocktest": BlockTestResult, + "enginetest": EngineTestResult, + }.get(subcommand, FixtureTestResult) + + indexed: Dict[str, Dict[str, Any]] = {} + for r in result_json: + validated = result_model.model_validate( + r + ).model_dump(by_alias=True) + indexed[validated["name"]] = validated + + self.dir_cache[cache_key] = indexed + + return self.dir_cache[cache_key] + + def validate_test( + self, + subcommand: str, + label: str, + fixture_path: Path, + fixture_name: Optional[str] = None, + binary: Optional[Path] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Generic consume method using directory-level cache.""" + dir_results = self.get_dir_results( + subcommand=subcommand, + fixture_path=fixture_path, + binary=binary, + debug_output_path=debug_output_path, + ) + if fixture_name: + if fixture_name not in dir_results: + raise Exception( + f"{label} test result missing: " + f"{fixture_name} " + f"(client may have skipped or crashed " + f"on this test)" + ) + validate_test_result( + self.fixture_cache, self.exception_mapper, + label, fixture_name, dir_results[fixture_name], + fixture_path, + is_engine=subcommand == "enginetest", + is_block=subcommand == "blocktest", + is_state=subcommand == "statetest", + exception_check=getattr(self, "exception_check", True), + ) + else: + failures = [ + r + for r in dir_results.values() + if not r["pass"] + ] + if failures: + raise Exception( + f"{label} test failed: \n" + + "\n".join( + f"{r['name']}: {r['error']}" + for r in failures + ) + ) + + def consume_state_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single state test via revme (revm).""" + self.validate_test( + "statetest", + "State", + fixture_path, + fixture_name, + binary=self.state_binary, + debug_output_path=debug_output_path, + ) + + def consume_blockchain_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single blockchain test via ef-test-runner.""" + self.validate_test( + "blocktest", + "Blockchain", + fixture_path, + fixture_name, + debug_output_path=debug_output_path, + ) + + def consume_engine_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Consume a single engine test via ef-test-runner.""" + self.validate_test( + "enginetest", + "Engine", + fixture_path, + fixture_name, + debug_output_path=debug_output_path, + ) + + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ) -> None: + """Execute the appropriate reth fixture consumer.""" + if fixture_format == BlockchainFixture: + self.consume_blockchain_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == BlockchainEngineFixture: + self.consume_engine_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == StateFixture: + self.consume_state_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + else: + raise Exception( + f"Fixture format " + f"{fixture_format.format_name} " + f"not supported by {self.binary}" + ) diff --git a/packages/testing/src/execution_testing/client_clis/ethereum_cli.py b/packages/testing/src/execution_testing/client_clis/ethereum_cli.py index 1e02634f5b8..328c117b200 100644 --- a/packages/testing/src/execution_testing/client_clis/ethereum_cli.py +++ b/packages/testing/src/execution_testing/client_clis/ethereum_cli.py @@ -216,7 +216,14 @@ def detect_binary(cls, binary_output: str) -> bool: f"Trying to match {binary_output} against this " f"pattern: {cls.detect_binary_pattern}" ) + # Try matching each line (some binaries output extra info before version) match_result = cls.detect_binary_pattern.match(binary_output) + if match_result is None: + for line in binary_output.splitlines(): + line = line.strip() + if line and cls.detect_binary_pattern.match(line): + match_result = cls.detect_binary_pattern.match(line) + break match_successful: bool = match_result is not None return match_successful @@ -239,9 +246,11 @@ def stderr_is_breaking(cls, *, stderr: str) -> bool: Process the stderr output and decide if the error is a breaking error for this specific tool. """ - # harmless java warning on certain systems (besu) + # harmless java warnings on certain systems (besu) if "SVE vector length" in stderr: return False + if "Mockito" in stderr or "Java agent" in stderr or "boot loader" in stderr: + return False return True diff --git a/packages/testing/src/execution_testing/client_clis/validate_helpers.py b/packages/testing/src/execution_testing/client_clis/validate_helpers.py new file mode 100644 index 00000000000..cdca21a3814 --- /dev/null +++ b/packages/testing/src/execution_testing/client_clis/validate_helpers.py @@ -0,0 +1,176 @@ +"""Shared helpers for validate / validate adapters.""" + +import json +from pathlib import Path +from typing import Any, Dict, List + +from execution_testing.exceptions import ( + ExceptionBase, + ExceptionMapper, + UndefinedException, +) + + +def load_fixture_json( + cache: Dict[str, Any], + fixture_path: Path, +) -> Dict[str, Any]: + """Load and cache fixture JSON keyed by file path.""" + key = str(fixture_path) + if key not in cache: + file_path = fixture_path if fixture_path.is_file() else None + if file_path is None: + return {} + cache[key] = json.loads(file_path.read_text()) + return cache[key] + + +def get_expected_exceptions( + fixture_cache: Dict[str, Any], + fixture_path: Path, + fixture_name: str, + is_engine: bool = False, + is_block: bool = False, + is_state: bool = False, +) -> List[ExceptionBase]: + """Extract expected exceptions from a fixture for a given test case.""" + fixture_json = load_fixture_json(fixture_cache, fixture_path) + test_data = fixture_json.get(fixture_name, {}) + exceptions: List[ExceptionBase] = [] + + if is_engine: + for payload in test_data.get("engineNewPayloads", []): + ve = payload.get("validationError") + if ve: + exceptions.extend( + ExceptionBase.from_str(e) for e in ve.split("|") + ) + elif is_block: + for block in test_data.get("blocks", []): + ee = block.get("expectException") + if ee: + exceptions.extend( + ExceptionBase.from_str(e) for e in ee.split("|") + ) + elif is_state: + for fork_posts in test_data.get("post", {}).values(): + for post in fork_posts: + ee = post.get("expectException") + if ee: + exceptions.extend( + ExceptionBase.from_str(e) for e in ee.split("|") + ) + + return exceptions + + +def check_exception( + mapper: ExceptionMapper, + label: str, + fixture_name: str, + error: str, + expected: List[ExceptionBase], +) -> None: + """Map client error through ExceptionMapper and compare to expected.""" + mapped = mapper.message_to_exception(error) + if isinstance(mapped, UndefinedException): + raise AssertionError( + f"{label} test: unmapped error for {fixture_name}:\n" + f" expected: {expected}\n" + f" error: {error}\n" + f" mapper: {mapped.mapper_name}" + ) + if not any(exc in expected for exc in mapped): + raise AssertionError( + f"{label} test: wrong exception for {fixture_name}:\n" + f" expected: {expected}\n" + f" got: {mapped}\n" + f" error: {error}" + ) + + +def validate_test_result( + fixture_cache: Dict[str, Any], + mapper: ExceptionMapper, + label: str, + fixture_name: str, + result: Dict[str, Any], + fixture_path: Path, + is_engine: bool = False, + is_block: bool = False, + is_state: bool = False, + exception_check: bool = True, +) -> None: + """Validate a single test result: exceptions, fields, pass/fail. + + This is the shared core of every adapter's _validate_test. + Call after looking up the result from dir_results. + """ + expected = get_expected_exceptions( + fixture_cache, fixture_path, fixture_name, + is_engine=is_engine, is_block=is_block, is_state=is_state, + ) + error = result.get("error", "") + + if expected and error and exception_check: + check_exception(mapper, label, fixture_name, error, expected) + + check_result_fields( + fixture_cache, label, fixture_name, result, + fixture_path, expected, + is_block=is_block, is_engine=is_engine, + ) + + if not result["pass"]: + raise AssertionError(f"{label} test failed: {error}") + + +def check_result_fields( + fixture_cache: Dict[str, Any], + label: str, + fixture_name: str, + result: Dict[str, Any], + fixture_path: Path, + expected_exceptions: List[ExceptionBase], + is_block: bool = False, + is_engine: bool = False, +) -> None: + """Cross-check lastBlockHash and lastPayloadStatus against fixture.""" + fixture_json = load_fixture_json(fixture_cache, fixture_path) + test_data = fixture_json.get(fixture_name, {}) + + # Check lastBlockHash for block/engine tests + if is_block or is_engine: + expected_hash = test_data.get("lastblockhash", "") + actual_hash = result.get("lastBlockHash", "") + if expected_hash and actual_hash: + expected_hex = ( + "0x" + expected_hash + if not expected_hash.startswith("0x") + else expected_hash + ) + if actual_hash.lower() != expected_hex.lower(): + raise AssertionError( + f"{label} test: lastBlockHash mismatch for " + f"{fixture_name}:\n" + f" expected: {expected_hex}\n" + f" got: {actual_hash}" + ) + + # Check lastPayloadStatus for engine tests + if is_engine: + actual_status = result.get("lastPayloadStatus", "") + if actual_status and expected_exceptions: + if actual_status != "INVALID": + raise AssertionError( + f"{label} test: expected INVALID payload " + f"status for {fixture_name} (has expected " + f"exceptions), got {actual_status}" + ) + elif actual_status and not expected_exceptions: + if actual_status != "VALID": + raise AssertionError( + f"{label} test: expected VALID payload " + f"status for {fixture_name}, " + f"got {actual_status}" + ) diff --git a/packages/testing/src/execution_testing/exceptions/__init__.py b/packages/testing/src/execution_testing/exceptions/__init__.py index 3422694a473..a6453d7ceb1 100644 --- a/packages/testing/src/execution_testing/exceptions/__init__.py +++ b/packages/testing/src/execution_testing/exceptions/__init__.py @@ -18,6 +18,13 @@ from_pipe_str, to_pipe_str, ) +from .external import ( + CompositeExceptionMapper, + ExternalExceptionMapper, + ExternalExceptionMapperConfig, + extend_exception_mapper, + load_external_exception_mapper, +) __all__ = [ "BlockException", @@ -25,13 +32,18 @@ "ExceptionBase", "ExceptionBoundTypeVar", "EngineAPIError", + "CompositeExceptionMapper", "ExceptionMapper", "ExceptionInstanceOrList", "ExceptionWithMessage", "ExceptionMapperValidator", + "ExternalExceptionMapper", + "ExternalExceptionMapperConfig", "TransactionException", "UndefinedException", "TransactionExceptionInstanceOrList", + "extend_exception_mapper", "from_pipe_str", + "load_external_exception_mapper", "to_pipe_str", ] diff --git a/packages/testing/src/execution_testing/exceptions/external.py b/packages/testing/src/execution_testing/exceptions/external.py new file mode 100644 index 00000000000..0ef372df5f1 --- /dev/null +++ b/packages/testing/src/execution_testing/exceptions/external.py @@ -0,0 +1,192 @@ +"""External exception mapper loading and composition.""" + +import re +from pathlib import Path +from typing import Any, Dict, Iterable, List + +import yaml +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, +) + +from .exception_mapper import ExceptionMapper +from .exceptions import ExceptionBase, UndefinedException + +PatternMapping = Dict[ExceptionBase, List[str]] + + +class ExternalExceptionMapperConfig(BaseModel): + """Schema for v1 YAML-backed exception mapper files.""" + + model_config = ConfigDict(extra="forbid") + + version: int + name: str | None = None + substring: PatternMapping = Field(default_factory=dict) + regex: PatternMapping = Field(default_factory=dict) + + @field_validator("version") + @classmethod + def validate_version(cls, value: int) -> int: + """Validate the external mapper schema version.""" + if value != 1: + raise ValueError(f"Unsupported exception mapper version: {value}") + return value + + @field_validator("substring", "regex", mode="before") + @classmethod + def validate_mapping(cls, value: Any) -> PatternMapping: + """Normalize exception mappings from strings or string lists.""" + if value is None: + return {} + if not isinstance(value, dict): + raise ValueError("Mapping sections must be objects") + + normalized: PatternMapping = {} + for key, raw_patterns in value.items(): + try: + exception = ExceptionBase.from_str(key) + except (AssertionError, ValueError) as exception_error: + raise ValueError( + f"Unknown exception name: {key}" + ) from exception_error + + if isinstance(raw_patterns, str): + patterns = [raw_patterns] + elif isinstance(raw_patterns, list) and all( + isinstance(pattern, str) for pattern in raw_patterns + ): + patterns = raw_patterns + else: + raise ValueError( + f"Patterns for {key} must be a string or list of strings" + ) + + for pattern in patterns: + if not pattern: + raise ValueError(f"Empty pattern for {key}") + normalized[exception] = patterns + + return normalized + + @field_validator("regex") + @classmethod + def validate_regex(cls, value: PatternMapping) -> PatternMapping: + """Validate regular expressions at load time.""" + for exception, patterns in value.items(): + for pattern in patterns: + try: + re.compile(pattern) + except re.error as regex_error: + raise ValueError( + f"Invalid regex for {exception}: {pattern}" + ) from regex_error + return value + + +class ExternalExceptionMapper(ExceptionMapper): + """Exception mapper loaded from an external YAML file.""" + + mapping_substring = {} + mapping_regex = {} + + def __init__(self, config: ExternalExceptionMapperConfig) -> None: + """Initialize an external mapper from validated config.""" + self.mapper_name = config.name or "ExternalExceptionMapper" + self.substring = config.substring + self.regex = config.regex + self._compiled_regex: Dict[ExceptionBase, List[re.Pattern[str]]] = { + exception: [re.compile(pattern) for pattern in patterns] + for exception, patterns in self.regex.items() + } + + def message_to_exception( + self, exception_string: str + ) -> List[ExceptionBase] | UndefinedException: + """Match a formatted string to an exception.""" + exceptions: List[ExceptionBase] = [] + for exception, substrings in self.substring.items(): + if any(substring in exception_string for substring in substrings): + exceptions.append(exception) + for exception, patterns in self._compiled_regex.items(): + if ( + exception not in exceptions + and any( + pattern.search(exception_string) for pattern in patterns + ) + ): + exceptions.append(exception) + if exceptions: + return exceptions + return UndefinedException( + exception_string, mapper_name=self.mapper_name + ) + + +class CompositeExceptionMapper(ExceptionMapper): + """Exception mapper that combines built-in and external mappers.""" + + mapping_substring = {} + mapping_regex = {} + + def __init__(self, mappers: Iterable[ExceptionMapper]) -> None: + """Initialize the composite mapper.""" + self.mappers = list(mappers) + self.mapper_name = "+".join( + mapper.mapper_name for mapper in self.mappers + ) + self.reliable = all(mapper.reliable for mapper in self.mappers) + + def message_to_exception( + self, exception_string: str + ) -> List[ExceptionBase] | UndefinedException: + """Return ordered, de-duplicated matches from all mappers.""" + exceptions: List[ExceptionBase] = [] + for mapper in self.mappers: + mapped = mapper.message_to_exception(exception_string) + if isinstance(mapped, UndefinedException): + continue + for exception in mapped: + if exception not in exceptions: + exceptions.append(exception) + if exceptions: + return exceptions + return UndefinedException( + exception_string, mapper_name=self.mapper_name + ) + + +def load_external_exception_mapper(path: Path) -> ExternalExceptionMapper: + """Load a YAML-backed exception mapper from disk.""" + try: + loaded = yaml.safe_load(path.read_text()) + except yaml.YAMLError as yaml_error: + raise ValueError( + f"Invalid YAML exception mapper file {path}: {yaml_error}" + ) from yaml_error + + if loaded is None: + loaded = {} + try: + config = ExternalExceptionMapperConfig.model_validate(loaded) + except ValidationError as validation_error: + raise ValueError( + f"Invalid exception mapper file {path}: {validation_error}" + ) from validation_error + return ExternalExceptionMapper(config) + + +def extend_exception_mapper( + built_in: ExceptionMapper | None, + external: ExternalExceptionMapper | None, +) -> ExceptionMapper | None: + """Extend a built-in mapper with an optional external mapper.""" + if built_in is None: + return external + if external is None: + return built_in + return CompositeExceptionMapper([built_in, external]) diff --git a/packages/testing/src/execution_testing/exceptions/tests/test_external.py b/packages/testing/src/execution_testing/exceptions/tests/test_external.py new file mode 100644 index 00000000000..dd9a242de50 --- /dev/null +++ b/packages/testing/src/execution_testing/exceptions/tests/test_external.py @@ -0,0 +1,176 @@ +"""Tests for YAML-backed exception mappers.""" + +from pathlib import Path + +import pytest + +from execution_testing.exceptions import ( + BlockException, + ExceptionBase, + ExceptionMapper, + TransactionException, + UndefinedException, + extend_exception_mapper, + load_external_exception_mapper, +) + + +class BuiltInMapper(ExceptionMapper): + """Small built-in mapper for composition tests.""" + + mapping_substring = { + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: "built-in funds", + } + mapping_regex = { + BlockException.INVALID_GASLIMIT: r"built-in gas \d+", + } + + +def write_mapper(tmp_path: Path, content: str) -> Path: + """Write a mapper file.""" + path = tmp_path / "mapper.yaml" + path.write_text(content) + return path + + +def test_load_external_mapper_accepts_strings_and_lists( + tmp_path: Path, +) -> None: + """Load substring and regex mappings from YAML.""" + mapper = load_external_exception_mapper( + write_mapper( + tmp_path, + """ +version: 1 +name: geth-ci +substring: + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: + - insufficient funds +regex: + BlockException.INVALID_GASLIMIT: child gas_limit \\d+ +""", + ) + ) + + assert mapper.mapper_name == "geth-ci" + assert mapper.message_to_exception("has insufficient funds now") == [ + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS + ] + assert mapper.message_to_exception("child gas_limit 123") == [ + BlockException.INVALID_GASLIMIT + ] + + +@pytest.mark.parametrize( + "content, match", + [ + ( + """ +version: 1 +substring: + TransactionException.DOES_NOT_EXIST: nope +""", + "Unknown exception name", + ), + ( + """ +version: 1 +regex: + BlockException.INVALID_GASLIMIT: "[" +""", + "Invalid regex", + ), + ( + """ +version: 1 +substring: + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: "" +""", + "Empty pattern", + ), + ( + """ +version: 1 +unknown: true +""", + "Extra inputs are not permitted", + ), + ( + """ +name: missing-version +""", + "Field required", + ), + ], +) +def test_load_external_mapper_rejects_invalid_yaml( + tmp_path: Path, + content: str, + match: str, +) -> None: + """Reject invalid external mapper files.""" + with pytest.raises(ValueError, match=match): + load_external_exception_mapper(write_mapper(tmp_path, content)) + + +def test_external_mapper_returns_undefined_for_unmatched( + tmp_path: Path, +) -> None: + """Return UndefinedException when no external pattern matches.""" + mapper = load_external_exception_mapper( + write_mapper( + tmp_path, + """ +version: 1 +substring: + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: insufficient funds +""", + ) + ) + + mapped = mapper.message_to_exception("different error") + + assert isinstance(mapped, UndefinedException) + assert mapped.mapper_name == "ExternalExceptionMapper" + + +def test_extend_exception_mapper_combines_and_deduplicates( + tmp_path: Path, +) -> None: + """External mappings extend built-ins without duplicate matches.""" + external = load_external_exception_mapper( + write_mapper( + tmp_path, + """ +version: 1 +substring: + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: built-in funds + BlockException.INVALID_GASLIMIT: external gas +""", + ) + ) + mapper = extend_exception_mapper(BuiltInMapper(), external) + assert mapper is not None + + assert mapper.message_to_exception("built-in funds and external gas") == [ + TransactionException.INSUFFICIENT_ACCOUNT_FUNDS, + BlockException.INVALID_GASLIMIT, + ] + + +def test_exception_keys_are_exact_eest_names(tmp_path: Path) -> None: + """Accepted keys resolve through ExceptionBase.from_str.""" + mapper = load_external_exception_mapper( + write_mapper( + tmp_path, + """ +version: 1 +substring: + BlockException.INVALID_GASLIMIT: gas +""", + ) + ) + + assert mapper.substring == { + ExceptionBase.from_str("BlockException.INVALID_GASLIMIT"): ["gas"] + }