feat(exceptions): support external client exception mappers

.gitignore

@@ -89,4 +89,7 @@ logs/
 
 env.yaml
 
+# Client binary config (user-specific paths)
+validate.toml
+
 site/

docs/running_tests/consume/exceptions.md

@@ -32,7 +32,56 @@ GETH_EXCEPTIONS = {
 }
 ```
 
-## Updating Client Exception Messages
+## External Client Exception Mappers
+
+Client repositories can extend EEST's built-in exception mappers with a
+client-owned YAML file. External mappings are additive: they do not remove or
+replace mappings maintained in EEST.
+
+```yaml
+version: 1
+name: geth-ci
+substring:
+  TransactionException.INSUFFICIENT_ACCOUNT_FUNDS:
+    - "insufficient funds for gas * price + value"
+regex:
+  BlockException.INVALID_GASLIMIT:
+    - 'child gas_limit \d+ .*'
+```
+
+Rules:
+
+- `version` must be `1`.
+- Exception keys must be exact EEST names such as
+  `TransactionException.INSUFFICIENT_ACCOUNT_FUNDS` or
+  `BlockException.INVALID_GASLIMIT`.
+- `substring` and `regex` values can be a string or a list of strings.
+- Empty patterns, unknown exception names, unknown top-level sections, and
+  invalid regular expressions are rejected when the file is loaded.
+
+For `validate`, add the mapper path to the relevant client section in
+`validate.toml`. Relative paths are resolved from the directory containing
+`validate.toml`.
+
+```toml
+[geth]
+bin = "../go-ethereum/build/bin/evm"
+exception-mapper = "../go-ethereum/eest-exceptions.yaml"
+```
+
+For Hive-backed `consume` commands, pass one or more mapper files on the
+command line:
+
+```bash
+uv run consume engine \
+  --input ./fixtures \
+  --exception-mapper geth=../go-ethereum/eest-exceptions.yaml
+```
+
+The `CLIENT` key is matched against Hive client names. For example, `geth`
+extends the mapper selected for `go-ethereum`.
+
+## Updating Built-In Client Exception Messages
 
 When clients change their error messages or you encounter unmapped exceptions:
 

packages/testing/pyproject.toml

@@ -84,6 +84,7 @@ checkfixtures = "execution_testing.cli.check_fixtures:check_fixtures"
 check_eip_versions = "execution_testing.cli.pytest_commands.check_eip_versions:check_eip_versions"
 consume = "execution_testing.cli.pytest_commands.consume:consume"
 protec = "execution_testing.cli.pytest_commands.consume:consume"
+validate = "execution_testing.cli.pytest_commands.validate:validate"
 checklist = "execution_testing.cli.pytest_commands.checklist:checklist"
 generate_checklist_stubs = "execution_testing.cli.generate_checklist_stubs:generate_checklist_stubs"
 genindex = "execution_testing.cli.gen_index:generate_fixtures_index_cli"
@@ -134,6 +135,7 @@ addopts = [
     "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/test_cache.py",
     "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/direct/",
     "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/simulators/",
+    "--ignore=src/execution_testing/cli/pytest_commands/plugins/validate/",
 ]
 markers = [
     "pre_alloc_group: Custom marker for pre-allocation grouping",

packages/testing/src/execution_testing/cli/pytest_commands/consume.py

@@ -55,10 +55,6 @@ def get_command_logic_test_paths(command_name: str) -> List[Path]:
         command_logic_test_paths = [
             base_path / "simulators" / "simulator_logic" / "test_via_sync.py"
         ]
-    elif command_name == "direct":
-        command_logic_test_paths = [
-            base_path / "direct" / "test_via_direct.py"
-        ]
     else:
         raise ValueError(f"Unexpected command: {command_name}.")
     return command_logic_test_paths
@@ -70,6 +66,8 @@ def consume() -> None:
     pass
 
 
+
+
 def consume_command(
     is_hive: bool = False,
 ) -> Callable[[Callable[..., Any]], click.Command]:
@@ -102,12 +100,6 @@ def command(pytest_args: List[str], **kwargs: Any) -> None:
     return decorator
 
 
-@consume_command(is_hive=False)
-def direct() -> None:
-    """Clients consume directly via the `blocktest` interface."""
-    pass
-
-
 @consume_command(is_hive=True)
 def rlp() -> None:
     """Client consumes RLP-encoded blocks on startup."""
@@ -144,3 +136,5 @@ def cache(pytest_args: List[str], **kwargs: Any) -> None:
         command_logic_test_paths=[], is_hive=False
     )
     cache_cmd.execute(list(pytest_args))
+
+

packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/exceptions.py

@@ -1,11 +1,17 @@
 """Pytest plugin that defines options and fixtures for client exceptions."""
 
+from pathlib import Path
 from typing import Dict, List
 
 import pytest
 from hive.client import ClientType
 
-from execution_testing.exceptions import ExceptionMapper
+from execution_testing.exceptions import (
+    ExceptionMapper,
+    ExternalExceptionMapper,
+    extend_exception_mapper,
+    load_external_exception_mapper,
+)
 from execution_testing.fixtures import (
     BlockchainFixtureCommon,
 )
@@ -28,6 +34,77 @@ def pytest_addoption(parser: pytest.Parser) -> None:
             "NOT use strict exception matching."
         ),
     )
+    consume_group.addoption(
+        "--exception-mapper",
+        action="append",
+        dest="external_exception_mappers",
+        default=[],
+        metavar="CLIENT=PATH",
+        help=(
+            "Extend a client's built-in exception mapper with a YAML mapper "
+            "file. Can be repeated, e.g. "
+            "--exception-mapper geth=../go-ethereum/eest-exceptions.yaml."
+        ),
+    )
+
+
+def _parse_external_exception_mapper_options(
+    options: List[str],
+) -> Dict[str, ExternalExceptionMapper]:
+    """Load external exception mapper files from CLIENT=PATH options."""
+    external_mappers: Dict[str, ExternalExceptionMapper] = {}
+    for option in options:
+        if "=" not in option:
+            raise ValueError(
+                "--exception-mapper must use CLIENT=PATH syntax"
+            )
+        client, path_string = option.split("=", 1)
+        client = client.strip().lower()
+        path_string = path_string.strip()
+        if not client or not path_string:
+            raise ValueError(
+                "--exception-mapper must use non-empty CLIENT=PATH values"
+            )
+        external_mappers[client] = load_external_exception_mapper(
+            Path(path_string).expanduser()
+        )
+    return external_mappers
+
+
+def _client_key_matches(
+    configured_key: str,
+    hive_client_name: str,
+    built_in_key: str,
+) -> bool:
+    """Return whether an external mapper key applies to a Hive client."""
+    configured_key = configured_key.lower()
+    hive_client_name = hive_client_name.lower()
+    built_in_key = built_in_key.lower()
+    return (
+        configured_key in hive_client_name
+        or configured_key == built_in_key
+        or (configured_key == "geth" and built_in_key == "go-ethereum")
+    )
+
+
+def get_configured_exception_mapper(
+    client_name: str,
+    external_exception_mappers: Dict[str, ExternalExceptionMapper],
+) -> ExceptionMapper | None:
+    """Return the built-in mapper extended with a matching external mapper."""
+    normalized_client_name = client_name.lower()
+    for client_key, built_in_mapper in EXCEPTION_MAPPERS.items():
+        if client_key not in normalized_client_name:
+            continue
+        external_mapper = None
+        for configured_key, mapper in external_exception_mappers.items():
+            if _client_key_matches(
+                configured_key, client_name, client_key
+            ):
+                external_mapper = mapper
+                break
+        return extend_exception_mapper(built_in_mapper, external_mapper)
+    return None
 
 
 @pytest.fixture(scope="session")
@@ -36,21 +113,30 @@ def client_exception_mapper_cache() -> Dict[str, ExceptionMapper | None]:
     return {}
 
 
+@pytest.fixture(scope="session")
+def external_exception_mappers(
+    request: pytest.FixtureRequest,
+) -> Dict[str, ExternalExceptionMapper]:
+    """Load external exception mapper files requested on the command line."""
+    return _parse_external_exception_mapper_options(
+        request.config.getoption("external_exception_mappers")
+    )
+
+
 @pytest.fixture(scope="function")
 def client_exception_mapper(
     client_type: ClientType,
     client_exception_mapper_cache: Dict[str, ExceptionMapper | None],
+    external_exception_mappers: Dict[str, ExternalExceptionMapper],
 ) -> ExceptionMapper | None:
     """Return the exception mapper for the client type, with caching."""
     if client_type.name not in client_exception_mapper_cache:
-        for client in EXCEPTION_MAPPERS:
-            if client in client_type.name:
-                client_exception_mapper_cache[client_type.name] = (
-                    EXCEPTION_MAPPERS[client]
-                )
-                break
-        else:
-            client_exception_mapper_cache[client_type.name] = None
+        client_exception_mapper_cache[client_type.name] = (
+            get_configured_exception_mapper(
+                client_type.name,
+                external_exception_mappers,
+            )
+        )
 
     return client_exception_mapper_cache[client_type.name]
 

packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py

@@ -15,7 +15,10 @@
 from hive.testing import HiveTest
 
 from execution_testing.base_types import to_json
-from execution_testing.exceptions import ExceptionMapper
+from execution_testing.exceptions import (
+    ExceptionMapper,
+    ExternalExceptionMapper,
+)
 from execution_testing.fixtures import BlockchainEngineSyncFixture
 from execution_testing.rpc import AdminRPC, EngineRPC, EthRPC, NetRPC
 
@@ -251,19 +254,18 @@ def sync_client(
 def sync_client_exception_mapper(
     sync_client_type: ClientType,
     client_exception_mapper_cache: Dict[str, ExceptionMapper | None],
+    external_exception_mappers: Dict[str, ExternalExceptionMapper],
 ) -> ExceptionMapper | None:
     """Return the exception mapper for the sync client type, with caching."""
     if sync_client_type.name not in client_exception_mapper_cache:
-        from ..exceptions import EXCEPTION_MAPPERS
+        from ..exceptions import get_configured_exception_mapper
 
-        for client in EXCEPTION_MAPPERS:
-            if client in sync_client_type.name:
-                client_exception_mapper_cache[sync_client_type.name] = (
-                    EXCEPTION_MAPPERS[client]
-                )
-                break
-        else:
-            client_exception_mapper_cache[sync_client_type.name] = None
+        client_exception_mapper_cache[sync_client_type.name] = (
+            get_configured_exception_mapper(
+                sync_client_type.name,
+                external_exception_mappers,
+            )
+        )
 
     return client_exception_mapper_cache[sync_client_type.name]
 

packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_exception_mappers.py

@@ -0,0 +1,84 @@
+"""Tests for consume exception mapper selection."""
+
+from pathlib import Path
+
+from execution_testing.cli.pytest_commands.plugins.consume.simulators import (
+    exceptions,
+)
+from execution_testing.exceptions import (
+    BlockException,
+    TransactionException,
+)
+
+
+def test_parse_exception_mapper_option(tmp_path: Path) -> None:
+    """Parse repeatable CLIENT=PATH mapper options."""
+    mapper_path = tmp_path / "mapper.yaml"
+    mapper_path.write_text(
+        """
+version: 1
+substring:
+  BlockException.INVALID_GASLIMIT: custom gas
+"""
+    )
+
+    mappers = exceptions._parse_external_exception_mapper_options(
+        [f"geth={mapper_path}"]
+    )
+
+    assert list(mappers) == ["geth"]
+    assert mappers["geth"].message_to_exception("custom gas") == [
+        BlockException.INVALID_GASLIMIT
+    ]
+
+
+def test_configured_exception_mapper_extends_matching_client(
+    tmp_path: Path,
+) -> None:
+    """Extend the selected built-in mapper for matching Hive client names."""
+    mapper_path = tmp_path / "mapper.yaml"
+    mapper_path.write_text(
+        """
+version: 1
+substring:
+  BlockException.INVALID_GASLIMIT: custom gas
+"""
+    )
+    external = exceptions._parse_external_exception_mapper_options(
+        [f"geth={mapper_path}"]
+    )
+
+    mapper = exceptions.get_configured_exception_mapper(
+        "go-ethereum", external
+    )
+
+    assert mapper is not None
+    assert mapper.message_to_exception("custom gas") == [
+        BlockException.INVALID_GASLIMIT
+    ]
+
+
+def test_unmatched_exception_mapper_option_is_ignored(
+    tmp_path: Path,
+) -> None:
+    """Do not apply external mappings to non-matching clients."""
+    mapper_path = tmp_path / "mapper.yaml"
+    mapper_path.write_text(
+        """
+version: 1
+substring:
+  TransactionException.INSUFFICIENT_ACCOUNT_FUNDS: custom funds
+"""
+    )
+    external = exceptions._parse_external_exception_mapper_options(
+        [f"besu={mapper_path}"]
+    )
+
+    mapper = exceptions.get_configured_exception_mapper(
+        "go-ethereum", external
+    )
+
+    assert mapper is not None
+    assert mapper.message_to_exception("custom funds") != [
+        TransactionException.INSUFFICIENT_ACCOUNT_FUNDS
+    ]