From fe335d05745b00883ecd0a1d01f388af583780bd Mon Sep 17 00:00:00 2001
From: Alex Ward <alxwrd@googlemail.com>
Date: Mon, 16 Mar 2026 22:48:24 +0000
Subject: [PATCH] allow creating dataset from file

Co-Authored-By: Claude <claude[bot]@users.noreply.github.com>
---
 src/pyeval/_core.py                   | 28 +++++++++++++++++++++------
 tests/evals/eval_from_file.py         | 20 +++++++++++++++++++
 tests/evals/eval_from_file_cases.yaml |  7 +++++++
 3 files changed, 49 insertions(+), 6 deletions(-)
 create mode 100644 tests/evals/eval_from_file.py
 create mode 100644 tests/evals/eval_from_file_cases.yaml

diff --git a/src/pyeval/_core.py b/src/pyeval/_core.py
index 06cb5e6..d50ffbf 100644
--- a/src/pyeval/_core.py
+++ b/src/pyeval/_core.py
@@ -5,10 +5,11 @@
 from collections.abc import Callable, Mapping
 from contextvars import ContextVar
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Any, TypeVar
 
 from pydantic import TypeAdapter
-from pydantic_evals import Case
+from pydantic_evals import Case, Dataset
 from pydantic_evals.evaluators import (
     EvaluationReason,
     EvaluationResult,
@@ -138,16 +139,20 @@ def execute(task: Callable[..., Any], case: Case) -> ExecutionResult:
 Func = TypeVar("Func", bound=Callable[..., Any])
 
 
-def dataset(*cases: Case) -> Callable[[Func], Func]:
+def dataset(*args: Case | str | Path) -> Callable[[Func], Func]:
     """Register evaluation cases for an eval function.
 
-    Attaches the provided :class:`~pydantic_evals.Case` instances to the decorated
-    function so that pytest-pyeval can discover and run each case as a separate test item.
+    Accepts either a file path (str or :class:`~pathlib.Path`) to load cases from,
+    or one or more :class:`~pydantic_evals.Case` instances directly.
+
+    When given a file path, the dataset is loaded via
+    :meth:`~pydantic_evals.Dataset.from_file`, which supports YAML and JSON formats.
 
     Args:
-        *cases: One or more :class:`~pydantic_evals.Case` instances to run against the function.
+        *args: Either a single file path (str or Path) or one or more
+            :class:`~pydantic_evals.Case` instances.
 
-    Example::
+    Example — inline cases::
 
         @dataset(
             Case(name="basic", inputs="hello", expected_output="HELLO"),
@@ -156,7 +161,18 @@ def dataset(*cases: Case) -> Callable[[Func], Func]:
         def eval_uppercase(case: Case) -> None:
             result = execute(str.upper, case)
             result.evaluate(EqualsExpected())
+
+    Example — from file::
+
+        @dataset("cases.yaml")
+        def eval_uppercase(case: Case) -> None:
+            result = execute(str.upper, case)
+            result.evaluate(EqualsExpected())
     """
+    if len(args) == 1 and isinstance(args[0], (str, Path)):
+        cases: tuple[Case, ...] = tuple(Dataset.from_file(args[0]).cases)
+    else:
+        cases = args  # type: ignore[assignment]
 
     def decorator(fn: Func) -> Func:
         fn.__eval_cases__ = cases
diff --git a/tests/evals/eval_from_file.py b/tests/evals/eval_from_file.py
new file mode 100644
index 0000000..a9dee26
--- /dev/null
+++ b/tests/evals/eval_from_file.py
@@ -0,0 +1,20 @@
+"""Eval functions that load their cases from a file."""
+
+from pathlib import Path
+
+from pyeval import Case, dataset, execute
+from pyeval.evaluators import EqualsExpected
+
+CASES_FILE = Path(__file__).parent / "eval_from_file_cases.yaml"
+
+
+@dataset(CASES_FILE)
+def eval_uppercase_from_file(case: Case) -> None:
+    result = execute(str.upper, case)
+    result.evaluate(EqualsExpected())
+
+
+@dataset(str(CASES_FILE))
+def eval_uppercase_from_file_str_path(case: Case) -> None:
+    result = execute(str.upper, case)
+    result.evaluate(EqualsExpected())
diff --git a/tests/evals/eval_from_file_cases.yaml b/tests/evals/eval_from_file_cases.yaml
new file mode 100644
index 0000000..0876839
--- /dev/null
+++ b/tests/evals/eval_from_file_cases.yaml
@@ -0,0 +1,7 @@
+cases:
+  - name: uppercase_basic
+    inputs: hello world
+    expected_output: HELLO WORLD
+  - name: uppercase_with_numbers
+    inputs: hello 123
+    expected_output: HELLO 123