diff --git a/CHANGELOG.md b/CHANGELOG.md index a3babc82..0349a2e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,8 @@ All notable changes to SkillOpt are documented here. This project adheres to - Skip step-level edits whose target runs into a protected slow-update or appendix region, so a `replace` or `delete` can no longer remove the start marker and expose the protected content (thanks @Shizoqua). +- Score LiveMathematicianBench answers written as `(B)`, `[B]`, or `**B**` as + the choice they name instead of marking them wrong (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/livemathematicianbench/evaluator.py b/skillopt/envs/livemathematicianbench/evaluator.py index d15db3e3..feb615fd 100644 --- a/skillopt/envs/livemathematicianbench/evaluator.py +++ b/skillopt/envs/livemathematicianbench/evaluator.py @@ -15,7 +15,8 @@ def extract_answer(text: str) -> str: def normalize_label(text: str) -> str: - return str(text).strip().upper().rstrip(".):") + """Normalize a choice label, accepting ``B``, ``B.``, ``(B)``, ``[B]`` and ``**B**``.""" + return str(text).strip().strip("*").strip().lstrip("([").upper().rstrip(".):]") def parse_choice_label(prediction_text: str, choices: list[dict]) -> str: diff --git a/tests/test_livemath_evaluator.py b/tests/test_livemath_evaluator.py new file mode 100644 index 00000000..5efa4762 --- /dev/null +++ b/tests/test_livemath_evaluator.py @@ -0,0 +1,33 @@ +import pytest + +from skillopt.envs.livemathematicianbench.evaluator import evaluate + +CHOICES = [ + {"label": "A", "text": "2"}, + {"label": "B", "text": "3"}, + {"label": "C", "text": "5"}, +] + + +@pytest.mark.parametrize( + "response", + [ + "B", + "B.", + "B)", + "(B)", + "[B]", + "**B**", + "(B) 3", + ], +) +def test_wrapped_choice_label_is_scored_correct(response): + result = evaluate(response, CHOICES[1], CHOICES) + + assert result["predicted_label"] == "B" + assert result["em"] == 1.0 + + +@pytest.mark.parametrize("response", ["(A)", "Option B", ""]) +def test_wrong_or_unparseable_label_is_not_scored_correct(response): + assert evaluate(response, CHOICES[1], CHOICES)["em"] == 0.0