From ebc20b792e0e54f7bd797e7000f3f3e5c62e1296 Mon Sep 17 00:00:00 2001
From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com>
Date: Sat, 7 Nov 2026 11:12:49 +0000
Subject: [PATCH] fix(livemath): score (B), [B] and **B** as the choice they
 name

normalize_label stripped trailing ".", ")" and ":" but nothing at the
front, so a model answering <answer>(B)</answer> was parsed as "(B" and
scored wrong even when B was correct. [B] and **B** failed the same way,
while B, B. and B) passed. hard is the exact-match result, so these
answers got zero reward during training and counted as misses in the
reported accuracy.

Strip surrounding Markdown bold and a leading "(" or "[" along with the
existing trailing punctuation. Wrong labels and free text such as
"Option B" are still not matched.
---
 CHANGELOG.md                                  |  2 ++
 .../envs/livemathematicianbench/evaluator.py  |  3 +-
 tests/test_livemath_evaluator.py              | 33 +++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_livemath_evaluator.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3babc82d..0349a2e63 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -147,6 +147,8 @@ All notable changes to SkillOpt are documented here. This project adheres to
 - Skip step-level edits whose target runs into a protected slow-update or
   appendix region, so a `replace` or `delete` can no longer remove the start
   marker and expose the protected content (thanks @Shizoqua).
+- Score LiveMathematicianBench answers written as `(B)`, `[B]`, or `**B**` as
+  the choice they name instead of marking them wrong (thanks @Shizoqua).
 
 ### Security
 - Prevent managed-identity credentials from being sent to non-Azure or
diff --git a/skillopt/envs/livemathematicianbench/evaluator.py b/skillopt/envs/livemathematicianbench/evaluator.py
index d15db3e3b..feb615fd5 100644
--- a/skillopt/envs/livemathematicianbench/evaluator.py
+++ b/skillopt/envs/livemathematicianbench/evaluator.py
@@ -15,7 +15,8 @@ def extract_answer(text: str) -> str:
 
 
 def normalize_label(text: str) -> str:
-    return str(text).strip().upper().rstrip(".):")
+    """Normalize a choice label, accepting ``B``, ``B.``, ``(B)``, ``[B]`` and ``**B**``."""
+    return str(text).strip().strip("*").strip().lstrip("([").upper().rstrip(".):]")
 
 
 def parse_choice_label(prediction_text: str, choices: list[dict]) -> str:
diff --git a/tests/test_livemath_evaluator.py b/tests/test_livemath_evaluator.py
new file mode 100644
index 000000000..5efa4762e
--- /dev/null
+++ b/tests/test_livemath_evaluator.py
@@ -0,0 +1,33 @@
+import pytest
+
+from skillopt.envs.livemathematicianbench.evaluator import evaluate
+
+CHOICES = [
+    {"label": "A", "text": "2"},
+    {"label": "B", "text": "3"},
+    {"label": "C", "text": "5"},
+]
+
+
+@pytest.mark.parametrize(
+    "response",
+    [
+        "<answer>B</answer>",
+        "<answer>B.</answer>",
+        "<answer>B)</answer>",
+        "<answer>(B)</answer>",
+        "<answer>[B]</answer>",
+        "<answer>**B**</answer>",
+        "<answer>(B) 3</answer>",
+    ],
+)
+def test_wrapped_choice_label_is_scored_correct(response):
+    result = evaluate(response, CHOICES[1], CHOICES)
+
+    assert result["predicted_label"] == "B"
+    assert result["em"] == 1.0
+
+
+@pytest.mark.parametrize("response", ["<answer>(A)</answer>", "<answer>Option B</answer>", "<answer></answer>"])
+def test_wrong_or_unparseable_label_is_not_scored_correct(response):
+    assert evaluate(response, CHOICES[1], CHOICES)["em"] == 0.0