diff --git a/CHANGELOG.md b/CHANGELOG.md index a3babc82..5b0317f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,9 @@ All notable changes to SkillOpt are documented here. This project adheres to - Skip step-level edits whose target runs into a protected slow-update or appendix region, so a `replace` or `delete` can no longer remove the start marker and expose the protected content (thanks @Shizoqua). +- Ignore sentence-ending periods when normalizing OfficeQA answers, so a + correct answer such as `42.` matches the gold `42` instead of scoring zero + exact match and F1 (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/officeqa/evaluator.py b/skillopt/envs/officeqa/evaluator.py index 124d25d5..37b7aa21 100644 --- a/skillopt/envs/officeqa/evaluator.py +++ b/skillopt/envs/officeqa/evaluator.py @@ -12,6 +12,8 @@ def normalize_answer(text: str) -> str: text = text.lower().strip() text = text.replace(",", "") text = "".join(ch for ch in text if ch not in string.punctuation or ch in _NUMERIC_CHARS or ch == "%") + # "." is kept for decimals; a period not followed by a digit is sentence punctuation. + text = re.sub(r"\.(?!\d)", "", text) text = re.sub(r"\b(million|millions|billion|billions|dollars|dollar|nominal)\b", " ", text) text = " ".join(text.split()) return text diff --git a/tests/test_officeqa_evaluator.py b/tests/test_officeqa_evaluator.py new file mode 100644 index 00000000..a4bfa9e3 --- /dev/null +++ b/tests/test_officeqa_evaluator.py @@ -0,0 +1,35 @@ +import pytest + +from skillopt.envs.officeqa.evaluator import evaluate + + +@pytest.mark.parametrize( + ("prediction", "gold"), + [ + ("42.", "42"), + ("-3.5.", "-3.5"), + ("$1,234.5 million.", "1234.5"), + ("12.5%.", "12.5%"), + ("U.S.", "US"), + ], +) +def test_sentence_period_does_not_break_the_match(prediction, gold): + result = evaluate(prediction, gold) + + assert result["em"] == 1.0 + assert result["f1"] == 1.0 + + +@pytest.mark.parametrize( + ("prediction", "gold"), + [("1.5", "15"), ("0.5", "5"), ("-3", "3"), ("42", "43")], +) +def test_decimal_point_and_sign_still_matter(prediction, gold): + assert evaluate(prediction, gold)["em"] == 0.0 + + +def test_partial_answer_with_trailing_period_gets_token_credit(): + result = evaluate("revenue was 42.", "42") + + assert result["em"] == 0.0 + assert result["f1"] == pytest.approx(0.5)