Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
- Skip step-level edits whose target runs into a protected slow-update or
appendix region, so a `replace` or `delete` can no longer remove the start
marker and expose the protected content (thanks @Shizoqua).
- Ignore sentence-ending periods when normalizing OfficeQA answers, so a
correct answer such as `42.` matches the gold `42` instead of scoring zero
exact match and F1 (thanks @Shizoqua).

### Security
- Prevent managed-identity credentials from being sent to non-Azure or
Expand Down
2 changes: 2 additions & 0 deletions skillopt/envs/officeqa/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ def normalize_answer(text: str) -> str:
text = text.lower().strip()
text = text.replace(",", "")
text = "".join(ch for ch in text if ch not in string.punctuation or ch in _NUMERIC_CHARS or ch == "%")
# "." is kept for decimals; a period not followed by a digit is sentence punctuation.
text = re.sub(r"\.(?!\d)", "", text)
text = re.sub(r"\b(million|millions|billion|billions|dollars|dollar|nominal)\b", " ", text)
text = " ".join(text.split())
return text
Expand Down
35 changes: 35 additions & 0 deletions tests/test_officeqa_evaluator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import pytest

from skillopt.envs.officeqa.evaluator import evaluate


@pytest.mark.parametrize(
("prediction", "gold"),
[
("42.", "42"),
("-3.5.", "-3.5"),
("$1,234.5 million.", "1234.5"),
("12.5%.", "12.5%"),
("U.S.", "US"),
],
)
def test_sentence_period_does_not_break_the_match(prediction, gold):
result = evaluate(prediction, gold)

assert result["em"] == 1.0
assert result["f1"] == 1.0


@pytest.mark.parametrize(
("prediction", "gold"),
[("1.5", "15"), ("0.5", "5"), ("-3", "3"), ("42", "43")],
)
def test_decimal_point_and_sign_still_matter(prediction, gold):
assert evaluate(prediction, gold)["em"] == 0.0


def test_partial_answer_with_trailing_period_gets_token_credit():
result = evaluate("revenue was 42.", "42")

assert result["em"] == 0.0
assert result["f1"] == pytest.approx(0.5)