diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3babc82..021b3217 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -161,6 +161,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
scores and gate actions (thanks @Tanmay9223, #96).
- Add focused coverage for the validation-gate decision core and edit-budget
schedulers (thanks @nankingjing, #122, #123).
+- Add focused coverage for the DocVQA ANLS evaluator, including the 0.5
+ threshold boundary, the maximum over gold answers, and the accepted gold
+ answer formats (thanks @Shizoqua).
### Acknowledgements 🙏
Thank you to the contributors behind this unreleased work:
diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py
new file mode 100644
index 00000000..a68519f0
--- /dev/null
+++ b/tests/test_docvqa_evaluator.py
@@ -0,0 +1,106 @@
+"""Unit tests for the DocVQA ANLS evaluator.
+
+ANLS (Biten et al., 2019): for each gold answer, NL is the Levenshtein distance
+divided by the longer string's length; the score is 1 - NL when NL < 0.5 and 0
+otherwise, and the item score is the maximum over gold answers.
+"""
+
+import pytest
+
+from skillopt.envs.docvqa.evaluator import (
+ DEFAULT_ANLS_THRESHOLD,
+ _extract_answer_strings,
+ _levenshtein_distance,
+ _score_single_answer,
+ evaluate,
+ extract_answer,
+)
+
+
+@pytest.mark.parametrize(
+ ("a", "b", "distance"),
+ [("kitten", "sitting", 3), ("flaw", "lawn", 2), ("abc", "abc", 0), ("", "abc", 3), ("abc", "", 3)],
+)
+def test_levenshtein_distance_known_values(a, b, distance):
+ assert _levenshtein_distance(a, b) == distance
+ assert _levenshtein_distance(b, a) == distance
+
+
+def test_threshold_is_the_standard_anls_value():
+ assert DEFAULT_ANLS_THRESHOLD == 0.5
+
+
+@pytest.mark.parametrize(
+ ("predicted", "target", "expected"),
+ [
+ ("kitten", "sitting", 1 - 3 / 7),
+ ("abcd", "abce", 0.75),
+ ("abcd", "ab", 0.0), # NL is exactly 0.5, which is not below the threshold
+ ("abcde", "abcd", 0.8),
+ ("ab", "cd", 0.0),
+ ],
+)
+def test_score_single_answer_follows_anls(predicted, target, expected):
+ assert _score_single_answer(predicted, target, DEFAULT_ANLS_THRESHOLD) == pytest.approx(expected)
+
+
+def test_score_single_answer_empty_strings():
+ assert _score_single_answer("", "", DEFAULT_ANLS_THRESHOLD) == 1.0
+ assert _score_single_answer(None, "", DEFAULT_ANLS_THRESHOLD) == 1.0
+ assert _score_single_answer("x", "", DEFAULT_ANLS_THRESHOLD) == 0.0
+ assert _score_single_answer("", "x", DEFAULT_ANLS_THRESHOLD) == 0.0
+
+
+def test_case_and_whitespace_are_normalized():
+ assert evaluate(" Total REVENUE ", ["total revenue"])["anls"] == 1.0
+
+
+@pytest.mark.parametrize("golds", [["2000", "1999", "abcd"], ["abcd", "1999", "2000"]])
+def test_item_score_is_the_max_over_gold_answers(golds):
+ result = evaluate("1999", golds)
+
+ assert result["anls"] == 1.0
+ assert result["predicted_answer"] == "1999"
+ assert result["gold_answers"] == golds
+
+
+def test_near_miss_gets_partial_credit_but_not_exact():
+ # The rollout sets hard = anls >= 0.999 and soft = anls.
+ anls = evaluate("abce", ["abcd"])["anls"]
+
+ assert anls == pytest.approx(0.75)
+ assert anls < 0.999
+
+
+@pytest.mark.parametrize(
+ ("raw", "expected"),
+ [
+ (["a", "b"], ["a", "b"]),
+ ('["a", "b"]', ["a", "b"]),
+ ("['a', 'b']", ["a", "b"]),
+ ({"answers": ["x"]}, ["x"]),
+ ({"ground_truth": "y"}, ["y"]),
+ ([{"text": "t"}, {"answer": "u"}, {"value": "v"}], ["t", "u", "v"]),
+ ("plain answer", ["plain answer"]),
+ ("[not json", ["[not json"]),
+ (5, ["5"]),
+ (None, [""]),
+ ("", [""]),
+ ([], [""]),
+ ],
+)
+def test_extract_answer_strings_accepts_dataset_shapes(raw, expected):
+ assert _extract_answer_strings(raw) == expected
+
+
+@pytest.mark.parametrize(
+ ("text", "expected"),
+ [
+ ("reasoning\nFoo", "Foo"),
+ ("x then y", "y"),
+ ("first line\n\nlast line ", "last line"),
+ ("", ""),
+ ],
+)
+def test_extract_answer(text, expected):
+ assert extract_answer(text) == expected