From 49b8439e3d089db4bdffb096cc27241e027a2792 Mon Sep 17 00:00:00 2001
From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com>
Date: Sat, 7 Nov 2026 11:34:09 +0000
Subject: [PATCH] test(docvqa): add unit tests for the ANLS evaluator

skillopt/envs/docvqa/evaluator.py scores every DocVQA rollout (soft is
the ANLS value and hard is ANLS >= 0.999) but had no direct tests.

Cover the metric as defined by Biten et al. (2019): Levenshtein distance
on known pairs, 1 - NL below the 0.5 threshold and 0 at or above it
(including the exact NL = 0.5 boundary), the maximum over gold answers
in any order, case and whitespace normalization, empty answers, the gold
answer shapes the loader accepts (lists, JSON and Python literal strings,
dicts, lists of dicts), and <answer> tag extraction.

No runtime code changes.
---
 CHANGELOG.md                   |   3 +
 tests/test_docvqa_evaluator.py | 106 +++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 tests/test_docvqa_evaluator.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3babc82..021b3217 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -161,6 +161,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
   scores and gate actions (thanks @Tanmay9223, #96).
 - Add focused coverage for the validation-gate decision core and edit-budget
   schedulers (thanks @nankingjing, #122, #123).
+- Add focused coverage for the DocVQA ANLS evaluator, including the 0.5
+  threshold boundary, the maximum over gold answers, and the accepted gold
+  answer formats (thanks @Shizoqua).
 
 ### Acknowledgements 🙏
 Thank you to the contributors behind this unreleased work:
diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py
new file mode 100644
index 00000000..a68519f0
--- /dev/null
+++ b/tests/test_docvqa_evaluator.py
@@ -0,0 +1,106 @@
+"""Unit tests for the DocVQA ANLS evaluator.
+
+ANLS (Biten et al., 2019): for each gold answer, NL is the Levenshtein distance
+divided by the longer string's length; the score is 1 - NL when NL < 0.5 and 0
+otherwise, and the item score is the maximum over gold answers.
+"""
+
+import pytest
+
+from skillopt.envs.docvqa.evaluator import (
+    DEFAULT_ANLS_THRESHOLD,
+    _extract_answer_strings,
+    _levenshtein_distance,
+    _score_single_answer,
+    evaluate,
+    extract_answer,
+)
+
+
+@pytest.mark.parametrize(
+    ("a", "b", "distance"),
+    [("kitten", "sitting", 3), ("flaw", "lawn", 2), ("abc", "abc", 0), ("", "abc", 3), ("abc", "", 3)],
+)
+def test_levenshtein_distance_known_values(a, b, distance):
+    assert _levenshtein_distance(a, b) == distance
+    assert _levenshtein_distance(b, a) == distance
+
+
+def test_threshold_is_the_standard_anls_value():
+    assert DEFAULT_ANLS_THRESHOLD == 0.5
+
+
+@pytest.mark.parametrize(
+    ("predicted", "target", "expected"),
+    [
+        ("kitten", "sitting", 1 - 3 / 7),
+        ("abcd", "abce", 0.75),
+        ("abcd", "ab", 0.0),  # NL is exactly 0.5, which is not below the threshold
+        ("abcde", "abcd", 0.8),
+        ("ab", "cd", 0.0),
+    ],
+)
+def test_score_single_answer_follows_anls(predicted, target, expected):
+    assert _score_single_answer(predicted, target, DEFAULT_ANLS_THRESHOLD) == pytest.approx(expected)
+
+
+def test_score_single_answer_empty_strings():
+    assert _score_single_answer("", "", DEFAULT_ANLS_THRESHOLD) == 1.0
+    assert _score_single_answer(None, "", DEFAULT_ANLS_THRESHOLD) == 1.0
+    assert _score_single_answer("x", "", DEFAULT_ANLS_THRESHOLD) == 0.0
+    assert _score_single_answer("", "x", DEFAULT_ANLS_THRESHOLD) == 0.0
+
+
+def test_case_and_whitespace_are_normalized():
+    assert evaluate("<answer>  Total   REVENUE </answer>", ["total revenue"])["anls"] == 1.0
+
+
+@pytest.mark.parametrize("golds", [["2000", "1999", "abcd"], ["abcd", "1999", "2000"]])
+def test_item_score_is_the_max_over_gold_answers(golds):
+    result = evaluate("<answer>1999</answer>", golds)
+
+    assert result["anls"] == 1.0
+    assert result["predicted_answer"] == "1999"
+    assert result["gold_answers"] == golds
+
+
+def test_near_miss_gets_partial_credit_but_not_exact():
+    # The rollout sets hard = anls >= 0.999 and soft = anls.
+    anls = evaluate("<answer>abce</answer>", ["abcd"])["anls"]
+
+    assert anls == pytest.approx(0.75)
+    assert anls < 0.999
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        (["a", "b"], ["a", "b"]),
+        ('["a", "b"]', ["a", "b"]),
+        ("['a', 'b']", ["a", "b"]),
+        ({"answers": ["x"]}, ["x"]),
+        ({"ground_truth": "y"}, ["y"]),
+        ([{"text": "t"}, {"answer": "u"}, {"value": "v"}], ["t", "u", "v"]),
+        ("plain answer", ["plain answer"]),
+        ("[not json", ["[not json"]),
+        (5, ["5"]),
+        (None, [""]),
+        ("", [""]),
+        ([], [""]),
+    ],
+)
+def test_extract_answer_strings_accepts_dataset_shapes(raw, expected):
+    assert _extract_answer_strings(raw) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("reasoning\n<ANSWER>Foo</ANSWER>", "Foo"),
+        ("<answer>x</answer> then <answer>y</answer>", "y"),
+        ("first line\n\nlast line  ", "last line"),
+        ("", ""),
+    ],
+)
+def test_extract_answer(text, expected):
+    assert extract_answer(text) == expected