From dfe627b07cbda84d4d95862ee06a21c8d8824d4d Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Wed, 24 Jun 2026 12:29:57 +0100 Subject: [PATCH 1/5] Fix reference authors missing from JATS-guided citation training data part of https://github.com/eLifePathways/ScienceBeam2.0/issues/105 The XPath for the REFERENCE_AUTHOR sub-field targeted elements outside , which does not exist in standard JATS. Authors live inside , and the not(ancestor::person-group) guard would have excluded them regardless. No reference author tokens were being annotated in generated citation training data. Fixed the XPath and added a unit test. --- .../training/jats/field_extractor.py | 2 +- tests/training/jats/test_field_extractor.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sciencebeam_parser/training/jats/field_extractor.py b/sciencebeam_parser/training/jats/field_extractor.py index bb0eef84..f63fad96 100644 --- a/sciencebeam_parser/training/jats/field_extractor.py +++ b/sciencebeam_parser/training/jats/field_extractor.py @@ -40,7 +40,7 @@ def _iter_sub_field_values( # Sub-field XPaths for references (relative to each element) _REFERENCE_SUB_FIELDS = [ (JatsSubFieldNames.REFERENCE_LABEL, './label'), - (JatsSubFieldNames.REFERENCE_AUTHOR, './/string-name[not(ancestor::person-group)]'), + (JatsSubFieldNames.REFERENCE_AUTHOR, './/person-group[@person-group-type="author"]'), (JatsSubFieldNames.REFERENCE_ARTICLE_TITLE, './/article-title'), (JatsSubFieldNames.REFERENCE_SOURCE, './/source'), (JatsSubFieldNames.REFERENCE_YEAR, './/year'), diff --git a/tests/training/jats/test_field_extractor.py b/tests/training/jats/test_field_extractor.py index df48821d..d102eb27 100644 --- a/tests/training/jats/test_field_extractor.py +++ b/tests/training/jats/test_field_extractor.py @@ -244,6 +244,22 @@ def test_extracts_reference_text(self): assert len(refs) >= 1 assert 'A Study' in refs[0].text + def test_extracts_reference_author_subfield(self): + fvs = _field_values_for( + '
' + '' + '' + 'SmithA' + 'JonesB C' + '' + '' + '
' + ) + sub = [v for v in fvs if v.sub_field_name == JatsSubFieldNames.REFERENCE_AUTHOR] + assert len(sub) == 1 + assert 'Smith' in sub[0].text + assert 'Jones' in sub[0].text + def test_extracts_reference_article_title_subfield(self): fvs = _field_values_for( '
' From 5da61bc8bab565ad1c53ad5356e224be582645ad Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Wed, 24 Jun 2026 12:46:01 +0100 Subject: [PATCH 2/5] Include "et al." in reference author annotation when is present carries no text content, so it was silently dropped from the author needle. The generated training data now includes "et al." in the span, matching the grobid training data convention. --- .../training/jats/field_extractor.py | 24 ++++++++++++++++++- tests/training/jats/test_field_extractor.py | 15 ++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/sciencebeam_parser/training/jats/field_extractor.py b/sciencebeam_parser/training/jats/field_extractor.py index f63fad96..5f0aa58a 100644 --- a/sciencebeam_parser/training/jats/field_extractor.py +++ b/sciencebeam_parser/training/jats/field_extractor.py @@ -40,7 +40,6 @@ def _iter_sub_field_values( # Sub-field XPaths for references (relative to each element) _REFERENCE_SUB_FIELDS = [ (JatsSubFieldNames.REFERENCE_LABEL, './label'), - (JatsSubFieldNames.REFERENCE_AUTHOR, './/person-group[@person-group-type="author"]'), (JatsSubFieldNames.REFERENCE_ARTICLE_TITLE, './/article-title'), (JatsSubFieldNames.REFERENCE_SOURCE, './/source'), (JatsSubFieldNames.REFERENCE_YEAR, './/year'), @@ -72,6 +71,28 @@ def _iter_sub_field_values( ] +def _person_group_text(pg_el: etree._Element) -> str: + """Collect author name text and append 'et al.' when is present.""" + text = ' '.join(' '.join(pg_el.itertext()).split()) + if pg_el.xpath('etal'): + text = (text + ' et al.').strip() + return text + + +def _iter_reference_author_values( + ref_el: etree._Element, + field_name: str, +) -> Iterator[JatsFieldValue]: + for pg_el in ref_el.xpath('.//person-group[@person-group-type="author"]'): + text = _person_group_text(pg_el) + if text: + yield JatsFieldValue( + text=text, + field_name=field_name, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + ) + + def _local_tag(el: etree._Element) -> str: tag = el.tag if isinstance(tag, str) and tag.startswith('{'): @@ -376,6 +397,7 @@ def _iter_back_reference_values(self, root: etree._Element) -> Iterator[JatsFiel text = _element_text(ref_el) if text: yield JatsFieldValue(text=text, field_name=JatsFieldNames.REFERENCE) + yield from _iter_reference_author_values(ref_el, JatsFieldNames.REFERENCE) yield from _iter_sub_field_values( ref_el, JatsFieldNames.REFERENCE, _REFERENCE_SUB_FIELDS ) diff --git a/tests/training/jats/test_field_extractor.py b/tests/training/jats/test_field_extractor.py index d102eb27..8944cc76 100644 --- a/tests/training/jats/test_field_extractor.py +++ b/tests/training/jats/test_field_extractor.py @@ -260,6 +260,21 @@ def test_extracts_reference_author_subfield(self): assert 'Smith' in sub[0].text assert 'Jones' in sub[0].text + def test_reference_author_includes_et_al(self): + fvs = _field_values_for( + '
' + '' + '' + 'SmithA' + '' + '' + '' + '
' + ) + sub = [v for v in fvs if v.sub_field_name == JatsSubFieldNames.REFERENCE_AUTHOR] + assert len(sub) == 1 + assert sub[0].text == 'Smith A et al.' + def test_extracts_reference_article_title_subfield(self): fvs = _field_values_for( '
' From 6acc5b9105ad6dd6d348cd33bfdb656c9894163c Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Thu, 25 Jun 2026 10:06:38 +0100 Subject: [PATCH 3/5] Improve reference sub-field alignment in JATS-guided citation training data - Sub-fields sorted longest-first so shorter needles (e.g. label "1") cannot displace longer ones (e.g. year "1987") at the same position - Full masking applied to all sub-field types, not only REFERENCE_AUTHOR - Within-gap mid-token guard advances prev_included_end so the gap chain is not broken by overlapping SW noise blocks - Tail extension off-by-one fixed (inclusive gap=3 boundary) - Surname-only fallback extended to also find given-name initials within gap=3 of the matched surname, in both mid-token and regular fallback paths - Gap fill tolerates unlabeled tokens (entry=None) so "et al." and interstitial periods no longer reset the fill state - Per-name author sub-fields: only REFERENCE_AUTHOR ranges are masked across names; other sub-fields are single-occurrence and do not need it --- sciencebeam_parser/training/jats/aligner.py | 382 ++++++++++++- .../training/jats/field_extractor.py | 34 +- tests/training/jats/test_aligner.py | 519 +++++++++++++++++- tests/training/jats/test_field_extractor.py | 16 +- 4 files changed, 922 insertions(+), 29 deletions(-) diff --git a/sciencebeam_parser/training/jats/aligner.py b/sciencebeam_parser/training/jats/aligner.py index 85bb5bb7..3393c0e9 100644 --- a/sciencebeam_parser/training/jats/aligner.py +++ b/sciencebeam_parser/training/jats/aligner.py @@ -7,7 +7,7 @@ from sciencebeam_parser.document.layout_document import LayoutDocument, LayoutToken from sciencebeam_parser.training.jats.annotated_document import JatsAnnotatedLayoutDocument from sciencebeam_parser.training.jats.field_extractor import JatsFieldValue -from sciencebeam_parser.training.jats.field_vocab import JatsFieldNames +from sciencebeam_parser.training.jats.field_vocab import JatsFieldNames, JatsSubFieldNames from sciencebeam_parser.training.jats.text_normalizer import normalize_for_alignment @@ -108,6 +108,9 @@ # fields are never silently dropped. _MIN_ANCHOR_BLOCK_SIZE = 5 _MAX_HAYSTACK_GAP_TO_FILL = 3 +# Maximum number of non-author tokens permitted between two author spans before +# the gap fill gives up. Covers up to ~5 initials with periods and separators. +_MAX_AUTHOR_GAP_TOKENS = 10 # Type alias for the return value of _fuzzy_match_field_value: # (abs_start, abs_end, [(block_start, block_end), ...]) @@ -230,6 +233,51 @@ def _match_quality( return matched / needle_len +def _extend_match_for_needle_tail( + window: str, + needle: str, + window_start: int, + abs_a_end: int, + matched_blocks: List[Tuple[int, int, int]], + abs_block_ranges: List[Tuple[int, int]], +) -> Tuple[int, List[Tuple[int, int]]]: + """Greedily extend the SW match to cover any unmatched needle suffix. + + Smith-Waterman terminates when extending the match would not increase the + score. A single-character suffix separated from the last block by one or + two gap characters produces a net-zero extension (e.g. two gap penalties + cancel one match bonus), so SW stops early. This function scans forward + within _MAX_HAYSTACK_GAP_TO_FILL characters of the current match end for + the first unmatched needle character and appends consecutive matches as an + extra block, mirroring the pre-anchor fill logic on the other end. + + Example: needle "brockmann d", match ends at "brockmann "; " , d" in the + haystack has "d" at gap 2, which is within the fill threshold. + """ + last_needle_end = max(bi + size for _, bi, size in matched_blocks) + needle_tail = needle[last_needle_end:] + if not needle_tail: + return abs_a_end, abs_block_ranges + + win_pos = abs_a_end - window_start + first_pos = window[win_pos: win_pos + _MAX_HAYSTACK_GAP_TO_FILL + 1].find(needle_tail[0]) + if first_pos == -1: + return abs_a_end, abs_block_ranges + + tail_start = win_pos + first_pos + match_count = 0 + for tc in needle_tail: + if tail_start + match_count >= len(window) or window[tail_start + match_count] != tc: + break + match_count += 1 + + if not match_count: + return abs_a_end, abs_block_ranges + + ext_start = window_start + tail_start + return ext_start + match_count, abs_block_ranges + [(ext_start, ext_start + match_count)] + + def _fuzzy_search_in_window( haystack: str, needle: str, @@ -241,6 +289,9 @@ def _fuzzy_search_in_window( Returns (abs_start, abs_end, [(block_start, block_end), ...]) if quality >= threshold, else None. Block ranges are in absolute haystack coordinates. + After the SW match, a greedy tail extension fills any unmatched needle + suffix whose first character falls within _MAX_HAYSTACK_GAP_TO_FILL chars + of the current match end (mirrors the pre-anchor fill on the trailing side). """ window = haystack[window_start:window_end] sm = LocalSequenceMatcher(a=window, b=needle, scoring=_SCORING) @@ -258,15 +309,45 @@ def _fuzzy_search_in_window( (ai + window_start, ai + size + window_start) for ai, _bi, size in matched_blocks ] + a_end, abs_block_ranges = _extend_match_for_needle_tail( + window, needle, window_start, a_end, matched_blocks, abs_block_ranges + ) return a_start, a_end, abs_block_ranges -def _fuzzy_match_field_value( +def _get_unmasked_segments( + search_start: int, + search_end: int, + masked_ranges: List[Tuple[int, int]], +) -> List[Tuple[int, int]]: + """Split [search_start, search_end) into segments that don't overlap masked_ranges. + + Masked boundaries are hard: SW runs on each segment independently and + cannot produce a match that spans across a masked span. + """ + clipped = sorted( + (max(m0, search_start), min(m1, search_end)) + for m0, m1 in masked_ranges + if m0 < search_end and m1 > search_start + ) + segments: List[Tuple[int, int]] = [] + cur = search_start + for m_start, m_end in clipped: + if cur < m_start: + segments.append((cur, m_start)) + cur = max(cur, m_end) + if cur < search_end: + segments.append((cur, search_end)) + return segments + + +def _fuzzy_match_field_value( # pylint: disable=too-many-locals token_index: _TokenIndex, field_value: JatsFieldValue, config: AlignmentConfig, search_start: int, search_end: Optional[int] = None, + masked_ranges: Optional[List[Tuple[int, int]]] = None, ) -> Optional[_MatchResult]: needle = normalize_for_alignment(field_value.text) if not needle: @@ -275,23 +356,29 @@ def _fuzzy_match_field_value( haystack = token_index.haystack hay_end = len(haystack) if search_end is None else min(search_end, len(haystack)) - need_len = len(needle) + segments: List[Tuple[int, int]] = ( + _get_unmasked_segments(search_start, hay_end, masked_ranges) + if masked_ranges + else [(search_start, hay_end)] + ) + need_len = len(needle) window_size = max( _DEFAULT_MIN_WINDOW, min(config.max_window, need_len * _WINDOW_NEEDLE_MULTIPLIER), ) stride = max(1, window_size - need_len - 20) - start = search_start - while start < hay_end: - end = min(start + window_size, hay_end) - result = _fuzzy_search_in_window(haystack, needle, start, end, config.threshold) - if result is not None: - return result - if end >= hay_end: - break - start += stride + for seg_start, seg_end in segments: + start = seg_start + while start < seg_end: + end = min(start + window_size, seg_end) + result = _fuzzy_search_in_window(haystack, needle, start, end, config.threshold) + if result is not None: + return result + if end >= seg_end: + break + start += stride return None @@ -380,6 +467,73 @@ def _is_haystack_token_start(token_index: _TokenIndex, pos: int) -> bool: return token_index.is_token_start(pos) +def _extend_match_with_given_names_tail( + match_range: '_MatchResult', + original_text: str, + fallback_text: str, + token_index: '_TokenIndex', +) -> '_MatchResult': + """After a surname-fallback match, try to extend it with the given-names portion. + + When the mid-token fallback preference selects the fallback (surname-only) match, + the given-names initial (e.g. "T" from "Guardian T") is missing. This function + searches within _MAX_HAYSTACK_GAP_TO_FILL chars of the fallback match end for the + first character of the given-names and extends the block list if found. + """ + given_tail = normalize_for_alignment(original_text)[ + len(normalize_for_alignment(fallback_text)): + ].lstrip() + if not given_tail: + return match_range + fb_end = match_range[1] + win = token_index.haystack[fb_end: fb_end + _MAX_HAYSTACK_GAP_TO_FILL + 1 + len(given_tail)] + idx = win.find(given_tail[0]) + if idx == -1 or idx > _MAX_HAYSTACK_GAP_TO_FILL: + return match_range + abs_tail_start = fb_end + idx + if not token_index.is_token_start(abs_tail_start): + return match_range + match_count = 0 + for i, char in enumerate(given_tail): + if abs_tail_start + i >= len(token_index.haystack): + break + if token_index.haystack[abs_tail_start + i] != char: + break + match_count += 1 + if not match_count: + return match_range + tail_end = abs_tail_start + match_count + return ( + match_range[0], + max(match_range[1], tail_end), + match_range[2] + [(abs_tail_start, tail_end)], + ) + + +def _has_mid_token_within_gap_blocks( + block_ranges: List[Tuple[int, int]], + token_index: '_TokenIndex', +) -> bool: + """Return True if block_ranges contains a non-anchor within-gap block that starts mid-token. + + This identifies SW matches that achieved quality only by reaching into the middle of a + word — e.g. 't' inside 'staff' when searching for author initial 'T'. When a fallback + needle (surname only) exists, the aligner can instead try the fallback to find the + correct earlier occurrence. + """ + prev_end: Optional[int] = None + for block_start, block_end in block_ranges: + is_anchor = (block_end - block_start) >= _MIN_ANCHOR_BLOCK_SIZE + within_gap = ( + prev_end is not None + and block_start - prev_end <= _MAX_HAYSTACK_GAP_TO_FILL + ) + if within_gap and not is_anchor and not token_index.is_token_start(block_start): + return True + prev_end = block_end + return False + + def _label_tokens_for_blocks( annotated: JatsAnnotatedLayoutDocument, token_index: _TokenIndex, @@ -420,6 +574,11 @@ def _label_tokens_for_blocks( token_index, block_start ): continue + if within_gap and not is_anchor and not _is_haystack_token_start( + token_index, block_start + ): + prev_included_end = block_end + continue fill_start = prev_included_end if within_gap else block_start assert fill_start is not None for token in token_index.tokens_in_range(fill_start, block_end): @@ -427,13 +586,130 @@ def _label_tokens_for_blocks( prev_included_end = block_end +def _fill_sub_field_gaps( + annotated: JatsAnnotatedLayoutDocument, + tokens: List[LayoutToken], + field_name: str, + sub_field_name: str, +) -> None: + """Fill token gaps between consecutive sub_field spans of the same instance. + + When author names are aligned per-name, separator tokens (commas, semicolons, + initials with periods) between consecutively matched names remain unlabeled. + This merges them into a single contiguous span so that all author tokens end + up in one element — matching the grobid training data convention. + + Gaps wider than _MAX_AUTHOR_GAP_TOKENS tokens are left unfilled to avoid + accidentally absorbing subsequent reference fields (title, year, etc.). + """ + last_instance: Optional[int] = None + pending: List[LayoutToken] = [] + + for token in tokens: + entry = annotated.token_label_by_id.get(id(token)) + if ( + entry is not None + and entry[0] == field_name + and entry[1] == sub_field_name + ): + if last_instance is not None and entry[2] == last_instance and pending: + for pt in pending: + annotated.set_token_label(pt, field_name, sub_field_name, last_instance) + last_instance = entry[2] + pending = [] + elif last_instance is not None and ( + entry is None + or (entry[0] == field_name and entry[2] == last_instance) + ): + # Include unlabeled tokens (entry is None) and same-instance reference tokens + # in the gap between two author spans. The parent bibl SW match does not cover + # separators like "." between an initial and "et al.", so those tokens have no + # label at this point. + if len(pending) < _MAX_AUTHOR_GAP_TOKENS: + pending.append(token) + else: + last_instance = None + pending = [] + else: + last_instance = None + pending = [] + + +def _attach_sub_field_trailing_periods( + annotated: JatsAnnotatedLayoutDocument, + tokens: List[LayoutToken], + field_name: str, + sub_field_name: str, +) -> None: + """Relabel a bare '.' token that immediately follows a sub_field-labeled token. + + The PDF tokeniser splits 'D.' into 'D' and '.'. The period is not in the + JATS text, so SW and gap-fill logic miss it. This pass attaches such periods, + mirroring reference_annotator.get_suffix_extended_token_tags in the old tool. + """ + last_was_sub = False + last_instance = 0 + + for token in tokens: + entry = annotated.token_label_by_id.get(id(token)) + if ( + entry is not None + and entry[0] == field_name + and entry[1] == sub_field_name + ): + last_was_sub = True + last_instance = entry[2] + elif ( + last_was_sub + and entry is not None + and entry[0] == field_name + and normalize_for_alignment(token.text or '') == '.' + ): + annotated.set_token_label(token, field_name, sub_field_name, last_instance) + else: + last_was_sub = False + + +def _sort_reference_sub_fields_by_length( + field_values: List[JatsFieldValue], +) -> List[JatsFieldValue]: + """Within each reference's sub-field group, sort by descending normalized-needle length. + + Processing longer needles first means a short REFERENCE_LABEL "1" cannot claim a + position that REFERENCE_YEAR "1987" should own: the year matches and masks first, so + the label either lands on the true citation number or fails gracefully. Authors and + other sub-fields follow the same rule — within each group the longest needle wins the + best position, and shorter needles fill remaining unmasked positions. + """ + result: List[JatsFieldValue] = [] + pending: List[JatsFieldValue] = [] + + def _flush() -> None: + if pending: + pending.sort( + key=lambda fv: len(normalize_for_alignment(fv.text)), + reverse=True, + ) + result.extend(pending) + pending.clear() + + for fv in field_values: + if fv.sub_field_name is None: + _flush() + result.append(fv) + else: + pending.append(fv) + _flush() + return result + + class LayoutDocumentJatsAligner: """Aligns JATS field values to LayoutDocument tokens via fuzzy text matching.""" def __init__(self, config: Optional[AlignmentConfig] = None) -> None: self.config = config or AlignmentConfig() - def align( # pylint: disable=too-many-locals,too-many-branches + def align( # pylint: disable=too-many-locals,too-many-branches,too-many-statements self, layout_document: LayoutDocument, field_values: List[JatsFieldValue], @@ -442,6 +718,11 @@ def align( # pylint: disable=too-many-locals,too-many-branches if not field_values: return annotated + # Within each reference, longer sub-field needles are matched first. + # This prevents short needles (e.g. REFERENCE_LABEL "1") from claiming a + # position that a longer needle (e.g. REFERENCE_YEAR "1987") should own. + field_values = _sort_reference_sub_fields_by_length(field_values) + token_index = _build_token_index(layout_document) if not token_index.haystack: return annotated @@ -456,6 +737,9 @@ def align( # pylint: disable=too-many-locals,too-many-branches missed_by_field: Dict[str, int] = {} matched_count = 0 instance_by_field: Dict[str, int] = {} + # Per-parent masked ranges: reset each time a new main-field match is + # established so that sub-fields of one parent don't bleed into the next. + sub_field_masked_ranges: Dict[str, List[Tuple[int, int]]] = {} for fv in field_values: search_start, search_end = _search_range( @@ -463,10 +747,45 @@ def align( # pylint: disable=too-many-locals,too-many-branches front_matter_end, keywords_floor, reference_floor, parent_match_by_field, ) + masked = ( + sub_field_masked_ranges.get(fv.field_name) + if fv.sub_field_name is not None + else None + ) match_range = _fuzzy_match_field_value( token_index, fv, self.config, search_start=search_start, search_end=search_end, + masked_ranges=masked, ) + # If primary match relied on a mid-token within-gap block (e.g. 't' + # inside 'staff' matching the initial 'T' in "Guardian T"), the SW + # found a false-positive at a later occurrence. Try the fallback + # (surname only) which ignores the ambiguous initial; if it lands + # earlier, prefer it. + if ( + match_range is not None + and fv.sub_field_name is not None + and fv.fallback_text + and _has_mid_token_within_gap_blocks(match_range[2], token_index) + ): + _fallback_fv = JatsFieldValue( + text=fv.fallback_text, + field_name=fv.field_name, + sub_field_name=fv.sub_field_name, + ) + _earlier_match = _fuzzy_match_field_value( + token_index, _fallback_fv, self.config, + search_start=search_start, search_end=search_end, + masked_ranges=masked, + ) + if _earlier_match is not None and _earlier_match[0] < match_range[0]: + match_range = _earlier_match + # Extend surname-only fallback match with given-names initial + # (e.g. "T" from "Guardian T") found within gap of surname end. + assert fv.fallback_text is not None + match_range = _extend_match_with_given_names_tail( + match_range, fv.text, fv.fallback_text, token_index, + ) # Front-matter region constraint is soft: if a field value (e.g. an # affiliation that appears near the end of the paper) is not found # within the preferred region, fall back to a global search. Sub-field @@ -491,6 +810,25 @@ def align( # pylint: disable=too-many-locals,too-many-branches token_index, fv, self.config, search_start=body_floor, search_end=None, ) + # Sub-field fallback: retry with fallback_text (e.g. surname only) + # when the primary JATS name text does not match the PDF text. + if match_range is None and fv.sub_field_name is not None and fv.fallback_text: + fallback_fv = JatsFieldValue( + text=fv.fallback_text, + field_name=fv.field_name, + sub_field_name=fv.sub_field_name, + ) + match_range = _fuzzy_match_field_value( + token_index, fallback_fv, self.config, + search_start=search_start, search_end=search_end, + masked_ranges=masked, + ) + if match_range is not None: + # Extend surname-only match with given-names initial from + # the original needle (e.g. "Y.-H." after "HSIEH"). + match_range = _extend_match_with_given_names_tail( + match_range, fv.text, fv.fallback_text, token_index, + ) if match_range is None: if fv.sub_field_name is None: missed_by_field[fv.field_name] = ( @@ -515,15 +853,33 @@ def align( # pylint: disable=too-many-locals,too-many-branches reference_floor = max(reference_floor, a_end) if fv.sub_field_name is None: parent_match_by_field[fv.field_name] = (a_start, a_end) + sub_field_masked_ranges[fv.field_name] = [] instance_by_field[fv.field_name] = ( instance_by_field.get(fv.field_name, 0) + 1 ) + else: + sub_field_masked_ranges.setdefault(fv.field_name, []).append( + (a_start, a_end) + ) instance_id = instance_by_field.get(fv.field_name, 0) _label_tokens_for_blocks( annotated, token_index, block_ranges, fv.field_name, fv.sub_field_name, instance_id, ) + # Merge per-name REFERENCE_AUTHOR spans into a single element. + # Separator tokens (commas, semicolons, initials with periods) between + # consecutively matched names remain unlabeled after per-name SW; these + # two passes fill the gaps and attach trailing periods on abbreviations. + _fill_sub_field_gaps( + annotated, token_index.tokens, + JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR, + ) + _attach_sub_field_trailing_periods( + annotated, token_index.tokens, + JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR, + ) + total = len(field_values) if missed_by_field: missed = sum(missed_by_field.values()) diff --git a/sciencebeam_parser/training/jats/field_extractor.py b/sciencebeam_parser/training/jats/field_extractor.py index 5f0aa58a..fea173b8 100644 --- a/sciencebeam_parser/training/jats/field_extractor.py +++ b/sciencebeam_parser/training/jats/field_extractor.py @@ -14,6 +14,7 @@ class JatsFieldValue: text: str field_name: str sub_field_name: Optional[str] = None + fallback_text: Optional[str] = None def _element_text(el: etree._Element) -> str: @@ -71,25 +72,38 @@ def _iter_sub_field_values( ] -def _person_group_text(pg_el: etree._Element) -> str: - """Collect author name text and append 'et al.' when is present.""" - text = ' '.join(' '.join(pg_el.itertext()).split()) - if pg_el.xpath('etal'): - text = (text + ' et al.').strip() - return text - - def _iter_reference_author_values( ref_el: etree._Element, field_name: str, ) -> Iterator[JatsFieldValue]: + """Yield one JatsFieldValue per in each author person-group, then + 'et al.' if is present. Per-name emission lets the aligner + match each author independently, so a JATS/PDF format mismatch on one + name (e.g. given-names abbreviation style) cannot cut off the next name. + + Each name value carries a fallback_text set to just the surname, so the + aligner can retry with surname-only matching when the full JATS name text + (e.g. 'Maier BE') does not match the PDF representation ('MAIER, B. F.'). + """ for pg_el in ref_el.xpath('.//person-group[@person-group-type="author"]'): - text = _person_group_text(pg_el) - if text: + for name_el in pg_el.xpath('name'): + text = _element_text(name_el) + if not text: + continue + surname_el = name_el.find('surname') + raw_surname = (surname_el.text or '').strip() if surname_el is not None else '' + fallback = raw_surname if raw_surname and raw_surname != text else None yield JatsFieldValue( text=text, field_name=field_name, sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text=fallback, + ) + if pg_el.xpath('etal'): + yield JatsFieldValue( + text='et al.', + field_name=field_name, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, ) diff --git a/tests/training/jats/test_aligner.py b/tests/training/jats/test_aligner.py index 25bbeb4a..e29e0012 100644 --- a/tests/training/jats/test_aligner.py +++ b/tests/training/jats/test_aligner.py @@ -7,6 +7,7 @@ LayoutPage, ) from sciencebeam_parser.training.jats.aligner import AlignmentConfig, LayoutDocumentJatsAligner +from sciencebeam_parser.training.jats.text_normalizer import normalize_for_alignment from sciencebeam_parser.training.jats.field_extractor import JatsFieldValue from sciencebeam_parser.training.jats.field_vocab import JatsFieldNames, JatsSubFieldNames @@ -20,7 +21,7 @@ def _fv(text: str, field: str = JatsFieldNames.BODY_SECTION_TITLE, sub: Optional return JatsFieldValue(text=text, field_name=field, sub_field_name=sub) -class TestLayoutDocumentJatsAligner: +class TestLayoutDocumentJatsAligner: # pylint: disable=too-many-public-methods def _align(self, doc, field_values, **kwargs): config = AlignmentConfig(**kwargs) if kwargs else None return LayoutDocumentJatsAligner(config).align(doc, field_values) @@ -367,3 +368,519 @@ def test_heading_label_not_overwritten_by_paragraph_mid_token_match(self): assert annotated.get_token_field(heading_token) == JatsFieldNames.BODY_SECTION_TITLE, ( f'Heading token label was overwritten to {annotated.get_token_field(heading_token)!r}' ) + + def test_per_name_authors_label_last_initial(self): + # Regression: when the whole person-group was one needle, a JATS/PDF + # format mismatch on an earlier name (e.g. "BE" vs "B. F.") could cause + # the SW to terminate before labelling the last author's given-name initial. + # Per-name emission gives each name its own independent SW run. + doc = _make_doc('MAIER, B. F.; BROCKMANN, D. Some title 2020') + fvs = [ + _fv('MAIER, B. F.; BROCKMANN, D. Some title 2020', JatsFieldNames.REFERENCE), + _fv('Maier BE', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + _fv('Brockmann D', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + d_token = next( + (t for t in tokens if normalize_for_alignment(t.text) == 'd'), + None, + ) + assert d_token is not None, 'No token matching "D" found in document' + assert annotated.get_token_sub_field(d_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'"D." token sub-field was {annotated.get_token_sub_field(d_token)!r}, ' + 'expected REFERENCE_AUTHOR' + ) + + def test_trailing_period_after_last_initial_is_labeled(self): + # The PDF tokeniser splits 'D.' into two tokens 'D' and '.'. + # The period is not in the JATS text, so it must be attached by the + # trailing-period pass rather than by SW. + doc = _make_doc('BROCKMANN, D. Some title 2020') + fvs = [ + _fv('BROCKMANN, D. Some title 2020', JatsFieldNames.REFERENCE), + _fv('Brockmann D', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + # '.' immediately after 'D' should be REFERENCE_AUTHOR + d_idx = next( + i for i, t in enumerate(tokens) if normalize_for_alignment(t.text) == 'd' + ) + period_token = tokens[d_idx + 1] + assert period_token.text == '.', 'Expected period token after D' + assert annotated.get_token_sub_field(period_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'Period after D should be REFERENCE_AUTHOR, got ' + f'{annotated.get_token_sub_field(period_token)!r}' + ) + + def test_gap_fill_merges_per_name_author_spans(self): + # Between two per-name author matches, separator tokens (comma, semicolon, + # initials with periods) should be filled in as REFERENCE_AUTHOR so that + # all author tokens form a single element. + doc = _make_doc('Smith A. B.; Jones C. D. Some title 2020') + fvs = [ + _fv('Smith A. B.; Jones C. D. Some title 2020', JatsFieldNames.REFERENCE), + _fv('Smith AB', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + _fv('Jones CD', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + # Every token up to and including the final 'D.' of Jones should be REFERENCE_AUTHOR + smith_idx = next(i for i, t in enumerate(tokens) if t.text == 'Smith') + jones_idx = next(i for i, t in enumerate(tokens) if t.text == 'Jones') + # Tokens between Smith and Jones (exclusive) are separator/initial tokens + between = tokens[smith_idx + 1: jones_idx] + assert len(between) > 0 + assert all( + annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + for t in between + ), 'Gap tokens between Smith and Jones should all be REFERENCE_AUTHOR' + + def test_surname_fallback_labels_initial_mismatch(self): + # When JATS given-names ('BE') do not match the PDF representation + # ('B. F.'), the aligner should fall back to surname-only matching so + # that at least the surname token is labeled REFERENCE_AUTHOR. The gap + # fill then extends to cover the initials between the two surnames. + doc = _make_doc('MAIER, B. F.; BROCKMANN, D. Some title 2020') + fvs = [ + _fv('MAIER, B. F.; BROCKMANN, D. Some title 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Maier BE', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Maier', + ), + _fv('Brockmann D', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + maier_token = next( + (t for t in tokens if normalize_for_alignment(t.text) == 'maier'), None + ) + assert maier_token is not None + assert annotated.get_token_sub_field(maier_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + 'MAIER token should be REFERENCE_AUTHOR via surname fallback' + ) + # Gap fill should also label the tokens between MAIER and BROCKMANN + brockmann_idx = next( + i for i, t in enumerate(tokens) + if normalize_for_alignment(t.text) == 'brockmann' + ) + maier_idx = next( + i for i, t in enumerate(tokens) + if normalize_for_alignment(t.text) == 'maier' + ) + gap_tokens = tokens[maier_idx + 1: brockmann_idx] + assert all( + annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + for t in gap_tokens + ), 'All tokens between MAIER and BROCKMANN should be REFERENCE_AUTHOR via gap fill' + + def test_mid_token_within_gap_block_not_labeled(self): + # When SW matches a short block that starts mid-token (e.g. 't' inside 'staff' + # when searching for author initial 'T'), that token must NOT be labeled. + # Regression: GUARDIAN T matched second 'guardian' in 'guardian staff' + # because 't' inside 'staff' was closer (gap 1) than the real 'T' (gap 2). + doc = _make_doc('GUARDIAN, T. Guardian staff 2020') + fvs = [ + _fv('GUARDIAN, T. Guardian staff 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Guardian T', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Guardian', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + staff_token = next((t for t in tokens if t.text == 'staff'), None) + assert staff_token is not None + assert annotated.get_token_sub_field(staff_token) != JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"staff" token should not be labeled REFERENCE_AUTHOR (mid-word "t" matched by SW)' + ) + # The correct first occurrence 'GUARDIAN' should be labeled + guardian_upper = next((t for t in tokens if t.text == 'GUARDIAN'), None) + assert guardian_upper is not None + assert annotated.get_token_sub_field(guardian_upper) == ( + JatsSubFieldNames.REFERENCE_AUTHOR + ), '"GUARDIAN" (first occurrence) should be labeled via surname fallback' + + def test_mid_token_fallback_extends_with_given_names(self): + # After the mid-token fallback selects the surname-only earlier match, + # the given-names initial should be found within the gap and also labeled. + # Regression: GUARDIAN, T. — only "GUARDIAN" was labeled, not ", T." + doc = _make_doc('GUARDIAN, T. Guardian staff 2020') + fvs = [ + _fv('GUARDIAN, T. Guardian staff 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Guardian T', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Guardian', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + t_token = next((t for t in tokens if t.text == 'T'), None) + assert t_token is not None + assert annotated.get_token_sub_field(t_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"T" initial should be labeled REFERENCE_AUTHOR after surname fallback extension' + ) + + def test_author_gap_fill_bridges_unlabeled_period_before_et_al(self): + # When "Chowell G" and "et al." are separate needles, the "." between "G" and + # "et al." has no label from SW (parent bibl text skips "et al."). The gap fill + # must bridge it so all tokens become one element. + doc = _make_doc('CHOWELL, G. et al. Phenomenological models 2016') + fvs = [ + _fv('CHOWELL G Phenomenological models 2016', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Chowell G', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Chowell', + ), + JatsFieldValue( + text='et al.', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + period_after_g = tokens[3] # 'CHOWELL'=0, ','=1, 'G'=2, '.'=3 + assert period_after_g.text == '.', f'Expected "." got {period_after_g.text!r}' + sub = annotated.get_token_sub_field(period_after_g) + assert sub == JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"." after G should be labeled via gap fill' + ) + + def test_author_gap_fill_bridges_middle_initial_before_et_al(self): + # "Vasconcelos GL" with "et al." as a separate needle — the ". L." between + # "G" and "et al." in the PDF are not in the JATS author text and must be + # bridged by gap fill so all tokens merge into one element. + doc = _make_doc('VASCONCELOS, G. L. et al. Modelling fatality 2020') + fvs = [ + _fv('VASCONCELOS GL Modelling fatality 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Vasconcelos GL', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Vasconcelos', + ), + JatsFieldValue( + text='et al.', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + l_token = next((t for t in tokens if t.text == 'L'), None) + assert l_token is not None + assert annotated.get_token_sub_field(l_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"L" middle initial should be labeled REFERENCE_AUTHOR via gap fill' + ) + + def test_reference_label_does_not_block_year(self): + # REFERENCE_LABEL "1" can match the "1" in "1987" if the citation number is + # outside the parent bibl match range. The masking that REFERENCE_LABEL adds + # must not block REFERENCE_YEAR from labeling "1987". + doc = _make_doc('Lewis RW Roberts PM Applied Scientific Research 1987') + fvs = [ + _fv('1 Lewis RW Roberts PM Applied Scientific Research 1987', + JatsFieldNames.REFERENCE), + _fv('1', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_LABEL), + _fv('Applied Scientific Research', JatsFieldNames.REFERENCE, + JatsSubFieldNames.REFERENCE_SOURCE), + _fv('1987', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + year_token = next((t for t in tokens if t.text == '1987'), None) + assert year_token is not None + sub = annotated.get_token_sub_field(year_token) + assert sub == JatsSubFieldNames.REFERENCE_YEAR, ( + f'"1987" should be reference-year, got {sub!r}' + ) + + def test_regular_fallback_extends_with_given_names(self): + # When the full JATS name (e.g. "Hsieh Y-H") fails SW quality against the PDF + # (e.g. "HSIEH, Y.-H.") and falls back to surname only ("Hsieh"), the + # given-names initial from the original needle ("Y") should still be labeled. + doc = _make_doc('HSIEH, Y.-H. Richards model 2009') + fvs = [ + _fv('HSIEH, Y.-H. Richards model 2009', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Hsieh Y-H', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Hsieh', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + y_token = next((t for t in tokens if t.text == 'Y'), None) + assert y_token is not None, 'Expected "Y" token in doc' + sub = annotated.get_token_sub_field(y_token) + assert sub == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'"Y" initial should be labeled REFERENCE_AUTHOR after fallback, got {sub!r}' + ) + + def test_masked_sub_field_prevents_duplicate_match(self): + # When the same author name appears twice in a reference, masking ensures + # the second sub-field value matches the second occurrence rather than + # re-matching the first (already-masked) position. + doc = _make_doc('Jones A and Jones A 2020') + fvs = [ + _fv('Jones A and Jones A 2020', JatsFieldNames.REFERENCE), + _fv('Jones A', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + _fv('Jones A', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + jones_tokens = [t for t in tokens if t.text == 'Jones'] + assert len(jones_tokens) == 2, f'Expected 2 "Jones" tokens, got {len(jones_tokens)}' + assert all( + annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + for t in jones_tokens + ), 'Both "Jones" tokens should be labeled REFERENCE_AUTHOR' + + # ── Comprehensive reference sub-field tests ────────────────────────────── + + def _ref_fv(self, text: str, sub: str, fallback: str = '') -> JatsFieldValue: + return JatsFieldValue( + text=text, + field_name=JatsFieldNames.REFERENCE, + sub_field_name=sub, + fallback_text=fallback or None, + ) + + def test_year_within_doi_not_relabeled_as_year(self): + # When the year digits only appear inside the DOI span, REFERENCE_YEAR must + # not claim them. The DOI (longer needle) is processed first via longest-first + # ordering, its range is masked, and REFERENCE_YEAR finds nothing. + doc = _make_doc( + 'Smith A article title A 10.12345/test.2001.56' + ) + doi = '10.12345/test.2001.56' + fvs = [ + _fv('Smith A article title A ' + doi, JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv(doi, JatsSubFieldNames.REFERENCE_DOI), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + sub_fields = [annotated.get_token_sub_field(t) for t in tokens] + assert JatsSubFieldNames.REFERENCE_YEAR not in sub_fields, ( + 'Year digits inside DOI should not be labeled as REFERENCE_YEAR' + ) + doi_tokens = [ + t.text for t in tokens + if annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_DOI + ] + assert doi_tokens, 'DOI should be labeled' + + def test_all_sub_fields_labeled_in_full_reference(self): + # End-to-end: all standard sub-fields of a single reference are labeled. + # Year appears standalone; DOI does not contain the year digits. + doc = _make_doc( + 'Smith A 1999 article title A source A 11 7 101 10.12345/test.paper.56' + ) + fvs = [ + _fv('Smith A 1999 article title A source A 11 7 101 10.12345/test.paper.56', + JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('source A', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('1999', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('11', JatsSubFieldNames.REFERENCE_VOLUME), + self._ref_fv('7', JatsSubFieldNames.REFERENCE_ISSUE), + self._ref_fv('101', JatsSubFieldNames.REFERENCE_FPAGE), + self._ref_fv('10.12345/test.paper.56', JatsSubFieldNames.REFERENCE_DOI), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('Smith') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('1999') == JatsSubFieldNames.REFERENCE_YEAR + assert by_text.get('source') == JatsSubFieldNames.REFERENCE_SOURCE + assert by_text.get('11') == JatsSubFieldNames.REFERENCE_VOLUME + assert by_text.get('7') == JatsSubFieldNames.REFERENCE_ISSUE + assert by_text.get('101') == JatsSubFieldNames.REFERENCE_FPAGE + + def test_multiple_authors_separated_by_comma_all_labeled(self): + # Both "Smith, A" and "Johnson, B" merged into one span. + doc = _make_doc('Smith A, Johnson B 2001 article title A') + fvs = [ + _fv('Smith A, Johnson B 2001 article title A', JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('Johnson B', JatsSubFieldNames.REFERENCE_AUTHOR, 'Johnson'), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('Smith') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('Johnson') == JatsSubFieldNames.REFERENCE_AUTHOR + # year and title must not bleed into author span + assert by_text.get('2001') == JatsSubFieldNames.REFERENCE_YEAR + + def test_dot_after_initial_included_in_author(self): + # "Smith, A." — the trailing dot after "A" should be labeled as author. + doc = _make_doc('Smith, A. 2001 article title A') + fvs = [ + _fv('Smith, A. 2001 article title A', JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + dot_after_a = next( + (t for i, t in enumerate(tokens) + if t.text == '.' and i > 0 and tokens[i - 1].text == 'A'), + None, + ) + assert dot_after_a is not None, 'Expected "." after "A" token' + sub = annotated.get_token_sub_field(dot_after_a) + assert sub == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'"." after initial should be REFERENCE_AUTHOR, got {sub!r}' + ) + + def test_et_al_included_in_author_span(self): + # "Smith A et al." — "et al." as separate needle merges with prior author. + doc = _make_doc('Smith A et al. article title A 2001') + fvs = [ + _fv('Smith A et al. article title A 2001', JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('et al.', JatsSubFieldNames.REFERENCE_AUTHOR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + et_token = next((t for t in tokens if t.text == 'et'), None) + al_token = next((t for t in tokens if t.text == 'al'), None) + assert et_token is not None and al_token is not None + assert annotated.get_token_sub_field(et_token) == JatsSubFieldNames.REFERENCE_AUTHOR + assert annotated.get_token_sub_field(al_token) == JatsSubFieldNames.REFERENCE_AUTHOR + + def test_multiple_references_no_cross_contamination(self): + # Sub-fields of reference 1 must not bleed into reference 2's tokens. + doc = _make_doc( + '1 Smith A title one source one 2001', + '2 Jones B title two source two 2002', + ) + ref1_parent = '1 Smith A title one source one 2001' + ref2_parent = '2 Jones B title two source two 2002' + fvs = [ + _fv(ref1_parent, JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('title one', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('source one', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + _fv(ref2_parent, JatsFieldNames.REFERENCE), + self._ref_fv('Jones B', JatsSubFieldNames.REFERENCE_AUTHOR, 'Jones'), + self._ref_fv('title two', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('source two', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('2002', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('Smith') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('Jones') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('2001') == JatsSubFieldNames.REFERENCE_YEAR + assert by_text.get('2002') == JatsSubFieldNames.REFERENCE_YEAR + + def test_label_with_same_digits_as_year_still_gets_year_labeled(self): + # If citation label is "2001" (same as year), length-ordering ensures the + # year sub-field wins its position (same length → original order preserved; + # both appear in different positions in the doc so no conflict). + doc = _make_doc('2001 Smith A title one 2001') + fvs = [ + _fv('2001 Smith A title one 2001', JatsFieldNames.REFERENCE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_LABEL), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('title one', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + year_tokens = [t for t in tokens if t.text == '2001'] + assert len(year_tokens) == 2 + # At least one of the two "2001" tokens should be labeled as reference-year + sub_fields = [annotated.get_token_sub_field(t) for t in year_tokens] + assert JatsSubFieldNames.REFERENCE_YEAR in sub_fields, ( + f'Expected reference-year in sub-fields, got {sub_fields}' + ) + + def test_varying_spaces_in_author_name_still_matched(self): + # The old tool's test: "Smith ,J .A ." (PDF format with odd spacing) should + # match JATS author "Smith J. A". + doc = _make_doc('Smith ,J .A . 2001 article title A') + fvs = [ + _fv('Smith ,J .A . 2001 article title A', JatsFieldNames.REFERENCE), + self._ref_fv('Smith J. A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + smith = next((t for t in tokens if t.text == 'Smith'), None) + assert smith is not None + assert annotated.get_token_sub_field(smith) == JatsSubFieldNames.REFERENCE_AUTHOR + + def test_sub_field_longer_than_label_wins_position(self): + # Structural regression guard: REFERENCE_YEAR (4 chars) must be processed before + # REFERENCE_LABEL (1 char) regardless of their order in the JATS document. + # Explicit ordering of fvs puts LABEL before YEAR to verify the sort is applied. + doc = _make_doc('Smith A source A 1987') + fvs = [ + _fv('1 Smith A source A 1987', JatsFieldNames.REFERENCE), + self._ref_fv('1', JatsSubFieldNames.REFERENCE_LABEL), # 1 char — before year + self._ref_fv('source A', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('1987', JatsSubFieldNames.REFERENCE_YEAR), # 4 chars + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + year_tok = next((t for t in tokens if t.text == '1987'), None) + assert year_tok is not None + assert annotated.get_token_sub_field(year_tok) == JatsSubFieldNames.REFERENCE_YEAR, ( + 'REFERENCE_YEAR should win "1987" even when REFERENCE_LABEL is listed first in fvs' + ) + + def test_hyphenated_surname_labeled_when_jats_has_extra_given_name(self): + # When JATS has an extra given name absent from the PDF, the SW produces + # overlapping mid-token blocks; the within-gap guard must advance + # prev_included_end through them to keep the hyphen within gap reach. + doc = _make_doc('Braun, Lena Silva-Braun, Meier') + fvs = [ + _fv('Braun, Lena Silva-Braun, Meier', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Braun Lena Kristina Silva-Braun', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Braun', + ), + JatsFieldValue( + text='Meier', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Meier', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + author_tokens = [ + t.text for t in tokens + if annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + ] + assert '-' in author_tokens, ( + f'Hyphen in "Silva-Braun" must be REFERENCE_AUTHOR; got: {author_tokens}' + ) diff --git a/tests/training/jats/test_field_extractor.py b/tests/training/jats/test_field_extractor.py index 8944cc76..bd403a32 100644 --- a/tests/training/jats/test_field_extractor.py +++ b/tests/training/jats/test_field_extractor.py @@ -256,9 +256,12 @@ def test_extracts_reference_author_subfield(self): '
' ) sub = [v for v in fvs if v.sub_field_name == JatsSubFieldNames.REFERENCE_AUTHOR] - assert len(sub) == 1 - assert 'Smith' in sub[0].text - assert 'Jones' in sub[0].text + assert len(sub) == 2 + assert sub[0].text == 'Smith A' + assert sub[1].text == 'Jones B C' + # fallback_text is set to surname only for per-name author matching fallback + assert sub[0].fallback_text == 'Smith' + assert sub[1].fallback_text == 'Jones' def test_reference_author_includes_et_al(self): fvs = _field_values_for( @@ -272,8 +275,11 @@ def test_reference_author_includes_et_al(self): '
' ) sub = [v for v in fvs if v.sub_field_name == JatsSubFieldNames.REFERENCE_AUTHOR] - assert len(sub) == 1 - assert sub[0].text == 'Smith A et al.' + assert len(sub) == 2 + assert sub[0].text == 'Smith A' + assert sub[1].text == 'et al.' + # et al. has no surname fallback + assert sub[1].fallback_text is None def test_extracts_reference_article_title_subfield(self): fvs = _field_values_for( From 39fa3317ca358bb62161a468d4603d7d5215edee Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Thu, 25 Jun 2026 10:39:29 +0100 Subject: [PATCH 4/5] Use exact whole-token matching for pure-numeric sub-field needles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SW character-level matching allowed "200" to match inside "2020" by consuming characters with a gap (2,0,_,0). For digit-only needles (years, page numbers, volumes, issues) the correct match is always an exact token — "200" cannot be a prefix or suffix of a longer number. _is_pure_number + _exact_number_match bypass SW for these needles, requiring the match to start at a token boundary and end at a token boundary. Two new _TokenIndex helper methods (is_in_token, is_token_boundary_after) expose the boundary checks without accessing the protected _token_index_at attribute directly. --- sciencebeam_parser/training/jats/aligner.py | 47 +++++++++++++++++++++ tests/training/jats/test_aligner.py | 15 +++++++ 2 files changed, 62 insertions(+) diff --git a/sciencebeam_parser/training/jats/aligner.py b/sciencebeam_parser/training/jats/aligner.py index 3393c0e9..ac993f5e 100644 --- a/sciencebeam_parser/training/jats/aligner.py +++ b/sciencebeam_parser/training/jats/aligner.py @@ -1,4 +1,5 @@ import logging +import re from dataclasses import dataclass from typing import Dict, FrozenSet, List, Optional, Set, Tuple @@ -198,6 +199,23 @@ def is_token_start(self, pos: int) -> bool: return False return self._token_index_at[pos - 1] != self._token_index_at[pos] + def is_in_token(self, pos: int) -> bool: + """Return True if pos is within a token (not a space between tokens).""" + if pos < 0 or pos >= len(self._token_index_at): + return False + return self._token_index_at[pos] != _NO_TOKEN_INDEX + + def is_token_boundary_after(self, pos: int) -> bool: + """Return True if the character at pos is the last in its token (or pos is past end).""" + if pos >= len(self._token_index_at): + return True + if not self.is_in_token(pos): + return True + next_pos = pos + 1 + if next_pos >= len(self._token_index_at): + return True + return self._token_index_at[pos] != self._token_index_at[next_pos] + def _build_token_index(layout_document: LayoutDocument) -> _TokenIndex: all_tokens: List[LayoutToken] = [] @@ -341,6 +359,32 @@ def _get_unmasked_segments( return segments +def _is_pure_number(text: str) -> bool: + return bool(re.fullmatch(r'\d+', text)) + + +def _exact_number_match( + token_index: _TokenIndex, + needle: str, + segments: List[Tuple[int, int]], +) -> Optional[_MatchResult]: + haystack = token_index.haystack + needle_len = len(needle) + for seg_start, seg_end in segments: + pos = seg_start + while pos <= seg_end - needle_len: + idx = haystack.find(needle, pos, seg_end) + if idx == -1: + break + end = idx + needle_len + if (token_index.is_in_token(idx) + and token_index.is_token_start(idx) + and token_index.is_token_boundary_after(end - 1)): + return idx, end, [(idx, end)] + pos = idx + 1 + return None + + def _fuzzy_match_field_value( # pylint: disable=too-many-locals token_index: _TokenIndex, field_value: JatsFieldValue, @@ -362,6 +406,9 @@ def _fuzzy_match_field_value( # pylint: disable=too-many-locals else [(search_start, hay_end)] ) + if _is_pure_number(needle): + return _exact_number_match(token_index, needle, segments) + need_len = len(needle) window_size = max( _DEFAULT_MIN_WINDOW, diff --git a/tests/training/jats/test_aligner.py b/tests/training/jats/test_aligner.py index e29e0012..d89e62c9 100644 --- a/tests/training/jats/test_aligner.py +++ b/tests/training/jats/test_aligner.py @@ -712,6 +712,21 @@ def test_all_sub_fields_labeled_in_full_reference(self): assert by_text.get('7') == JatsSubFieldNames.REFERENCE_ISSUE assert by_text.get('101') == JatsSubFieldNames.REFERENCE_FPAGE + def test_page_number_not_matched_inside_year(self): + # "200" (lpage) must match the exact token "200", not the "200" inside "2020". + doc = _make_doc('Author A 181-200, acesso em 2020') + fvs = [ + _fv('Author A 181-200, acesso em 2020', JatsFieldNames.REFERENCE), + self._ref_fv('Author A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Author'), + self._ref_fv('181', JatsSubFieldNames.REFERENCE_FPAGE), + self._ref_fv('200', JatsSubFieldNames.REFERENCE_LPAGE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('200') == JatsSubFieldNames.REFERENCE_LPAGE + assert by_text.get('2020') != JatsSubFieldNames.REFERENCE_LPAGE + def test_multiple_authors_separated_by_comma_all_labeled(self): # Both "Smith, A" and "Johnson, B" merged into one span. doc = _make_doc('Smith A, Johnson B 2001 article title A') From 91c6e8864213d9124691da0888fd53a95b52eaa5 Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Thu, 25 Jun 2026 11:27:44 +0100 Subject: [PATCH 5/5] Prefer exact SW match over earlier gap match in field alignment When multiple windows produce a hit, the first match was returned regardless of whether it was exact (single contiguous block) or gapped (SW used character-level gaps to cover the needle). This caused "2020b" to be labeled at the first plain "2020" occurrence via a 0.8-quality gap match, while the true "2020b" token later in the text was ignored. _fuzzy_match_field_value now stores the first gap match as a fallback and continues scanning, returning immediately only when an exact contiguous match is found. The existing pure-number path (_exact_number_match) is unaffected; this change covers all other tokens. --- sciencebeam_parser/training/jats/aligner.py | 14 ++++++++++++-- tests/training/jats/test_aligner.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/sciencebeam_parser/training/jats/aligner.py b/sciencebeam_parser/training/jats/aligner.py index ac993f5e..bf2bd211 100644 --- a/sciencebeam_parser/training/jats/aligner.py +++ b/sciencebeam_parser/training/jats/aligner.py @@ -363,6 +363,12 @@ def _is_pure_number(text: str) -> bool: return bool(re.fullmatch(r'\d+', text)) +def _is_exact_sw_match(result: _MatchResult, needle_len: int) -> bool: + """True when SW found the needle as one contiguous block (no gaps).""" + _, _, blocks = result + return len(blocks) == 1 and (blocks[0][1] - blocks[0][0]) == needle_len + + def _exact_number_match( token_index: _TokenIndex, needle: str, @@ -416,18 +422,22 @@ def _fuzzy_match_field_value( # pylint: disable=too-many-locals ) stride = max(1, window_size - need_len - 20) + gap_match: Optional[_MatchResult] = None for seg_start, seg_end in segments: start = seg_start while start < seg_end: end = min(start + window_size, seg_end) result = _fuzzy_search_in_window(haystack, needle, start, end, config.threshold) if result is not None: - return result + if _is_exact_sw_match(result, need_len): + return result + if gap_match is None: + gap_match = result if end >= seg_end: break start += stride - return None + return gap_match def _search_range( diff --git a/tests/training/jats/test_aligner.py b/tests/training/jats/test_aligner.py index d89e62c9..af6e0cb4 100644 --- a/tests/training/jats/test_aligner.py +++ b/tests/training/jats/test_aligner.py @@ -727,6 +727,21 @@ def test_page_number_not_matched_inside_year(self): assert by_text.get('200') == JatsSubFieldNames.REFERENCE_LPAGE assert by_text.get('2020') != JatsSubFieldNames.REFERENCE_LPAGE + def test_exact_sw_match_preferred_over_earlier_gap_match(self): + # Needle "2020b" gap-matches "2020" (quality 0.8) before the exact "2020b". + # The exact contiguous match must win even though it appears later. + doc = _make_doc('Author A 2020 title source 2020b') + fvs = [ + _fv('Author A 2020 title source 2020b', JatsFieldNames.REFERENCE), + self._ref_fv('Author A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Author'), + self._ref_fv('2020b', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('2020b') == JatsSubFieldNames.REFERENCE_YEAR + assert by_text.get('2020') != JatsSubFieldNames.REFERENCE_YEAR + def test_multiple_authors_separated_by_comma_all_labeled(self): # Both "Smith, A" and "Johnson, B" merged into one span. doc = _make_doc('Smith A, Johnson B 2001 article title A')