diff --git a/sciencebeam_parser/training/jats/aligner.py b/sciencebeam_parser/training/jats/aligner.py index 85bb5bb7..bf2bd211 100644 --- a/sciencebeam_parser/training/jats/aligner.py +++ b/sciencebeam_parser/training/jats/aligner.py @@ -1,4 +1,5 @@ import logging +import re from dataclasses import dataclass from typing import Dict, FrozenSet, List, Optional, Set, Tuple @@ -7,7 +8,7 @@ from sciencebeam_parser.document.layout_document import LayoutDocument, LayoutToken from sciencebeam_parser.training.jats.annotated_document import JatsAnnotatedLayoutDocument from sciencebeam_parser.training.jats.field_extractor import JatsFieldValue -from sciencebeam_parser.training.jats.field_vocab import JatsFieldNames +from sciencebeam_parser.training.jats.field_vocab import JatsFieldNames, JatsSubFieldNames from sciencebeam_parser.training.jats.text_normalizer import normalize_for_alignment @@ -108,6 +109,9 @@ # fields are never silently dropped. _MIN_ANCHOR_BLOCK_SIZE = 5 _MAX_HAYSTACK_GAP_TO_FILL = 3 +# Maximum number of non-author tokens permitted between two author spans before +# the gap fill gives up. Covers up to ~5 initials with periods and separators. +_MAX_AUTHOR_GAP_TOKENS = 10 # Type alias for the return value of _fuzzy_match_field_value: # (abs_start, abs_end, [(block_start, block_end), ...]) @@ -195,6 +199,23 @@ def is_token_start(self, pos: int) -> bool: return False return self._token_index_at[pos - 1] != self._token_index_at[pos] + def is_in_token(self, pos: int) -> bool: + """Return True if pos is within a token (not a space between tokens).""" + if pos < 0 or pos >= len(self._token_index_at): + return False + return self._token_index_at[pos] != _NO_TOKEN_INDEX + + def is_token_boundary_after(self, pos: int) -> bool: + """Return True if the character at pos is the last in its token (or pos is past end).""" + if pos >= len(self._token_index_at): + return True + if not self.is_in_token(pos): + return True + next_pos = pos + 1 + if next_pos >= len(self._token_index_at): + return True + return self._token_index_at[pos] != self._token_index_at[next_pos] + def _build_token_index(layout_document: LayoutDocument) -> _TokenIndex: all_tokens: List[LayoutToken] = [] @@ -230,6 +251,51 @@ def _match_quality( return matched / needle_len +def _extend_match_for_needle_tail( + window: str, + needle: str, + window_start: int, + abs_a_end: int, + matched_blocks: List[Tuple[int, int, int]], + abs_block_ranges: List[Tuple[int, int]], +) -> Tuple[int, List[Tuple[int, int]]]: + """Greedily extend the SW match to cover any unmatched needle suffix. + + Smith-Waterman terminates when extending the match would not increase the + score. A single-character suffix separated from the last block by one or + two gap characters produces a net-zero extension (e.g. two gap penalties + cancel one match bonus), so SW stops early. This function scans forward + within _MAX_HAYSTACK_GAP_TO_FILL characters of the current match end for + the first unmatched needle character and appends consecutive matches as an + extra block, mirroring the pre-anchor fill logic on the other end. + + Example: needle "brockmann d", match ends at "brockmann "; " , d" in the + haystack has "d" at gap 2, which is within the fill threshold. + """ + last_needle_end = max(bi + size for _, bi, size in matched_blocks) + needle_tail = needle[last_needle_end:] + if not needle_tail: + return abs_a_end, abs_block_ranges + + win_pos = abs_a_end - window_start + first_pos = window[win_pos: win_pos + _MAX_HAYSTACK_GAP_TO_FILL + 1].find(needle_tail[0]) + if first_pos == -1: + return abs_a_end, abs_block_ranges + + tail_start = win_pos + first_pos + match_count = 0 + for tc in needle_tail: + if tail_start + match_count >= len(window) or window[tail_start + match_count] != tc: + break + match_count += 1 + + if not match_count: + return abs_a_end, abs_block_ranges + + ext_start = window_start + tail_start + return ext_start + match_count, abs_block_ranges + [(ext_start, ext_start + match_count)] + + def _fuzzy_search_in_window( haystack: str, needle: str, @@ -241,6 +307,9 @@ def _fuzzy_search_in_window( Returns (abs_start, abs_end, [(block_start, block_end), ...]) if quality >= threshold, else None. Block ranges are in absolute haystack coordinates. + After the SW match, a greedy tail extension fills any unmatched needle + suffix whose first character falls within _MAX_HAYSTACK_GAP_TO_FILL chars + of the current match end (mirrors the pre-anchor fill on the trailing side). """ window = haystack[window_start:window_end] sm = LocalSequenceMatcher(a=window, b=needle, scoring=_SCORING) @@ -258,15 +327,77 @@ def _fuzzy_search_in_window( (ai + window_start, ai + size + window_start) for ai, _bi, size in matched_blocks ] + a_end, abs_block_ranges = _extend_match_for_needle_tail( + window, needle, window_start, a_end, matched_blocks, abs_block_ranges + ) return a_start, a_end, abs_block_ranges -def _fuzzy_match_field_value( +def _get_unmasked_segments( + search_start: int, + search_end: int, + masked_ranges: List[Tuple[int, int]], +) -> List[Tuple[int, int]]: + """Split [search_start, search_end) into segments that don't overlap masked_ranges. + + Masked boundaries are hard: SW runs on each segment independently and + cannot produce a match that spans across a masked span. + """ + clipped = sorted( + (max(m0, search_start), min(m1, search_end)) + for m0, m1 in masked_ranges + if m0 < search_end and m1 > search_start + ) + segments: List[Tuple[int, int]] = [] + cur = search_start + for m_start, m_end in clipped: + if cur < m_start: + segments.append((cur, m_start)) + cur = max(cur, m_end) + if cur < search_end: + segments.append((cur, search_end)) + return segments + + +def _is_pure_number(text: str) -> bool: + return bool(re.fullmatch(r'\d+', text)) + + +def _is_exact_sw_match(result: _MatchResult, needle_len: int) -> bool: + """True when SW found the needle as one contiguous block (no gaps).""" + _, _, blocks = result + return len(blocks) == 1 and (blocks[0][1] - blocks[0][0]) == needle_len + + +def _exact_number_match( + token_index: _TokenIndex, + needle: str, + segments: List[Tuple[int, int]], +) -> Optional[_MatchResult]: + haystack = token_index.haystack + needle_len = len(needle) + for seg_start, seg_end in segments: + pos = seg_start + while pos <= seg_end - needle_len: + idx = haystack.find(needle, pos, seg_end) + if idx == -1: + break + end = idx + needle_len + if (token_index.is_in_token(idx) + and token_index.is_token_start(idx) + and token_index.is_token_boundary_after(end - 1)): + return idx, end, [(idx, end)] + pos = idx + 1 + return None + + +def _fuzzy_match_field_value( # pylint: disable=too-many-locals token_index: _TokenIndex, field_value: JatsFieldValue, config: AlignmentConfig, search_start: int, search_end: Optional[int] = None, + masked_ranges: Optional[List[Tuple[int, int]]] = None, ) -> Optional[_MatchResult]: needle = normalize_for_alignment(field_value.text) if not needle: @@ -275,25 +406,38 @@ def _fuzzy_match_field_value( haystack = token_index.haystack hay_end = len(haystack) if search_end is None else min(search_end, len(haystack)) - need_len = len(needle) + segments: List[Tuple[int, int]] = ( + _get_unmasked_segments(search_start, hay_end, masked_ranges) + if masked_ranges + else [(search_start, hay_end)] + ) + + if _is_pure_number(needle): + return _exact_number_match(token_index, needle, segments) + need_len = len(needle) window_size = max( _DEFAULT_MIN_WINDOW, min(config.max_window, need_len * _WINDOW_NEEDLE_MULTIPLIER), ) stride = max(1, window_size - need_len - 20) - start = search_start - while start < hay_end: - end = min(start + window_size, hay_end) - result = _fuzzy_search_in_window(haystack, needle, start, end, config.threshold) - if result is not None: - return result - if end >= hay_end: - break - start += stride + gap_match: Optional[_MatchResult] = None + for seg_start, seg_end in segments: + start = seg_start + while start < seg_end: + end = min(start + window_size, seg_end) + result = _fuzzy_search_in_window(haystack, needle, start, end, config.threshold) + if result is not None: + if _is_exact_sw_match(result, need_len): + return result + if gap_match is None: + gap_match = result + if end >= seg_end: + break + start += stride - return None + return gap_match def _search_range( @@ -380,6 +524,73 @@ def _is_haystack_token_start(token_index: _TokenIndex, pos: int) -> bool: return token_index.is_token_start(pos) +def _extend_match_with_given_names_tail( + match_range: '_MatchResult', + original_text: str, + fallback_text: str, + token_index: '_TokenIndex', +) -> '_MatchResult': + """After a surname-fallback match, try to extend it with the given-names portion. + + When the mid-token fallback preference selects the fallback (surname-only) match, + the given-names initial (e.g. "T" from "Guardian T") is missing. This function + searches within _MAX_HAYSTACK_GAP_TO_FILL chars of the fallback match end for the + first character of the given-names and extends the block list if found. + """ + given_tail = normalize_for_alignment(original_text)[ + len(normalize_for_alignment(fallback_text)): + ].lstrip() + if not given_tail: + return match_range + fb_end = match_range[1] + win = token_index.haystack[fb_end: fb_end + _MAX_HAYSTACK_GAP_TO_FILL + 1 + len(given_tail)] + idx = win.find(given_tail[0]) + if idx == -1 or idx > _MAX_HAYSTACK_GAP_TO_FILL: + return match_range + abs_tail_start = fb_end + idx + if not token_index.is_token_start(abs_tail_start): + return match_range + match_count = 0 + for i, char in enumerate(given_tail): + if abs_tail_start + i >= len(token_index.haystack): + break + if token_index.haystack[abs_tail_start + i] != char: + break + match_count += 1 + if not match_count: + return match_range + tail_end = abs_tail_start + match_count + return ( + match_range[0], + max(match_range[1], tail_end), + match_range[2] + [(abs_tail_start, tail_end)], + ) + + +def _has_mid_token_within_gap_blocks( + block_ranges: List[Tuple[int, int]], + token_index: '_TokenIndex', +) -> bool: + """Return True if block_ranges contains a non-anchor within-gap block that starts mid-token. + + This identifies SW matches that achieved quality only by reaching into the middle of a + word — e.g. 't' inside 'staff' when searching for author initial 'T'. When a fallback + needle (surname only) exists, the aligner can instead try the fallback to find the + correct earlier occurrence. + """ + prev_end: Optional[int] = None + for block_start, block_end in block_ranges: + is_anchor = (block_end - block_start) >= _MIN_ANCHOR_BLOCK_SIZE + within_gap = ( + prev_end is not None + and block_start - prev_end <= _MAX_HAYSTACK_GAP_TO_FILL + ) + if within_gap and not is_anchor and not token_index.is_token_start(block_start): + return True + prev_end = block_end + return False + + def _label_tokens_for_blocks( annotated: JatsAnnotatedLayoutDocument, token_index: _TokenIndex, @@ -420,6 +631,11 @@ def _label_tokens_for_blocks( token_index, block_start ): continue + if within_gap and not is_anchor and not _is_haystack_token_start( + token_index, block_start + ): + prev_included_end = block_end + continue fill_start = prev_included_end if within_gap else block_start assert fill_start is not None for token in token_index.tokens_in_range(fill_start, block_end): @@ -427,13 +643,130 @@ def _label_tokens_for_blocks( prev_included_end = block_end +def _fill_sub_field_gaps( + annotated: JatsAnnotatedLayoutDocument, + tokens: List[LayoutToken], + field_name: str, + sub_field_name: str, +) -> None: + """Fill token gaps between consecutive sub_field spans of the same instance. + + When author names are aligned per-name, separator tokens (commas, semicolons, + initials with periods) between consecutively matched names remain unlabeled. + This merges them into a single contiguous span so that all author tokens end + up in one element — matching the grobid training data convention. + + Gaps wider than _MAX_AUTHOR_GAP_TOKENS tokens are left unfilled to avoid + accidentally absorbing subsequent reference fields (title, year, etc.). + """ + last_instance: Optional[int] = None + pending: List[LayoutToken] = [] + + for token in tokens: + entry = annotated.token_label_by_id.get(id(token)) + if ( + entry is not None + and entry[0] == field_name + and entry[1] == sub_field_name + ): + if last_instance is not None and entry[2] == last_instance and pending: + for pt in pending: + annotated.set_token_label(pt, field_name, sub_field_name, last_instance) + last_instance = entry[2] + pending = [] + elif last_instance is not None and ( + entry is None + or (entry[0] == field_name and entry[2] == last_instance) + ): + # Include unlabeled tokens (entry is None) and same-instance reference tokens + # in the gap between two author spans. The parent bibl SW match does not cover + # separators like "." between an initial and "et al.", so those tokens have no + # label at this point. + if len(pending) < _MAX_AUTHOR_GAP_TOKENS: + pending.append(token) + else: + last_instance = None + pending = [] + else: + last_instance = None + pending = [] + + +def _attach_sub_field_trailing_periods( + annotated: JatsAnnotatedLayoutDocument, + tokens: List[LayoutToken], + field_name: str, + sub_field_name: str, +) -> None: + """Relabel a bare '.' token that immediately follows a sub_field-labeled token. + + The PDF tokeniser splits 'D.' into 'D' and '.'. The period is not in the + JATS text, so SW and gap-fill logic miss it. This pass attaches such periods, + mirroring reference_annotator.get_suffix_extended_token_tags in the old tool. + """ + last_was_sub = False + last_instance = 0 + + for token in tokens: + entry = annotated.token_label_by_id.get(id(token)) + if ( + entry is not None + and entry[0] == field_name + and entry[1] == sub_field_name + ): + last_was_sub = True + last_instance = entry[2] + elif ( + last_was_sub + and entry is not None + and entry[0] == field_name + and normalize_for_alignment(token.text or '') == '.' + ): + annotated.set_token_label(token, field_name, sub_field_name, last_instance) + else: + last_was_sub = False + + +def _sort_reference_sub_fields_by_length( + field_values: List[JatsFieldValue], +) -> List[JatsFieldValue]: + """Within each reference's sub-field group, sort by descending normalized-needle length. + + Processing longer needles first means a short REFERENCE_LABEL "1" cannot claim a + position that REFERENCE_YEAR "1987" should own: the year matches and masks first, so + the label either lands on the true citation number or fails gracefully. Authors and + other sub-fields follow the same rule — within each group the longest needle wins the + best position, and shorter needles fill remaining unmasked positions. + """ + result: List[JatsFieldValue] = [] + pending: List[JatsFieldValue] = [] + + def _flush() -> None: + if pending: + pending.sort( + key=lambda fv: len(normalize_for_alignment(fv.text)), + reverse=True, + ) + result.extend(pending) + pending.clear() + + for fv in field_values: + if fv.sub_field_name is None: + _flush() + result.append(fv) + else: + pending.append(fv) + _flush() + return result + + class LayoutDocumentJatsAligner: """Aligns JATS field values to LayoutDocument tokens via fuzzy text matching.""" def __init__(self, config: Optional[AlignmentConfig] = None) -> None: self.config = config or AlignmentConfig() - def align( # pylint: disable=too-many-locals,too-many-branches + def align( # pylint: disable=too-many-locals,too-many-branches,too-many-statements self, layout_document: LayoutDocument, field_values: List[JatsFieldValue], @@ -442,6 +775,11 @@ def align( # pylint: disable=too-many-locals,too-many-branches if not field_values: return annotated + # Within each reference, longer sub-field needles are matched first. + # This prevents short needles (e.g. REFERENCE_LABEL "1") from claiming a + # position that a longer needle (e.g. REFERENCE_YEAR "1987") should own. + field_values = _sort_reference_sub_fields_by_length(field_values) + token_index = _build_token_index(layout_document) if not token_index.haystack: return annotated @@ -456,6 +794,9 @@ def align( # pylint: disable=too-many-locals,too-many-branches missed_by_field: Dict[str, int] = {} matched_count = 0 instance_by_field: Dict[str, int] = {} + # Per-parent masked ranges: reset each time a new main-field match is + # established so that sub-fields of one parent don't bleed into the next. + sub_field_masked_ranges: Dict[str, List[Tuple[int, int]]] = {} for fv in field_values: search_start, search_end = _search_range( @@ -463,10 +804,45 @@ def align( # pylint: disable=too-many-locals,too-many-branches front_matter_end, keywords_floor, reference_floor, parent_match_by_field, ) + masked = ( + sub_field_masked_ranges.get(fv.field_name) + if fv.sub_field_name is not None + else None + ) match_range = _fuzzy_match_field_value( token_index, fv, self.config, search_start=search_start, search_end=search_end, + masked_ranges=masked, ) + # If primary match relied on a mid-token within-gap block (e.g. 't' + # inside 'staff' matching the initial 'T' in "Guardian T"), the SW + # found a false-positive at a later occurrence. Try the fallback + # (surname only) which ignores the ambiguous initial; if it lands + # earlier, prefer it. + if ( + match_range is not None + and fv.sub_field_name is not None + and fv.fallback_text + and _has_mid_token_within_gap_blocks(match_range[2], token_index) + ): + _fallback_fv = JatsFieldValue( + text=fv.fallback_text, + field_name=fv.field_name, + sub_field_name=fv.sub_field_name, + ) + _earlier_match = _fuzzy_match_field_value( + token_index, _fallback_fv, self.config, + search_start=search_start, search_end=search_end, + masked_ranges=masked, + ) + if _earlier_match is not None and _earlier_match[0] < match_range[0]: + match_range = _earlier_match + # Extend surname-only fallback match with given-names initial + # (e.g. "T" from "Guardian T") found within gap of surname end. + assert fv.fallback_text is not None + match_range = _extend_match_with_given_names_tail( + match_range, fv.text, fv.fallback_text, token_index, + ) # Front-matter region constraint is soft: if a field value (e.g. an # affiliation that appears near the end of the paper) is not found # within the preferred region, fall back to a global search. Sub-field @@ -491,6 +867,25 @@ def align( # pylint: disable=too-many-locals,too-many-branches token_index, fv, self.config, search_start=body_floor, search_end=None, ) + # Sub-field fallback: retry with fallback_text (e.g. surname only) + # when the primary JATS name text does not match the PDF text. + if match_range is None and fv.sub_field_name is not None and fv.fallback_text: + fallback_fv = JatsFieldValue( + text=fv.fallback_text, + field_name=fv.field_name, + sub_field_name=fv.sub_field_name, + ) + match_range = _fuzzy_match_field_value( + token_index, fallback_fv, self.config, + search_start=search_start, search_end=search_end, + masked_ranges=masked, + ) + if match_range is not None: + # Extend surname-only match with given-names initial from + # the original needle (e.g. "Y.-H." after "HSIEH"). + match_range = _extend_match_with_given_names_tail( + match_range, fv.text, fv.fallback_text, token_index, + ) if match_range is None: if fv.sub_field_name is None: missed_by_field[fv.field_name] = ( @@ -515,15 +910,33 @@ def align( # pylint: disable=too-many-locals,too-many-branches reference_floor = max(reference_floor, a_end) if fv.sub_field_name is None: parent_match_by_field[fv.field_name] = (a_start, a_end) + sub_field_masked_ranges[fv.field_name] = [] instance_by_field[fv.field_name] = ( instance_by_field.get(fv.field_name, 0) + 1 ) + else: + sub_field_masked_ranges.setdefault(fv.field_name, []).append( + (a_start, a_end) + ) instance_id = instance_by_field.get(fv.field_name, 0) _label_tokens_for_blocks( annotated, token_index, block_ranges, fv.field_name, fv.sub_field_name, instance_id, ) + # Merge per-name REFERENCE_AUTHOR spans into a single element. + # Separator tokens (commas, semicolons, initials with periods) between + # consecutively matched names remain unlabeled after per-name SW; these + # two passes fill the gaps and attach trailing periods on abbreviations. + _fill_sub_field_gaps( + annotated, token_index.tokens, + JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR, + ) + _attach_sub_field_trailing_periods( + annotated, token_index.tokens, + JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR, + ) + total = len(field_values) if missed_by_field: missed = sum(missed_by_field.values()) diff --git a/sciencebeam_parser/training/jats/field_extractor.py b/sciencebeam_parser/training/jats/field_extractor.py index bb0eef84..fea173b8 100644 --- a/sciencebeam_parser/training/jats/field_extractor.py +++ b/sciencebeam_parser/training/jats/field_extractor.py @@ -14,6 +14,7 @@ class JatsFieldValue: text: str field_name: str sub_field_name: Optional[str] = None + fallback_text: Optional[str] = None def _element_text(el: etree._Element) -> str: @@ -40,7 +41,6 @@ def _iter_sub_field_values( # Sub-field XPaths for references (relative to each element) _REFERENCE_SUB_FIELDS = [ (JatsSubFieldNames.REFERENCE_LABEL, './label'), - (JatsSubFieldNames.REFERENCE_AUTHOR, './/string-name[not(ancestor::person-group)]'), (JatsSubFieldNames.REFERENCE_ARTICLE_TITLE, './/article-title'), (JatsSubFieldNames.REFERENCE_SOURCE, './/source'), (JatsSubFieldNames.REFERENCE_YEAR, './/year'), @@ -72,6 +72,41 @@ def _iter_sub_field_values( ] +def _iter_reference_author_values( + ref_el: etree._Element, + field_name: str, +) -> Iterator[JatsFieldValue]: + """Yield one JatsFieldValue per in each author person-group, then + 'et al.' if is present. Per-name emission lets the aligner + match each author independently, so a JATS/PDF format mismatch on one + name (e.g. given-names abbreviation style) cannot cut off the next name. + + Each name value carries a fallback_text set to just the surname, so the + aligner can retry with surname-only matching when the full JATS name text + (e.g. 'Maier BE') does not match the PDF representation ('MAIER, B. F.'). + """ + for pg_el in ref_el.xpath('.//person-group[@person-group-type="author"]'): + for name_el in pg_el.xpath('name'): + text = _element_text(name_el) + if not text: + continue + surname_el = name_el.find('surname') + raw_surname = (surname_el.text or '').strip() if surname_el is not None else '' + fallback = raw_surname if raw_surname and raw_surname != text else None + yield JatsFieldValue( + text=text, + field_name=field_name, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text=fallback, + ) + if pg_el.xpath('etal'): + yield JatsFieldValue( + text='et al.', + field_name=field_name, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + ) + + def _local_tag(el: etree._Element) -> str: tag = el.tag if isinstance(tag, str) and tag.startswith('{'): @@ -376,6 +411,7 @@ def _iter_back_reference_values(self, root: etree._Element) -> Iterator[JatsFiel text = _element_text(ref_el) if text: yield JatsFieldValue(text=text, field_name=JatsFieldNames.REFERENCE) + yield from _iter_reference_author_values(ref_el, JatsFieldNames.REFERENCE) yield from _iter_sub_field_values( ref_el, JatsFieldNames.REFERENCE, _REFERENCE_SUB_FIELDS ) diff --git a/tests/training/jats/test_aligner.py b/tests/training/jats/test_aligner.py index 25bbeb4a..af6e0cb4 100644 --- a/tests/training/jats/test_aligner.py +++ b/tests/training/jats/test_aligner.py @@ -7,6 +7,7 @@ LayoutPage, ) from sciencebeam_parser.training.jats.aligner import AlignmentConfig, LayoutDocumentJatsAligner +from sciencebeam_parser.training.jats.text_normalizer import normalize_for_alignment from sciencebeam_parser.training.jats.field_extractor import JatsFieldValue from sciencebeam_parser.training.jats.field_vocab import JatsFieldNames, JatsSubFieldNames @@ -20,7 +21,7 @@ def _fv(text: str, field: str = JatsFieldNames.BODY_SECTION_TITLE, sub: Optional return JatsFieldValue(text=text, field_name=field, sub_field_name=sub) -class TestLayoutDocumentJatsAligner: +class TestLayoutDocumentJatsAligner: # pylint: disable=too-many-public-methods def _align(self, doc, field_values, **kwargs): config = AlignmentConfig(**kwargs) if kwargs else None return LayoutDocumentJatsAligner(config).align(doc, field_values) @@ -367,3 +368,549 @@ def test_heading_label_not_overwritten_by_paragraph_mid_token_match(self): assert annotated.get_token_field(heading_token) == JatsFieldNames.BODY_SECTION_TITLE, ( f'Heading token label was overwritten to {annotated.get_token_field(heading_token)!r}' ) + + def test_per_name_authors_label_last_initial(self): + # Regression: when the whole person-group was one needle, a JATS/PDF + # format mismatch on an earlier name (e.g. "BE" vs "B. F.") could cause + # the SW to terminate before labelling the last author's given-name initial. + # Per-name emission gives each name its own independent SW run. + doc = _make_doc('MAIER, B. F.; BROCKMANN, D. Some title 2020') + fvs = [ + _fv('MAIER, B. F.; BROCKMANN, D. Some title 2020', JatsFieldNames.REFERENCE), + _fv('Maier BE', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + _fv('Brockmann D', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + d_token = next( + (t for t in tokens if normalize_for_alignment(t.text) == 'd'), + None, + ) + assert d_token is not None, 'No token matching "D" found in document' + assert annotated.get_token_sub_field(d_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'"D." token sub-field was {annotated.get_token_sub_field(d_token)!r}, ' + 'expected REFERENCE_AUTHOR' + ) + + def test_trailing_period_after_last_initial_is_labeled(self): + # The PDF tokeniser splits 'D.' into two tokens 'D' and '.'. + # The period is not in the JATS text, so it must be attached by the + # trailing-period pass rather than by SW. + doc = _make_doc('BROCKMANN, D. Some title 2020') + fvs = [ + _fv('BROCKMANN, D. Some title 2020', JatsFieldNames.REFERENCE), + _fv('Brockmann D', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + # '.' immediately after 'D' should be REFERENCE_AUTHOR + d_idx = next( + i for i, t in enumerate(tokens) if normalize_for_alignment(t.text) == 'd' + ) + period_token = tokens[d_idx + 1] + assert period_token.text == '.', 'Expected period token after D' + assert annotated.get_token_sub_field(period_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'Period after D should be REFERENCE_AUTHOR, got ' + f'{annotated.get_token_sub_field(period_token)!r}' + ) + + def test_gap_fill_merges_per_name_author_spans(self): + # Between two per-name author matches, separator tokens (comma, semicolon, + # initials with periods) should be filled in as REFERENCE_AUTHOR so that + # all author tokens form a single element. + doc = _make_doc('Smith A. B.; Jones C. D. Some title 2020') + fvs = [ + _fv('Smith A. B.; Jones C. D. Some title 2020', JatsFieldNames.REFERENCE), + _fv('Smith AB', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + _fv('Jones CD', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + # Every token up to and including the final 'D.' of Jones should be REFERENCE_AUTHOR + smith_idx = next(i for i, t in enumerate(tokens) if t.text == 'Smith') + jones_idx = next(i for i, t in enumerate(tokens) if t.text == 'Jones') + # Tokens between Smith and Jones (exclusive) are separator/initial tokens + between = tokens[smith_idx + 1: jones_idx] + assert len(between) > 0 + assert all( + annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + for t in between + ), 'Gap tokens between Smith and Jones should all be REFERENCE_AUTHOR' + + def test_surname_fallback_labels_initial_mismatch(self): + # When JATS given-names ('BE') do not match the PDF representation + # ('B. F.'), the aligner should fall back to surname-only matching so + # that at least the surname token is labeled REFERENCE_AUTHOR. The gap + # fill then extends to cover the initials between the two surnames. + doc = _make_doc('MAIER, B. F.; BROCKMANN, D. Some title 2020') + fvs = [ + _fv('MAIER, B. F.; BROCKMANN, D. Some title 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Maier BE', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Maier', + ), + _fv('Brockmann D', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + maier_token = next( + (t for t in tokens if normalize_for_alignment(t.text) == 'maier'), None + ) + assert maier_token is not None + assert annotated.get_token_sub_field(maier_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + 'MAIER token should be REFERENCE_AUTHOR via surname fallback' + ) + # Gap fill should also label the tokens between MAIER and BROCKMANN + brockmann_idx = next( + i for i, t in enumerate(tokens) + if normalize_for_alignment(t.text) == 'brockmann' + ) + maier_idx = next( + i for i, t in enumerate(tokens) + if normalize_for_alignment(t.text) == 'maier' + ) + gap_tokens = tokens[maier_idx + 1: brockmann_idx] + assert all( + annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + for t in gap_tokens + ), 'All tokens between MAIER and BROCKMANN should be REFERENCE_AUTHOR via gap fill' + + def test_mid_token_within_gap_block_not_labeled(self): + # When SW matches a short block that starts mid-token (e.g. 't' inside 'staff' + # when searching for author initial 'T'), that token must NOT be labeled. + # Regression: GUARDIAN T matched second 'guardian' in 'guardian staff' + # because 't' inside 'staff' was closer (gap 1) than the real 'T' (gap 2). + doc = _make_doc('GUARDIAN, T. Guardian staff 2020') + fvs = [ + _fv('GUARDIAN, T. Guardian staff 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Guardian T', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Guardian', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + staff_token = next((t for t in tokens if t.text == 'staff'), None) + assert staff_token is not None + assert annotated.get_token_sub_field(staff_token) != JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"staff" token should not be labeled REFERENCE_AUTHOR (mid-word "t" matched by SW)' + ) + # The correct first occurrence 'GUARDIAN' should be labeled + guardian_upper = next((t for t in tokens if t.text == 'GUARDIAN'), None) + assert guardian_upper is not None + assert annotated.get_token_sub_field(guardian_upper) == ( + JatsSubFieldNames.REFERENCE_AUTHOR + ), '"GUARDIAN" (first occurrence) should be labeled via surname fallback' + + def test_mid_token_fallback_extends_with_given_names(self): + # After the mid-token fallback selects the surname-only earlier match, + # the given-names initial should be found within the gap and also labeled. + # Regression: GUARDIAN, T. — only "GUARDIAN" was labeled, not ", T." + doc = _make_doc('GUARDIAN, T. Guardian staff 2020') + fvs = [ + _fv('GUARDIAN, T. Guardian staff 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Guardian T', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Guardian', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + t_token = next((t for t in tokens if t.text == 'T'), None) + assert t_token is not None + assert annotated.get_token_sub_field(t_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"T" initial should be labeled REFERENCE_AUTHOR after surname fallback extension' + ) + + def test_author_gap_fill_bridges_unlabeled_period_before_et_al(self): + # When "Chowell G" and "et al." are separate needles, the "." between "G" and + # "et al." has no label from SW (parent bibl text skips "et al."). The gap fill + # must bridge it so all tokens become one element. + doc = _make_doc('CHOWELL, G. et al. Phenomenological models 2016') + fvs = [ + _fv('CHOWELL G Phenomenological models 2016', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Chowell G', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Chowell', + ), + JatsFieldValue( + text='et al.', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + period_after_g = tokens[3] # 'CHOWELL'=0, ','=1, 'G'=2, '.'=3 + assert period_after_g.text == '.', f'Expected "." got {period_after_g.text!r}' + sub = annotated.get_token_sub_field(period_after_g) + assert sub == JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"." after G should be labeled via gap fill' + ) + + def test_author_gap_fill_bridges_middle_initial_before_et_al(self): + # "Vasconcelos GL" with "et al." as a separate needle — the ". L." between + # "G" and "et al." in the PDF are not in the JATS author text and must be + # bridged by gap fill so all tokens merge into one element. + doc = _make_doc('VASCONCELOS, G. L. et al. Modelling fatality 2020') + fvs = [ + _fv('VASCONCELOS GL Modelling fatality 2020', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Vasconcelos GL', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Vasconcelos', + ), + JatsFieldValue( + text='et al.', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + l_token = next((t for t in tokens if t.text == 'L'), None) + assert l_token is not None + assert annotated.get_token_sub_field(l_token) == JatsSubFieldNames.REFERENCE_AUTHOR, ( + '"L" middle initial should be labeled REFERENCE_AUTHOR via gap fill' + ) + + def test_reference_label_does_not_block_year(self): + # REFERENCE_LABEL "1" can match the "1" in "1987" if the citation number is + # outside the parent bibl match range. The masking that REFERENCE_LABEL adds + # must not block REFERENCE_YEAR from labeling "1987". + doc = _make_doc('Lewis RW Roberts PM Applied Scientific Research 1987') + fvs = [ + _fv('1 Lewis RW Roberts PM Applied Scientific Research 1987', + JatsFieldNames.REFERENCE), + _fv('1', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_LABEL), + _fv('Applied Scientific Research', JatsFieldNames.REFERENCE, + JatsSubFieldNames.REFERENCE_SOURCE), + _fv('1987', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + year_token = next((t for t in tokens if t.text == '1987'), None) + assert year_token is not None + sub = annotated.get_token_sub_field(year_token) + assert sub == JatsSubFieldNames.REFERENCE_YEAR, ( + f'"1987" should be reference-year, got {sub!r}' + ) + + def test_regular_fallback_extends_with_given_names(self): + # When the full JATS name (e.g. "Hsieh Y-H") fails SW quality against the PDF + # (e.g. "HSIEH, Y.-H.") and falls back to surname only ("Hsieh"), the + # given-names initial from the original needle ("Y") should still be labeled. + doc = _make_doc('HSIEH, Y.-H. Richards model 2009') + fvs = [ + _fv('HSIEH, Y.-H. Richards model 2009', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Hsieh Y-H', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Hsieh', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + y_token = next((t for t in tokens if t.text == 'Y'), None) + assert y_token is not None, 'Expected "Y" token in doc' + sub = annotated.get_token_sub_field(y_token) + assert sub == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'"Y" initial should be labeled REFERENCE_AUTHOR after fallback, got {sub!r}' + ) + + def test_masked_sub_field_prevents_duplicate_match(self): + # When the same author name appears twice in a reference, masking ensures + # the second sub-field value matches the second occurrence rather than + # re-matching the first (already-masked) position. + doc = _make_doc('Jones A and Jones A 2020') + fvs = [ + _fv('Jones A and Jones A 2020', JatsFieldNames.REFERENCE), + _fv('Jones A', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + _fv('Jones A', JatsFieldNames.REFERENCE, JatsSubFieldNames.REFERENCE_AUTHOR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + jones_tokens = [t for t in tokens if t.text == 'Jones'] + assert len(jones_tokens) == 2, f'Expected 2 "Jones" tokens, got {len(jones_tokens)}' + assert all( + annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + for t in jones_tokens + ), 'Both "Jones" tokens should be labeled REFERENCE_AUTHOR' + + # ── Comprehensive reference sub-field tests ────────────────────────────── + + def _ref_fv(self, text: str, sub: str, fallback: str = '') -> JatsFieldValue: + return JatsFieldValue( + text=text, + field_name=JatsFieldNames.REFERENCE, + sub_field_name=sub, + fallback_text=fallback or None, + ) + + def test_year_within_doi_not_relabeled_as_year(self): + # When the year digits only appear inside the DOI span, REFERENCE_YEAR must + # not claim them. The DOI (longer needle) is processed first via longest-first + # ordering, its range is masked, and REFERENCE_YEAR finds nothing. + doc = _make_doc( + 'Smith A article title A 10.12345/test.2001.56' + ) + doi = '10.12345/test.2001.56' + fvs = [ + _fv('Smith A article title A ' + doi, JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv(doi, JatsSubFieldNames.REFERENCE_DOI), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + sub_fields = [annotated.get_token_sub_field(t) for t in tokens] + assert JatsSubFieldNames.REFERENCE_YEAR not in sub_fields, ( + 'Year digits inside DOI should not be labeled as REFERENCE_YEAR' + ) + doi_tokens = [ + t.text for t in tokens + if annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_DOI + ] + assert doi_tokens, 'DOI should be labeled' + + def test_all_sub_fields_labeled_in_full_reference(self): + # End-to-end: all standard sub-fields of a single reference are labeled. + # Year appears standalone; DOI does not contain the year digits. + doc = _make_doc( + 'Smith A 1999 article title A source A 11 7 101 10.12345/test.paper.56' + ) + fvs = [ + _fv('Smith A 1999 article title A source A 11 7 101 10.12345/test.paper.56', + JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('source A', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('1999', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('11', JatsSubFieldNames.REFERENCE_VOLUME), + self._ref_fv('7', JatsSubFieldNames.REFERENCE_ISSUE), + self._ref_fv('101', JatsSubFieldNames.REFERENCE_FPAGE), + self._ref_fv('10.12345/test.paper.56', JatsSubFieldNames.REFERENCE_DOI), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('Smith') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('1999') == JatsSubFieldNames.REFERENCE_YEAR + assert by_text.get('source') == JatsSubFieldNames.REFERENCE_SOURCE + assert by_text.get('11') == JatsSubFieldNames.REFERENCE_VOLUME + assert by_text.get('7') == JatsSubFieldNames.REFERENCE_ISSUE + assert by_text.get('101') == JatsSubFieldNames.REFERENCE_FPAGE + + def test_page_number_not_matched_inside_year(self): + # "200" (lpage) must match the exact token "200", not the "200" inside "2020". + doc = _make_doc('Author A 181-200, acesso em 2020') + fvs = [ + _fv('Author A 181-200, acesso em 2020', JatsFieldNames.REFERENCE), + self._ref_fv('Author A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Author'), + self._ref_fv('181', JatsSubFieldNames.REFERENCE_FPAGE), + self._ref_fv('200', JatsSubFieldNames.REFERENCE_LPAGE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('200') == JatsSubFieldNames.REFERENCE_LPAGE + assert by_text.get('2020') != JatsSubFieldNames.REFERENCE_LPAGE + + def test_exact_sw_match_preferred_over_earlier_gap_match(self): + # Needle "2020b" gap-matches "2020" (quality 0.8) before the exact "2020b". + # The exact contiguous match must win even though it appears later. + doc = _make_doc('Author A 2020 title source 2020b') + fvs = [ + _fv('Author A 2020 title source 2020b', JatsFieldNames.REFERENCE), + self._ref_fv('Author A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Author'), + self._ref_fv('2020b', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('2020b') == JatsSubFieldNames.REFERENCE_YEAR + assert by_text.get('2020') != JatsSubFieldNames.REFERENCE_YEAR + + def test_multiple_authors_separated_by_comma_all_labeled(self): + # Both "Smith, A" and "Johnson, B" merged into one span. + doc = _make_doc('Smith A, Johnson B 2001 article title A') + fvs = [ + _fv('Smith A, Johnson B 2001 article title A', JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('Johnson B', JatsSubFieldNames.REFERENCE_AUTHOR, 'Johnson'), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('Smith') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('Johnson') == JatsSubFieldNames.REFERENCE_AUTHOR + # year and title must not bleed into author span + assert by_text.get('2001') == JatsSubFieldNames.REFERENCE_YEAR + + def test_dot_after_initial_included_in_author(self): + # "Smith, A." — the trailing dot after "A" should be labeled as author. + doc = _make_doc('Smith, A. 2001 article title A') + fvs = [ + _fv('Smith, A. 2001 article title A', JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + dot_after_a = next( + (t for i, t in enumerate(tokens) + if t.text == '.' and i > 0 and tokens[i - 1].text == 'A'), + None, + ) + assert dot_after_a is not None, 'Expected "." after "A" token' + sub = annotated.get_token_sub_field(dot_after_a) + assert sub == JatsSubFieldNames.REFERENCE_AUTHOR, ( + f'"." after initial should be REFERENCE_AUTHOR, got {sub!r}' + ) + + def test_et_al_included_in_author_span(self): + # "Smith A et al." — "et al." as separate needle merges with prior author. + doc = _make_doc('Smith A et al. article title A 2001') + fvs = [ + _fv('Smith A et al. article title A 2001', JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('et al.', JatsSubFieldNames.REFERENCE_AUTHOR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + et_token = next((t for t in tokens if t.text == 'et'), None) + al_token = next((t for t in tokens if t.text == 'al'), None) + assert et_token is not None and al_token is not None + assert annotated.get_token_sub_field(et_token) == JatsSubFieldNames.REFERENCE_AUTHOR + assert annotated.get_token_sub_field(al_token) == JatsSubFieldNames.REFERENCE_AUTHOR + + def test_multiple_references_no_cross_contamination(self): + # Sub-fields of reference 1 must not bleed into reference 2's tokens. + doc = _make_doc( + '1 Smith A title one source one 2001', + '2 Jones B title two source two 2002', + ) + ref1_parent = '1 Smith A title one source one 2001' + ref2_parent = '2 Jones B title two source two 2002' + fvs = [ + _fv(ref1_parent, JatsFieldNames.REFERENCE), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('title one', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('source one', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + _fv(ref2_parent, JatsFieldNames.REFERENCE), + self._ref_fv('Jones B', JatsSubFieldNames.REFERENCE_AUTHOR, 'Jones'), + self._ref_fv('title two', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('source two', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('2002', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + by_text = {t.text: annotated.get_token_sub_field(t) for t in tokens} + assert by_text.get('Smith') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('Jones') == JatsSubFieldNames.REFERENCE_AUTHOR + assert by_text.get('2001') == JatsSubFieldNames.REFERENCE_YEAR + assert by_text.get('2002') == JatsSubFieldNames.REFERENCE_YEAR + + def test_label_with_same_digits_as_year_still_gets_year_labeled(self): + # If citation label is "2001" (same as year), length-ordering ensures the + # year sub-field wins its position (same length → original order preserved; + # both appear in different positions in the doc so no conflict). + doc = _make_doc('2001 Smith A title one 2001') + fvs = [ + _fv('2001 Smith A title one 2001', JatsFieldNames.REFERENCE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_LABEL), + self._ref_fv('Smith A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('title one', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + year_tokens = [t for t in tokens if t.text == '2001'] + assert len(year_tokens) == 2 + # At least one of the two "2001" tokens should be labeled as reference-year + sub_fields = [annotated.get_token_sub_field(t) for t in year_tokens] + assert JatsSubFieldNames.REFERENCE_YEAR in sub_fields, ( + f'Expected reference-year in sub-fields, got {sub_fields}' + ) + + def test_varying_spaces_in_author_name_still_matched(self): + # The old tool's test: "Smith ,J .A ." (PDF format with odd spacing) should + # match JATS author "Smith J. A". + doc = _make_doc('Smith ,J .A . 2001 article title A') + fvs = [ + _fv('Smith ,J .A . 2001 article title A', JatsFieldNames.REFERENCE), + self._ref_fv('Smith J. A', JatsSubFieldNames.REFERENCE_AUTHOR, 'Smith'), + self._ref_fv('2001', JatsSubFieldNames.REFERENCE_YEAR), + self._ref_fv('article title A', JatsSubFieldNames.REFERENCE_ARTICLE_TITLE), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + smith = next((t for t in tokens if t.text == 'Smith'), None) + assert smith is not None + assert annotated.get_token_sub_field(smith) == JatsSubFieldNames.REFERENCE_AUTHOR + + def test_sub_field_longer_than_label_wins_position(self): + # Structural regression guard: REFERENCE_YEAR (4 chars) must be processed before + # REFERENCE_LABEL (1 char) regardless of their order in the JATS document. + # Explicit ordering of fvs puts LABEL before YEAR to verify the sort is applied. + doc = _make_doc('Smith A source A 1987') + fvs = [ + _fv('1 Smith A source A 1987', JatsFieldNames.REFERENCE), + self._ref_fv('1', JatsSubFieldNames.REFERENCE_LABEL), # 1 char — before year + self._ref_fv('source A', JatsSubFieldNames.REFERENCE_SOURCE), + self._ref_fv('1987', JatsSubFieldNames.REFERENCE_YEAR), # 4 chars + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + year_tok = next((t for t in tokens if t.text == '1987'), None) + assert year_tok is not None + assert annotated.get_token_sub_field(year_tok) == JatsSubFieldNames.REFERENCE_YEAR, ( + 'REFERENCE_YEAR should win "1987" even when REFERENCE_LABEL is listed first in fvs' + ) + + def test_hyphenated_surname_labeled_when_jats_has_extra_given_name(self): + # When JATS has an extra given name absent from the PDF, the SW produces + # overlapping mid-token blocks; the within-gap guard must advance + # prev_included_end through them to keep the hyphen within gap reach. + doc = _make_doc('Braun, Lena Silva-Braun, Meier') + fvs = [ + _fv('Braun, Lena Silva-Braun, Meier', JatsFieldNames.REFERENCE), + JatsFieldValue( + text='Braun Lena Kristina Silva-Braun', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Braun', + ), + JatsFieldValue( + text='Meier', + field_name=JatsFieldNames.REFERENCE, + sub_field_name=JatsSubFieldNames.REFERENCE_AUTHOR, + fallback_text='Meier', + ), + ] + annotated = self._align(doc, fvs) + tokens = list(doc.iter_all_tokens()) + author_tokens = [ + t.text for t in tokens + if annotated.get_token_sub_field(t) == JatsSubFieldNames.REFERENCE_AUTHOR + ] + assert '-' in author_tokens, ( + f'Hyphen in "Silva-Braun" must be REFERENCE_AUTHOR; got: {author_tokens}' + ) diff --git a/tests/training/jats/test_field_extractor.py b/tests/training/jats/test_field_extractor.py index df48821d..bd403a32 100644 --- a/tests/training/jats/test_field_extractor.py +++ b/tests/training/jats/test_field_extractor.py @@ -244,6 +244,43 @@ def test_extracts_reference_text(self): assert len(refs) >= 1 assert 'A Study' in refs[0].text + def test_extracts_reference_author_subfield(self): + fvs = _field_values_for( + '
' + '' + '' + 'SmithA' + 'JonesB C' + '' + '' + '
' + ) + sub = [v for v in fvs if v.sub_field_name == JatsSubFieldNames.REFERENCE_AUTHOR] + assert len(sub) == 2 + assert sub[0].text == 'Smith A' + assert sub[1].text == 'Jones B C' + # fallback_text is set to surname only for per-name author matching fallback + assert sub[0].fallback_text == 'Smith' + assert sub[1].fallback_text == 'Jones' + + def test_reference_author_includes_et_al(self): + fvs = _field_values_for( + '
' + '' + '' + 'SmithA' + '' + '' + '' + '
' + ) + sub = [v for v in fvs if v.sub_field_name == JatsSubFieldNames.REFERENCE_AUTHOR] + assert len(sub) == 2 + assert sub[0].text == 'Smith A' + assert sub[1].text == 'et al.' + # et al. has no surname fallback + assert sub[1].fallback_text is None + def test_extracts_reference_article_title_subfield(self): fvs = _field_values_for( '
'