Fix generate_delft_data round-trip for citation model training TEI

sciencebeam_parser/models/citation/training_data.py

@@ -3,7 +3,8 @@
 
 from lxml import etree
 
-from sciencebeam_parser.document.tei.common import tei_xpath
+from sciencebeam_parser.document.semantic_document import SemanticExternalIdentifierTypes
+from sciencebeam_parser.document.tei.common import TEI_NS_PREFIX, tei_xpath
 from sciencebeam_parser.models.training_data import (
     AbstractTeiTrainingDataGenerator,
     AbstractTrainingTeiParser
@@ -13,6 +14,15 @@
 )
 from sciencebeam_parser.utils.xml import get_text_content
 
+# get_post_processed_xml_root tags <idno> elements with any detected identifier type
+# (not just DOI); all but DOI should still resolve back to the <pubnum> label
+_PUBNUM_EXTERNAL_IDENTIFIER_TYPES = (
+    SemanticExternalIdentifierTypes.ARXIV,
+    SemanticExternalIdentifierTypes.PII,
+    SemanticExternalIdentifierTypes.PMCID,
+    SemanticExternalIdentifierTypes.PMID,
+)
+
 # Matches "388 - 412", "281-282", "1199 -1207" etc.
 _PAGE_RANGE_RE = re.compile(r'^(\S+)\s*[-–]\s*(\S+)$')
 
@@ -91,3 +101,7 @@ def __init__(self) -> None:
             ),
             use_tei_namespace=True
         )
+        for external_identifier_type in _PUBNUM_EXTERNAL_IDENTIFIER_TYPES:
+            self.label_by_relative_element_path_map[
+                ('{}idno[@type="{}"]'.format(TEI_NS_PREFIX, external_identifier_type),)
+            ] = '<pubnum>'

sciencebeam_parser/models/training_data.py

@@ -375,12 +375,20 @@ def get_training_tei_xml_for_model_data_iterable(
 }
 
 
+_NON_PATH_ATTRIBUTES = frozenset({'from', 'to'})
+
+
 def _get_tag_expression_for_element(element: etree.ElementBase) -> str:
-    if not element.attrib:
+    attrib = {
+        key: value
+        for key, value in element.attrib.items()
+        if key not in _NON_PATH_ATTRIBUTES
+    }
+    if not attrib:
         return element.tag
-    if len(element.attrib) > 1:
+    if len(attrib) > 1:
         raise ValueError('only supporting up to one attribute')
-    key, value = list(element.attrib.items())[0]
+    key, value = list(attrib.items())[0]
     return '{tag}[@{key}="{value}"]'.format(tag=element.tag, key=key, value=value)
 
 

tests/models/citation/training_data_test.py

@@ -446,6 +446,55 @@ def test_should_parse_single_label_with_multiple_tokens_on_multiple_lines(self):
             (TOKEN_4, 'I-<author>')
         ]]
 
+    @pytest.mark.parametrize(
+        "external_identifier_type",
+        [
+            SemanticExternalIdentifierTypes.ARXIV,
+            SemanticExternalIdentifierTypes.PII,
+            SemanticExternalIdentifierTypes.PMCID,
+            SemanticExternalIdentifierTypes.PMID
+        ]
+    )
+    def test_should_parse_non_doi_idno_type_as_pubnum(
+        self,
+        external_identifier_type: str
+    ):
+        # get_post_processed_xml_root tags <pubnum>-derived <idno> elements with the
+        # detected identifier type (e.g. PMCID); parsing must still recover <pubnum>
+        tei_root = _get_training_tei_with_references([
+            TEI_E('bibl', *[
+                TEI_E('idno', {'type': external_identifier_type}, TOKEN_1, TEI_E('lb')),
+                '\n'
+            ])
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<pubnum>')
+        ]]
+
+    def test_should_round_trip_pubnum_with_detected_external_identifier_type(self):
+        label_and_layout_line_list = [
+            ('<pubnum>', get_next_layout_line_for_text('PMC1234567'))
+        ]
+        labeled_model_data_list = get_labeled_model_data_list(
+            label_and_layout_line_list,
+            data_generator=get_data_generator()
+        )
+        xml_root = get_training_tei_xml_for_model_data_iterable(
+            labeled_model_data_list
+        )
+        assert get_tei_xpath_text_content_list(
+            xml_root, f'{BIBL_XPATH}/tei:idno[@type="{SemanticExternalIdentifierTypes.PMCID}"]'
+        ) == ['PMC1234567']
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            xml_root
+        )
+        assert tag_result == [[
+            ('PMC1234567', 'B-<pubnum>')
+        ]]
+
     @pytest.mark.parametrize(
         "tei_label,element_path",
         list(TRAINING_XML_ELEMENT_PATH_BY_LABEL.items())

tests/models/training_data_test.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Sequence, Union
 
+import pytest
 from lxml import etree
 from lxml.builder import E
 
@@ -45,7 +46,8 @@ def test_should_return_false_for_different_parent(self):
 
 TRAINING_XML_ELEMENT_PATH_BY_LABEL = {
     '<head>': ROOT_TRAINING_XML_ELEMENT_PATH + ['head'],
-    '<paragraph>': ROOT_TRAINING_XML_ELEMENT_PATH + ['p']
+    '<paragraph>': ROOT_TRAINING_XML_ELEMENT_PATH + ['p'],
+    '<page>': ROOT_TRAINING_XML_ELEMENT_PATH + ['biblScope[@unit="page"]']
 }
 
 
@@ -144,6 +146,31 @@ def test_should_parse_multi_line_labelled_token_with_diff_line_meta(self):
             != labeled_layout_tokens[1].layout_token.line_meta
         )
 
+    def test_should_ignore_from_and_to_attributes_when_matching_element_path(self):
+        # e.g. <biblScope unit="page" from="1619" to="1621"> added by
+        # CitationTeiTrainingDataGenerator.get_post_processed_xml_root for sciencebeam-judge;
+        # from/to must not affect which label the element resolves to
+        tei_root = _get_training_tei_with_text([
+            E('biblScope', {'unit': 'page', 'from': '1619', 'to': '1621'}, TOKEN_1, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<page>')
+        ]]
+
+    def test_should_raise_error_for_element_with_more_than_one_matching_attribute(self):
+        tei_root = _get_training_tei_with_text([
+            E('biblScope', {'unit': 'page', 'other': 'value'}, TOKEN_1, E('lb')),
+            '\n'
+        ])
+        with pytest.raises(ValueError):
+            SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+                tei_root
+            )
+
     def test_unannotated_tokens_after_annotated_span_use_O(self):
         # training_data.py emits plain 'O' for all unannotated tokens regardless of
         # position. The I-<other> GROBID convention is applied later by