diff --git a/sciencebeam_parser/models/citation/training_data.py b/sciencebeam_parser/models/citation/training_data.py index 65210abe..d84b6a1f 100644 --- a/sciencebeam_parser/models/citation/training_data.py +++ b/sciencebeam_parser/models/citation/training_data.py @@ -3,7 +3,8 @@ from lxml import etree -from sciencebeam_parser.document.tei.common import tei_xpath +from sciencebeam_parser.document.semantic_document import SemanticExternalIdentifierTypes +from sciencebeam_parser.document.tei.common import TEI_NS_PREFIX, tei_xpath from sciencebeam_parser.models.training_data import ( AbstractTeiTrainingDataGenerator, AbstractTrainingTeiParser @@ -13,6 +14,15 @@ ) from sciencebeam_parser.utils.xml import get_text_content +# get_post_processed_xml_root tags elements with any detected identifier type +# (not just DOI); all but DOI should still resolve back to the label +_PUBNUM_EXTERNAL_IDENTIFIER_TYPES = ( + SemanticExternalIdentifierTypes.ARXIV, + SemanticExternalIdentifierTypes.PII, + SemanticExternalIdentifierTypes.PMCID, + SemanticExternalIdentifierTypes.PMID, +) + # Matches "388 - 412", "281-282", "1199 -1207" etc. _PAGE_RANGE_RE = re.compile(r'^(\S+)\s*[-–]\s*(\S+)$') @@ -91,3 +101,7 @@ def __init__(self) -> None: ), use_tei_namespace=True ) + for external_identifier_type in _PUBNUM_EXTERNAL_IDENTIFIER_TYPES: + self.label_by_relative_element_path_map[ + ('{}idno[@type="{}"]'.format(TEI_NS_PREFIX, external_identifier_type),) + ] = '' diff --git a/sciencebeam_parser/models/training_data.py b/sciencebeam_parser/models/training_data.py index 33045c41..874bd9cf 100644 --- a/sciencebeam_parser/models/training_data.py +++ b/sciencebeam_parser/models/training_data.py @@ -375,12 +375,20 @@ def get_training_tei_xml_for_model_data_iterable( } +_NON_PATH_ATTRIBUTES = frozenset({'from', 'to'}) + + def _get_tag_expression_for_element(element: etree.ElementBase) -> str: - if not element.attrib: + attrib = { + key: value + for key, value in element.attrib.items() + if key not in _NON_PATH_ATTRIBUTES + } + if not attrib: return element.tag - if len(element.attrib) > 1: + if len(attrib) > 1: raise ValueError('only supporting up to one attribute') - key, value = list(element.attrib.items())[0] + key, value = list(attrib.items())[0] return '{tag}[@{key}="{value}"]'.format(tag=element.tag, key=key, value=value) diff --git a/tests/models/citation/training_data_test.py b/tests/models/citation/training_data_test.py index 789e6838..3119e98e 100644 --- a/tests/models/citation/training_data_test.py +++ b/tests/models/citation/training_data_test.py @@ -446,6 +446,55 @@ def test_should_parse_single_label_with_multiple_tokens_on_multiple_lines(self): (TOKEN_4, 'I-') ]] + @pytest.mark.parametrize( + "external_identifier_type", + [ + SemanticExternalIdentifierTypes.ARXIV, + SemanticExternalIdentifierTypes.PII, + SemanticExternalIdentifierTypes.PMCID, + SemanticExternalIdentifierTypes.PMID + ] + ) + def test_should_parse_non_doi_idno_type_as_pubnum( + self, + external_identifier_type: str + ): + # get_post_processed_xml_root tags -derived elements with the + # detected identifier type (e.g. PMCID); parsing must still recover + tei_root = _get_training_tei_with_references([ + TEI_E('bibl', *[ + TEI_E('idno', {'type': external_identifier_type}, TOKEN_1, TEI_E('lb')), + '\n' + ]) + ]) + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-') + ]] + + def test_should_round_trip_pubnum_with_detected_external_identifier_type(self): + label_and_layout_line_list = [ + ('', get_next_layout_line_for_text('PMC1234567')) + ] + labeled_model_data_list = get_labeled_model_data_list( + label_and_layout_line_list, + data_generator=get_data_generator() + ) + xml_root = get_training_tei_xml_for_model_data_iterable( + labeled_model_data_list + ) + assert get_tei_xpath_text_content_list( + xml_root, f'{BIBL_XPATH}/tei:idno[@type="{SemanticExternalIdentifierTypes.PMCID}"]' + ) == ['PMC1234567'] + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + xml_root + ) + assert tag_result == [[ + ('PMC1234567', 'B-') + ]] + @pytest.mark.parametrize( "tei_label,element_path", list(TRAINING_XML_ELEMENT_PATH_BY_LABEL.items()) diff --git a/tests/models/training_data_test.py b/tests/models/training_data_test.py index c336f6eb..d4ddb323 100644 --- a/tests/models/training_data_test.py +++ b/tests/models/training_data_test.py @@ -1,6 +1,7 @@ import logging from typing import Sequence, Union +import pytest from lxml import etree from lxml.builder import E @@ -45,7 +46,8 @@ def test_should_return_false_for_different_parent(self): TRAINING_XML_ELEMENT_PATH_BY_LABEL = { '': ROOT_TRAINING_XML_ELEMENT_PATH + ['head'], - '': ROOT_TRAINING_XML_ELEMENT_PATH + ['p'] + '': ROOT_TRAINING_XML_ELEMENT_PATH + ['p'], + '': ROOT_TRAINING_XML_ELEMENT_PATH + ['biblScope[@unit="page"]'] } @@ -144,6 +146,31 @@ def test_should_parse_multi_line_labelled_token_with_diff_line_meta(self): != labeled_layout_tokens[1].layout_token.line_meta ) + def test_should_ignore_from_and_to_attributes_when_matching_element_path(self): + # e.g. added by + # CitationTeiTrainingDataGenerator.get_post_processed_xml_root for sciencebeam-judge; + # from/to must not affect which label the element resolves to + tei_root = _get_training_tei_with_text([ + E('biblScope', {'unit': 'page', 'from': '1619', 'to': '1621'}, TOKEN_1, E('lb')), + '\n' + ]) + tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-') + ]] + + def test_should_raise_error_for_element_with_more_than_one_matching_attribute(self): + tei_root = _get_training_tei_with_text([ + E('biblScope', {'unit': 'page', 'other': 'value'}, TOKEN_1, E('lb')), + '\n' + ]) + with pytest.raises(ValueError): + SampleTrainingTeiParser().parse_training_tei_to_tag_result( + tei_root + ) + def test_unannotated_tokens_after_annotated_span_use_O(self): # training_data.py emits plain 'O' for all unannotated tokens regardless of # position. The I- GROBID convention is applied later by