fix(webvtt): deal with HTML entities in cue text spans

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

Author: ceberam

docling_core/types/doc/webvtt.py

@@ -98,10 +98,23 @@ class _WebVTTCueTextSpan(BaseModel):
     text: str
     span_type: Literal["text"] = "text"
 
+    _valid_entities: ClassVar[set] = {"amp", "lt", "gt", "lrm", "rlm", "nbsp"}
+    _entity_pattern: ClassVar[re.Pattern] = re.compile(r"&([a-zA-Z0-9]+);")
+
     @field_validator("text", mode="after")
     @classmethod
     def validate_text(cls, value: str) -> str:
-        if any(ch in value for ch in {"\n", "\r", "&", "<"}):
+        for match in cls._entity_pattern.finditer(value):
+            entity = match.group(1)
+            if entity not in cls._valid_entities:
+                raise ValueError(
+                    f"Cue text span contains an invalid HTML entity: &{entity};"
+                )
+        if "&" in re.sub(cls._entity_pattern, "", value):
+            raise ValueError(
+                "Found '&' not part of a valid entity in the cue text span"
+            )
+        if any(ch in value for ch in {"\n", "\r", "<"}):
             raise ValueError("Cue text span contains invalid characters")
         if len(value) == 0:
             raise ValueError("Cue text span cannot be empty")

test/test_webvtt.py

@@ -96,6 +96,12 @@ def test_vtt_cue_commponents():
     invalid_text = "This cue text span contains &."
     with pytest.raises(ValidationError):
         _WebVTTCueTextSpan(text=invalid_text)
+    invalid_text = "An invalid &foo; entity"
+    with pytest.raises(ValidationError):
+        _WebVTTCueTextSpan(text=invalid_text)
+    valid_text = "My favorite book is Pride & Prejudice"
+    span = _WebVTTCueTextSpan(text=valid_text)
+    assert span.text == valid_text
 
     """Test with text containing less-than sign."""
     invalid_text = "This cue text span contains <."