Handle invalid surrogate characters in HTML conversion

martinburchell · martinburchell · commit 51e9295292f5 · 2025-05-13T16:51:18.000+01:00
diff --git a/cardinal_pythonlib/extract_text.py b/cardinal_pythonlib/extract_text.py
@@ -1149,7 +1149,15 @@ def convert_html_to_text(
 
     with get_filelikeobject(filename, blob) as fp:
         soup = bs4.BeautifulSoup(fp, "html.parser")
-        return soup.get_text()
+
+        # In the real world we can end up with UTF-16 characters embedded as
+        # numbered entities in Windows-1252 encoded HTML such as
+        # &#55357;&#56898; "Slightly smiling face". Replacing these here
+        # avoids "UnicodeEncodeError: 'utf-8' codec can't encode characters in
+        # position ... surrogates not allowed".
+        text = soup.get_text().encode(errors="replace").decode()
+
+        return text
 
 
 # =============================================================================
diff --git a/cardinal_pythonlib/tests/extract_text_tests.py b/cardinal_pythonlib/tests/extract_text_tests.py
@@ -531,6 +531,36 @@ def test_eml_with_illegal_multibyte_sequence_replaced(self) -> None:
 
         self.assertEqual(text.strip(), "??")
 
+    def test_eml_invalid_surrogate_characters_replaced(self) -> None:
+        content = """From: bar@example.org
+Subject: Invalid surrogate characters
+To: foo@example.org
+Mime-Version: 1.0
+Content-Type: multipart/mixed;boundary="==="
+
+--===
+Content-Type: text/html; charset="windows-1252"
+Content-Transfer-Encoding: quoted-printable
+
+<html><head>
+<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3DWindows-1=
+252">
+</head>
+<body>
+&#55357;&#56898;
+</body>
+</html>
+--===--
+"""
+        message = message_from_string(content, policy=policy.default)
+        blob = message.as_bytes()
+
+        text = document_to_text(
+            blob=blob, extension=".eml", config=self.config
+        )
+
+        self.assertEqual(text.strip(), "??")
+
     def test_unsupported_converted(self) -> None:
         with mock.patch.multiple(
             "cardinal_pythonlib.extract_text.subprocess",
