Default to UTF-8 when no charset in emails

Author: martinburchell

cardinal_pythonlib/extract_text.py

@@ -1286,7 +1286,7 @@ def _get_email_content(
     if ext is not None and ext in ext_map:
         content = message.get_content()
         if isinstance(content, str):
-            charset = message["Content-Type"].params["charset"]
+            charset = message["Content-Type"].params.get("charset", "utf-8")
             blob = content.encode(charset)
         elif isinstance(content, EmailMessage):
             blob = content.as_bytes()

cardinal_pythonlib/tests/extract_text_tests.py

@@ -424,6 +424,33 @@ def test_eml_latin1_html_decoded_correctly(self) -> None:
 
         self.assertIn("Café", text)
 
+    def test_eml_with_no_charset_converted(self) -> None:
+        text_content = self.fake.paragraph(nb_sentences=10)
+
+        content = f"""From: bar@example.org
+Subject: No charset
+To: foo@example.org
+Mime-Version: 1.0
+Content-Type: multipart/mixed;boundary="==="
+
+--===
+Content-Type: text/plain
+
+{text_content}
+
+--===--
+
+"""
+
+        message = message_from_string(content, policy=policy.default)
+        blob = message.as_bytes()
+
+        text = document_to_text(
+            blob=blob, extension=".eml", config=self.config
+        )
+
+        self.assertIn(text_content, text)
+
     def test_unsupported_converted(self) -> None:
         with mock.patch.multiple(
             "cardinal_pythonlib.extract_text.subprocess",