Merge pull request #38 from RudolfCardinal/outlook-msg-text-extraction

Support text extraction from Outlook msg files

Author: martinburchell

cardinal_pythonlib/extract_text.py

@@ -100,13 +100,15 @@
     Iterator,
     List,
     Optional,
+    TYPE_CHECKING,
 )
 from xml.etree import ElementTree as ElementTree
 import zipfile
 
 import bs4
 import chardet
 from chardet.universaldetector import UniversalDetector
+from extract_msg import openMsg
 import pdfminer  # pip install pdfminer.six
 import pdfminer.pdfinterp
 import pdfminer.converter
@@ -117,6 +119,8 @@
 
 from cardinal_pythonlib.logs import get_brace_style_log_with_null_handler
 
+if TYPE_CHECKING:
+    from extract_msg import MSGFile
 
 log = get_brace_style_log_with_null_handler(__name__)
 
@@ -1320,6 +1324,60 @@ def _get_email_content(
     return None
 
 
+# =============================================================================
+# MSG (Outlook binary format)
+# =============================================================================
+
+
+def convert_msg_to_text(
+    filename: str = None,
+    blob: bytes = None,
+    config: TextProcessingConfig = _DEFAULT_CONFIG,
+) -> str:
+    message_content_list: list[str] = []
+
+    if not filename and blob is None:
+        raise ValueError("convert_msg_to_text: no filename and no blob")
+    if filename and blob:
+        raise ValueError(
+            "convert_msg_to_text: specify either filename or blob"
+        )
+
+    if blob is not None:
+        filename_or_blob = blob
+    else:
+        filename_or_blob = filename
+
+    message = openMsg(filename_or_blob, delayAttachments=False)
+    for message_content in _gen_msg_content(message, config=config):
+        if message_content_list is not None:
+            message_content_list.append(message_content)
+
+    text = "\n".join(message_content_list)
+
+    return text
+
+
+def _gen_msg_content(
+    message: "MSGFile", config: TextProcessingConfig
+) -> Generator[Optional[str], None, None]:
+    if message.body is not None:
+        yield message.body
+    elif message.htmlBody is not None:
+        yield document_to_text(
+            blob=message.htmlBody, extension=".htm", config=config
+        )
+
+    for attachment in message.attachments:
+        # null termination seen in the real world
+        # https://github.com/TeamMsgExtractor/msg-extractor/issues/464
+        ext = attachment.extension.replace("\x00", "")
+        if ext is not None and ext in ext_map:
+            yield document_to_text(
+                blob=attachment.data, extension=ext, config=config
+            )
+
+
 # =============================================================================
 # Anything
 # =============================================================================
@@ -1371,12 +1429,7 @@ def availability_anything() -> bool:
     ".htm": {CONVERTER: convert_html_to_text, AVAILABILITY: True},
     ".html": {CONVERTER: convert_html_to_text, AVAILABILITY: True},
     ".log": {CONVERTER: get_file_contents_text, AVAILABILITY: True},
-    # .msg is often Outlook binary, not text
-    #
-    # '.msg': {
-    #     CONVERTER: get_file_contents_text,
-    #     AVAILABILITY: True,
-    # },
+    ".msg": {CONVERTER: convert_msg_to_text, AVAILABILITY: True},
     ".odt": {CONVERTER: convert_odt_to_text, AVAILABILITY: True},
     ".pdf": {CONVERTER: convert_pdf_to_txt, AVAILABILITY: availability_pdf},
     ".rtf": {CONVERTER: convert_rtf_to_text, AVAILABILITY: availability_rtf},

cardinal_pythonlib/tests/extract_text_tests.py

@@ -27,6 +27,7 @@
 
 from email import message_from_string, policy
 from email.message import EmailMessage
+from io import BytesIO
 import os
 import subprocess
 from tempfile import mkdtemp, NamedTemporaryFile
@@ -44,18 +45,26 @@
 from faker_file.providers.xml_file import XmlFileProvider
 
 from cardinal_pythonlib.extract_text import (
+    convert_msg_to_text,
     document_to_text,
     TextProcessingConfig,
     update_external_tools,
 )
 
 
-class DocumentToTextTests(TestCase):
+class ExtractTextTestCase(TestCase):
     def setUp(self) -> None:
+        self.config = TextProcessingConfig()
+        self.fake = Faker("en-US")  # en-US to avoid Lorem Ipsum from en-GB
+        self.fake.seed_instance(12345)
+
+
+class DocumentToTextTests(ExtractTextTestCase):
+    def setUp(self) -> None:
+        super().setUp()
         self.empty_dir = mkdtemp()
 
         self._replace_external_tools_with_fakes()
-        self.config = TextProcessingConfig()
         self._create_mock_objects()
         self._register_faker_providers()
 
@@ -69,8 +78,6 @@ def _create_mock_objects(self) -> None:
         )
 
     def _register_faker_providers(self) -> None:
-        self.fake = Faker("en-US")  # To avoid Lorem Ipsum
-        self.fake.seed_instance(12345)
         self.fake.add_provider(DocxFileProvider)
         self.fake.add_provider(EmlFileProvider)
         self.fake.add_provider(OdtFileProvider)
@@ -109,13 +116,13 @@ def test_raises_when_filename_empty(self) -> None:
 
     def test_raises_when_filename_and_blob(self) -> None:
         with self.assertRaises(ValueError) as cm:
-            document_to_text(filename="foo", blob="bar")
+            document_to_text(filename="foo", blob=b"bar")
 
         self.assertIn("specify either filename or blob", str(cm.exception))
 
     def test_raises_when_blob_but_no_extension(self) -> None:
         with self.assertRaises(ValueError) as cm:
-            document_to_text(blob="bar")
+            document_to_text(blob=b"bar")
 
         self.assertIn("need extension hint for blob", str(cm.exception))
 
@@ -586,3 +593,125 @@ def test_unsupported_will_be_converted_with_strings(self) -> None:
             ),
         ]
         self.mock_popen.assert_has_calls(expected_calls)
+
+
+class ConvertMsgToTextTests(ExtractTextTestCase):
+    # There is no easy way to create test Outlook msg files and we don't want
+    # to store real ones so we mock the interface to extract-msg and assume the
+    # library itself is working correctly.
+    def setUp(self) -> None:
+        super().setUp()
+        self.dummy_filename = "dummy_filename.msg"
+        self.dummy_blob = b"dummy blob"
+
+    def test_raises_when_no_filename_or_blob(self) -> None:
+        with self.assertRaises(ValueError) as cm:
+            convert_msg_to_text()
+
+        self.assertIn("no filename and no blob", str(cm.exception))
+
+    def test_raises_when_filename_and_blob(self) -> None:
+        with self.assertRaises(ValueError) as cm:
+            convert_msg_to_text(filename="foo", blob=b"bar")
+
+        self.assertIn("specify either filename or blob", str(cm.exception))
+
+    def test_blob_passed_to_openmsg(self) -> None:
+        content = self.fake.paragraph(nb_sentences=10)
+
+        mock_msgfile = mock.Mock(body=content, htmlBody=None, attachments=[])
+        mock_openmsg = mock.Mock(return_value=mock_msgfile)
+        with mock.patch.multiple(
+            "cardinal_pythonlib.extract_text",
+            openMsg=mock_openmsg,
+        ):
+            convert_msg_to_text(blob=self.dummy_blob, config=self.config)
+
+        expected_calls = [mock.call(self.dummy_blob, delayAttachments=False)]
+        mock_openmsg.assert_has_calls(expected_calls)
+
+    def test_file_passed_to_openmsg(self) -> None:
+        content = self.fake.paragraph(nb_sentences=10)
+
+        mock_msgfile = mock.Mock(body=content, htmlBody=None, attachments=[])
+        mock_openmsg = mock.Mock(return_value=mock_msgfile)
+        with mock.patch.multiple(
+            "cardinal_pythonlib.extract_text",
+            openMsg=mock_openmsg,
+        ):
+            convert_msg_to_text(
+                filename=self.dummy_filename, config=self.config
+            )
+
+        expected_calls = [
+            mock.call(self.dummy_filename, delayAttachments=False)
+        ]
+        mock_openmsg.assert_has_calls(expected_calls)
+
+    def test_text_body_converted(self) -> None:
+        content = self.fake.paragraph(nb_sentences=10)
+
+        mock_msgfile = mock.Mock(body=content, htmlBody=None, attachments=[])
+        mock_openmsg = mock.Mock(return_value=mock_msgfile)
+        with mock.patch.multiple(
+            "cardinal_pythonlib.extract_text",
+            openMsg=mock_openmsg,
+        ):
+            converted = convert_msg_to_text(
+                filename=self.dummy_filename, config=self.config
+            )
+
+        self.assertEqual(converted, content)
+
+    def test_html_body_converted(self) -> None:
+        content = self.fake.paragraph(nb_sentences=10)
+        html = f"""
+<!DOCTYPE html>
+<html>
+<head>
+</head>
+<body>
+{content}
+</body>
+</html>
+"""
+
+        mock_msgfile = mock.Mock(
+            body=None, htmlBody=html.encode("utf-8"), attachments=[]
+        )
+        mock_openmsg = mock.Mock(return_value=mock_msgfile)
+        with mock.patch.multiple(
+            "cardinal_pythonlib.extract_text",
+            openMsg=mock_openmsg,
+        ):
+            converted = convert_msg_to_text(
+                filename=self.dummy_filename, config=self.config
+            )
+
+        self.assertEqual(converted.strip(), content)
+
+    def test_attachment_converted(self) -> None:
+        self.fake.add_provider(DocxFileProvider)
+
+        dummy_filename = "dummy_filename.msg"
+
+        content = self.fake.paragraph(nb_sentences=10)
+        docx = self.fake.docx_file(content=content, raw=True)
+        mock_attachment = mock.Mock(
+            # null termination seen in the real world
+            # https://github.com/TeamMsgExtractor/msg-extractor/issues/464
+            extension=".docx\x00",
+            data=BytesIO(docx).read(),
+        )
+        mock_msgfile = mock.Mock(
+            body=None, htmlBody=None, attachments=[mock_attachment]
+        )
+        mock_openmsg = mock.Mock(return_value=mock_msgfile)
+        with mock.patch.multiple(
+            "cardinal_pythonlib.extract_text",
+            openMsg=mock_openmsg,
+        ):
+            self.config.width = 0
+            converted = convert_msg_to_text(dummy_filename, config=self.config)
+
+        self.assertEqual(converted.strip(), content)

docs/source/changelog.rst

@@ -902,3 +902,9 @@ Quick links:
 - Add support for ``.eml`` files with attachments processed by supported
   document converters (``.docx``, ``.pdf``, ``.odt`` etc.) to
   :func:`cardinal_pythonlib.extract_text.document_to_text`.
+
+**2.1.1**
+
+- Add support for Outlook ``.msg`` files with attachments processed by supported
+  document converters (``.docx``, ``.pdf``, ``.odt`` etc.) to
+  :func:`cardinal_pythonlib.extract_text.document_to_text`.

setup.py

@@ -53,6 +53,7 @@
     "beautifulsoup4",  # "import bs4" or "from bs4 import ..."
     "chardet>=5.0.0",
     "colorlog",
+    "extract_msg",
     "isodate>=0.5.4",
     "numba",  # just-in-time compilation
     "numpy>=1.20.0,<2.0",  # 1.20.0 required for numpy.typing