diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py index 7717f62d8..ca060cabe 100644 --- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py +++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py @@ -1,8 +1,13 @@ +import io import sys from typing import Any, Union, BinaryIO + +from charset_normalizer import from_bytes + from .._stream_info import StreamInfo from .._base_converter import DocumentConverter, DocumentConverterResult from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE +from ._html_converter import HtmlConverter # Try loading optional (but in this case, required) dependencies # Save reporting of any exceptions for later @@ -26,9 +31,14 @@ class OutlookMsgConverter(DocumentConverter): Uses the olefile package to parse the .msg file structure and extract: - Email headers (From, To, Subject) - - Email body content + - Email body content, falling back to the HTML body for messages that + carry no plain-text part """ + def __init__(self): + super().__init__() + self._html_converter = HtmlConverter() + def accepts( self, file_stream: BinaryIO, @@ -100,7 +110,7 @@ def convert( # Get headers headers = { - "From": self._get_stream_data(msg, "__substg1.0_0C1F001F"), + "From": self._get_sender(msg), "To": self._get_stream_data(msg, "__substg1.0_0E04001F"), "Subject": self._get_stream_data(msg, "__substg1.0_0037001F"), } @@ -112,8 +122,11 @@ def convert( md_content += "\n## Content\n\n" - # Get email body + # Get email body. HTML-only messages leave PR_BODY empty and carry + # their content in PR_HTML instead. body = self._get_stream_data(msg, "__substg1.0_1000001F") + if not body: + body = self._get_html_body(msg) if body: md_content += body @@ -124,8 +137,87 @@ def convert( title=headers.get("Subject"), ) + def _get_sender(self, msg: Any) -> Union[str, None]: + """Helper to resolve the sender's address, preferring SMTP over an Exchange DN. + + PR_SENDER_EMAIL_ADDRESS (0x0C1F) only holds an email address when + PR_SENDER_ADDRTYPE (0x0C1E) is "SMTP". Messages sent through Exchange + set the address type to "EX" and store a Distinguished Name there + instead (e.g. "/O=EXCHANGELABS/OU=.../CN=RECIPIENTS/CN=..."), keeping + the real address in PR_SENDER_SMTP_ADDRESS (0x5D01). + """ + # PR_SENDER_SMTP_ADDRESS, then PR_SENT_REPRESENTING_SMTP_ADDRESS. + for address_stream in ("__substg1.0_5D01001F", "__substg1.0_5D02001F"): + address = self._get_stream_data(msg, address_stream) + if address: + return address + + # PR_SENDER_EMAIL_ADDRESS / PR_SENT_REPRESENTING_EMAIL_ADDRESS, each + # usable only when its address type is not the Exchange DN form. + for address_stream, addrtype_stream in ( + ("__substg1.0_0C1F001F", "__substg1.0_0C1E001F"), + ("__substg1.0_0065001F", "__substg1.0_0064001F"), + ): + address = self._get_stream_data(msg, address_stream) + addrtype = self._get_stream_data(msg, addrtype_stream) or "" + if address and addrtype.upper() != "EX": + return address + + # Nothing better on file: report the DN rather than dropping the header. + return self._get_stream_data(msg, "__substg1.0_0C1F001F") + + def _get_html_body(self, msg: Any) -> Union[str, None]: + """Helper to convert the HTML body (PR_HTML, 0x1013) to markdown. + + Used for messages that carry no plain-text body, where reading only + PR_BODY would silently drop the entire message content. + """ + charset: Union[str, None] = None + + # PR_HTML is normally a binary stream; some clients write the string variant. + html_bytes = self._read_stream(msg, "__substg1.0_10130102") + if not html_bytes: + html = self._get_stream_data(msg, "__substg1.0_1013001F") + if not html: + return None + html_bytes = html.encode("utf-8") + charset = "utf-8" + else: + # The stream carries no encoding of its own, so detect it rather + # than assuming UTF-8 and mangling non-ASCII text. + detected = from_bytes(html_bytes).best() + if detected is not None: + charset = detected.encoding + + return self._html_converter.convert( + io.BytesIO(html_bytes), + StreamInfo(mimetype="text/html", extension=".html", charset=charset), + ).markdown + def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]: """Helper to safely extract and decode stream data from the MSG file.""" + data = self._read_stream(msg, stream_path) + if data is None: + return None + + # Try UTF-16 first (common for .msg files) + try: + return data.decode("utf-16-le").strip() + except UnicodeDecodeError: + # Fall back to UTF-8 + try: + return data.decode("utf-8").strip() + except UnicodeDecodeError: + # Last resort - ignore errors + return data.decode("utf-8", errors="ignore").strip() + + def _read_stream(self, msg: Any, stream_path: str) -> Union[bytes, None]: + """Helper to safely read the raw bytes of a stream from the MSG file. + + Binary properties (such as PR_HTML) must not go through + `_get_stream_data`: arbitrary bytes rarely fail a UTF-16 decode, so + they would come back as mojibake instead of raising. + """ assert olefile is not None assert isinstance( msg, olefile.OleFileIO @@ -133,17 +225,8 @@ def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]: try: if msg.exists(stream_path): - data = msg.openstream(stream_path).read() - # Try UTF-16 first (common for .msg files) - try: - return data.decode("utf-16-le").strip() - except UnicodeDecodeError: - # Fall back to UTF-8 - try: - return data.decode("utf-8").strip() - except UnicodeDecodeError: - # Last resort - ignore errors - return data.decode("utf-8", errors="ignore").strip() + data: bytes = msg.openstream(stream_path).read() + return data except Exception: pass return None diff --git a/packages/markitdown/tests/test_outlook_msg.py b/packages/markitdown/tests/test_outlook_msg.py new file mode 100644 index 000000000..2414c4fc5 --- /dev/null +++ b/packages/markitdown/tests/test_outlook_msg.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 -m pytest +"""Tests for .msg files saved from Outlook 365 / Exchange. + +Two properties of such files break the naive reads: + +* ``PR_SENDER_EMAIL_ADDRESS`` (0x0C1F) holds an Exchange Distinguished Name + rather than an email address whenever ``PR_SENDER_ADDRTYPE`` (0x0C1E) is + ``EX``; the address lives in ``PR_SENDER_SMTP_ADDRESS`` (0x5D01). +* HTML-only messages leave ``PR_BODY`` (0x1000) empty and store their content + in ``PR_HTML`` (0x1013), so reading only ``PR_BODY`` drops the whole message. + +A .msg file is an OLE2 compound file and nothing in the dependency set can +write that container, so these tests serve the MAPI streams through a stand-in +for ``olefile.OleFileIO`` instead of a binary fixture. +""" +import io + +import olefile + +from markitdown._stream_info import StreamInfo +from markitdown.converters._outlook_msg_converter import OutlookMsgConverter + +EXCHANGE_DN = ( + "/O=EXCHANGELABS/OU=EXCHANGE ADMINISTRATIVE GROUP" + " (FYDIBOHF23SPDLT)/CN=RECIPIENTS/CN=ab12cd34" +) + + +class _FakeMsg(olefile.OleFileIO): + """Serves a fixed set of MSG streams. ``_get_stream_data`` asserts the type.""" + + def __init__(self, streams): + self._streams = streams + # OleFileIO.__del__ reads this; we deliberately skip super().__init__. + self._we_opened_fp = False + + def exists(self, path): + return path in self._streams + + def openstream(self, path): + return io.BytesIO(self._streams[path]) + + def close(self): + pass + + +def _unicode_stream(value: str) -> bytes: + """MAPI PT_UNICODE properties are stored little-endian UTF-16.""" + return value.encode("utf-16-le") + + +def _patch_olefile(monkeypatch, streams) -> None: + """Make ``olefile.OleFileIO(...)`` open the given streams instead of a file.""" + + class _PatchedOleFileIO(_FakeMsg): + def __init__(self, *_args, **_kwargs): + super().__init__(streams) + + monkeypatch.setattr(olefile, "OleFileIO", _PatchedOleFileIO) + + +def test_sender_prefers_smtp_address_over_exchange_dn() -> None: + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_5D01001F": _unicode_stream("sender@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "sender@example.com" + + +def test_sender_prefers_sent_representing_smtp_address() -> None: + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_5D02001F": _unicode_stream("delegate@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "delegate@example.com" + + +def test_sender_skips_exchange_dn_for_sent_representing_address() -> None: + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_0064001F": _unicode_stream("SMTP"), + "__substg1.0_0065001F": _unicode_stream("onbehalf@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "onbehalf@example.com" + + +def test_sender_keeps_smtp_addrtype_address() -> None: + # The common non-Exchange case must keep behaving exactly as before. + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("SMTP"), + "__substg1.0_0C1F001F": _unicode_stream("plain@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "plain@example.com" + + +def test_sender_falls_back_to_dn_when_no_smtp_address_exists() -> None: + # Nothing better on file: keep the DN rather than dropping the header. + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == EXCHANGE_DN + + +def test_html_only_body_is_converted_to_markdown() -> None: + msg = _FakeMsg( + { + "__substg1.0_10130102": ( + b"

Status

Hello world

" + ), + } + ) + + body = OutlookMsgConverter()._get_html_body(msg) + + assert body is not None + assert "# Status" in body + assert "Hello **world**" in body + # The HTML must be converted, not passed through verbatim. + assert "

" not in body + + +def test_html_body_reads_the_string_property_variant() -> None: + msg = _FakeMsg( + { + "__substg1.0_1013001F": _unicode_stream( + "

String variant

" + ), + } + ) + + assert "String variant" in (OutlookMsgConverter()._get_html_body(msg) or "") + + +def test_html_body_decodes_non_utf8_bytes() -> None: + # PR_HTML carries no encoding of its own; assuming UTF-8 mangles this. + msg = _FakeMsg( + { + "__substg1.0_10130102": ( + "

Grüße aus München

".encode("cp1252") + ), + } + ) + + assert "Grüße aus München" in (OutlookMsgConverter()._get_html_body(msg) or "") + + +def test_html_body_absent_returns_none() -> None: + assert OutlookMsgConverter()._get_html_body(_FakeMsg({})) is None + + +def test_html_body_skips_an_empty_binary_property() -> None: + # An empty binary stream must not mask a populated string variant. + msg = _FakeMsg( + { + "__substg1.0_10130102": b"", + "__substg1.0_1013001F": _unicode_stream( + "

Fallback body

" + ), + } + ) + + assert "Fallback body" in (OutlookMsgConverter()._get_html_body(msg) or "") + + +def test_plain_text_body_wins_over_html(monkeypatch) -> None: + streams = { + "__substg1.0_0037001F": _unicode_stream("Q3 planning"), + "__substg1.0_0E04001F": _unicode_stream("recipient@example.com"), + "__substg1.0_0C1E001F": _unicode_stream("SMTP"), + "__substg1.0_0C1F001F": _unicode_stream("sender@example.com"), + "__substg1.0_1000001F": _unicode_stream("The plain text body."), + "__substg1.0_10130102": b"

The HTML body.

", + } + _patch_olefile(monkeypatch, streams) + + result = OutlookMsgConverter().convert( + io.BytesIO(b""), StreamInfo(extension=".msg") + ) + + assert "The plain text body." in result.markdown + assert "The HTML body." not in result.markdown + + +def test_convert_renders_exchange_message(monkeypatch) -> None: + # End-to-end shape of the message reported in #1188: Exchange sender, no + # plain-text body. + streams = { + "__substg1.0_0037001F": _unicode_stream("Q3 planning"), + "__substg1.0_0E04001F": _unicode_stream("recipient@example.com"), + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_5D01001F": _unicode_stream("sender@example.com"), + "__substg1.0_10130102": b"

Hello world

", + } + _patch_olefile(monkeypatch, streams) + + result = OutlookMsgConverter().convert( + io.BytesIO(b""), StreamInfo(extension=".msg") + ) + + assert "**From:** sender@example.com" in result.markdown + assert "EXCHANGELABS" not in result.markdown + assert "Hello **world**" in result.markdown + assert result.title == "Q3 planning"