From a58cc89785c0ca93ce98db13998fb8442f357937 Mon Sep 17 00:00:00 2001 From: Gaurav Dubey Date: Mon, 27 Jul 2026 00:46:39 +0530 Subject: [PATCH 1/2] Fix Outlook 365 sender address and HTML-only message bodies OutlookMsgConverter read the sender only from PR_SENDER_EMAIL_ADDRESS (0x0C1F). That property holds an email address only when PR_SENDER_ADDRTYPE (0x0C1E) is "SMTP"; messages sent through Exchange set it to "EX" and store a Distinguished Name there instead, so From rendered as "/O=EXCHANGELABS/OU=.../CN=RECIPIENTS/CN=...". Resolve the sender from PR_SENDER_SMTP_ADDRESS / PR_SENT_REPRESENTING_SMTP_ADDRESS first, use the legacy properties only when their address type is not the DN form, and keep the DN as a last resort so the header is never dropped. The body was likewise read only from PR_BODY (0x1000), which HTML-only messages leave empty, so the whole message silently vanished from "## Content". Fall back to PR_HTML (0x1013) and convert it with the existing HtmlConverter, handling both the binary and string variants of the property and detecting the byte stream's charset rather than assuming UTF-8. Binary streams are read through a new _read_stream helper because _get_stream_data decodes UTF-16 first, which rarely fails on arbitrary bytes and would return mojibake instead of raising. Fixes #1188 --- .../converters/_outlook_msg_converter.py | 108 +++++++-- packages/markitdown/tests/test_outlook_msg.py | 209 ++++++++++++++++++ 2 files changed, 304 insertions(+), 13 deletions(-) create mode 100644 packages/markitdown/tests/test_outlook_msg.py diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py index 7717f62d8..370f6fc77 100644 --- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py +++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py @@ -1,8 +1,13 @@ +import io import sys from typing import Any, Union, BinaryIO + +from charset_normalizer import from_bytes + from .._stream_info import StreamInfo from .._base_converter import DocumentConverter, DocumentConverterResult from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE +from ._html_converter import HtmlConverter # Try loading optional (but in this case, required) dependencies # Save reporting of any exceptions for later @@ -29,6 +34,10 @@ class OutlookMsgConverter(DocumentConverter): - Email body content """ + def __init__(self): + super().__init__() + self._html_converter = HtmlConverter() + def accepts( self, file_stream: BinaryIO, @@ -100,7 +109,7 @@ def convert( # Get headers headers = { - "From": self._get_stream_data(msg, "__substg1.0_0C1F001F"), + "From": self._get_sender(msg), "To": self._get_stream_data(msg, "__substg1.0_0E04001F"), "Subject": self._get_stream_data(msg, "__substg1.0_0037001F"), } @@ -112,8 +121,11 @@ def convert( md_content += "\n## Content\n\n" - # Get email body + # Get email body. HTML-only messages leave PR_BODY empty and carry + # their content in PR_HTML instead. body = self._get_stream_data(msg, "__substg1.0_1000001F") + if not body: + body = self._get_html_body(msg) if body: md_content += body @@ -124,8 +136,87 @@ def convert( title=headers.get("Subject"), ) + def _get_sender(self, msg: Any) -> Union[str, None]: + """Helper to resolve the sender's address, preferring SMTP over an Exchange DN. + + PR_SENDER_EMAIL_ADDRESS (0x0C1F) only holds an email address when + PR_SENDER_ADDRTYPE (0x0C1E) is "SMTP". Messages sent through Exchange + set the address type to "EX" and store a Distinguished Name there + instead (e.g. "/O=EXCHANGELABS/OU=.../CN=RECIPIENTS/CN=..."), keeping + the real address in PR_SENDER_SMTP_ADDRESS (0x5D01). + """ + # PR_SENDER_SMTP_ADDRESS, then PR_SENT_REPRESENTING_SMTP_ADDRESS. + for address_stream in ("__substg1.0_5D01001F", "__substg1.0_5D02001F"): + address = self._get_stream_data(msg, address_stream) + if address: + return address + + # PR_SENDER_EMAIL_ADDRESS / PR_SENT_REPRESENTING_EMAIL_ADDRESS, each + # usable only when its address type is not the Exchange DN form. + for address_stream, addrtype_stream in ( + ("__substg1.0_0C1F001F", "__substg1.0_0C1E001F"), + ("__substg1.0_0065001F", "__substg1.0_0064001F"), + ): + address = self._get_stream_data(msg, address_stream) + addrtype = self._get_stream_data(msg, addrtype_stream) or "" + if address and addrtype.upper() != "EX": + return address + + # Nothing better on file: report the DN rather than dropping the header. + return self._get_stream_data(msg, "__substg1.0_0C1F001F") + + def _get_html_body(self, msg: Any) -> Union[str, None]: + """Helper to convert the HTML body (PR_HTML, 0x1013) to markdown. + + Used for messages that carry no plain-text body, where reading only + PR_BODY would silently drop the entire message content. + """ + charset: Union[str, None] = None + + # PR_HTML is normally a binary stream; some clients write the string variant. + html_bytes = self._read_stream(msg, "__substg1.0_10130102") + if html_bytes is None: + html = self._get_stream_data(msg, "__substg1.0_1013001F") + if not html: + return None + html_bytes = html.encode("utf-8") + charset = "utf-8" + else: + # The stream carries no encoding of its own, so detect it rather + # than assuming UTF-8 and mangling non-ASCII text. + detected = from_bytes(html_bytes).best() + if detected is not None: + charset = detected.encoding + + return self._html_converter.convert( + io.BytesIO(html_bytes), + StreamInfo(mimetype="text/html", extension=".html", charset=charset), + ).markdown + def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]: """Helper to safely extract and decode stream data from the MSG file.""" + data = self._read_stream(msg, stream_path) + if data is None: + return None + + # Try UTF-16 first (common for .msg files) + try: + return data.decode("utf-16-le").strip() + except UnicodeDecodeError: + # Fall back to UTF-8 + try: + return data.decode("utf-8").strip() + except UnicodeDecodeError: + # Last resort - ignore errors + return data.decode("utf-8", errors="ignore").strip() + + def _read_stream(self, msg: Any, stream_path: str) -> Union[bytes, None]: + """Helper to safely read the raw bytes of a stream from the MSG file. + + Binary properties (such as PR_HTML) must not go through + `_get_stream_data`: arbitrary bytes rarely fail a UTF-16 decode, so + they would come back as mojibake instead of raising. + """ assert olefile is not None assert isinstance( msg, olefile.OleFileIO @@ -133,17 +224,8 @@ def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]: try: if msg.exists(stream_path): - data = msg.openstream(stream_path).read() - # Try UTF-16 first (common for .msg files) - try: - return data.decode("utf-16-le").strip() - except UnicodeDecodeError: - # Fall back to UTF-8 - try: - return data.decode("utf-8").strip() - except UnicodeDecodeError: - # Last resort - ignore errors - return data.decode("utf-8", errors="ignore").strip() + data: bytes = msg.openstream(stream_path).read() + return data except Exception: pass return None diff --git a/packages/markitdown/tests/test_outlook_msg.py b/packages/markitdown/tests/test_outlook_msg.py new file mode 100644 index 000000000..28bfd15b5 --- /dev/null +++ b/packages/markitdown/tests/test_outlook_msg.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 -m pytest +"""Tests for .msg files saved from Outlook 365 / Exchange. + +Two properties of such files break the naive reads: + +* ``PR_SENDER_EMAIL_ADDRESS`` (0x0C1F) holds an Exchange Distinguished Name + rather than an email address whenever ``PR_SENDER_ADDRTYPE`` (0x0C1E) is + ``EX``; the address lives in ``PR_SENDER_SMTP_ADDRESS`` (0x5D01). +* HTML-only messages leave ``PR_BODY`` (0x1000) empty and store their content + in ``PR_HTML`` (0x1013), so reading only ``PR_BODY`` drops the whole message. + +A .msg file is an OLE2 compound file and nothing in the dependency set can +write that container, so these tests serve the MAPI streams through a stand-in +for ``olefile.OleFileIO`` instead of a binary fixture. +""" +import io + +import olefile + +from markitdown._stream_info import StreamInfo +from markitdown.converters._outlook_msg_converter import OutlookMsgConverter + +EXCHANGE_DN = ( + "/O=EXCHANGELABS/OU=EXCHANGE ADMINISTRATIVE GROUP" + " (FYDIBOHF23SPDLT)/CN=RECIPIENTS/CN=ab12cd34" +) + + +class _FakeMsg(olefile.OleFileIO): + """Serves a fixed set of MSG streams. ``_get_stream_data`` asserts the type.""" + + def __init__(self, streams): + self._streams = streams + # OleFileIO.__del__ reads this; we deliberately skip super().__init__. + self._we_opened_fp = False + + def exists(self, path): + return path in self._streams + + def openstream(self, path): + return io.BytesIO(self._streams[path]) + + def close(self): + pass + + +def _unicode_stream(value: str) -> bytes: + """MAPI PT_UNICODE properties are stored little-endian UTF-16.""" + return value.encode("utf-16-le") + + +def _patch_olefile(monkeypatch, streams) -> None: + """Make ``olefile.OleFileIO(...)`` open the given streams instead of a file.""" + + class _PatchedOleFileIO(_FakeMsg): + def __init__(self, *_args, **_kwargs): + super().__init__(streams) + + monkeypatch.setattr(olefile, "OleFileIO", _PatchedOleFileIO) + + +def test_sender_prefers_smtp_address_over_exchange_dn() -> None: + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_5D01001F": _unicode_stream("sender@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "sender@example.com" + + +def test_sender_prefers_sent_representing_smtp_address() -> None: + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_5D02001F": _unicode_stream("delegate@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "delegate@example.com" + + +def test_sender_skips_exchange_dn_for_sent_representing_address() -> None: + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_0064001F": _unicode_stream("SMTP"), + "__substg1.0_0065001F": _unicode_stream("onbehalf@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "onbehalf@example.com" + + +def test_sender_keeps_smtp_addrtype_address() -> None: + # The common non-Exchange case must keep behaving exactly as before. + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("SMTP"), + "__substg1.0_0C1F001F": _unicode_stream("plain@example.com"), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == "plain@example.com" + + +def test_sender_falls_back_to_dn_when_no_smtp_address_exists() -> None: + # Nothing better on file: keep the DN rather than dropping the header. + msg = _FakeMsg( + { + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + } + ) + + assert OutlookMsgConverter()._get_sender(msg) == EXCHANGE_DN + + +def test_html_only_body_is_converted_to_markdown() -> None: + msg = _FakeMsg( + { + "__substg1.0_10130102": ( + b"

Status

Hello world

" + ), + } + ) + + body = OutlookMsgConverter()._get_html_body(msg) + + assert body is not None + assert "# Status" in body + assert "Hello **world**" in body + # The HTML must be converted, not passed through verbatim. + assert "

" not in body + + +def test_html_body_reads_the_string_property_variant() -> None: + msg = _FakeMsg( + { + "__substg1.0_1013001F": _unicode_stream( + "

String variant

" + ), + } + ) + + assert "String variant" in (OutlookMsgConverter()._get_html_body(msg) or "") + + +def test_html_body_decodes_non_utf8_bytes() -> None: + # PR_HTML carries no encoding of its own; assuming UTF-8 mangles this. + msg = _FakeMsg( + { + "__substg1.0_10130102": ( + "

Grüße aus München

".encode("cp1252") + ), + } + ) + + assert "Grüße aus München" in (OutlookMsgConverter()._get_html_body(msg) or "") + + +def test_html_body_absent_returns_none() -> None: + assert OutlookMsgConverter()._get_html_body(_FakeMsg({})) is None + + +def test_plain_text_body_wins_over_html(monkeypatch) -> None: + streams = { + "__substg1.0_0037001F": _unicode_stream("Q3 planning"), + "__substg1.0_0E04001F": _unicode_stream("recipient@example.com"), + "__substg1.0_0C1E001F": _unicode_stream("SMTP"), + "__substg1.0_0C1F001F": _unicode_stream("sender@example.com"), + "__substg1.0_1000001F": _unicode_stream("The plain text body."), + "__substg1.0_10130102": b"

The HTML body.

", + } + _patch_olefile(monkeypatch, streams) + + result = OutlookMsgConverter().convert( + io.BytesIO(b""), StreamInfo(extension=".msg") + ) + + assert "The plain text body." in result.markdown + assert "The HTML body." not in result.markdown + + +def test_convert_renders_exchange_message(monkeypatch) -> None: + # End-to-end shape of the message reported in #1188: Exchange sender, no + # plain-text body. + streams = { + "__substg1.0_0037001F": _unicode_stream("Q3 planning"), + "__substg1.0_0E04001F": _unicode_stream("recipient@example.com"), + "__substg1.0_0C1E001F": _unicode_stream("EX"), + "__substg1.0_0C1F001F": _unicode_stream(EXCHANGE_DN), + "__substg1.0_5D01001F": _unicode_stream("sender@example.com"), + "__substg1.0_10130102": b"

Hello world

", + } + _patch_olefile(monkeypatch, streams) + + result = OutlookMsgConverter().convert( + io.BytesIO(b""), StreamInfo(extension=".msg") + ) + + assert "**From:** sender@example.com" in result.markdown + assert "EXCHANGELABS" not in result.markdown + assert "Hello **world**" in result.markdown + assert result.title == "Q3 planning" From 3e1db6aee7500a7946b8064d5637927d742c838e Mon Sep 17 00:00:00 2001 From: Gaurav Dubey Date: Mon, 27 Jul 2026 00:58:17 +0530 Subject: [PATCH 2/2] Skip an empty PR_HTML binary stream and note the HTML fallback in the class docstring An empty 0x1013 binary property must not mask a populated string variant of the same property. --- .../converters/_outlook_msg_converter.py | 5 +++-- packages/markitdown/tests/test_outlook_msg.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py index 370f6fc77..ca060cabe 100644 --- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py +++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py @@ -31,7 +31,8 @@ class OutlookMsgConverter(DocumentConverter): Uses the olefile package to parse the .msg file structure and extract: - Email headers (From, To, Subject) - - Email body content + - Email body content, falling back to the HTML body for messages that + carry no plain-text part """ def __init__(self): @@ -175,7 +176,7 @@ def _get_html_body(self, msg: Any) -> Union[str, None]: # PR_HTML is normally a binary stream; some clients write the string variant. html_bytes = self._read_stream(msg, "__substg1.0_10130102") - if html_bytes is None: + if not html_bytes: html = self._get_stream_data(msg, "__substg1.0_1013001F") if not html: return None diff --git a/packages/markitdown/tests/test_outlook_msg.py b/packages/markitdown/tests/test_outlook_msg.py index 28bfd15b5..2414c4fc5 100644 --- a/packages/markitdown/tests/test_outlook_msg.py +++ b/packages/markitdown/tests/test_outlook_msg.py @@ -167,6 +167,20 @@ def test_html_body_absent_returns_none() -> None: assert OutlookMsgConverter()._get_html_body(_FakeMsg({})) is None +def test_html_body_skips_an_empty_binary_property() -> None: + # An empty binary stream must not mask a populated string variant. + msg = _FakeMsg( + { + "__substg1.0_10130102": b"", + "__substg1.0_1013001F": _unicode_stream( + "

Fallback body

" + ), + } + ) + + assert "Fallback body" in (OutlookMsgConverter()._get_html_body(msg) or "") + + def test_plain_text_body_wins_over_html(monkeypatch) -> None: streams = { "__substg1.0_0037001F": _unicode_stream("Q3 planning"),