diff --git a/README.md b/README.md index 5c5a0ccaa5..8f7a9759cc 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ MarkItDown currently supports the conversion from: - PowerPoint - Word - Excel +- OpenDocument (ODT, ODS, ODP, and the flat FODT/FODS/FODP variants) - Images (EXIF metadata and OCR) - Audio (EXIF metadata and speech transcription) - HTML diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 888d1e6260..8d636fe7fb 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -39,6 +39,7 @@ OutlookMsgConverter, ZipConverter, EpubConverter, + OdfConverter, DocumentIntelligenceConverter, ContentUnderstandingConverter, CsvConverter, @@ -249,6 +250,7 @@ def enable_builtins(self, **kwargs) -> None: self.register_converter(PdfConverter()) self.register_converter(OutlookMsgConverter()) self.register_converter(EpubConverter()) + self.register_converter(OdfConverter()) self.register_converter(CsvConverter()) # Register Document Intelligence converter at the top of the stack if endpoint is provided diff --git a/packages/markitdown/src/markitdown/converters/__init__.py b/packages/markitdown/src/markitdown/converters/__init__.py index 77f8b1acd1..76dc0538e7 100644 --- a/packages/markitdown/src/markitdown/converters/__init__.py +++ b/packages/markitdown/src/markitdown/converters/__init__.py @@ -26,6 +26,7 @@ ContentUnderstandingFileType, ) from ._epub_converter import EpubConverter +from ._odf_converter import OdfConverter from ._csv_converter import CsvConverter __all__ = [ @@ -50,5 +51,6 @@ "ContentUnderstandingConverter", "ContentUnderstandingFileType", "EpubConverter", + "OdfConverter", "CsvConverter", ] diff --git a/packages/markitdown/src/markitdown/converters/_odf_converter.py b/packages/markitdown/src/markitdown/converters/_odf_converter.py new file mode 100644 index 0000000000..76c07856dc --- /dev/null +++ b/packages/markitdown/src/markitdown/converters/_odf_converter.py @@ -0,0 +1,712 @@ +"""Converts OpenDocument files (ODT / ODS / ODP) to Markdown. + +Why this belongs in the repository rather than in a plugin +--------------------------------------------------------- +New formats are added sparingly, and the stated reason is dependencies: a +converter that pulls in a parser makes the base install heavier for everyone. +This one adds **no dependency at all**. OpenDocument is a ZIP of XML parts, so it +is read with :mod:`zipfile` from the standard library and :mod:`defusedxml`, +which ``markitdown`` already requires. ``defusedxml`` is the right parser here +rather than a convenience: ``content.xml`` arrives from an untrusted file, and +the hardened parser blocks entity expansion and external entity resolution that +plain :mod:`xml.dom.minidom` allows. + +OpenDocument is what LibreOffice, OpenOffice and most public-sector suites write +by default, and it is the one common office format MarkItDown could not read. +The flat variants (``.fodt``, ``.fods``, ``.fodp``) are the same XML with the ZIP +removed and are handled by the same code. + +Three properties of the format drive most of the code below, and all three are +things a straightforward implementation gets wrong: + +1. **Repetition is a compression scheme, not data.** Every row of a spreadsheet + ends with something like + ```` for the empty + tail of the sheet, and a *populated* cell can be repeated too. Expanding + repeats blindly turns a three-column file into a 16384-column Markdown table, + so the width of a table is the last column that holds a *value* and repeats are + capped (:data:`MAX_REPEATED_CELLS`, :data:`MAX_COLUMNS`) so a hostile file + cannot exhaust memory. +2. **Whitespace is markup.** ```` is three spaces and + ```` is a tab; runs of literal whitespace between elements are not + significant. Reading ``nodeValue`` alone silently drops both. +3. **A cell is a container, not a string.** A cell holds ``text:p`` paragraphs and + a table holds cells; flattening either one to text produces a table whose rows + do not line up with its header. + +Where the format has a choice, this converter follows :class:`XlsxConverter` so +that ``.ods`` and ``.xlsx`` of the same data produce the same Markdown: each sheet +becomes a ``##`` heading followed by one table, and the first row is treated as +the header, which is what ``pandas.DataFrame.to_html(index=False)`` does. + +Limitations, stated rather than implied: + +- Embedded images are not extracted. Their ``xlink:href`` values are + package-internal paths (``Pictures/1000000000000001.png``) that mean nothing + outside the file, and a Markdown link to one would be a broken link. +- Footnotes and endnotes are dropped rather than emitted as ``[^n]`` references + with no definition. Losing a footnote is better than producing a document that + does not render. +- Change tracking is not resolved: revisions are rendered as they appear in + ``content.xml``. +- Ordered lists render as bullets. Whether a list is numbered is a *style* + property stored in ``styles.xml``, not an attribute of the list itself, so + resolving it would mean resolving the whole style cascade. +""" + +import io +import re +import zipfile +from typing import Any, BinaryIO, Iterator, List, Optional, Sequence, Tuple +from xml.dom import Node + +from defusedxml import minidom + +from .._base_converter import DocumentConverter, DocumentConverterResult +from .._stream_info import StreamInfo + +# --------------------------------------------------------------------------- +# Namespaces. Matching on namespace URI rather than on the literal prefix is +# what makes this work on files written by something other than LibreOffice: +# the prefix is arbitrary, the URI is not. +# --------------------------------------------------------------------------- + +OFFICE_NS = "urn:oasis:names:tc:opendocument:xmlns:office:1.0" +TEXT_NS = "urn:oasis:names:tc:opendocument:xmlns:text:1.0" +TABLE_NS = "urn:oasis:names:tc:opendocument:xmlns:table:1.0" +DRAW_NS = "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" +PRESENTATION_NS = "urn:oasis:names:tc:opendocument:xmlns:presentation:1.0" +DC_NS = "http://purl.org/dc/elements/1.1/" +XLINK_NS = "http://www.w3.org/1999/xlink" + +ACCEPTED_MIME_TYPE_PREFIXES = [ + "application/vnd.oasis.opendocument", +] + +ACCEPTED_FILE_EXTENSIONS = [".odt", ".ods", ".odp", ".fodt", ".fods", ".fodp"] + +#: Flat OpenDocument: the same document as a single uncompressed XML file. +FLAT_EXTENSIONS = frozenset({".fodt", ".fods", ".fodp"}) + +#: Cap on one ``number-columns-repeated`` / ``number-rows-repeated`` value. A +#: populated cell legitimately repeats a handful of times; a value in the +#: thousands is a compression artefact of the empty tail, and honouring it is how +#: a converter turns a 4 KiB file into a 200 MiB string. +MAX_REPEATED_CELLS = 256 +MAX_REPEATED_ROWS = 64 + +#: Cap on the width of a generated table, applied after repeats are resolved. +MAX_COLUMNS = 1024 + +#: Cap on the uncompressed size of ``content.xml``. ``defusedxml`` blocks entity +#: expansion but not a plain 10 GiB member, and the ZIP central directory is +#: attacker-controlled. +MAX_CONTENT_XML_BYTES = 64 * 1024 * 1024 + +#: A pipe together with any run of backslashes in front of it, so the run can be +#: doubled before the pipe is escaped. Same rule as the CSV converter, because a +#: cell is a cell whichever format it came from. +_PIPE_ESCAPE_RE = re.compile(r"(? Iterator[Node]: + """Element children of ``node``, in document order.""" + for child in node.childNodes: + if child.nodeType == Node.ELEMENT_NODE: + yield child + + +def _children(node: Node, namespace: str, local_name: str) -> List[Node]: + return [ + child + for child in _elements(node) + if child.namespaceURI == namespace and child.localName == local_name + ] + + +def _first(node: Node, namespace: str, local_name: str) -> Optional[Node]: + for child in _elements(node): + if child.namespaceURI == namespace and child.localName == local_name: + return child + return None + + +def _attr(node: Node, namespace: str, local_name: str, default: str = "") -> str: + value = node.getAttributeNS(namespace, local_name) + return value if value else default + + +def _positive_int(value: str, default: int) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return default + return number if number > 0 else default + + +def _outline_level(node: Node, default: int = 1) -> int: + level = _positive_int(_attr(node, TEXT_NS, "outline-level", ""), default) + return max(1, min(6, level)) + + +# --------------------------------------------------------------------------- +# Inline rendering +# --------------------------------------------------------------------------- + + +def _render_inline(node: Node) -> str: + """Render the inline content of a paragraph-like node to Markdown text. + + Whitespace inside a text node is insignificant, so it collapses to a single + space; ``text:s`` and ``text:tab`` are *not* whitespace, they are elements + that happen to mean whitespace, so they survive the collapse and are only + de-duplicated afterwards. That ordering is the whole trick: collapsing the + assembled string instead would turn ``ab`` and ``ab`` into the same + output. + """ + parts: List[str] = [] + + for child in node.childNodes: + if child.nodeType == Node.TEXT_NODE: + parts.append(_WHITESPACE_RE.sub(" ", child.nodeValue or "")) + continue + if child.nodeType != Node.ELEMENT_NODE: + continue + + namespace, name = child.namespaceURI, child.localName + + if namespace == TEXT_NS: + if name == "s": + parts.append(" " * _positive_int(_attr(child, TEXT_NS, "c", "1"), 1)) + continue + if name == "tab": + parts.append("\t") + continue + if name == "line-break": + parts.append("
") + continue + if name == "a": + inner = _render_inline(child) + href = _attr(child, XLINK_NS, "href") + parts.append(f"[{inner}]({href})" if inner and href else inner) + continue + if name in ( + "span", + "bookmark-ref", + "reference-ref", + "note-ref", + "meta-field", + ): + parts.append(_render_inline(child)) + continue + if name in ( + "note", + "annotation", + "bookmark", + "soft-page-break", + "index-mark", + ): + # A footnote reference without its body would render as a broken + # link; see the module docstring for why they are dropped. + continue + parts.append(_render_inline(child)) + continue + + if namespace == OFFICE_NS and name == "annotation": + continue + # draw:frame and draw:text-box appear inline in text documents, and their + # text:p children are the anchor's content. + parts.append(_render_inline(child)) + + return _REPEATED_SPACE_RE.sub(" ", "".join(parts)).strip() + + +# --------------------------------------------------------------------------- +# Tables +# --------------------------------------------------------------------------- + + +def _escape_cell(value: str) -> str: + r"""Escape a value so it survives inside a Markdown table cell. + + A pipe is a column separator, so it is escaped; a paragraph break inside a + cell becomes ``
`` rather than a newline, because a newline would end the + row and shift every following cell by one column. + """ + value = _PIPE_ESCAPE_RE.sub(lambda match: match.group(1) * 2 + r"\|", value) + return value.replace("\r\n", "
").replace("\n", "
").replace("\r", "
") + + +def _cell_text(cell: Node) -> str: + """The visible text of one cell. + + A cell is a container: its paragraphs are joined with ``
`` so a + multi-paragraph cell keeps both its lines and its row. When there is no + display text at all, the stored ``office:value`` is used, which is the only + content a formula cell carries when the producer omitted the cached text. + """ + paragraphs = [ + child + for child in _elements(cell) + if child.namespaceURI == TEXT_NS and child.localName in ("p", "h") + ] + text = "
".join(part for part in (_render_inline(p) for p in paragraphs) if part) + + if not text: + value_type = _attr(cell, OFFICE_NS, "value-type") + if value_type in ("float", "percentage", "currency"): + text = _attr(cell, OFFICE_NS, "value") + elif value_type == "date": + text = _attr(cell, OFFICE_NS, "date-value") + elif value_type == "time": + text = _attr(cell, OFFICE_NS, "time-value") + elif value_type == "boolean": + raw = _attr(cell, OFFICE_NS, "boolean-value") + text = "TRUE" if raw == "true" else "FALSE" if raw else "" + + return _escape_cell(text) + + +def _iter_rows(node: Node) -> Iterator[Node]: + """Rows of a table, descending through row groups but never into a nested table. + + Rows live under ``table:table-header-rows`` or ``table:table-row-group`` as + often as directly under ``table:table``, and a cell can contain a whole nested + table whose rows are not part of this grid. + """ + for child in _elements(node): + if child.namespaceURI != TABLE_NS: + continue + if child.localName == "table-row": + yield child + elif child.localName in ("table-header-rows", "table-row-group", "table-rows"): + yield from _iter_rows(child) + + +def _row_cells(row: Node) -> List[Tuple[str, int]]: + """``(text, repeat)`` for each cell of a row, in column order.""" + cells: List[Tuple[str, int]] = [] + for child in _elements(row): + if child.namespaceURI != TABLE_NS: + continue + if child.localName == "table-cell": + repeat = _positive_int( + _attr(child, TABLE_NS, "number-columns-repeated", "1"), 1 + ) + cells.append((_cell_text(child), repeat)) + elif child.localName == "covered-table-cell": + # A cell swallowed by a span still occupies a column, so it becomes an + # empty one. Markdown has no merged cells; leaving the column in place + # keeps every other row aligned. + repeat = _positive_int( + _attr(child, TABLE_NS, "number-columns-repeated", "1"), 1 + ) + cells.append(("", repeat)) + return cells + + +def _table_width(rows: Sequence[Sequence[Tuple[str, int]]]) -> int: + """The width of the table: the last column that holds a value, plus its repeat. + + Empty cells contribute nothing. They still *occupy* a column when the row is + laid out, but they never define the grid, and that is the whole point: every + LibreOffice row ends with an empty cell repeated to the sheet's maximum width, + so a rule of "the rightmost cell defines the width" turns a two-column file + into a 16384-column table. The cost is that a genuinely blank trailing column + is dropped, which is not information a Markdown table can carry anyway. + """ + width = 0 + for cells in rows: + column = 0 + for text, repeat in cells: + repeat = min(repeat, MAX_REPEATED_CELLS) + if text: + width = max(width, column + repeat) + column += repeat + return min(width, MAX_COLUMNS) + + +def _expand_row(cells: Sequence[Tuple[str, int]], width: int) -> List[str]: + row: List[str] = [] + column = 0 + for text, repeat in cells: + if column >= width: + break + take = min(repeat, width - column, MAX_REPEATED_CELLS) + row.extend([text] * take) + column += take + row.extend([""] * (width - len(row))) + return row[:width] + + +def _grid_to_markdown(grid: Sequence[Sequence[str]]) -> List[str]: + """A GitHub-flavoured Markdown table whose first row is the header.""" + header = list(grid[0]) + lines = [ + "| " + " | ".join(header) + " |", + "| " + " | ".join(["---"] * len(header)) + " |", + ] + for row in grid[1:]: + lines.append("| " + " | ".join(row) + " |") + return lines + + +def _render_table(node: Node, title: Optional[str]) -> List[str]: + """A table as Markdown lines, preceded by ``## title`` when one is given.""" + raw_rows = [ + ( + _row_cells(row), + _positive_int(_attr(row, TABLE_NS, "number-rows-repeated", "1"), 1), + ) + for row in _iter_rows(node) + ] + if not raw_rows: + return [] + + width = _table_width([cells for cells, _ in raw_rows]) + if width == 0: + return [] + + grid: List[List[str]] = [] + for cells, repeat in raw_rows: + dense = _expand_row(cells, width) + if not any(dense): + # A blank row carries no data and Markdown cannot express one, so it + # is dropped rather than rendered as a row of empty cells. This also + # absorbs the "row repeated 1048576 times, all empty" tail. + continue + grid.extend([dense] * min(repeat, MAX_REPEATED_ROWS)) + + if not grid: + return [] + + lines: List[str] = [] + if title: + lines.extend([f"## {title}", ""]) + lines.extend(_grid_to_markdown(grid)) + return lines + + +# --------------------------------------------------------------------------- +# Text documents +# --------------------------------------------------------------------------- + + +def _render_list(node: Node, indent: int) -> List[str]: + """A list as Markdown lines, nested two spaces per level.""" + pad = " " * indent + lines: List[str] = [] + + for item in _children(node, TEXT_NS, "list-item"): + item_lines: List[str] = [] + for child in _elements(item): + namespace, name = child.namespaceURI, child.localName + if namespace == TEXT_NS and name in ("p", "h"): + text = _render_inline(child) + if text: + # A paragraph after the first is a continuation of the same + # item, so it is indented under the bullet instead of getting + # a bullet of its own. + item_lines.append(text if not item_lines else f"{pad} {text}") + elif namespace == TEXT_NS and name == "list": + item_lines.extend(_render_list(child, indent + 1)) + elif namespace == TABLE_NS and name == "table": + item_lines.extend(_render_table(child, None)) + elif namespace == TEXT_NS and name in ("soft-page-break", "bookmark"): + continue + else: + for block in _render_blocks(child): + item_lines.extend(block.split("\n")) + + if not item_lines: + lines.append(f"{pad}-") + elif item_lines[0].startswith((" ", "#")): + # The item opened with something that carries its own layout (a nested + # list, a heading): prefixing a bullet would produce "- - ...". + lines.extend(item_lines) + else: + lines.append(f"{pad}- {item_lines[0]}") + lines.extend(item_lines[1:]) + + return lines + + +def _render_blocks(node: Node) -> List[str]: + """Render a container's block-level children as a list of Markdown blocks.""" + blocks: List[str] = [] + + for child in _elements(node): + namespace, name = child.namespaceURI, child.localName + + if namespace == TEXT_NS: + if name == "h": + text = _render_inline(child) + if text: + blocks.append("#" * _outline_level(child) + " " + text) + continue + if name == "p": + text = _render_inline(child) + if text: + blocks.append(text) + continue + if name == "list": + lines = _render_list(child, 0) + if lines: + blocks.append("\n".join(lines)) + continue + if name in _TOC_ELEMENTS: + # A table of contents is derived from the headings, which are + # already in the output; emitting the stale copy as well would + # duplicate every heading. + continue + if name == "section": + blocks.extend(_render_blocks(child)) + continue + blocks.extend(_render_blocks(child)) + continue + + if namespace == TABLE_NS and name == "table": + lines = _render_table(child, None) + if lines: + blocks.append("\n".join(lines)) + continue + + if namespace == DRAW_NS and name == "frame": + text = _render_inline(child) + if text: + blocks.append(text) + continue + + blocks.extend(_render_blocks(child)) + + return blocks + + +def _render_spreadsheet(body: Node) -> List[str]: + blocks: List[str] = [] + for sheet in _children(body, TABLE_NS, "table"): + lines = _render_table(sheet, _attr(sheet, TABLE_NS, "name") or "Sheet") + if lines: + blocks.append("\n".join(lines)) + return blocks + + +def _iter_slide_text(node: Node) -> Iterator[Node]: + """``text:p`` / ``text:h`` under a slide, in order, skipping speaker notes. + + A ``text:p`` is not descended into: its nested frames would otherwise be + emitted twice. + """ + for child in _elements(node): + if child.namespaceURI == PRESENTATION_NS and child.localName == "notes": + continue + if child.namespaceURI == TEXT_NS and child.localName in ("p", "h"): + yield child + else: + yield from _iter_slide_text(child) + + +def _render_presentation(body: Node) -> List[str]: + blocks: List[str] = [] + for index, page in enumerate(_children(body, DRAW_NS, "page"), start=1): + name = _attr(page, DRAW_NS, "name") + blocks.append(f"## {name}" if name else f"## Slide {index}") + + for node in _iter_slide_text(page): + text = _render_inline(node) + if not text: + continue + if node.localName == "h": + # A slide's own heading level would compete with the slide title, + # so content headings start below it. + level = max(3, _outline_level(node)) + blocks.append("#" * level + " " + text) + else: + blocks.append(text) + + notes = _first(page, PRESENTATION_NS, "notes") + if notes is not None: + note_text = "\n".join( + text + for text in (_render_inline(node) for node in _iter_slide_text(notes)) + if text + ) + if note_text: + blocks.append("\n".join("> " + line for line in note_text.split("\n"))) + + return blocks + + +# --------------------------------------------------------------------------- +# Package access +# --------------------------------------------------------------------------- + + +def _parse_xml(data: Any, origin: str) -> Node: + """Parse XML from bytes, text or an open stream, with a named error. + + ``minidom.parse`` takes a *file object or a filename*, so bytes have to be + wrapped: passing them straight through makes expat interpret the payload as a + path and report a confusing I/O error instead of the XML problem. + """ + if isinstance(data, bytes): + source: Any = io.BytesIO(data) + elif isinstance(data, str): + source = io.StringIO(data) + else: + source = data + try: + return minidom.parse(source) + except Exception as exc: + raise ValueError(f"{origin} is not well-formed XML: {exc}") from exc + + +def _root_element(document: Node) -> Node: + """The root element of a parsed document. + + ``minidom.parse`` returns a ``Document``, whose only child is the root + element; every lookup below wants the element, not the wrapper. + """ + return getattr(document, "documentElement", None) or document + + +def _document_title(root: Node) -> Optional[str]: + meta = _first(root, OFFICE_NS, "meta") + if meta is None: + return None + title = _first(meta, DC_NS, "title") + if title is None: + return None + text = "".join( + child.nodeValue or "" + for child in title.childNodes + if child.nodeType == Node.TEXT_NODE + ).strip() + return text or None + + +class OdfConverter(DocumentConverter): + """Converts OpenDocument text, spreadsheet and presentation files to Markdown. + + Handles the packaged (``.odt`` / ``.ods`` / ``.odp``) and flat + (``.fodt`` / ``.fods`` / ``.fodp``) variants. The body is dispatched on which + of ``office:text``, ``office:spreadsheet`` or ``office:presentation`` the + document actually has, not on the file extension, so a mislabelled file still + converts. + """ + + def accepts( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, # Options to pass to the converter + ) -> bool: + mimetype = (stream_info.mimetype or "").lower() + extension = (stream_info.extension or "").lower() + + if extension in ACCEPTED_FILE_EXTENSIONS: + return True + + for prefix in ACCEPTED_MIME_TYPE_PREFIXES: + if mimetype.startswith(prefix): + return True + + return False + + def convert( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, # Options to pass to the converter + ) -> DocumentConverterResult: + extension = (stream_info.extension or "").lower() + + if extension in FLAT_EXTENSIONS: + # Flat ODF is the same document as one uncompressed XML file, so there + # is no package to open and the root element is office:document rather + # than office:document-content. + document = _root_element( + _parse_xml(file_stream, "the flat OpenDocument file") + ) + else: + document = self._read_packaged(file_stream) + + body = _first(document, OFFICE_NS, "body") + if body is None: + raise ValueError( + "not an OpenDocument file: no office:body element was found" + ) + + blocks: List[str] = [] + if _first(body, OFFICE_NS, "text") is not None: + blocks = _render_blocks(_first(body, OFFICE_NS, "text")) + elif _first(body, OFFICE_NS, "spreadsheet") is not None: + blocks = _render_spreadsheet(_first(body, OFFICE_NS, "spreadsheet")) + elif _first(body, OFFICE_NS, "presentation") is not None: + blocks = _render_presentation(_first(body, OFFICE_NS, "presentation")) + else: + raise ValueError( + "unsupported OpenDocument body: expected office:text, " + "office:spreadsheet or office:presentation" + ) + + markdown = "\n\n".join(block for block in blocks if block.strip()).strip() + return DocumentConverterResult( + markdown=markdown, title=_document_title(document) + ) + + def _read_packaged(self, file_stream: BinaryIO) -> Node: + """Parse ``content.xml`` out of an OpenDocument ZIP, with the title if present.""" + try: + with zipfile.ZipFile(file_stream, "r") as archive: + names = set(archive.namelist()) + if "content.xml" not in names: + raise ValueError( + "not an OpenDocument file: the package has no content.xml" + ) + + info = archive.getinfo("content.xml") + if info.file_size > MAX_CONTENT_XML_BYTES: + raise ValueError( + f"refusing to parse a {info.file_size} byte content.xml " + f"(limit {MAX_CONTENT_XML_BYTES})" + ) + + document = _root_element( + _parse_xml(archive.read("content.xml"), "content.xml") + ) + + # meta.xml carries dc:title. It is a separate part, so the title is + # grafted onto the body document to keep _document_title() simple. + if "meta.xml" in names: + try: + meta_document = _parse_xml(archive.read("meta.xml"), "meta.xml") + except ValueError: + meta_document = None + if meta_document is not None: + meta = _first(_root_element(meta_document), OFFICE_NS, "meta") + if meta is not None: + document.appendChild(meta) + except zipfile.BadZipFile as exc: + raise ValueError(f"not a valid OpenDocument package: {exc}") from exc + + return document diff --git a/packages/markitdown/tests/_test_vectors.py b/packages/markitdown/tests/_test_vectors.py index 74fa9bd0ac..454fd48d28 100644 --- a/packages/markitdown/tests/_test_vectors.py +++ b/packages/markitdown/tests/_test_vectors.py @@ -234,6 +234,40 @@ class FileTestVector(object): ], must_not_include=[], ), + FileTestVector( + filename="test.odt", + mimetype="application/vnd.oasis.opendocument.text", + charset=None, + url=None, + must_include=[ + "# Release notes", + "[changelog](https://example.com/changelog)", + "- First item", + " - Nested", + "| Component | Status |", + r"| parser | ready \| shipped |", + "Trailing paragraph.", + ], + must_not_include=[ + # The empty tail every LibreOffice row carries must not become columns. + "16384", + ], + ), + FileTestVector( + filename="test.ods", + mimetype="application/vnd.oasis.opendocument.spreadsheet", + charset=None, + url=None, + must_include=[ + "## Inventory", + "| Item | Qty | Price |", + "| Widget | 3 | 1.50 |", + "| Gadget | 1 | 9.99 |", + ], + must_not_include=[ + "16384", + ], + ), ] diff --git a/packages/markitdown/tests/test_files/test.ods b/packages/markitdown/tests/test_files/test.ods new file mode 100644 index 0000000000..cd1993a3d9 Binary files /dev/null and b/packages/markitdown/tests/test_files/test.ods differ diff --git a/packages/markitdown/tests/test_files/test.odt b/packages/markitdown/tests/test_files/test.odt new file mode 100644 index 0000000000..ffab686ad8 Binary files /dev/null and b/packages/markitdown/tests/test_files/test.odt differ diff --git a/packages/markitdown/tests/test_odf_converter.py b/packages/markitdown/tests/test_odf_converter.py new file mode 100644 index 0000000000..5269f64bdf --- /dev/null +++ b/packages/markitdown/tests/test_odf_converter.py @@ -0,0 +1,529 @@ +"""Tests for the OpenDocument converter (``_odf_converter.py``). + +Fixtures are assembled in memory rather than checked in as binaries, because the +interesting cases are all *structural* and a hand-written XML fragment shows what +is being asserted. Two of them are worth reading before the assertions: + +* the ``number-columns-repeated="16384"`` tail that every LibreOffice row carries, + which is what a naive implementation turns into a 16384-column table; +* a cell whose text contains a pipe, which silently adds a column if it is not + escaped. +""" + +import io +import zipfile + +import pytest + +from markitdown import MarkItDown, StreamInfo +from markitdown.converters import OdfConverter + +OFFICE_NS = "urn:oasis:names:tc:opendocument:xmlns:office:1.0" +TEXT_NS = "urn:oasis:names:tc:opendocument:xmlns:text:1.0" +TABLE_NS = "urn:oasis:names:tc:opendocument:xmlns:table:1.0" +DRAW_NS = "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" +PRESENTATION_NS = "urn:oasis:names:tc:opendocument:xmlns:presentation:1.0" + +DOCUMENT_HEADER = ( + '\n' + "' +) + + +def _package( + body: str, *, title: str = "Test Document", meta: bool = True +) -> io.BytesIO: + """Build a minimal but structurally valid OpenDocument package.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("mimetype", "application/vnd.oasis.opendocument.text") + archive.writestr( + "content.xml", DOCUMENT_HEADER + body + "" + ) + if meta: + archive.writestr( + "meta.xml", + '\n' + f'' + f"{title}" + "", + ) + buffer.seek(0) + return buffer + + +def _flat(body: str) -> io.BytesIO: + """Build a flat OpenDocument file: the same XML, no package.""" + text = ( + '\n' + "' + "Flat" + + body + + "" + ) + return io.BytesIO(text.encode("utf-8")) + + +def _text_body(inner: str) -> str: + return f"{inner}" + + +def _sheet_body(inner: str) -> str: + return ( + f"{inner}" + ) + + +def _slide_body(inner: str) -> str: + return ( + f"{inner}" + ) + + +def _convert(stream: io.BytesIO, extension: str = ".odt"): + return OdfConverter().convert( + stream, StreamInfo(extension=extension, mimetype="application/octet-stream") + ) + + +class TestAccepts: + @pytest.mark.parametrize( + "extension", [".odt", ".ods", ".odp", ".fodt", ".fods", ".fodp"] + ) + def test_accepts_by_extension(self, extension): + converter = OdfConverter() + assert converter.accepts(io.BytesIO(), StreamInfo(extension=extension)) + + @pytest.mark.parametrize( + "mimetype", + [ + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.text-template", + ], + ) + def test_accepts_by_mimetype(self, mimetype): + assert OdfConverter().accepts(io.BytesIO(), StreamInfo(mimetype=mimetype)) + + @pytest.mark.parametrize("extension", [".docx", ".zip", ".epub", ".csv", ""]) + def test_rejects_other_extensions(self, extension): + assert not OdfConverter().accepts( + io.BytesIO(), StreamInfo(extension=extension, mimetype="application/zip") + ) + + def test_zip_converter_does_not_steal_odt(self): + """A .odt is a ZIP; the ODF converter must be tried before the ZIP one.""" + stream = _package(_text_body("from the odf converter")) + result = MarkItDown().convert_stream( + stream, stream_info=StreamInfo(extension=".odt", filename="doc.odt") + ) + assert "from the odf converter" in result.markdown + + +class TestTextDocument: + def test_headings_use_the_outline_level(self): + body = _text_body( + 'One' + 'Three' + ) + markdown = _convert(_package(body)).markdown + assert "# One" in markdown + assert "### Three" in markdown + + def test_outline_level_is_clamped_and_defaults_to_one(self): + body = _text_body( + 'Too deepNo level' + ) + markdown = _convert(_package(body)).markdown + assert "###### Too deep" in markdown + assert "# No level" in markdown + + def test_paragraphs_are_separated_by_a_blank_line(self): + body = _text_body("firstsecond") + assert _convert(_package(body)).markdown == "first\n\nsecond" + + def test_whitespace_markup_survives_collapsing(self): + """`text:s` and `text:tab` are elements, not whitespace, so they survive. + + The XML below also contains a newline and indentation between the + elements, which must *not* survive. + """ + body = _text_body( + 'ab\n' + " cd\n" + " ef\n" + ) + markdown = _convert(_package(body)).markdown + assert "a b" in markdown + assert "c d" in markdown + assert "e\tf" in markdown + + def test_line_break_becomes_br(self): + body = _text_body("updown") + assert "up
down" in _convert(_package(body)).markdown + + def test_hyperlink_is_rendered_as_a_markdown_link(self): + body = _text_body( + 'the link' + ) + assert "[the link](https://example.com/x)" in _convert(_package(body)).markdown + + def test_link_without_href_keeps_its_text(self): + body = _text_body("bare") + markdown = _convert(_package(body)).markdown + assert "bare" in markdown + assert "[](" not in markdown + + def test_footnote_is_dropped_rather_than_dangling(self): + body = _text_body( + 'claim' + "the note body" + "" + ) + markdown = _convert(_package(body)).markdown + assert markdown == "claim" + assert "the note body" not in markdown + + def test_table_of_content_is_not_duplicated(self): + body = _text_body( + '' + "" + 'Real heading' + ) + markdown = _convert(_package(body)).markdown + assert markdown.count("Real heading") == 1 + + def test_unknown_inline_element_keeps_its_text(self): + body = _text_body( + "keepthis" + ) + assert "keepthis" in _convert(_package(body)).markdown + + +class TestLists: + def test_bullet_list(self): + body = _text_body( + '' + "one" + "two" + "" + ) + assert _convert(_package(body)).markdown == "- one\n- two" + + def test_nested_list_is_indented(self): + body = _text_body( + '' + "outer" + 'inner' + "" + "" + ) + assert _convert(_package(body)).markdown == "- outer\n - inner" + + def test_second_paragraph_of_an_item_is_a_continuation(self): + body = _text_body( + '' + "first linesecond line" + "" + ) + assert _convert(_package(body)).markdown == "- first line\n second line" + + def test_empty_item_does_not_lose_its_bullet(self): + body = _text_body( + '' + "kept" + "" + ) + assert _convert(_package(body)).markdown == "-\n- kept" + + +class TestTables: + def test_text_table_has_no_title(self): + body = _text_body( + '' + "" + "h1" + "h2" + "" + "" + "a" + "b" + "" + "" + ) + markdown = _convert(_package(body)).markdown + assert markdown == "| h1 | h2 |\n| --- | --- |\n| a | b |" + assert "Table1" not in markdown + + def test_pipe_in_a_cell_is_escaped(self): + body = _sheet_body( + '' + "a|b" + "c" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + row = next(line for line in markdown.split("\n") if "a" in line) + assert row == r"| a\|b | c |" + # The escaped pipe must not add a column: the separator row still declares + # two columns, so the table is not ragged. + separator = markdown.split("\n")[3] + assert separator == "| --- | --- |" + + def test_multi_paragraph_cell_stays_on_one_row(self): + body = _sheet_body( + '' + "topbottom" + "x" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert "| top
bottom | x |" in markdown + + def test_covered_cell_keeps_the_column(self): + body = _sheet_body( + '' + "" + 'merged' + "" + "third" + "" + "" + "1" + '' + "3" + "" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + rows = [line for line in markdown.split("\n") if line.startswith("|")] + assert rows == [ + "| merged | third | |", + "| --- | --- | --- |", + "| 1 | | 3 |", + ] + + +class TestSpreadsheetRepetition: + def test_trailing_repeated_empty_cells_do_not_set_the_width(self): + """The case that makes a naive implementation produce a 16384-column table.""" + body = _sheet_body( + '' + "" + "name" + "count" + '' + "" + "" + "widgets" + "3" + '' + "" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert markdown == "## S\n\n| name | count |\n| --- | --- |\n| widgets | 3 |" + + def test_repeated_populated_cell_is_expanded(self): + body = _sheet_body( + '' + "" + 'same' + "" + "tail" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert markdown.split("\n")[2] == "| same | same | same | tail |" + + def test_repeated_row_is_expanded(self): + body = _sheet_body( + '' + "h" + "" + '' + "r" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert markdown.count("| r |") == 3 + + def test_repeated_empty_rows_are_dropped(self): + body = _sheet_body( + '' + "h" + "" + '' + '' + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert markdown == "## S\n\n| h |\n| --- |" + + def test_row_group_rows_are_included(self): + body = _sheet_body( + '' + "header" + "" + "" + "body" + "" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert "| header |" in markdown + assert "| body |" in markdown + + def test_empty_sheet_is_skipped(self): + body = _sheet_body( + '' + "" + '' + "" + '' + "v" + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert "Empty" not in markdown + assert "## Real" in markdown + + def test_typed_value_is_used_when_there_is_no_display_text(self): + body = _sheet_body( + '' + '' + '' + '' + "" + ) + markdown = _convert(_package(body), ".ods").markdown + assert markdown.split("\n")[2] == "| 1.5 | 2024-01-02 | TRUE |" + + +class TestPresentation: + def test_slides_and_frames(self): + body = _slide_body( + '' + "Welcome" + "" + '' + "" + 'Step onedo the thing' + "" + "" + ) + markdown = _convert(_package(body), ".odp").markdown + assert "## Intro" in markdown + assert "Welcome" in markdown + assert "## Details" in markdown + # A content heading starts below the slide title, not at the same level. + assert "### Step one" in markdown + assert "do the thing" in markdown + + def test_unnamed_slide_gets_a_number(self): + body = _slide_body( + "" + "only" + "" + ) + assert "## Slide 1" in _convert(_package(body), ".odp").markdown + + def test_speaker_notes_become_a_blockquote(self): + body = _slide_body( + '' + "slide text" + "" + "remember this" + "" + ) + markdown = _convert(_package(body), ".odp").markdown + assert "> remember this" in markdown + # The note must not also appear as slide content. + assert markdown.count("remember this") == 1 + + +class TestMetadataAndErrors: + def test_title_comes_from_meta_xml(self): + result = _convert(_package(_text_body("x"), title="Quarterly")) + assert result.title == "Quarterly" + + def test_missing_meta_is_not_an_error(self): + result = _convert(_package(_text_body("x"), meta=False)) + assert result.title is None + assert result.markdown == "x" + + def test_flat_document_has_no_package(self): + result = _convert(_flat(_text_body("flat body")), ".fodt") + assert result.markdown == "flat body" + assert result.title == "Flat" + + def test_flat_spreadsheet(self): + body = _sheet_body( + '' + "f" + "" + ) + markdown = _convert(_flat(body), ".fods").markdown + assert "## S" in markdown and "| f |" in markdown + + def test_body_kind_is_detected_not_assumed(self): + """A spreadsheet mislabelled as .odt still converts as a spreadsheet.""" + body = _sheet_body( + '' + "v" + "" + ) + markdown = _convert(_package(body), ".odt").markdown + assert "## S" in markdown + + def test_zip_without_content_xml_is_refused(self): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("something.txt", "hi") + buffer.seek(0) + with pytest.raises(ValueError, match="content.xml"): + _convert(buffer) + + def test_not_a_zip_is_refused(self): + with pytest.raises(ValueError, match="not a valid OpenDocument package"): + _convert(io.BytesIO(b"this is not a zip file at all")) + + def test_malformed_xml_is_refused(self): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("content.xml", "") + buffer.seek(0) + with pytest.raises(ValueError, match="well-formed"): + _convert(buffer) + + def test_unknown_body_is_refused(self): + body = f"" + with pytest.raises(ValueError, match="unsupported OpenDocument body"): + _convert(_package(body)) + + def test_oversized_content_xml_is_refused(self, monkeypatch): + import markitdown.converters._odf_converter as module + + monkeypatch.setattr(module, "MAX_CONTENT_XML_BYTES", 32) + body = _text_body("" + "x" * 200 + "") + with pytest.raises(ValueError, match="refusing to parse"): + _convert(_package(body))