diff --git a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py index 6bc1c1673f..817f457bca 100644 --- a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py +++ b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py @@ -1,5 +1,6 @@ from typing import BinaryIO, Any import json +import re from .._base_converter import DocumentConverter, DocumentConverterResult from .._exceptions import FileConversionException @@ -69,23 +70,31 @@ def _convert(self, notebook_content: dict) -> DocumentConverterResult: for cell in notebook_content.get("cells", []): cell_type = cell.get("cell_type", "") - source_lines = cell.get("source", []) + # nbformat's `multiline_string` is a string *or* a list of lines, and + # both are valid. Joining the list and taking the string as-is give + # the same text. + source = cell.get("source", []) + source_text = source if isinstance(source, str) else "".join(source) if cell_type == "markdown": - md_output.append("".join(source_lines)) + md_output.append(source_text) - # Extract the first # heading as title if not already found + # Extract the first # heading as title if not already found. + # Split the text rather than walking the raw value: a string + # source would otherwise be iterated one character at a time, + # and a list entry holding several lines would hide a heading + # that is not on its first one. if title is None: - for line in source_lines: + for line in re.split(r"\r\n|\r|\n", source_text): if line.startswith("# "): title = line.removeprefix("# ").strip() break elif cell_type == "code": # Code cells are wrapped in Markdown code blocks - md_output.append(f"```python\n{''.join(source_lines)}\n```") + md_output.append(f"```python\n{source_text}\n```") elif cell_type == "raw": - md_output.append(f"```\n{''.join(source_lines)}\n```") + md_output.append(f"```\n{source_text}\n```") md_text = "\n\n".join(md_output) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d6c9be59b..044445ab62 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -552,6 +552,124 @@ def test_ipynb_accepts_non_ascii() -> None: assert result is True +def test_ipynb_string_source_is_read_as_text_not_characters() -> None: + """`source` may be a single string, and the title must still be found. + + nbformat's `multiline_string` is a string or a list of lines, and both + validate. Walking the raw value iterated a string one character at a time, + so no character ever started with "# " and the notebook came out titleless. + """ + from markitdown.converters._ipynb_converter import IpynbConverter + + notebook = { + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {}, + "cells": [ + { + "cell_type": "markdown", + "source": "# My Notebook\nintro text\n", + "metadata": {}, + }, + {"cell_type": "code", "source": "print('hello')\n", "metadata": {}}, + ], + } + + result = IpynbConverter()._convert(notebook) + + assert result.title == "My Notebook" + assert "# My Notebook" in result.markdown + assert "```python\nprint('hello')" in result.markdown + + +def test_ipynb_string_and_list_sources_agree() -> None: + """The two spellings of the same notebook convert to the same thing.""" + from markitdown.converters._ipynb_converter import IpynbConverter + + def notebook(markdown_source, code_source): + return { + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {}, + "cells": [ + { + "cell_type": "markdown", + "source": markdown_source, + "metadata": {}, + }, + {"cell_type": "code", "source": code_source, "metadata": {}}, + ], + } + + as_list = IpynbConverter()._convert( + notebook(["# Title\n", "body\n"], ["print(1)\n"]) + ) + as_string = IpynbConverter()._convert(notebook("# Title\nbody\n", "print(1)\n")) + + assert as_list.title == as_string.title == "Title" + assert as_list.markdown == as_string.markdown + + +def test_ipynb_heading_below_the_first_line_of_a_source_entry() -> None: + """A list entry can hold several lines; a heading on a later one still counts.""" + from markitdown.converters._ipynb_converter import IpynbConverter + + notebook = { + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {}, + "cells": [ + { + "cell_type": "markdown", + "source": ["intro paragraph\n# My Notebook\n"], + "metadata": {}, + } + ], + } + + assert IpynbConverter()._convert(notebook).title == "My Notebook" + + +def test_ipynb_heading_only_counts_at_the_start_of_a_line() -> None: + """Only \\n, \\r\\n and \\r end a line, so "# " after U+2028 is not a heading.""" + from markitdown.converters._ipynb_converter import IpynbConverter + + notebook = { + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {}, + "cells": [ + { + "cell_type": "markdown", + "source": ["pasted text\u2028# not a title\n"], + "metadata": {}, + } + ], + } + + assert IpynbConverter()._convert(notebook).title is None + + +def test_ipynb_heading_with_cr_line_endings() -> None: + """A source written with \\r line endings still yields just the heading.""" + from markitdown.converters._ipynb_converter import IpynbConverter + + notebook = { + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {}, + "cells": [ + { + "cell_type": "markdown", + "source": ["# My Notebook\r", "intro text\r"], + "metadata": {}, + } + ], + } + + assert IpynbConverter()._convert(notebook).title == "My Notebook" + + # File URI validation UNC_URI_PATHS = [