From 20b92625a3bcd4252a4b4a4430dc74cd9e5dd1ca Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 09:57:55 +0000 Subject: [PATCH 1/4] XLSX/XLS: keep repeated header names instead of Total.1, Item.1 pandas renames duplicate column labels. When a label looks renamed, re-read the sheet's raw first row and use the original labels. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VVn4SeAXPsUajXT2rtHQfo --- .../markitdown/converters/_xlsx_converter.py | 33 +++++++++++++++++++ packages/markitdown/tests/test_xlsx.py | 14 ++++++++ 2 files changed, 47 insertions(+) diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py index fbe7ab4365..be8d8a3217 100644 --- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py @@ -52,10 +52,43 @@ def _read_excel_sheets(stream: BinaryIO, engine: str) -> dict[str, Any]: object keeps the value each cell holds; infer_objects() then restores the usual column dtypes for genuinely numeric, boolean and date columns. """ + start_pos = stream.tell() sheets = pd.read_excel(stream, sheet_name=None, engine=engine, dtype=object) + for name, sheet in sheets.items(): + if _may_have_renamed_duplicates(sheet.columns): + stream.seek(start_pos) + raw = pd.read_excel( + stream, sheet_name=name, engine=engine, dtype=object, header=None + ) + sheet.columns = _restore_header(sheet.columns, raw.iloc[0]) return {name: sheet.infer_objects() for name, sheet in sheets.items()} +# pandas renames a repeated header ("Total", "Total") to "Total", "Total.1", and +# pandas 2 removed the option to turn that off. +_RENAMED_DUPLICATE = re.compile(r"^(.*)\.\d+$", re.DOTALL) + + +def _may_have_renamed_duplicates(columns: Any) -> bool: + names = {str(column) for column in columns} + for column in columns: + match = _RENAMED_DUPLICATE.match(str(column)) + if match and match.group(1) in names: + return True + return False + + +def _restore_header(columns: Any, first_row: Any) -> list[Any]: + """Take the header text from the sheet's first row, as the cells hold it. + + A header cell pandas reads as missing (blank, or text such as "NA") keeps the + name pandas gave it, so only the renamed duplicates change. + """ + return [ + column if pd.isna(value) else value for column, value in zip(columns, first_row) + ] + + @contextmanager def _read_xlsx_sheets( file_stream: BinaryIO, diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index 8bced4c21b..5ccf9c7b45 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -584,5 +584,19 @@ def test_xlsx_numeric_cells_render_as_before() -> None: ] +def test_xlsx_repeated_header_names_are_kept() -> None: + markdown = _convert_rows( + [ + ["Item", "Total", "Total", "Item", "Total.1", "NA", None], + ["x", 1, 2, "y", 3, 4, 5], + ] + ) + + assert markdown.splitlines()[1] == ( + "| Item | Total | Total | Item | Total.1 | NA | Unnamed: 6 |" + ) + assert markdown.splitlines()[3] == "| x | 1 | 2 | y | 3 | 4 | 5 |" + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) From 0e905d9c92650219b2ccca7086015dc89ef27971 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 09:57:55 +0000 Subject: [PATCH 2/4] XLSX/XLS: keep text cells that look numeric as text pd.read_excel parsed cells stored as text into numbers, so 02134 became 2134, +15551234 became 15551234 and 007 became 7.0. Read with dtype=object and infer_objects() so text stays text and numeric columns render as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VVn4SeAXPsUajXT2rtHQfo --- .../tests/test_xlsx_inheritance.py | 2 +- .../markitdown/converters/_xlsx_converter.py | 18 ++++++-- packages/markitdown/tests/test_xlsx.py | 43 +++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py index c94c8fac5f..4a18a5255f 100644 --- a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py @@ -283,7 +283,7 @@ def fixed_read(*args: Any, **kwargs: Any): assert "Future shared fix" in result and "" not in result assert result.count("[Image OCR]") == 2 repair.assert_called_once() - assert calls == [{"sheet_name": None, "engine": "openpyxl"}] * 2 + assert calls == [{"sheet_name": None, "engine": "openpyxl", "dtype": object}] * 2 service.extract_text.assert_called_once() diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py index 9f794a3b77..fbe7ab4365 100644 --- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py @@ -44,6 +44,18 @@ _SHOW_ZEROES_ATTRIBUTE = re.compile(rb"(?<=[\s])showZeroes(\s*=)") +def _read_excel_sheets(stream: BinaryIO, engine: str) -> dict[str, Any]: + """Read every sheet, keeping text cells as text. + + By default pandas parses a text cell that looks numeric into a number, so a + ZIP code "02134" becomes 2134 and "+15551234" becomes 15551234. Reading as + object keeps the value each cell holds; infer_objects() then restores the + usual column dtypes for genuinely numeric, boolean and date columns. + """ + sheets = pd.read_excel(stream, sheet_name=None, engine=engine, dtype=object) + return {name: sheet.infer_objects() for name, sheet in sheets.items()} + + @contextmanager def _read_xlsx_sheets( file_stream: BinaryIO, @@ -52,12 +64,12 @@ def _read_xlsx_sheets( repaired_stream = None try: try: - sheets = pd.read_excel(file_stream, sheet_name=None, engine="openpyxl") + sheets = _read_excel_sheets(file_stream, "openpyxl") except TypeError as exc: if "showZeroes" not in str(exc): raise repaired_stream = _repair_sheetview_show_zeroes(file_stream, start_pos) - sheets = pd.read_excel(repaired_stream, sheet_name=None, engine="openpyxl") + sheets = _read_excel_sheets(repaired_stream, "openpyxl") yield sheets, repaired_stream if repaired_stream is not None else file_stream finally: if repaired_stream is not None: @@ -237,7 +249,7 @@ def convert( _xls_dependency_exc_info[2] ) - sheets = pd.read_excel(file_stream, sheet_name=None, engine="xlrd") + sheets = _read_excel_sheets(file_stream, "xlrd") md_content = "" for s in sheets: md_content += f"## {s}\n" diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index bb071c868f..8bced4c21b 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -541,5 +541,48 @@ def test_xlsx_show_zeroes_rename_is_scoped_to_sheet_view_tags() -> None: assert b'' in repaired +# Cell values + + +def _convert_rows(rows: list[list[Any]]) -> str: + workbook = openpyxl.Workbook() + for row in rows: + workbook.active.append(row) + stream = io.BytesIO() + workbook.save(stream) + stream.seek(0) + return MarkItDown().convert_stream(stream, stream_info=_INFO).markdown + + +def test_xlsx_text_cells_that_look_numeric_stay_text() -> None: + markdown = _convert_rows( + [ + ["zip", "phone", "account"], + ["02134", "+15551234", "12345678901234567890"], + ["00501", "0612345678", "007"], + ] + ) + + assert markdown.splitlines()[-2:] == [ + "| 02134 | +15551234 | 12345678901234567890 |", + "| 00501 | 0612345678 | 007 |", + ] + + +def test_xlsx_numeric_cells_render_as_before() -> None: + markdown = _convert_rows( + [ + ["int", "sparse", "float", "flag"], + [1, 2, 1.5, True], + [3, None, 0.25, False], + ] + ) + + assert markdown.splitlines()[-2:] == [ + "| 1 | 2.0 | 1.50 | True |", + "| 3 | NaN | 0.25 | False |", + ] + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) From 2cee2ab1c24a1ec05dc0a2fe3b8058679bbcfc29 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:25:59 +0000 Subject: [PATCH 3/4] XLSX/XLS: keep repeated NA-like header names; read only the header row The header re-read used pandas' default NA parsing, so repeated headers such as "NA", "N/A" or "NULL" were read as missing and kept their renamed NA.1 / N/A.1 / NULL.1 names. Re-read only the first row with NA parsing off; blank header cells still keep pandas' "Unnamed: n" names, including when the first row is narrower than the sheet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VVn4SeAXPsUajXT2rtHQfo --- .../markitdown/converters/_xlsx_converter.py | 48 +++++++++++++++---- packages/markitdown/tests/test_xlsx.py | 18 +++++++ 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py index be8d8a3217..614d302b8d 100644 --- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py @@ -57,10 +57,9 @@ def _read_excel_sheets(stream: BinaryIO, engine: str) -> dict[str, Any]: for name, sheet in sheets.items(): if _may_have_renamed_duplicates(sheet.columns): stream.seek(start_pos) - raw = pd.read_excel( - stream, sheet_name=name, engine=engine, dtype=object, header=None + sheet.columns = _restore_header( + sheet.columns, _read_header_row(stream, engine, name) ) - sheet.columns = _restore_header(sheet.columns, raw.iloc[0]) return {name: sheet.infer_objects() for name, sheet in sheets.items()} @@ -78,15 +77,44 @@ def _may_have_renamed_duplicates(columns: Any) -> bool: return False -def _restore_header(columns: Any, first_row: Any) -> list[Any]: - """Take the header text from the sheet's first row, as the cells hold it. +def _read_header_row(stream: BinaryIO, engine: str, sheet_name: str) -> list[Any]: + """Read only the sheet's first row, as the cells hold it. - A header cell pandas reads as missing (blank, or text such as "NA") keeps the - name pandas gave it, so only the renamed duplicates change. + NA parsing is off so header text such as "NA", "N/A" or "NULL" stays text; + blank cells come back as "". """ - return [ - column if pd.isna(value) else value for column, value in zip(columns, first_row) - ] + raw = pd.read_excel( + stream, + sheet_name=sheet_name, + engine=engine, + dtype=object, + header=None, + nrows=1, + na_filter=False, + ) + return list(raw.iloc[0]) if len(raw) else [] + + +def _restore_header(columns: Any, first_row: list[Any]) -> list[Any]: + """Take the header text from the sheet's first row. + + A blank header cell keeps the name pandas gave it ("Unnamed: 3"), so only the + renamed duplicates change. The first row can be narrower than the sheet when + its trailing cells are empty. + """ + restored = [] + for index, column in enumerate(columns): + value = first_row[index] if index < len(first_row) else "" + restored.append(column if _is_blank(value) else value) + return restored + + +def _is_blank(value: Any) -> bool: + return ( + value is None + or (isinstance(value, str) and value == "") + or (isinstance(value, float) and value != value) + ) @contextmanager diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index 5ccf9c7b45..804dcabe27 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -598,5 +598,23 @@ def test_xlsx_repeated_header_names_are_kept() -> None: assert markdown.splitlines()[3] == "| x | 1 | 2 | y | 3 | 4 | 5 |" +@pytest.mark.parametrize("label", ["NA", "N/A", "NULL", "nan", ""]) +def test_xlsx_repeated_na_like_header_names_are_kept(label: str) -> None: + markdown = _convert_rows([[label, label, "x"], [1, 2, 3]]) + + if label: + assert markdown.splitlines()[1] == f"| {label} | {label} | x |" + else: + # openpyxl writes "" as an empty cell, so pandas names both columns. + assert markdown.splitlines()[1] == "| Unnamed: 0 | Unnamed: 1 | x |" + + +def test_xlsx_repeated_headers_with_a_narrow_first_row() -> None: + markdown = _convert_rows([["Total", "Total"], [1, 2, 3, 4]]) + + assert markdown.splitlines()[1] == "| Total | Total | Unnamed: 2 | Unnamed: 3 |" + assert markdown.splitlines()[3] == "| 1 | 2 | 3 | 4 |" + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) From 36422a3cc4cb83a6f393d8e6f1aba35c2e9d1b67 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 14:24:53 +0000 Subject: [PATCH 4/4] XLSX/XLS: pin that pandas' missing-value markers are unchanged Keeping numeric-looking text as text does not change how pandas reads "NA", "N/A", "NULL" or "nan" data cells: they were missing before and still are. Say so in the helper docstring and add a test with a kept text cell next to them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VVn4SeAXPsUajXT2rtHQfo --- .../markitdown/converters/_xlsx_converter.py | 3 +++ packages/markitdown/tests/test_xlsx.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py index fbe7ab4365..819901b756 100644 --- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py @@ -51,6 +51,9 @@ def _read_excel_sheets(stream: BinaryIO, engine: str) -> dict[str, Any]: ZIP code "02134" becomes 2134 and "+15551234" becomes 15551234. Reading as object keeps the value each cell holds; infer_objects() then restores the usual column dtypes for genuinely numeric, boolean and date columns. + + pandas' missing-value markers are unchanged: text such as "NA", "N/A", + "NULL" or "nan" in a data cell is still read as missing, as before. """ sheets = pd.read_excel(stream, sheet_name=None, engine=engine, dtype=object) return {name: sheet.infer_objects() for name, sheet in sheets.items()} diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index 8bced4c21b..1afc01117c 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -569,6 +569,29 @@ def test_xlsx_text_cells_that_look_numeric_stay_text() -> None: ] +def test_xlsx_missing_value_markers_are_unchanged() -> None: + # Keeping numeric-looking text does not change pandas' NA markers: these + # cells were missing before and still are, next to text that is kept. + markdown = _convert_rows( + [ + ["zip", "marker"], + ["02134", "NA"], + ["00501", "N/A"], + ["00601", "NULL"], + ["00701", "nan"], + ["00801", "kept"], + ] + ) + + assert markdown.splitlines()[-5:] == [ + "| 02134 | NaN |", + "| 00501 | NaN |", + "| 00601 | NaN |", + "| 00701 | NaN |", + "| 00801 | kept |", + ] + + def test_xlsx_numeric_cells_render_as_before() -> None: markdown = _convert_rows( [