XLSX/XLS: keep repeated header names instead of Total.1, Item.1

packages/markitdown-ocr/tests/test_xlsx_inheritance.py

@@ -283,7 +283,7 @@ def fixed_read(*args: Any, **kwargs: Any):
     assert "Future shared fix" in result and "<native>" not in result
     assert result.count("[Image OCR]") == 2
     repair.assert_called_once()
-    assert calls == [{"sheet_name": None, "engine": "openpyxl"}] * 2
+    assert calls == [{"sheet_name": None, "engine": "openpyxl", "dtype": object}] * 2
     service.extract_text.assert_called_once()
 
 

packages/markitdown/src/markitdown/converters/_xlsx_converter.py

@@ -44,6 +44,82 @@
 _SHOW_ZEROES_ATTRIBUTE = re.compile(rb"(?<=[\s])showZeroes(\s*=)")
 
 
+def _read_excel_sheets(stream: BinaryIO, engine: str) -> dict[str, Any]:
+    """Read every sheet, keeping text cells as text.
+
+    By default pandas parses a text cell that looks numeric into a number, so a
+    ZIP code "02134" becomes 2134 and "+15551234" becomes 15551234. Reading as
+    object keeps the value each cell holds; infer_objects() then restores the
+    usual column dtypes for genuinely numeric, boolean and date columns.
+
+    pandas' missing-value markers are unchanged: text such as "NA", "N/A",
+    "NULL" or "nan" in a data cell is still read as missing, as before.
+    """
+    start_pos = stream.tell()
+    sheets = pd.read_excel(stream, sheet_name=None, engine=engine, dtype=object)
+    for name, sheet in sheets.items():
+        if _may_have_renamed_duplicates(sheet.columns):
+            stream.seek(start_pos)
+            sheet.columns = _restore_header(
+                sheet.columns, _read_header_row(stream, engine, name)
+            )
+    return {name: sheet.infer_objects() for name, sheet in sheets.items()}
+
+
+# pandas renames a repeated header ("Total", "Total") to "Total", "Total.1", and
+# pandas 2 removed the option to turn that off.
+_RENAMED_DUPLICATE = re.compile(r"^(.*)\.\d+$", re.DOTALL)
+
+
+def _may_have_renamed_duplicates(columns: Any) -> bool:
+    names = {str(column) for column in columns}
+    for column in columns:
+        match = _RENAMED_DUPLICATE.match(str(column))
+        if match and match.group(1) in names:
+            return True
+    return False
+
+
+def _read_header_row(stream: BinaryIO, engine: str, sheet_name: str) -> list[Any]:
+    """Read only the sheet's first row, as the cells hold it.
+
+    NA parsing is off so header text such as "NA", "N/A" or "NULL" stays text;
+    blank cells come back as "".
+    """
+    raw = pd.read_excel(
+        stream,
+        sheet_name=sheet_name,
+        engine=engine,
+        dtype=object,
+        header=None,
+        nrows=1,
+        na_filter=False,
+    )
+    return list(raw.iloc[0]) if len(raw) else []
+
+
+def _restore_header(columns: Any, first_row: list[Any]) -> list[Any]:
+    """Take the header text from the sheet's first row.
+
+    A blank header cell keeps the name pandas gave it ("Unnamed: 3"), so only the
+    renamed duplicates change. The first row can be narrower than the sheet when
+    its trailing cells are empty.
+    """
+    restored = []
+    for index, column in enumerate(columns):
+        value = first_row[index] if index < len(first_row) else ""
+        restored.append(column if _is_blank(value) else value)
+    return restored
+
+
+def _is_blank(value: Any) -> bool:
+    return (
+        value is None
+        or (isinstance(value, str) and value == "")
+        or (isinstance(value, float) and value != value)
+    )
+
+
 @contextmanager
 def _read_xlsx_sheets(
     file_stream: BinaryIO,
@@ -52,12 +128,12 @@ def _read_xlsx_sheets(
     repaired_stream = None
     try:
         try:
-            sheets = pd.read_excel(file_stream, sheet_name=None, engine="openpyxl")
+            sheets = _read_excel_sheets(file_stream, "openpyxl")
         except TypeError as exc:
             if "showZeroes" not in str(exc):
                 raise
             repaired_stream = _repair_sheetview_show_zeroes(file_stream, start_pos)
-            sheets = pd.read_excel(repaired_stream, sheet_name=None, engine="openpyxl")
+            sheets = _read_excel_sheets(repaired_stream, "openpyxl")
         yield sheets, repaired_stream if repaired_stream is not None else file_stream
     finally:
         if repaired_stream is not None:
@@ -237,7 +313,7 @@ def convert(
                 _xls_dependency_exc_info[2]
             )
 
-        sheets = pd.read_excel(file_stream, sheet_name=None, engine="xlrd")
+        sheets = _read_excel_sheets(file_stream, "xlrd")
         md_content = ""
         for s in sheets:
             md_content += f"## {s}\n"

packages/markitdown/tests/test_xlsx.py

@@ -541,5 +541,103 @@ def test_xlsx_show_zeroes_rename_is_scoped_to_sheet_view_tags() -> None:
     assert b'<customSheetView showZeroes="0"/>' in repaired
 
 
+# Cell values
+
+
+def _convert_rows(rows: list[list[Any]]) -> str:
+    workbook = openpyxl.Workbook()
+    for row in rows:
+        workbook.active.append(row)
+    stream = io.BytesIO()
+    workbook.save(stream)
+    stream.seek(0)
+    return MarkItDown().convert_stream(stream, stream_info=_INFO).markdown
+
+
+def test_xlsx_text_cells_that_look_numeric_stay_text() -> None:
+    markdown = _convert_rows(
+        [
+            ["zip", "phone", "account"],
+            ["02134", "+15551234", "12345678901234567890"],
+            ["00501", "0612345678", "007"],
+        ]
+    )
+
+    assert markdown.splitlines()[-2:] == [
+        "| 02134 | +15551234 | 12345678901234567890 |",
+        "| 00501 | 0612345678 | 007 |",
+    ]
+
+
+def test_xlsx_missing_value_markers_are_unchanged() -> None:
+    # Keeping numeric-looking text does not change pandas' NA markers: these
+    # cells were missing before and still are, next to text that is kept.
+    markdown = _convert_rows(
+        [
+            ["zip", "marker"],
+            ["02134", "NA"],
+            ["00501", "N/A"],
+            ["00601", "NULL"],
+            ["00701", "nan"],
+            ["00801", "kept"],
+        ]
+    )
+
+    assert markdown.splitlines()[-5:] == [
+        "| 02134 | NaN |",
+        "| 00501 | NaN |",
+        "| 00601 | NaN |",
+        "| 00701 | NaN |",
+        "| 00801 | kept |",
+    ]
+
+
+def test_xlsx_numeric_cells_render_as_before() -> None:
+    markdown = _convert_rows(
+        [
+            ["int", "sparse", "float", "flag"],
+            [1, 2, 1.5, True],
+            [3, None, 0.25, False],
+        ]
+    )
+
+    assert markdown.splitlines()[-2:] == [
+        "| 1 | 2.0 | 1.50 | True |",
+        "| 3 | NaN | 0.25 | False |",
+    ]
+
+
+def test_xlsx_repeated_header_names_are_kept() -> None:
+    markdown = _convert_rows(
+        [
+            ["Item", "Total", "Total", "Item", "Total.1", "NA", None],
+            ["x", 1, 2, "y", 3, 4, 5],
+        ]
+    )
+
+    assert markdown.splitlines()[1] == (
+        "| Item | Total | Total | Item | Total.1 | NA | Unnamed: 6 |"
+    )
+    assert markdown.splitlines()[3] == "| x | 1 | 2 | y | 3 | 4 | 5 |"
+
+
+@pytest.mark.parametrize("label", ["NA", "N/A", "NULL", "nan", ""])
+def test_xlsx_repeated_na_like_header_names_are_kept(label: str) -> None:
+    markdown = _convert_rows([[label, label, "x"], [1, 2, 3]])
+
+    if label:
+        assert markdown.splitlines()[1] == f"| {label} | {label} | x |"
+    else:
+        # openpyxl writes "" as an empty cell, so pandas names both columns.
+        assert markdown.splitlines()[1] == "| Unnamed: 0 | Unnamed: 1 | x |"
+
+
+def test_xlsx_repeated_headers_with_a_narrow_first_row() -> None:
+    markdown = _convert_rows([["Total", "Total"], [1, 2, 3, 4]])
+
+    assert markdown.splitlines()[1] == "| Total | Total | Unnamed: 2 | Unnamed: 3 |"
+    assert markdown.splitlines()[3] == "| 1 | 2 | 3 | 4 |"
+
+
 if __name__ == "__main__":
     raise SystemExit(pytest.main([__file__]))