diff --git a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py index c94c8fac5f..4a18a5255f 100644 --- a/packages/markitdown-ocr/tests/test_xlsx_inheritance.py +++ b/packages/markitdown-ocr/tests/test_xlsx_inheritance.py @@ -283,7 +283,7 @@ def fixed_read(*args: Any, **kwargs: Any): assert "Future shared fix" in result and "" not in result assert result.count("[Image OCR]") == 2 repair.assert_called_once() - assert calls == [{"sheet_name": None, "engine": "openpyxl"}] * 2 + assert calls == [{"sheet_name": None, "engine": "openpyxl", "dtype": object}] * 2 service.extract_text.assert_called_once() diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py index 9f794a3b77..7624a005b0 100644 --- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py @@ -44,6 +44,82 @@ _SHOW_ZEROES_ATTRIBUTE = re.compile(rb"(?<=[\s])showZeroes(\s*=)") +def _read_excel_sheets(stream: BinaryIO, engine: str) -> dict[str, Any]: + """Read every sheet, keeping text cells as text. + + By default pandas parses a text cell that looks numeric into a number, so a + ZIP code "02134" becomes 2134 and "+15551234" becomes 15551234. Reading as + object keeps the value each cell holds; infer_objects() then restores the + usual column dtypes for genuinely numeric, boolean and date columns. + + pandas' missing-value markers are unchanged: text such as "NA", "N/A", + "NULL" or "nan" in a data cell is still read as missing, as before. + """ + start_pos = stream.tell() + sheets = pd.read_excel(stream, sheet_name=None, engine=engine, dtype=object) + for name, sheet in sheets.items(): + if _may_have_renamed_duplicates(sheet.columns): + stream.seek(start_pos) + sheet.columns = _restore_header( + sheet.columns, _read_header_row(stream, engine, name) + ) + return {name: sheet.infer_objects() for name, sheet in sheets.items()} + + +# pandas renames a repeated header ("Total", "Total") to "Total", "Total.1", and +# pandas 2 removed the option to turn that off. +_RENAMED_DUPLICATE = re.compile(r"^(.*)\.\d+$", re.DOTALL) + + +def _may_have_renamed_duplicates(columns: Any) -> bool: + names = {str(column) for column in columns} + for column in columns: + match = _RENAMED_DUPLICATE.match(str(column)) + if match and match.group(1) in names: + return True + return False + + +def _read_header_row(stream: BinaryIO, engine: str, sheet_name: str) -> list[Any]: + """Read only the sheet's first row, as the cells hold it. + + NA parsing is off so header text such as "NA", "N/A" or "NULL" stays text; + blank cells come back as "". + """ + raw = pd.read_excel( + stream, + sheet_name=sheet_name, + engine=engine, + dtype=object, + header=None, + nrows=1, + na_filter=False, + ) + return list(raw.iloc[0]) if len(raw) else [] + + +def _restore_header(columns: Any, first_row: list[Any]) -> list[Any]: + """Take the header text from the sheet's first row. + + A blank header cell keeps the name pandas gave it ("Unnamed: 3"), so only the + renamed duplicates change. The first row can be narrower than the sheet when + its trailing cells are empty. + """ + restored = [] + for index, column in enumerate(columns): + value = first_row[index] if index < len(first_row) else "" + restored.append(column if _is_blank(value) else value) + return restored + + +def _is_blank(value: Any) -> bool: + return ( + value is None + or (isinstance(value, str) and value == "") + or (isinstance(value, float) and value != value) + ) + + @contextmanager def _read_xlsx_sheets( file_stream: BinaryIO, @@ -52,12 +128,12 @@ def _read_xlsx_sheets( repaired_stream = None try: try: - sheets = pd.read_excel(file_stream, sheet_name=None, engine="openpyxl") + sheets = _read_excel_sheets(file_stream, "openpyxl") except TypeError as exc: if "showZeroes" not in str(exc): raise repaired_stream = _repair_sheetview_show_zeroes(file_stream, start_pos) - sheets = pd.read_excel(repaired_stream, sheet_name=None, engine="openpyxl") + sheets = _read_excel_sheets(repaired_stream, "openpyxl") yield sheets, repaired_stream if repaired_stream is not None else file_stream finally: if repaired_stream is not None: @@ -237,7 +313,7 @@ def convert( _xls_dependency_exc_info[2] ) - sheets = pd.read_excel(file_stream, sheet_name=None, engine="xlrd") + sheets = _read_excel_sheets(file_stream, "xlrd") md_content = "" for s in sheets: md_content += f"## {s}\n" diff --git a/packages/markitdown/tests/test_xlsx.py b/packages/markitdown/tests/test_xlsx.py index bb071c868f..4b1ba2d57d 100644 --- a/packages/markitdown/tests/test_xlsx.py +++ b/packages/markitdown/tests/test_xlsx.py @@ -541,5 +541,103 @@ def test_xlsx_show_zeroes_rename_is_scoped_to_sheet_view_tags() -> None: assert b'' in repaired +# Cell values + + +def _convert_rows(rows: list[list[Any]]) -> str: + workbook = openpyxl.Workbook() + for row in rows: + workbook.active.append(row) + stream = io.BytesIO() + workbook.save(stream) + stream.seek(0) + return MarkItDown().convert_stream(stream, stream_info=_INFO).markdown + + +def test_xlsx_text_cells_that_look_numeric_stay_text() -> None: + markdown = _convert_rows( + [ + ["zip", "phone", "account"], + ["02134", "+15551234", "12345678901234567890"], + ["00501", "0612345678", "007"], + ] + ) + + assert markdown.splitlines()[-2:] == [ + "| 02134 | +15551234 | 12345678901234567890 |", + "| 00501 | 0612345678 | 007 |", + ] + + +def test_xlsx_missing_value_markers_are_unchanged() -> None: + # Keeping numeric-looking text does not change pandas' NA markers: these + # cells were missing before and still are, next to text that is kept. + markdown = _convert_rows( + [ + ["zip", "marker"], + ["02134", "NA"], + ["00501", "N/A"], + ["00601", "NULL"], + ["00701", "nan"], + ["00801", "kept"], + ] + ) + + assert markdown.splitlines()[-5:] == [ + "| 02134 | NaN |", + "| 00501 | NaN |", + "| 00601 | NaN |", + "| 00701 | NaN |", + "| 00801 | kept |", + ] + + +def test_xlsx_numeric_cells_render_as_before() -> None: + markdown = _convert_rows( + [ + ["int", "sparse", "float", "flag"], + [1, 2, 1.5, True], + [3, None, 0.25, False], + ] + ) + + assert markdown.splitlines()[-2:] == [ + "| 1 | 2.0 | 1.50 | True |", + "| 3 | NaN | 0.25 | False |", + ] + + +def test_xlsx_repeated_header_names_are_kept() -> None: + markdown = _convert_rows( + [ + ["Item", "Total", "Total", "Item", "Total.1", "NA", None], + ["x", 1, 2, "y", 3, 4, 5], + ] + ) + + assert markdown.splitlines()[1] == ( + "| Item | Total | Total | Item | Total.1 | NA | Unnamed: 6 |" + ) + assert markdown.splitlines()[3] == "| x | 1 | 2 | y | 3 | 4 | 5 |" + + +@pytest.mark.parametrize("label", ["NA", "N/A", "NULL", "nan", ""]) +def test_xlsx_repeated_na_like_header_names_are_kept(label: str) -> None: + markdown = _convert_rows([[label, label, "x"], [1, 2, 3]]) + + if label: + assert markdown.splitlines()[1] == f"| {label} | {label} | x |" + else: + # openpyxl writes "" as an empty cell, so pandas names both columns. + assert markdown.splitlines()[1] == "| Unnamed: 0 | Unnamed: 1 | x |" + + +def test_xlsx_repeated_headers_with_a_narrow_first_row() -> None: + markdown = _convert_rows([["Total", "Total"], [1, 2, 3, 4]]) + + assert markdown.splitlines()[1] == "| Total | Total | Unnamed: 2 | Unnamed: 3 |" + assert markdown.splitlines()[3] == "| 1 | 2 | 3 | 4 |" + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__]))