From 9806737c97c29ff27e4d8b75c19ff80e8dd2a040 Mon Sep 17 00:00:00 2001
From: ANSHUL SINGH <72524975+ekanshul@users.noreply.github.com>
Date: Sun, 20 Sep 2026 17:35:16 +0000
Subject: [PATCH] Fall back to utf-8 when chardet's encoding cannot decode a
 file

--hard-encoding-detection crashed if chardet picked an encoding that
failed while reading, because only open() was wrapped and there was
no fallback. Catch UnicodeDecodeError and LookupError from the
chardet path, warn, and reuse the utf-8 / iso-8859-1 opener.

Fixes #3985
---
 codespell_lib/_codespell.py       | 31 ++++++++++++++---------------
 codespell_lib/tests/test_basic.py | 33 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/codespell_lib/_codespell.py b/codespell_lib/_codespell.py
index 1ec09fdf8f..ac85fb04e3 100644
--- a/codespell_lib/_codespell.py
+++ b/codespell_lib/_codespell.py
@@ -247,7 +247,16 @@ def init_chardet(self) -> None:
 
     def open(self, filename: str) -> tuple[list[tuple[bool, int, list[str]]], str]:
         if self.use_chardet:
-            return self.open_with_chardet(filename)
+            try:
+                return self.open_with_chardet(filename)
+            except (UnicodeDecodeError, LookupError):
+                encoding = getattr(self, "detected_encoding", None)
+                if not self.quiet_level & QuietLevels.ENCODING:
+                    print(
+                        f'WARNING: Cannot decode file using encoding "{encoding}": '
+                        f"{filename}",
+                        file=sys.stderr,
+                    )
         return self.open_with_internal(filename)
 
     def open_with_chardet(
@@ -261,23 +270,13 @@ def open_with_chardet(
                     break
         self.encdetector.close()
         encoding = self.encdetector.result["encoding"]
+        self.detected_encoding = encoding
+        if not encoding:
+            raise LookupError(encoding)
 
-        try:
-            f = open(filename, encoding=encoding, newline="")
-        except UnicodeDecodeError:
-            print(f"ERROR: Could not detect encoding: {filename}", file=sys.stderr)
-            raise
-        except LookupError:
-            print(
-                f"ERROR: Don't know how to handle encoding {encoding}: {filename}",
-                file=sys.stderr,
-            )
-            raise
-        else:
+        with open(filename, encoding=encoding, newline="") as f:
             lines = self.get_lines(f)
-            f.close()
-
-        return lines, f.encoding
+            return lines, f.encoding
 
     def open_with_internal(
         self, filename: str
diff --git a/codespell_lib/tests/test_basic.py b/codespell_lib/tests/test_basic.py
index 0127f57013..01ab604390 100644
--- a/codespell_lib/tests/test_basic.py
+++ b/codespell_lib/tests/test_basic.py
@@ -20,6 +20,7 @@
     EX_DATAERR,
     EX_OK,
     EX_USAGE,
+    FileOpener,
     _builtin_dictionaries,
     uri_regex_def,
 )
@@ -693,6 +694,38 @@ def test_unknown_encoding_chardet(
     assert cs.main("--hard-encoding-detection", fname) == 0
 
 
+def test_chardet_falls_back_when_detected_encoding_fails(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """If chardet picks an encoding that cannot decode the file, try utf-8."""
+    fname = tmp_path / "tmp"
+    fname.write_bytes("naïve\nspeling\n".encode())
+    opener = FileOpener(False, 0, None)
+    opener.use_chardet = True
+
+    class FakeDetector:
+        done = True
+        result = {"encoding": "ascii"}
+
+        def reset(self) -> None:
+            return None
+
+        def feed(self, _line: bytes) -> None:
+            return None
+
+        def close(self) -> None:
+            return None
+
+    opener.encdetector = FakeDetector()
+    lines, encoding = opener.open(str(fname))
+    assert encoding == "utf-8"
+    assert any("speling" in "".join(chunk) for _ignored, _lineno, chunk in lines)
+    captured = capsys.readouterr()
+    assert "ascii" in captured.err
+    assert "WARNING" in captured.err
+
+
 def test_ignore(
     tmp_path: Path,
     capsys: pytest.CaptureFixture[str],