Fall back to utf-8 when chardet's encoding cannot decode a file

codespell_lib/_codespell.py

@@ -247,7 +247,16 @@ def init_chardet(self) -> None:
 
     def open(self, filename: str) -> tuple[list[tuple[bool, int, list[str]]], str]:
         if self.use_chardet:
-            return self.open_with_chardet(filename)
+            try:
+                return self.open_with_chardet(filename)
+            except (UnicodeDecodeError, LookupError):
+                encoding = getattr(self, "detected_encoding", None)
+                if not self.quiet_level & QuietLevels.ENCODING:
+                    print(
+                        f'WARNING: Cannot decode file using encoding "{encoding}": '
+                        f"{filename}",
+                        file=sys.stderr,
+                    )
         return self.open_with_internal(filename)
 
     def open_with_chardet(
@@ -261,23 +270,13 @@ def open_with_chardet(
                     break
         self.encdetector.close()
         encoding = self.encdetector.result["encoding"]
+        self.detected_encoding = encoding
+        if not encoding:
+            raise LookupError(encoding)
 
-        try:
-            f = open(filename, encoding=encoding, newline="")
-        except UnicodeDecodeError:
-            print(f"ERROR: Could not detect encoding: {filename}", file=sys.stderr)
-            raise
-        except LookupError:
-            print(
-                f"ERROR: Don't know how to handle encoding {encoding}: {filename}",
-                file=sys.stderr,
-            )
-            raise
-        else:
+        with open(filename, encoding=encoding, newline="") as f:
             lines = self.get_lines(f)
-            f.close()
-
-        return lines, f.encoding
+            return lines, f.encoding
 
     def open_with_internal(
         self, filename: str

codespell_lib/tests/test_basic.py

@@ -20,6 +20,7 @@
     EX_DATAERR,
     EX_OK,
     EX_USAGE,
+    FileOpener,
     _builtin_dictionaries,
     uri_regex_def,
 )
@@ -693,6 +694,38 @@ def test_unknown_encoding_chardet(
     assert cs.main("--hard-encoding-detection", fname) == 0
 
 
+def test_chardet_falls_back_when_detected_encoding_fails(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """If chardet picks an encoding that cannot decode the file, try utf-8."""
+    fname = tmp_path / "tmp"
+    fname.write_bytes("naïve\nspeling\n".encode())
+    opener = FileOpener(False, 0, None)
+    opener.use_chardet = True
+
+    class FakeDetector:
+        done = True
+        result = {"encoding": "ascii"}
+
+        def reset(self) -> None:
+            return None
+
+        def feed(self, _line: bytes) -> None:
+            return None
+
+        def close(self) -> None:
+            return None
+
+    opener.encdetector = FakeDetector()
+    lines, encoding = opener.open(str(fname))
+    assert encoding == "utf-8"
+    assert any("speling" in "".join(chunk) for _ignored, _lineno, chunk in lines)
+    captured = capsys.readouterr()
+    assert "ascii" in captured.err
+    assert "WARNING" in captured.err
+
+
 def test_ignore(
     tmp_path: Path,
     capsys: pytest.CaptureFixture[str],