diff --git a/codespell_lib/_codespell.py b/codespell_lib/_codespell.py index 1ec09fdf8f..ac85fb04e3 100644 --- a/codespell_lib/_codespell.py +++ b/codespell_lib/_codespell.py @@ -247,7 +247,16 @@ def init_chardet(self) -> None: def open(self, filename: str) -> tuple[list[tuple[bool, int, list[str]]], str]: if self.use_chardet: - return self.open_with_chardet(filename) + try: + return self.open_with_chardet(filename) + except (UnicodeDecodeError, LookupError): + encoding = getattr(self, "detected_encoding", None) + if not self.quiet_level & QuietLevels.ENCODING: + print( + f'WARNING: Cannot decode file using encoding "{encoding}": ' + f"{filename}", + file=sys.stderr, + ) return self.open_with_internal(filename) def open_with_chardet( @@ -261,23 +270,13 @@ def open_with_chardet( break self.encdetector.close() encoding = self.encdetector.result["encoding"] + self.detected_encoding = encoding + if not encoding: + raise LookupError(encoding) - try: - f = open(filename, encoding=encoding, newline="") - except UnicodeDecodeError: - print(f"ERROR: Could not detect encoding: {filename}", file=sys.stderr) - raise - except LookupError: - print( - f"ERROR: Don't know how to handle encoding {encoding}: {filename}", - file=sys.stderr, - ) - raise - else: + with open(filename, encoding=encoding, newline="") as f: lines = self.get_lines(f) - f.close() - - return lines, f.encoding + return lines, f.encoding def open_with_internal( self, filename: str diff --git a/codespell_lib/tests/test_basic.py b/codespell_lib/tests/test_basic.py index 0127f57013..01ab604390 100644 --- a/codespell_lib/tests/test_basic.py +++ b/codespell_lib/tests/test_basic.py @@ -20,6 +20,7 @@ EX_DATAERR, EX_OK, EX_USAGE, + FileOpener, _builtin_dictionaries, uri_regex_def, ) @@ -693,6 +694,38 @@ def test_unknown_encoding_chardet( assert cs.main("--hard-encoding-detection", fname) == 0 +def test_chardet_falls_back_when_detected_encoding_fails( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """If chardet picks an encoding that cannot decode the file, try utf-8.""" + fname = tmp_path / "tmp" + fname.write_bytes("naïve\nspeling\n".encode()) + opener = FileOpener(False, 0, None) + opener.use_chardet = True + + class FakeDetector: + done = True + result = {"encoding": "ascii"} + + def reset(self) -> None: + return None + + def feed(self, _line: bytes) -> None: + return None + + def close(self) -> None: + return None + + opener.encdetector = FakeDetector() + lines, encoding = opener.open(str(fname)) + assert encoding == "utf-8" + assert any("speling" in "".join(chunk) for _ignored, _lineno, chunk in lines) + captured = capsys.readouterr() + assert "ascii" in captured.err + assert "WARNING" in captured.err + + def test_ignore( tmp_path: Path, capsys: pytest.CaptureFixture[str],