diff --git a/AUTHORS b/AUTHORS index ba5672d4c51..6d56abccbfa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -45,6 +45,7 @@ Andrzej Ostrowski Andy Freeland Anita Hammer Anna Tasiopoulou +Ant Newman Anthon van der Neut Anthony Shaw Anthony Sottile diff --git a/changelog/7623.bugfix.rst b/changelog/7623.bugfix.rst new file mode 100644 index 00000000000..f7410cc445d --- /dev/null +++ b/changelog/7623.bugfix.rst @@ -0,0 +1 @@ +:meth:`pytester.run ` no longer raises ``UnicodeDecodeError`` when a command writes output which is not valid UTF-8, such as a Python subprocess on a host whose locale encoding is not UTF-8; the undecodable bytes are now escaped instead. diff --git a/src/_pytest/pytester.py b/src/_pytest/pytester.py index 80c417843bf..99b41726118 100644 --- a/src/_pytest/pytester.py +++ b/src/_pytest/pytester.py @@ -1405,7 +1405,7 @@ def run( """Run a command with arguments. Run a process using :py:class:`subprocess.Popen` saving the stdout and - stderr. + stderr. The output is decoded as UTF-8, with undecodable bytes escaped. :param cmdargs: The sequence of arguments to pass to :py:class:`subprocess.Popen`, @@ -1471,7 +1471,12 @@ def handle_timeout() -> None: f1.flush() f2.flush() - with p1.open(encoding="utf8") as f1, p2.open(encoding="utf8") as f2: + # The command may write bytes which are not valid UTF-8. Escaping them + # keeps the byte values, and keeps the result printable by _dump_lines. + with ( + p1.open(encoding="utf8", errors="backslashreplace") as f1, + p2.open(encoding="utf8", errors="backslashreplace") as f2, + ): out = f1.read().splitlines() err = f2.read().splitlines() diff --git a/testing/test_pytester.py b/testing/test_pytester.py index e9a20f1063c..6e8d70aea1a 100644 --- a/testing/test_pytester.py +++ b/testing/test_pytester.py @@ -704,6 +704,38 @@ def test_inner(pytester): assert result.ret == 0 +def test_run_utf8_output(pytester: Pytester) -> None: + """UTF-8 output from a command is decoded as UTF-8""" + p1 = pytester.makepyfile( + r""" + import sys + + sys.stdout.buffer.write(b"h\xc3\xb6llo\n") + sys.stderr.buffer.write(b"w\xc3\xb6rld\n") + """ + ) + result = pytester.runpython(p1) + assert result.ret == 0 + assert result.stdout.lines == ["höllo"] + assert result.stderr.lines == ["wörld"] + + +def test_run_undecodable_output(pytester: Pytester) -> None: + """Output which is not valid UTF-8 is escaped instead of raising (#7623)""" + p1 = pytester.makepyfile( + r""" + import sys + + sys.stdout.buffer.write(b"before \xf6 after\n") + sys.stderr.buffer.write(b"\xf6\n") + """ + ) + result = pytester.runpython(p1) + assert result.ret == 0 + assert result.stdout.lines == [r"before \xf6 after"] + assert result.stderr.lines == [r"\xf6"] + + def test_spawn_uses_tmphome(pytester: Pytester) -> None: tmphome = str(pytester.path) assert os.environ.get("HOME") == tmphome