pypy support: fix incompatibilities and a severe cpyext memory leak, see #1755

src/borg/archive.py

@@ -16,7 +16,7 @@
 from collections.abc import Iterator
 from shutil import get_terminal_size
 
-from .platformflags import is_win32
+from .platformflags import is_win32, is_pypy
 from .logger import create_logger
 
 logger = create_logger()
@@ -831,7 +831,19 @@ def extract_helper(self, item, path, hlm, *, dry_run=False):
                     # backup_io would not catch) on platforms where os.link ignores follow_symlinks.
                     with backup_io("link"):
                         if os.link in os.supports_follow_symlinks:
-                            os.link(link_target, path, follow_symlinks=False)
+                            try:
+                                os.link(link_target, path, follow_symlinks=False)
+                            except OSError as os_error:
+                                # pypy advertises follow_symlinks support, but raises EINVAL when
+                                # the parameter is actually used (pypy bug) - emulate the secure
+                                # follow_symlinks=False behaviour documented above.
+                                if not (is_pypy and os_error.errno == errno.EINVAL):
+                                    raise
+                                if os.path.islink(link_target):
+                                    # we cannot hardlink the symlink itself, so make an equal symlink.
+                                    os.symlink(os.readlink(link_target), path)
+                                else:
+                                    os.link(link_target, path)
                         else:
                             os.link(link_target, path)
                 hardlink_set = True

src/borg/crypto/low_level.pyx

@@ -38,6 +38,8 @@ import hashlib
 import hmac
 from math import ceil
 
+from ..platformflags import is_pypy
+
 from cpython cimport PyMem_Malloc, PyMem_Free
 from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
 from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString
@@ -658,6 +660,12 @@ cdef class CHACHA20_POLY1305(_AEAD_BASE):
 
 
 def hmac_sha256(key, data):
+    if is_pypy:
+        # pypy's hmac/_hashlib only accepts bytes (no memoryview, no bytearray)
+        if not isinstance(key, bytes):
+            key = bytes(key)
+        if not isinstance(data, bytes):
+            data = bytes(data)
     return hmac.digest(key, data, 'sha256')
 
 

src/borg/platform/darwin.pyx

@@ -269,6 +269,9 @@ def set_flags(path, bsd_flags, fd=None):
 import errno as errno_mod
 import fcntl as fcntl_mod
 
+# pypy's fcntl module does not expose F_FULLFSYNC (value 51 in macOS headers)
+F_FULLFSYNC = getattr(fcntl_mod, "F_FULLFSYNC", 51)
+
 
 def fdatasync(fd):
     """macOS fdatasync using F_FULLFSYNC for true data durability.
@@ -278,7 +281,7 @@ def fdatasync(fd):
     Falls back to os.fsync() if F_FULLFSYNC is not supported (e.g. network fs).
     """
     try:
-        fcntl_mod.fcntl(fd, fcntl_mod.F_FULLFSYNC)
+        fcntl_mod.fcntl(fd, F_FULLFSYNC)
     except OSError:
         # F_FULLFSYNC not supported (e.g. network filesystem), fall back
         os.fsync(fd)

src/borg/platformflags.py

@@ -19,3 +19,6 @@
 
 # MSYS2 (on Windows)
 is_msystem = is_win32 and "MSYSTEM" in os.environ
+
+# Python implementation
+is_pypy = sys.implementation.name == "pypy"

src/borg/testsuite/archiver/extract_cmd_test.py

@@ -21,7 +21,7 @@
 from .. import changedir, same_ts_ns, granularity_sleep
 from .. import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported, is_birthtime_fully_supported
 from ...platform import get_birthtime_ns
-from ...platformflags import is_darwin, is_freebsd, is_win32
+from ...platformflags import is_darwin, is_freebsd, is_win32, is_pypy
 from . import (
     RK_ENCRYPTION,
     requires_hardlinks,
@@ -132,7 +132,10 @@ def test_extract_hardlinked_symlink_does_not_leak_target(archivers, request):
     h_path = os.path.join(archiver.output_path, "h")
     # "h" is a hardlink to the symlink itself (same inode as "s"), a faithful restore ...
     assert os.path.islink(h_path)
-    assert os.stat(h_path, follow_symlinks=False).st_ino == os.stat(s_path, follow_symlinks=False).st_ino
+    if not is_pypy:
+        # pypy's os.link(follow_symlinks=False) is broken, borg emulates it there by
+        # creating an equal symlink, which shares no inode with "s".
+        assert os.stat(h_path, follow_symlinks=False).st_ino == os.stat(s_path, follow_symlinks=False).st_ino
     # ... and NOT a hardlink to the external victim file's inode (that would be the leak).
     assert os.stat(h_path, follow_symlinks=False).st_ino != os.stat(victim_file).st_ino
     # the out-of-tree victim is untouched

src/borg/testsuite/archiver/lock_cmds_test.py

@@ -28,13 +28,14 @@ def test_with_lock(tmp_path):
     print("sys.path: %r" % sys.path)
     print("PYTHONPATH: %s" % env.get("PYTHONPATH", ""))
     print("PATH: %s" % env.get("PATH", ""))
-    command0 = "python3", "-m", "borg", "repo-create", "--encryption=none"
+    python = sys.executable or "python3"
+    command0 = python, "-m", "borg", "repo-create", "--encryption=none"
     # Timings must be adjusted so that command1 keeps running while command2 tries to get the lock,
     # so that lock acquisition for command2 fails as the test expects it.
     lock_wait = 2
-    command1 = ("python3", "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()')
-    command2 = "python3", "-c", 'print("second command - should never get executed")'
-    borgwl = "python3", "-m", "borg", "with-lock", f"--lock-wait={lock_wait}"
+    command1 = (python, "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()')
+    command2 = python, "-c", 'print("second command - should never get executed")'
+    borgwl = python, "-m", "borg", "with-lock", f"--lock-wait={lock_wait}"
     popen_options = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env)
     subprocess.run(command0, env=env, check=True, text=True, capture_output=True)
     assert repo_path.exists()
@@ -50,7 +51,8 @@ def test_with_lock(tmp_path):
             assert "Failed to create/acquire the lock" in err_out
             assert p2.returncode == 73  # LockTimeout: could not acquire the lock, p1 already has it
         out, err_out = p1.communicate(input="")  # Unblock command1 and read output
-        assert not err_out
+        # ignore the pure-python msgpack warning borg emits on pypy
+        assert not [line for line in err_out.splitlines() if "pure-python msgpack" not in line]
         assert p1.returncode == 0
 
 

src/borg/testsuite/helpers/msgpack_test.py

@@ -13,6 +13,9 @@ def expected_py_mp_slow_combination():
     # msgpack is slow on Cygwin
     if is_cygwin:
         return True
+    # pypy only has the pure-python msgpack (which pypy's jit hopefully makes fast enough)
+    if sys.implementation.name == "pypy":
+        return True
     # msgpack < 1.0.6 did not have Python 3.12 wheels
     if sys.version_info[:2] == (3, 12) and msgpack.version < (1, 0, 6):
         return True

src/borg/testsuite/item_test.py

@@ -4,6 +4,7 @@
 from ..item import Item, chunks_contents_equal
 from ..helpers import StableDict
 from ..helpers.msgpack import Timestamp
+from ..platformflags import is_pypy
 
 
 def test_item_empty():
@@ -131,6 +132,7 @@ def test_item_dict_property():
     assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}}
 
 
+@pytest.mark.xfail(is_pypy, reason="setting undeclared attributes on cdef class instances is not blocked on pypy")
 def test_unknown_property():
     # We do not want the user to be able to set unknown attributes —
     # they will not appear in the .as_dict() result dictionary.

src/borg/testsuite/platform/darwin_test.py

@@ -68,7 +68,7 @@ def mock_fcntl(fd, cmd, *args):
         tmp.flush()
         darwin.fdatasync(tmp.fileno())
 
-    assert any(cmd == fcntl_mod.F_FULLFSYNC for _, cmd in calls), "fdatasync should call fcntl with F_FULLFSYNC"
+    assert any(cmd == darwin.F_FULLFSYNC for _, cmd in calls), "fdatasync should call fcntl with F_FULLFSYNC"
 
 
 def test_fdatasync_falls_back_to_fsync(monkeypatch):
@@ -79,7 +79,7 @@ def test_fdatasync_falls_back_to_fsync(monkeypatch):
     fsync_calls = []
 
     def mock_fcntl(fd, cmd, *args):
-        if cmd == fcntl_mod.F_FULLFSYNC:
+        if cmd == darwin.F_FULLFSYNC:
             raise OSError("F_FULLFSYNC not supported")
         return 0