hash/defrag: support the blake3 algorithm

.github/workflows/ci.yml

@@ -202,13 +202,13 @@ jobs:
         pip install -r requirements.d/dev.txt
     - name: Install borgstore (Linux)
       if: runner.os == 'Linux'
-      run: pip install -ve ".[s3,sftp,rest,rclone]"
+      run: pip install -ve ".[s3,sftp,rest,rclone,blake3]"
     - name: Install borgstore (Windows)
       if: runner.os == 'Windows'
-      run: pip install -ve ".[rest]"
+      run: pip install -ve ".[rest,blake3]"
     - name: Install borgstore (macOS)
       if: runner.os == 'macOS'
-      run: pip install -ve ".[rest,rclone]"
+      run: pip install -ve ".[rest,rclone,blake3]"
     - name: run tox envs
       run: tox -e ${{ matrix.toxenv }}
     - name: Display diagnostics on failure

README.rst

@@ -39,7 +39,7 @@ Backend features
 - new backends are simple to implement
 - key validation
 - partial loads / range requests
-- stored object hashing
+- stored object hashing (hashlib algorithms, e.g. sha256, and optionally blake3)
 - stored object defragmentation
 - quota support (only `posixfs`)
 - permissions checking (only `posixfs`)

docs/backends.rst

@@ -116,6 +116,7 @@ Use storage on an SFTP server:
 - Namespaces: directories
 - Values: in key-named files
 - hash: runs the hexdigest computation server-side (if server supports check-file).
+  "blake3" is not part of the check-file extension, so it is always computed client-side.
 
 rclone
 ------
@@ -168,6 +169,7 @@ Use a storage backend running inside a BorgStore REST server process:
 - Namespaces: depends on backend used by the server
 - Values: depends on backend used by the server
 - Authentication: Optional Basic Auth is supported.
-- hash: runs the hexdigest computation server-side.
+- hash: runs the hexdigest computation server-side. Using algorithm "blake3" requires
+  the optional ``blake3`` package to be installed **on the server**.
 - defrag: runs the defragmentation helper server-side.
 - atime: supported (if backend used by server supports it).

docs/changes.rst

@@ -1,6 +1,18 @@
 Changelog
 =========
 
+Version 0.5.6 (not released yet)
+--------------------------------
+
+New features:
+
+- hash / defrag: support the "blake3" algorithm (in addition to all hashlib algorithms).
+  Needs the optional "blake3" package: pip install 'borgstore[blake3]'.
+  For backends that hash server-side, it needs to be installed on the server.
+  Note: sftp always computes blake3 client-side, because the SFTP "check-file"
+  extension does not support blake3.
+
+
 Version 0.5.5 (2026-07-10)
 --------------------------
 

docs/installation.rst

@@ -18,10 +18,14 @@ To also enable other backends or other optional features, use:
 
 .. code-block:: bash
 
-    pip install 'borgstore[rest,rclone,sftp,s3]'
+    pip install 'borgstore[rest,rclone,sftp,s3,blake3]'
 
 For the available optional dependencies, see ``pyproject.toml``, section ``[project.optional-dependencies]``.
 
+The ``blake3`` extra is not about a backend: it enables the "blake3" hash algorithm
+for hash computations (blake3 is not part of ``hashlib``). For remote backends that
+compute hashes server-side, it needs to be installed on the server.
+
 
 Running the demo
 ----------------

docs/servers.rst

@@ -10,7 +10,8 @@ cloud storage servers:
 - enforcing permissions
 - server rejects store operation if content hashsum does not match expected
   hashsum (from http header X-Content-hash-sha256)
-- server-side hash computation (e.g. sha256) for item content
+- server-side hash computation (e.g. sha256, or blake3 if the optional ``blake3``
+  package is installed on the server) for item content
 - server-side defragmentation helper (copies blocks to new items)
 
 Running the server on host:port

docs/store.rst

@@ -16,10 +16,15 @@ API can be much simpler:
   an offset and/or size are supported.
 - info: get information about an item via its key (exists, size, ...).
 - hash: computes the hexdigest for the content of an item (given its key).
+  Supported algorithms are all algorithms supported by ``hashlib`` (e.g.
+  "sha256", the default) and also "blake3" (requires the optional ``blake3``
+  package, see :doc:`installation`).
 - delete: immediately remove an item from the store (given its key).
 - move: implements renaming, soft delete/undelete, and moving to the current
   nesting level.
-- defrag: general purpose defragmentation helper (copies blocks to new items)
+- defrag: general purpose defragmentation helper (copies blocks to new items).
+  If the target name is computed from the content, the same algorithms as for
+  hash are supported.
 - quota: return quota limit and usage (-1 if quotas not enabled or not supported)
 - stats: API call counters, time spent in API methods, data volume/throughput.
 - latency/bandwidth emulator: see :ref:`store-latency-bandwidth-emulator`.

pyproject.toml

@@ -37,6 +37,9 @@ sftp = [
 s3 = [
     "boto3",
 ]
+blake3 = [
+    "blake3 >= 1.0",  # optional "blake3" hash algorithm support for hash computations
+]
 none = []
 
 [project.scripts]

src/borgstore/backends/_base.py

@@ -4,12 +4,12 @@
 Docs that are not backend-specific are also found here.
 """
 
-import hashlib
 from abc import ABC, abstractmethod
 from collections import namedtuple
 from typing import Iterator
 
 from ..constants import MAX_NAME_LENGTH, TMP_SUFFIX, HID_SUFFIX
+from ..utils import hashing
 
 # atime is the last read access UNIX timestamp [s] or 0 if not implemented
 ItemInfo = namedtuple("ItemInfo", "name exists size directory atime", defaults=(0,))
@@ -126,6 +126,9 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, levels
         - source and target item names are with namespace.
         - if levels > 0, source and target item names are nested.
 
+        <algorithm> can be any algorithm supported by hashlib or "blake3"
+        (the latter requires the optional "blake3" package).
+
         Returns the target item name.
         """
         # default implementation: slow, but works for all backends.
@@ -145,10 +148,7 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, levels
         if target is None:
             if algorithm is None:
                 raise ValueError("Either target or algorithm must be given for defrag")
-            try:
-                h = hashlib.new(algorithm)
-            except (ValueError, TypeError):
-                raise ValueError(f"Unsupported hash algorithm: {algorithm}")
+            h = hashing.new(algorithm)
             h.update(data)
             target = h.hexdigest()
             if namespace:
@@ -159,13 +159,14 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, levels
         return target
 
     def hash(self, name: str, algorithm: str = "sha256") -> str:
-        """compute full-file hex digest of <name> content using <algorithm>"""
+        """compute full-file hex digest of <name> content using <algorithm>
+
+        <algorithm> can be any algorithm supported by hashlib or "blake3"
+        (the latter requires the optional "blake3" package).
+        """
         # default implementation: slow, but works for all backends.
         # might be overridden for performance.
-        try:
-            h = hashlib.new(algorithm)
-        except ValueError:
-            raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None
+        h = hashing.new(algorithm)
         h.update(self.load(name))
         return h.hexdigest()
 

src/borgstore/backends/posixfs.py

@@ -2,7 +2,6 @@
 Filesystem-based backend implementation - uses files in directories below a base path.
 """
 
-import hashlib
 import os
 import re
 import sys
@@ -26,6 +25,7 @@
 from .errors import BackendError, BackendAlreadyExists, BackendDoesNotExist, BackendMustNotBeOpen, BackendMustBeOpen
 from .errors import ObjectNotFound, PermissionDenied, QuotaExceeded
 from ..constants import TMP_SUFFIX, QUOTA_STORE_NAME, QUOTA_PERSIST_DELTA, QUOTA_PERSIST_INTERVAL
+from ..utils import hashing
 
 
 def get_file_backend(url, permissions=None, quota=None):
@@ -324,13 +324,10 @@ def hash(self, name: str, algorithm: str = "sha256") -> str:
             raise BackendMustBeOpen()
         path = self._validate_join(name)
         self._check_permission(name, "r")
-        try:
-            hashlib.new(algorithm)
-        except ValueError:
-            raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None
+        hashing.new(algorithm)  # validate the algorithm before opening the file
         try:
             with path.open("rb") as f:
-                h = hashlib.file_digest(f, algorithm)
+                h = hashing.file_digest(f, algorithm)
         except FileNotFoundError:
             raise ObjectNotFound(name) from None
         return h.hexdigest()

src/borgstore/backends/sftp.py

@@ -23,6 +23,7 @@
 from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists
 from .errors import ObjectNotFound
 from ..constants import TMP_SUFFIX
+from ..utils import hashing
 
 logger = logging.getLogger(__name__)
 
@@ -470,6 +471,10 @@ def delete(self, name):
     def _sftp_hash(self, name: str, algorithm: str) -> str | None:
         # Sadly, as of 2026-03-28, this is not supported by OpenSSH,
         # but by some less popular SFTP servers.
+        if algorithm == hashing.BLAKE3:
+            # the check-file extension only defines md5 / sha* algorithms. asking for blake3
+            # would just fail and, worse, make us give up on check-file for all later calls.
+            return None
         if self.check_file_supported:
             try:
                 with self.client.open(name) as f:

src/borgstore/store.py

@@ -633,6 +633,13 @@ def _list(self, name: str, deleted: bool = False) -> Iterator[ItemInfo]:
                     yield info
 
     def hash(self, name: str, algorithm: str = "sha256", *, deleted: bool = False) -> str:
+        """
+        compute the hex digest of the content of item <name> using <algorithm>.
+
+        algorithm can be any algorithm supported by hashlib (e.g. "sha256") or "blake3".
+        blake3 needs the optional "blake3" package to be installed - for backends that
+        compute the hash remotely, it must be installed on the server side.
+        """
         with self._stats_updater("hash", f"hash({name!r}, algorithm={algorithm!r}, deleted={deleted})"):
             return self._backend_call(
                 lambda: self.backend.hash(self.find(name, deleted=deleted), algorithm=algorithm), volume=0
@@ -647,7 +654,8 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, delete
         in order of appearance in the list and their contents will be appended to the target item.
 
         if the target name is not given, algorithm must be given to compute the target name
-        as hash(algorithm, target_content).hexdigest().
+        as hash(algorithm, target_content).hexdigest(). the supported algorithms are the same
+        as for the hash method, see there.
 
         returns the target name.
         """

src/borgstore/utils/hashing.py

@@ -0,0 +1,45 @@
+"""
+Hashing support: all algorithms supported by hashlib, plus "blake3".
+
+blake3 is not part of hashlib, it is provided by the optional "blake3" package
+(pip install 'borgstore[blake3]'). It is usually much faster than the sha2
+family, so it is an attractive choice for hashing stored object contents.
+"""
+
+import hashlib
+from typing import Any, Callable, Optional
+
+blake3: Optional[Callable[..., Any]]
+try:
+    from blake3 import blake3  # type: ignore[import-not-found,no-redef]
+except ImportError:
+    blake3 = None
+
+BLAKE3 = "blake3"
+
+
+def _blake3_factory() -> Callable[..., Any]:
+    """return the blake3 hash object factory (or raise, if it is not available)"""
+    if blake3 is None:
+        raise ValueError(f"Unsupported hash algorithm: {BLAKE3} (pip install 'borgstore[blake3]')")
+    return blake3
+
+
+def new(algorithm: str, data: bytes = b"") -> Any:
+    """like hashlib.new, but also supports "blake3"."""
+    if algorithm == BLAKE3:
+        return _blake3_factory()(data)
+    try:
+        return hashlib.new(algorithm, data)
+    except (ValueError, TypeError):
+        raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None
+
+
+def file_digest(fileobj, algorithm: str) -> Any:
+    """like hashlib.file_digest, but also supports "blake3"."""
+    if algorithm == BLAKE3:
+        return hashlib.file_digest(fileobj, _blake3_factory())
+    try:
+        return hashlib.file_digest(fileobj, algorithm)
+    except (ValueError, TypeError):
+        raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None

tests/test_backends.py

@@ -19,6 +19,13 @@
 except ImportError:
     requests = None
 
+try:
+    from blake3 import blake3
+except ImportError:
+    blake3 = None
+
+blake3_is_available = blake3 is not None
+
 from borgstore.backends._base import ItemInfo
 from borgstore.backends.errors import (
     BackendAlreadyExists,
@@ -733,6 +740,26 @@ def test_hash(tested_backends, request):
         backend.hash("test/item")
 
 
+@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed")
+def test_hash_blake3(tested_backends, request):
+    backend = get_backend_from_fixture(tested_backends, request)
+    with backend:
+        data = b"hash me"
+        expected_hash = blake3(data).hexdigest()
+        backend.store("test/item", data)
+        assert backend.hash("test/item", algorithm="blake3") == expected_hash
+
+        # Large-ish data to test chunking
+        large_data = b"a" * (2 * 1024 * 1024)
+        expected_large_hash = blake3(large_data).hexdigest()
+        backend.store("test/large_item", large_data)
+        assert backend.hash("test/large_item", algorithm="blake3") == expected_large_hash
+
+        # Test error for nonexistent object
+        with pytest.raises(ObjectNotFound):
+            backend.hash("test/nonexistent", algorithm="blake3")
+
+
 @pytest.mark.skipif(not (rest1_is_available and rest2_is_available), reason="REST1 and REST2 backends not available")
 def test_nginx_dispatch(rest1_backend_created, rest2_backend_created):
     """

tests/test_hashing.py

@@ -0,0 +1,54 @@
+"""
+Tests for the hashing utils (hashlib algorithms plus optional blake3).
+"""
+
+import hashlib
+import io
+
+import pytest
+
+from borgstore.utils import hashing
+
+blake3_is_available = hashing.blake3 is not None
+
+
+def test_new_hashlib():
+    data = b"hash me"
+    assert hashing.new("sha256", data).hexdigest() == hashlib.sha256(data).hexdigest()
+    h = hashing.new("sha256")
+    h.update(data)
+    assert h.hexdigest() == hashlib.sha256(data).hexdigest()
+
+
+def test_file_digest_hashlib():
+    data = b"hash me"
+    assert hashing.file_digest(io.BytesIO(data), "sha256").hexdigest() == hashlib.sha256(data).hexdigest()
+
+
+@pytest.mark.parametrize("algorithm", ["invalid_algo", "", None, 42])
+def test_unsupported_algorithm(algorithm):
+    with pytest.raises(ValueError, match="Unsupported hash algorithm"):
+        hashing.new(algorithm)
+    with pytest.raises(ValueError, match="Unsupported hash algorithm"):
+        hashing.file_digest(io.BytesIO(b"hash me"), algorithm)
+
+
+@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed")
+def test_blake3():
+    data = b"hash me"
+    expected = hashing.blake3(data).hexdigest()
+    # known value, so we notice if the algorithm/output encoding ever changes
+    assert expected == "e02b7e4520e277d4ef287f09c74fec4dd1df095b8b41a4d61dd4ed1589c4a281"
+    assert hashing.new("blake3", data).hexdigest() == expected
+    h = hashing.new("blake3")
+    h.update(data)
+    assert h.hexdigest() == expected
+    assert hashing.file_digest(io.BytesIO(data), "blake3").hexdigest() == expected
+
+
+@pytest.mark.skipif(blake3_is_available, reason="blake3 package is installed")
+def test_blake3_not_available():
+    with pytest.raises(ValueError, match="Unsupported hash algorithm: blake3"):
+        hashing.new("blake3")
+    with pytest.raises(ValueError, match="Unsupported hash algorithm: blake3"):
+        hashing.file_digest(io.BytesIO(b"hash me"), "blake3")