From 5db95c322a0d057ff2eae11f05b61f6891bb095a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 28 Jul 2026 00:40:56 +0200 Subject: [PATCH] hash/defrag: support the blake3 algorithm blake3 is not part of hashlib, so it comes from the optional "blake3" package: pip install 'borgstore[blake3]'. The new borgstore.utils.hashing module wraps hashlib and adds blake3 for hashing purposes, so all the backends (and the REST server) support it in hash and defrag. For backends computing hashes remotely, the package needs to be installed on the server side. sftp always computes blake3 client-side, because the SFTP "check-file" extension does not support blake3 (and asking for it would make us give up on check-file for all later calls). Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 6 ++-- README.rst | 2 +- docs/backends.rst | 4 ++- docs/changes.rst | 12 +++++++ docs/installation.rst | 6 +++- docs/servers.rst | 3 +- docs/store.rst | 7 +++- pyproject.toml | 3 ++ src/borgstore/backends/_base.py | 21 ++++++------ src/borgstore/backends/posixfs.py | 9 ++---- src/borgstore/backends/sftp.py | 5 +++ src/borgstore/store.py | 10 +++++- src/borgstore/utils/hashing.py | 45 ++++++++++++++++++++++++++ tests/test_backends.py | 27 ++++++++++++++++ tests/test_hashing.py | 54 +++++++++++++++++++++++++++++++ tests/test_server_rest.py | 43 ++++++++++++++++++++++++ tests/test_store.py | 33 +++++++++++++++++++ tox.ini | 1 + 18 files changed, 266 insertions(+), 25 deletions(-) create mode 100644 src/borgstore/utils/hashing.py create mode 100644 tests/test_hashing.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aecdcb2..dfd6d0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,13 +202,13 @@ jobs: pip install -r requirements.d/dev.txt - name: Install borgstore (Linux) if: runner.os == 'Linux' - run: pip install -ve ".[s3,sftp,rest,rclone]" + run: pip install -ve ".[s3,sftp,rest,rclone,blake3]" - name: Install borgstore (Windows) if: runner.os == 'Windows' - run: pip install -ve ".[rest]" + run: pip install -ve ".[rest,blake3]" - name: Install borgstore (macOS) if: runner.os == 'macOS' - run: pip install -ve ".[rest,rclone]" + run: pip install -ve ".[rest,rclone,blake3]" - name: run tox envs run: tox -e ${{ matrix.toxenv }} - name: Display diagnostics on failure diff --git a/README.rst b/README.rst index 84763b3..268ff4f 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ Backend features - new backends are simple to implement - key validation - partial loads / range requests -- stored object hashing +- stored object hashing (hashlib algorithms, e.g. sha256, and optionally blake3) - stored object defragmentation - quota support (only `posixfs`) - permissions checking (only `posixfs`) diff --git a/docs/backends.rst b/docs/backends.rst index 4ba3305..575a8a6 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -116,6 +116,7 @@ Use storage on an SFTP server: - Namespaces: directories - Values: in key-named files - hash: runs the hexdigest computation server-side (if server supports check-file). + "blake3" is not part of the check-file extension, so it is always computed client-side. rclone ------ @@ -168,6 +169,7 @@ Use a storage backend running inside a BorgStore REST server process: - Namespaces: depends on backend used by the server - Values: depends on backend used by the server - Authentication: Optional Basic Auth is supported. -- hash: runs the hexdigest computation server-side. +- hash: runs the hexdigest computation server-side. Using algorithm "blake3" requires + the optional ``blake3`` package to be installed **on the server**. - defrag: runs the defragmentation helper server-side. - atime: supported (if backend used by server supports it). diff --git a/docs/changes.rst b/docs/changes.rst index 1d86e3a..09b3eb6 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,18 @@ Changelog ========= +Version 0.5.6 (not released yet) +-------------------------------- + +New features: + +- hash / defrag: support the "blake3" algorithm (in addition to all hashlib algorithms). + Needs the optional "blake3" package: pip install 'borgstore[blake3]'. + For backends that hash server-side, it needs to be installed on the server. + Note: sftp always computes blake3 client-side, because the SFTP "check-file" + extension does not support blake3. + + Version 0.5.5 (2026-07-10) -------------------------- diff --git a/docs/installation.rst b/docs/installation.rst index c6efc77..b90907e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -18,10 +18,14 @@ To also enable other backends or other optional features, use: .. code-block:: bash - pip install 'borgstore[rest,rclone,sftp,s3]' + pip install 'borgstore[rest,rclone,sftp,s3,blake3]' For the available optional dependencies, see ``pyproject.toml``, section ``[project.optional-dependencies]``. +The ``blake3`` extra is not about a backend: it enables the "blake3" hash algorithm +for hash computations (blake3 is not part of ``hashlib``). For remote backends that +compute hashes server-side, it needs to be installed on the server. + Running the demo ---------------- diff --git a/docs/servers.rst b/docs/servers.rst index 2bb18f0..c034466 100644 --- a/docs/servers.rst +++ b/docs/servers.rst @@ -10,7 +10,8 @@ cloud storage servers: - enforcing permissions - server rejects store operation if content hashsum does not match expected hashsum (from http header X-Content-hash-sha256) -- server-side hash computation (e.g. sha256) for item content +- server-side hash computation (e.g. sha256, or blake3 if the optional ``blake3`` + package is installed on the server) for item content - server-side defragmentation helper (copies blocks to new items) Running the server on host:port diff --git a/docs/store.rst b/docs/store.rst index 9959a30..f0c2914 100644 --- a/docs/store.rst +++ b/docs/store.rst @@ -16,10 +16,15 @@ API can be much simpler: an offset and/or size are supported. - info: get information about an item via its key (exists, size, ...). - hash: computes the hexdigest for the content of an item (given its key). + Supported algorithms are all algorithms supported by ``hashlib`` (e.g. + "sha256", the default) and also "blake3" (requires the optional ``blake3`` + package, see :doc:`installation`). - delete: immediately remove an item from the store (given its key). - move: implements renaming, soft delete/undelete, and moving to the current nesting level. -- defrag: general purpose defragmentation helper (copies blocks to new items) +- defrag: general purpose defragmentation helper (copies blocks to new items). + If the target name is computed from the content, the same algorithms as for + hash are supported. - quota: return quota limit and usage (-1 if quotas not enabled or not supported) - stats: API call counters, time spent in API methods, data volume/throughput. - latency/bandwidth emulator: see :ref:`store-latency-bandwidth-emulator`. diff --git a/pyproject.toml b/pyproject.toml index b042725..cde9444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ sftp = [ s3 = [ "boto3", ] +blake3 = [ + "blake3 >= 1.0", # optional "blake3" hash algorithm support for hash computations +] none = [] [project.scripts] diff --git a/src/borgstore/backends/_base.py b/src/borgstore/backends/_base.py index a71f5df..dced871 100644 --- a/src/borgstore/backends/_base.py +++ b/src/borgstore/backends/_base.py @@ -4,12 +4,12 @@ Docs that are not backend-specific are also found here. """ -import hashlib from abc import ABC, abstractmethod from collections import namedtuple from typing import Iterator from ..constants import MAX_NAME_LENGTH, TMP_SUFFIX, HID_SUFFIX +from ..utils import hashing # atime is the last read access UNIX timestamp [s] or 0 if not implemented ItemInfo = namedtuple("ItemInfo", "name exists size directory atime", defaults=(0,)) @@ -126,6 +126,9 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, levels - source and target item names are with namespace. - if levels > 0, source and target item names are nested. + can be any algorithm supported by hashlib or "blake3" + (the latter requires the optional "blake3" package). + Returns the target item name. """ # default implementation: slow, but works for all backends. @@ -145,10 +148,7 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, levels if target is None: if algorithm is None: raise ValueError("Either target or algorithm must be given for defrag") - try: - h = hashlib.new(algorithm) - except (ValueError, TypeError): - raise ValueError(f"Unsupported hash algorithm: {algorithm}") + h = hashing.new(algorithm) h.update(data) target = h.hexdigest() if namespace: @@ -159,13 +159,14 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, levels return target def hash(self, name: str, algorithm: str = "sha256") -> str: - """compute full-file hex digest of content using """ + """compute full-file hex digest of content using + + can be any algorithm supported by hashlib or "blake3" + (the latter requires the optional "blake3" package). + """ # default implementation: slow, but works for all backends. # might be overridden for performance. - try: - h = hashlib.new(algorithm) - except ValueError: - raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None + h = hashing.new(algorithm) h.update(self.load(name)) return h.hexdigest() diff --git a/src/borgstore/backends/posixfs.py b/src/borgstore/backends/posixfs.py index 341e2dd..e3622f3 100644 --- a/src/borgstore/backends/posixfs.py +++ b/src/borgstore/backends/posixfs.py @@ -2,7 +2,6 @@ Filesystem-based backend implementation - uses files in directories below a base path. """ -import hashlib import os import re import sys @@ -26,6 +25,7 @@ from .errors import BackendError, BackendAlreadyExists, BackendDoesNotExist, BackendMustNotBeOpen, BackendMustBeOpen from .errors import ObjectNotFound, PermissionDenied, QuotaExceeded from ..constants import TMP_SUFFIX, QUOTA_STORE_NAME, QUOTA_PERSIST_DELTA, QUOTA_PERSIST_INTERVAL +from ..utils import hashing def get_file_backend(url, permissions=None, quota=None): @@ -324,13 +324,10 @@ def hash(self, name: str, algorithm: str = "sha256") -> str: raise BackendMustBeOpen() path = self._validate_join(name) self._check_permission(name, "r") - try: - hashlib.new(algorithm) - except ValueError: - raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None + hashing.new(algorithm) # validate the algorithm before opening the file try: with path.open("rb") as f: - h = hashlib.file_digest(f, algorithm) + h = hashing.file_digest(f, algorithm) except FileNotFoundError: raise ObjectNotFound(name) from None return h.hexdigest() diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index dc9d02f..3b483a0 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -23,6 +23,7 @@ from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists from .errors import ObjectNotFound from ..constants import TMP_SUFFIX +from ..utils import hashing logger = logging.getLogger(__name__) @@ -470,6 +471,10 @@ def delete(self, name): def _sftp_hash(self, name: str, algorithm: str) -> str | None: # Sadly, as of 2026-03-28, this is not supported by OpenSSH, # but by some less popular SFTP servers. + if algorithm == hashing.BLAKE3: + # the check-file extension only defines md5 / sha* algorithms. asking for blake3 + # would just fail and, worse, make us give up on check-file for all later calls. + return None if self.check_file_supported: try: with self.client.open(name) as f: diff --git a/src/borgstore/store.py b/src/borgstore/store.py index 3789813..cd229a3 100644 --- a/src/borgstore/store.py +++ b/src/borgstore/store.py @@ -633,6 +633,13 @@ def _list(self, name: str, deleted: bool = False) -> Iterator[ItemInfo]: yield info def hash(self, name: str, algorithm: str = "sha256", *, deleted: bool = False) -> str: + """ + compute the hex digest of the content of item using . + + algorithm can be any algorithm supported by hashlib (e.g. "sha256") or "blake3". + blake3 needs the optional "blake3" package to be installed - for backends that + compute the hash remotely, it must be installed on the server side. + """ with self._stats_updater("hash", f"hash({name!r}, algorithm={algorithm!r}, deleted={deleted})"): return self._backend_call( lambda: self.backend.hash(self.find(name, deleted=deleted), algorithm=algorithm), volume=0 @@ -647,7 +654,8 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, delete in order of appearance in the list and their contents will be appended to the target item. if the target name is not given, algorithm must be given to compute the target name - as hash(algorithm, target_content).hexdigest(). + as hash(algorithm, target_content).hexdigest(). the supported algorithms are the same + as for the hash method, see there. returns the target name. """ diff --git a/src/borgstore/utils/hashing.py b/src/borgstore/utils/hashing.py new file mode 100644 index 0000000..c1b10c4 --- /dev/null +++ b/src/borgstore/utils/hashing.py @@ -0,0 +1,45 @@ +""" +Hashing support: all algorithms supported by hashlib, plus "blake3". + +blake3 is not part of hashlib, it is provided by the optional "blake3" package +(pip install 'borgstore[blake3]'). It is usually much faster than the sha2 +family, so it is an attractive choice for hashing stored object contents. +""" + +import hashlib +from typing import Any, Callable, Optional + +blake3: Optional[Callable[..., Any]] +try: + from blake3 import blake3 # type: ignore[import-not-found,no-redef] +except ImportError: + blake3 = None + +BLAKE3 = "blake3" + + +def _blake3_factory() -> Callable[..., Any]: + """return the blake3 hash object factory (or raise, if it is not available)""" + if blake3 is None: + raise ValueError(f"Unsupported hash algorithm: {BLAKE3} (pip install 'borgstore[blake3]')") + return blake3 + + +def new(algorithm: str, data: bytes = b"") -> Any: + """like hashlib.new, but also supports "blake3".""" + if algorithm == BLAKE3: + return _blake3_factory()(data) + try: + return hashlib.new(algorithm, data) + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None + + +def file_digest(fileobj, algorithm: str) -> Any: + """like hashlib.file_digest, but also supports "blake3".""" + if algorithm == BLAKE3: + return hashlib.file_digest(fileobj, _blake3_factory()) + try: + return hashlib.file_digest(fileobj, algorithm) + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash algorithm: {algorithm}") from None diff --git a/tests/test_backends.py b/tests/test_backends.py index 5c567ca..eceb76b 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -19,6 +19,13 @@ except ImportError: requests = None +try: + from blake3 import blake3 +except ImportError: + blake3 = None + +blake3_is_available = blake3 is not None + from borgstore.backends._base import ItemInfo from borgstore.backends.errors import ( BackendAlreadyExists, @@ -733,6 +740,26 @@ def test_hash(tested_backends, request): backend.hash("test/item") +@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed") +def test_hash_blake3(tested_backends, request): + backend = get_backend_from_fixture(tested_backends, request) + with backend: + data = b"hash me" + expected_hash = blake3(data).hexdigest() + backend.store("test/item", data) + assert backend.hash("test/item", algorithm="blake3") == expected_hash + + # Large-ish data to test chunking + large_data = b"a" * (2 * 1024 * 1024) + expected_large_hash = blake3(large_data).hexdigest() + backend.store("test/large_item", large_data) + assert backend.hash("test/large_item", algorithm="blake3") == expected_large_hash + + # Test error for nonexistent object + with pytest.raises(ObjectNotFound): + backend.hash("test/nonexistent", algorithm="blake3") + + @pytest.mark.skipif(not (rest1_is_available and rest2_is_available), reason="REST1 and REST2 backends not available") def test_nginx_dispatch(rest1_backend_created, rest2_backend_created): """ diff --git a/tests/test_hashing.py b/tests/test_hashing.py new file mode 100644 index 0000000..10b461c --- /dev/null +++ b/tests/test_hashing.py @@ -0,0 +1,54 @@ +""" +Tests for the hashing utils (hashlib algorithms plus optional blake3). +""" + +import hashlib +import io + +import pytest + +from borgstore.utils import hashing + +blake3_is_available = hashing.blake3 is not None + + +def test_new_hashlib(): + data = b"hash me" + assert hashing.new("sha256", data).hexdigest() == hashlib.sha256(data).hexdigest() + h = hashing.new("sha256") + h.update(data) + assert h.hexdigest() == hashlib.sha256(data).hexdigest() + + +def test_file_digest_hashlib(): + data = b"hash me" + assert hashing.file_digest(io.BytesIO(data), "sha256").hexdigest() == hashlib.sha256(data).hexdigest() + + +@pytest.mark.parametrize("algorithm", ["invalid_algo", "", None, 42]) +def test_unsupported_algorithm(algorithm): + with pytest.raises(ValueError, match="Unsupported hash algorithm"): + hashing.new(algorithm) + with pytest.raises(ValueError, match="Unsupported hash algorithm"): + hashing.file_digest(io.BytesIO(b"hash me"), algorithm) + + +@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed") +def test_blake3(): + data = b"hash me" + expected = hashing.blake3(data).hexdigest() + # known value, so we notice if the algorithm/output encoding ever changes + assert expected == "e02b7e4520e277d4ef287f09c74fec4dd1df095b8b41a4d61dd4ed1589c4a281" + assert hashing.new("blake3", data).hexdigest() == expected + h = hashing.new("blake3") + h.update(data) + assert h.hexdigest() == expected + assert hashing.file_digest(io.BytesIO(data), "blake3").hexdigest() == expected + + +@pytest.mark.skipif(blake3_is_available, reason="blake3 package is installed") +def test_blake3_not_available(): + with pytest.raises(ValueError, match="Unsupported hash algorithm: blake3"): + hashing.new("blake3") + with pytest.raises(ValueError, match="Unsupported hash algorithm: blake3"): + hashing.file_digest(io.BytesIO(b"hash me"), "blake3") diff --git a/tests/test_server_rest.py b/tests/test_server_rest.py index ae12c45..253f5aa 100644 --- a/tests/test_server_rest.py +++ b/tests/test_server_rest.py @@ -10,6 +10,13 @@ except ImportError: pytest.skip("requests is not installed", allow_module_level=True) +try: + from blake3 import blake3 +except ImportError: + blake3 = None + +blake3_is_available = blake3 is not None + from borgstore.constants import DEL_SUFFIX from borgstore.server.rest import BorgStoreRESTServer from borgstore.backends.rest import get_rest_backend @@ -286,6 +293,24 @@ def test_rest_server_hash(rest_server_with_auth): be.close() +@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed") +def test_rest_server_hash_blake3(rest_server_with_auth): + be = rest_server_with_auth + be.create() + be.open() + try: + data = b"hash me" + expected_hash = blake3(data).hexdigest() + be.store("test/item", data) + assert be.hash("test/item", algorithm="blake3") == expected_hash + + # Test error for nonexistent object + with pytest.raises(ObjectNotFound): + be.hash("test/nonexistent", algorithm="blake3") + finally: + be.close() + + def test_rest_server_defrag(tmp_path): backend_url = tmp_path.as_uri() address, port = "127.0.0.1", 0 @@ -445,6 +470,24 @@ def test_rest_backend_defrag(rest_server_with_auth): be.close() +@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed") +def test_rest_backend_defrag_blake3(rest_server_with_auth): + be = rest_server_with_auth + be.create() + be.open() + try: + be.store("file1", b"0123456789") + be.store("file2", b"abcdefghij") + + sources = [("file1", 2, 3), ("file2", 5, 2)] + expected_hash = blake3(b"234fg").hexdigest() + res = be.defrag(sources, algorithm="blake3") + assert res == expected_hash + assert be.load(expected_hash) == b"234fg" + finally: + be.close() + + def test_rest_content_hash_verification(rest_server_with_auth): be = rest_server_with_auth base_url = be.base_url + "/" diff --git a/tests/test_store.py b/tests/test_store.py index 29c8263..735ea14 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -11,6 +11,7 @@ from .test_backends import get_sftp_test_backend, sftp_is_available # noqa from .test_backends import get_rclone_test_backend, rclone_is_available # noqa from .test_backends import get_s3_test_backend, s3_is_available # noqa +from .test_backends import blake3, blake3_is_available from borgstore.constants import ROOTNS from borgstore.store import Store, ItemInfo, ReadRangeError @@ -128,6 +129,22 @@ def test_defrag_nested(posixfs_store_created): assert "requested 20 bytes" in str(exc_info.value) +@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed") +def test_defrag_nested_blake3(posixfs_store_created): + ns = "two" # nested! CONFIG has {"two/": {"levels": [2]}} + v1 = b"0123456789" + v2 = b"abcdefghij" + with posixfs_store_created as store: + store.store(ns + "/file1", v1) + store.store(ns + "/file2", v2) + + sources = [("file1", 2, 3), ("file2", 5, 2)] + expected_data = b"234fg" + res = store.defrag(sources, algorithm="blake3", namespace=ns) + assert res == blake3(expected_data).hexdigest() + assert store.load(ns + "/" + res) == expected_data + + def test_hash(posixfs_store_created): ns = "two" k0 = key(0) @@ -148,6 +165,22 @@ def test_hash(posixfs_store_created): store.hash(nsk0, algorithm="invalid_algo", deleted=True) +@pytest.mark.skipif(not blake3_is_available, reason="blake3 package is not installed") +def test_hash_blake3(posixfs_store_created): + ns = "two" + k0 = key(0) + v0 = b"value0" + nsk0 = ns + "/" + k0 + expected_hash = blake3(v0).hexdigest() + with posixfs_store_created as store: + store.store(nsk0, v0) + assert store.hash(nsk0, algorithm="blake3") == expected_hash + + # Test with soft-deletion + store.move(nsk0, delete=True) + assert store.hash(nsk0, algorithm="blake3", deleted=True) == expected_hash + + @pytest.mark.parametrize("namespace,count", [("zero", 100), ("one", 1000)]) def test_scalability_count(posixfs_store_created, namespace, count): with posixfs_store_created as store: diff --git a/tox.ini b/tox.ini index 24272ed..ef3df51 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ deps = requests boto3 paramiko + blake3 commands = pytest -v -rs tests pass_env = BORGSTORE_TEST_*_URL