diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8e1479b5d800..308688006c9d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,12 @@ Changelog Cobblestone-128 and Cobblestone-256 instantiations of the `C2SP chunked-encryption specification `_ for streaming authenticated - encryption of large messages. + encryption of large messages. It also provides + ``Cobblestone128RangeDecryptor`` and ``Cobblestone256RangeDecryptor`` + for authenticated random access into a message, reading ciphertext + through a ``RangeReader`` (``BufferReader`` and ``FileReader`` are + provided, and remote sources such as object storage can implement the + protocol themselves). * Parsing a Signed Certificate Timestamp list now rejects encodings that carry trailing bytes after the list or after an individual SCT, instead of silently ignoring them. diff --git a/docs/cobblestone.rst b/docs/cobblestone.rst index 35c98116b8eb..6144d0b023ef 100644 --- a/docs/cobblestone.rst +++ b/docs/cobblestone.rst @@ -182,4 +182,195 @@ that mandate 256-bit keys). messages produced by :class:`Cobblestone256Encryptor` with a 32-byte key. +.. class:: Cobblestone128RangeDecryptor(key, context) + + .. versionadded:: 50.0.0 + + Decrypts arbitrary ranges of a message encrypted by + :class:`Cobblestone128Encryptor`, reading only the ciphertext that + covers each requested range rather than the whole message. See + :ref:`random access ` below. + + This is a separate class from :class:`Cobblestone128Decryptor`: + random access and streaming are distinct modes and cannot be + interleaved on one object. Unlike the streaming decryptor, an + instance is not consumed and may be used for any number of ranges. + + :param key: The 16-byte key the message was encrypted with. + :type key: :term:`bytes-like` + :param context: The context value the message was encrypted with. + :type context: :term:`bytes-like` + :raises ValueError: If ``key`` is not 16 bytes. + + .. method:: decrypt_range(reader, offset, length) + + Decrypts and returns the ``length`` plaintext bytes beginning at + ``offset``. + + The requested range is silently widened to whole 16 KiB chunk + boundaries so that every chunk it touches can be authenticated + by its tag; the requested sub-range is then sliced out of the + authenticated plaintext. Unauthenticated bytes are never + returned. + + .. warning:: + + A range read authenticates the bytes it returns, but **not + the message as a whole**. It cannot detect that chunks beyond + the requested range were removed, so it provides no + protection against truncation of the overall message. The + total plaintext length must come from a trusted source; do + not infer it from the size of the ciphertext. Whole-message + truncation protection is only provided by streaming through + :meth:`Cobblestone128Decryptor.finalize`. + + :param reader: The ciphertext, as a :class:`RangeReader` — + typically a :class:`BufferReader` or :class:`FileReader`. + :param int offset: The plaintext offset to start at. + :param int length: The number of plaintext bytes to return. + :return bytes: The authenticated plaintext for the requested + range. + :raises cryptography.exceptions.InvalidTag: If a covering chunk + was encrypted with a different key or context, has been + modified, or is missing because the ciphertext is truncated + within the requested range (which includes reading past the + end of the message). + :raises ValueError: If ``offset`` or ``length`` is negative, or + the range exceeds the maximum message length. + +.. class:: Cobblestone256RangeDecryptor(key, context) + + .. versionadded:: 50.0.0 + + Exactly like :class:`Cobblestone128RangeDecryptor`, but decrypts + ranges of messages produced by :class:`Cobblestone256Encryptor` with + a 32-byte key. + +.. _cobblestone-random-access: + +Random-access decryption +------------------------- + +Because each 16 KiB chunk is encrypted independently, a range of a large +message can be decrypted without processing everything before it. A +:class:`Cobblestone128RangeDecryptor` reads only the chunks covering the +requested range, authenticates each of them, and returns just the bytes +asked for. + +Ciphertext is read through a :class:`RangeReader`. Wrap whatever holds +the ciphertext in the matching reader and pass it to +:meth:`~Cobblestone128RangeDecryptor.decrypt_range`: + +.. doctest:: + + >>> from cryptography.cobblestone import ( + ... BufferReader, Cobblestone128Encryptor, + ... Cobblestone128RangeDecryptor + ... ) + >>> key = Cobblestone128Encryptor.generate_key() + >>> encryptor = Cobblestone128Encryptor(key, context=b"ranged") + >>> plaintext = b"the quick brown fox" * 10000 + >>> ciphertext = encryptor.update(plaintext) + encryptor.finalize() + >>> decryptor = Cobblestone128RangeDecryptor(key, context=b"ranged") + >>> decryptor.decrypt_range(BufferReader(ciphertext), 40005, 9) + b'brown fox' + >>> _ == plaintext[40005:40014] + True + +.. class:: RangeReader + + .. versionadded:: 50.0.0 + + The interface a range decryptor reads ciphertext through. This is a + :class:`typing.Protocol`: implement it to read from a source this + module does not provide a reader for, such as an object store. + + .. method:: read_at(offset, length) + + Returns up to ``length`` bytes of ciphertext starting at + ``offset``. + + Reads are *positional*: this method must not depend on, or + disturb, any cursor, so that concurrent calls on one reader + cannot interfere with each other. + + Returning fewer than ``length`` bytes signals the end of the + ciphertext, so a short read must not be used for any other + reason. The decryptor will ask again for whatever remains. + + :param int offset: The ciphertext offset to read from. + :param int length: The maximum number of bytes to return. + :return: The bytes read. + :rtype: :term:`bytes-like` + +.. class:: BufferReader(data) + + .. versionadded:: 50.0.0 + + A :class:`RangeReader` over an in-memory buffer. + + :param data: The ciphertext. This may be any :term:`bytes-like` + object, including an :class:`mmap.mmap` of a file, which allows + reading ranges out of a large file without loading it into + memory. + +.. class:: FileReader(fileobj) + + .. versionadded:: 50.0.0 + + A :class:`RangeReader` over an open binary file. + + Where the platform provides ``pread`` (via :func:`os.pread`) the + file's cursor is neither used nor modified, so one reader can serve + concurrent range requests; elsewhere reads are serialized around a + seek. + + :param fileobj: A file object opened in binary mode, which must have + a file descriptor (:meth:`~io.IOBase.fileno`). + +Reading from remote storage +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For ciphertext held remotely, implement :class:`RangeReader` in terms of +whatever ranged read the service offers — an HTTP ``Range`` request, or +an object-storage ranged ``GET``. Because ``read_at`` takes the offset as +an argument, no cursor is kept and the reader is safe to share: + +.. code-block:: python + + class S3Reader: + def __init__(self, client, bucket, key): + self._client = client + self._bucket = bucket + self._key = key + + def read_at(self, offset, length): + if length == 0: + return b"" + end = offset + length - 1 + response = self._client.get_object( + Bucket=self._bucket, + Key=self._key, + Range=f"bytes={offset}-{end}", + ) + return response["Body"].read() + + decryptor = Cobblestone128RangeDecryptor(key, context=b"ranged") + reader = S3Reader(client, "my-bucket", "backup.bin") + data = decryptor.decrypt_range(reader, 40000, 4096) + +Each call makes two reads: one for the 56-byte header, which binds the +plaintext to this particular message, and one contiguous read covering +every chunk the range touches. + +.. warning:: + + A range read authenticates the bytes it returns, but not the + message as a whole: it cannot detect that chunks beyond the + requested range were removed. Obtain the total plaintext length from + a trusted source rather than inferring it from the ciphertext, and + rely on streaming through + :meth:`Cobblestone128Decryptor.finalize` when you need to verify + that a whole message is intact and has not been truncated. + .. _`C2SP chunked-encryption specification`: https://c2sp.org/chunked-encryption diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index df4c2e037e92..51f1fcaa909e 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -43,6 +43,7 @@ Decapsulate Decapsulation declaratively decrypt +decryptor Decryptor decrypts Decrypts diff --git a/src/cryptography/cobblestone.py b/src/cryptography/cobblestone.py index 09f67af9b1c0..14b9ba04d637 100644 --- a/src/cryptography/cobblestone.py +++ b/src/cryptography/cobblestone.py @@ -4,18 +4,92 @@ from __future__ import annotations +import mmap +import os +import threading +import typing + from cryptography.hazmat.bindings._rust import ( cobblestone as _cobblestone, ) +from cryptography.utils import Buffer Cobblestone128Decryptor = _cobblestone.Cobblestone128Decryptor Cobblestone128Encryptor = _cobblestone.Cobblestone128Encryptor +Cobblestone128RangeDecryptor = _cobblestone.Cobblestone128RangeDecryptor Cobblestone256Decryptor = _cobblestone.Cobblestone256Decryptor Cobblestone256Encryptor = _cobblestone.Cobblestone256Encryptor +Cobblestone256RangeDecryptor = _cobblestone.Cobblestone256RangeDecryptor + +# os.pread is POSIX-only. Where it exists a range can be read without a cursor +# at all, so one reader can serve concurrent requests. +_HAS_PREAD = hasattr(os, "pread") + + +class RangeReader(typing.Protocol): + """The interface a range decryptor reads ciphertext through. + + Implement this to decrypt ranges out of a source this module does not + provide a reader for, such as an object store. ``read_at`` is positional: + it must not depend on, or disturb, any cursor, so that concurrent calls + cannot interfere with each other. + """ + + def read_at(self, offset: int, length: int) -> Buffer: + """Returns up to ``length`` bytes starting at ``offset``. + + Returning fewer than ``length`` bytes signals the end of the + ciphertext, so a short read must not be used for any other reason. + """ + + +class BufferReader: + """Reads ranges out of an in-memory buffer. + + ``data`` may be any bytes-like object, including an :class:`mmap.mmap` of + a file, which allows reading ranges out of a large file without loading it + into memory. + """ + + # mmap.mmap supports the buffer protocol but is not one of the concrete + # types Buffer names, so it is spelled out here. + def __init__(self, data: Buffer | mmap.mmap) -> None: + self._data = memoryview(data).cast("B") + + def read_at(self, offset: int, length: int) -> Buffer: + return self._data[offset : offset + length] + + +class FileReader: + """Reads ranges out of an open binary file. + + ``fileobj`` must be open in binary mode and have a file descriptor. Where + the platform provides ``pread`` the file's cursor is neither used nor + modified, so a single reader can serve concurrent range requests; + elsewhere reads are serialized around a seek. + """ + + def __init__(self, fileobj: typing.BinaryIO) -> None: + self._fileobj = fileobj + self._fd = fileobj.fileno() + self._lock = threading.Lock() + + def read_at(self, offset: int, length: int) -> Buffer: + if _HAS_PREAD: + return os.pread(self._fd, length, offset) + with self._lock: + self._fileobj.seek(offset) + return self._fileobj.read(length) + __all__ = [ + "BufferReader", "Cobblestone128Decryptor", "Cobblestone128Encryptor", + "Cobblestone128RangeDecryptor", "Cobblestone256Decryptor", "Cobblestone256Encryptor", + "Cobblestone256RangeDecryptor", + "FileReader", + "RangeReader", ] diff --git a/src/cryptography/hazmat/bindings/_rust/cobblestone.pyi b/src/cryptography/hazmat/bindings/_rust/cobblestone.pyi index 95e958382904..261bb5eec08d 100644 --- a/src/cryptography/hazmat/bindings/_rust/cobblestone.pyi +++ b/src/cryptography/hazmat/bindings/_rust/cobblestone.pyi @@ -2,8 +2,15 @@ # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. +import typing + from cryptography.utils import Buffer +class _RangeReader(typing.Protocol): + # Structural mirror of cryptography.cobblestone.RangeReader, kept local so + # this stub does not import back into the package it is a binding for. + def read_at(self, offset: int, length: int) -> Buffer: ... + class Cobblestone128Encryptor: def __init__(self, key: Buffer, context: Buffer) -> None: ... @staticmethod @@ -18,6 +25,12 @@ class Cobblestone128Decryptor: def update_into(self, data: Buffer, buf: Buffer) -> int: ... def finalize(self) -> bytes: ... +class Cobblestone128RangeDecryptor: + def __init__(self, key: Buffer, context: Buffer) -> None: ... + def decrypt_range( + self, reader: _RangeReader, offset: int, length: int + ) -> bytes: ... + class Cobblestone256Encryptor: def __init__(self, key: Buffer, context: Buffer) -> None: ... @staticmethod @@ -31,3 +44,9 @@ class Cobblestone256Decryptor: def update(self, data: Buffer) -> bytes: ... def update_into(self, data: Buffer, buf: Buffer) -> int: ... def finalize(self) -> bytes: ... + +class Cobblestone256RangeDecryptor: + def __init__(self, key: Buffer, context: Buffer) -> None: ... + def decrypt_range( + self, reader: _RangeReader, offset: int, length: int + ) -> bytes: ... diff --git a/src/rust/src/cobblestone.rs b/src/rust/src/cobblestone.rs index 24fc6a66b1b7..d71f076657f4 100644 --- a/src/rust/src/cobblestone.rs +++ b/src/rust/src/cobblestone.rs @@ -71,6 +71,38 @@ fn check_key_length(params: &AeadParams, key: &[u8]) -> CryptographyResult<()> { Ok(()) } +fn truncated_error() -> CryptographyError { + CryptographyError::from(exceptions::InvalidTag::new_err(())) +} + +// Reads up to `want` bytes at byte offset `offset` from a reader, by calling +// its `read_at(offset, length)` method. The read is positional: the reader +// holds no cursor, so a single reader can serve concurrent range requests. +// Returns fewer than `want` bytes only at the end of the ciphertext. +fn range_read( + py: pyo3::Python<'_>, + reader: &pyo3::Bound<'_, pyo3::PyAny>, + offset: u64, + want: usize, +) -> CryptographyResult> { + let mut out = Vec::with_capacity(want); + while out.len() < want { + let result = reader.call_method1( + pyo3::intern!(py, "read_at"), + (offset + out.len() as u64, want - out.len()), + )?; + let read = result.extract::>()?; + let read_bytes = read.as_bytes(); + // A short read means the end of the ciphertext was reached. + if read_bytes.is_empty() { + break; + } + let take = std::cmp::min(read_bytes.len(), want - out.len()); + out.extend_from_slice(&read_bytes[..take]); + } + Ok(out) +} + struct DerivedKeys<'p> { key: pyo3::Bound<'p, pyo3::types::PyBytes>, base_nonce: [u8; NONCE_LEN], @@ -131,15 +163,22 @@ impl ChunkNonces { Ok(()) } - fn next(&mut self) -> CryptographyResult<[u8; NONCE_LEN]> { - self.check_capacity(1)?; + // The nonce for an arbitrary chunk index: the base nonce XOR'd with the + // index as a big-endian integer. This is stateless, so it serves both the + // sequential streaming path (via `next`) and random-access decryption. + fn nonce_for(&self, index: u64) -> CryptographyResult<[u8; NONCE_LEN]> { + if index >= MAX_CHUNK_COUNT { + return Err(message_too_large_error()); + } let mut nonce = self.base_nonce; - for (n, c) in nonce[NONCE_LEN - 8..] - .iter_mut() - .zip(self.counter.to_be_bytes()) - { + for (n, c) in nonce[NONCE_LEN - 8..].iter_mut().zip(index.to_be_bytes()) { *n ^= c; } + Ok(nonce) + } + + fn next(&mut self) -> CryptographyResult<[u8; NONCE_LEN]> { + let nonce = self.nonce_for(self.counter)?; self.counter += 1; Ok(nonce) } @@ -187,6 +226,29 @@ impl ChunkCipher { Ok(()) } + // Decrypts the chunk at an explicit `index`, without advancing the + // sequential counter, for random-access decryption. `ciphertext` includes + // the trailing tag; `out` must be exactly `ciphertext.len() - self.tag_len` + // bytes. Returns InvalidTag if the chunk does not authenticate (e.g. it was + // tampered with, or belongs at a different index). + fn decrypt_chunk_at( + &self, + py: pyo3::Python<'_>, + index: u64, + ciphertext: &[u8], + out: &mut [u8], + ) -> CryptographyResult<()> { + let nonce = self.nonces.nonce_for(index)?; + self.aead.decrypt_into( + py, + CffiBuf::from_bytes(py, &nonce), + CffiBuf::from_bytes(py, ciphertext), + None, + CffiMutBuf::from_bytes(py, out), + )?; + Ok(()) + } + // `ciphertext` includes the trailing tag; `out` must be exactly // `ciphertext.len() - self.tag_len` bytes. fn decrypt_chunk( @@ -547,6 +609,146 @@ impl ChunkedDecryptor { } } +// Random-access decryption of a single message. Unlike ChunkedDecryptor this +// holds no per-message stream state: each call reads, authenticates, and +// returns an independent range. +struct RangeDecryptor { + params: &'static AeadParams, + key: Vec, + context: Vec, + // The derived cipher, cached alongside the header it came from so that + // repeated ranges of one message skip the key derivation while a reader + // for a *different* message is still handled correctly. + cached: Option<([u8; HEADER_LEN], ChunkCipher)>, +} + +impl RangeDecryptor { + fn new(params: &'static AeadParams, key: &[u8], context: &[u8]) -> CryptographyResult { + check_key_length(params, key)?; + Ok(RangeDecryptor { + params, + key: key.to_vec(), + context: context.to_vec(), + cached: None, + }) + } + + // Derives the message's keys from `header` and verifies the commitment, + // reusing the cached cipher when the header is unchanged. + fn ensure_cipher( + &mut self, + py: pyo3::Python<'_>, + header: &[u8], + ) -> CryptographyResult<&ChunkCipher> { + let fresh = match &self.cached { + Some((cached_header, _)) => cached_header != header, + None => true, + }; + if fresh { + let (salt, commitment) = header.split_at(SALT_LEN); + let keys = derive_keys(py, self.params, &self.key, salt, &self.context)?; + // The commitment is checked before any plaintext is produced. + if !cryptography_crypto::constant_time::bytes_eq(&keys.commitment, commitment) { + return Err(CryptographyError::from(exceptions::InvalidTag::new_err(()))); + } + let cipher = ChunkCipher::new(py, self.params, &keys)?; + let mut owned = [0; HEADER_LEN]; + owned.copy_from_slice(header); + self.cached = Some((owned, cipher)); + } + Ok(&self.cached.as_ref().expect("just populated").1) + } + + // Returns the authenticated plaintext bytes in `[offset, offset + length)`. + // + // The requested range is silently widened to whole chunk boundaries so that + // every chunk it touches is authenticated by its AEAD tag; only then is the + // requested sub-range sliced out. Unauthenticated bytes are never returned. + // + // Note: a range read authenticates the bytes it returns, but not the + // message as a whole. It cannot detect truncation of chunks beyond the + // requested range, so the total plaintext length must come from a trusted + // source, not be inferred from the ciphertext. + fn decrypt_range<'p>( + &mut self, + py: pyo3::Python<'p>, + reader: &pyo3::Bound<'p, pyo3::PyAny>, + offset: u64, + length: usize, + ) -> CryptographyResult> { + if length == 0 { + return Ok(pyo3::types::PyBytes::new(py, &[])); + } + + let params = self.params; + let wire = params.wire_chunk_size(); + let first_chunk = offset / CHUNK_SIZE as u64; + // Bound the range to the maximum message length (which also guards + // against `offset + length` overflowing for a hostile offset). This + // keeps every chunk index below MAX_CHUNK_COUNT. + let last_byte = offset + .checked_add(length as u64) + .filter(|end| *end <= MAX_CHUNK_COUNT * CHUNK_SIZE as u64) + .ok_or_else(message_too_large_error)? + - 1; + let last_chunk = last_byte / CHUNK_SIZE as u64; + let n_chunks = (last_chunk - first_chunk + 1) as usize; + + // The header is re-read on every call: it is what binds the returned + // plaintext to this particular message. + let header = range_read(py, reader, 0, HEADER_LEN)?; + if header.len() < HEADER_LEN { + return Err(truncated_error()); + } + let cipher = self.ensure_cipher(py, &header)?; + + // Read the contiguous ciphertext window covering the requested chunks + // in one request, so a multi-chunk range is a single round trip. + let ct_start = HEADER_LEN as u64 + first_chunk * wire as u64; + let data = range_read(py, reader, ct_start, n_chunks * wire)?; + + // Decrypt whole chunks, releasing plaintext only for those that + // authenticate. Any trailing partial wire chunk is the message's short + // final chunk. + let mut plaintext = Vec::with_capacity(n_chunks * CHUNK_SIZE); + let mut consumed = 0; + let mut index = first_chunk; + while consumed + wire <= data.len() { + let start = plaintext.len(); + plaintext.resize(start + CHUNK_SIZE, 0); + cipher.decrypt_chunk_at( + py, + index, + &data[consumed..consumed + wire], + &mut plaintext[start..], + )?; + consumed += wire; + index += 1; + } + let remainder = data.len() - consumed; + if remainder > 0 { + if remainder < params.tag_len { + // Too short to be a valid chunk: the message is truncated. + return Err(truncated_error()); + } + let start = plaintext.len(); + plaintext.resize(start + remainder - params.tag_len, 0); + cipher.decrypt_chunk_at(py, index, &data[consumed..], &mut plaintext[start..])?; + } + + // Slice out the requested range. If the authenticated plaintext does + // not reach `offset + length`, the message is shorter than the request + // (truncated, or the caller read past the end); either way there are no + // authenticated bytes to return there. + let intra = (offset - first_chunk * CHUNK_SIZE as u64) as usize; + let end = intra + length; + if end > plaintext.len() { + return Err(truncated_error()); + } + Ok(pyo3::types::PyBytes::new(py, &plaintext[intra..end])) + } +} + #[pyo3::pyclass(module = "cryptography.hazmat.bindings._rust.cobblestone")] pub(crate) struct Cobblestone128Encryptor { inner: ChunkedEncryptor, @@ -729,13 +931,63 @@ impl Cobblestone256Decryptor { } } +#[pyo3::pyclass(module = "cryptography.hazmat.bindings._rust.cobblestone")] +pub(crate) struct Cobblestone128RangeDecryptor { + inner: RangeDecryptor, +} + +#[pyo3::pymethods] +impl Cobblestone128RangeDecryptor { + #[new] + fn new(key: CffiBuf<'_>, context: CffiBuf<'_>) -> CryptographyResult { + Ok(Cobblestone128RangeDecryptor { + inner: RangeDecryptor::new(&AES_128_GCM, key.as_bytes(), context.as_bytes())?, + }) + } + + fn decrypt_range<'p>( + &mut self, + py: pyo3::Python<'p>, + reader: pyo3::Bound<'p, pyo3::PyAny>, + offset: u64, + length: usize, + ) -> CryptographyResult> { + self.inner.decrypt_range(py, &reader, offset, length) + } +} + +#[pyo3::pyclass(module = "cryptography.hazmat.bindings._rust.cobblestone")] +pub(crate) struct Cobblestone256RangeDecryptor { + inner: RangeDecryptor, +} + +#[pyo3::pymethods] +impl Cobblestone256RangeDecryptor { + #[new] + fn new(key: CffiBuf<'_>, context: CffiBuf<'_>) -> CryptographyResult { + Ok(Cobblestone256RangeDecryptor { + inner: RangeDecryptor::new(&AES_256_GCM, key.as_bytes(), context.as_bytes())?, + }) + } + + fn decrypt_range<'p>( + &mut self, + py: pyo3::Python<'p>, + reader: pyo3::Bound<'p, pyo3::PyAny>, + offset: u64, + length: usize, + ) -> CryptographyResult> { + self.inner.decrypt_range(py, &reader, offset, length) + } +} + #[pyo3::pymodule(gil_used = false)] #[pyo3(name = "cobblestone")] pub(crate) mod cobblestone_mod { #[pymodule_export] use super::{ - Cobblestone128Decryptor, Cobblestone128Encryptor, Cobblestone256Decryptor, - Cobblestone256Encryptor, + Cobblestone128Decryptor, Cobblestone128Encryptor, Cobblestone128RangeDecryptor, + Cobblestone256Decryptor, Cobblestone256Encryptor, Cobblestone256RangeDecryptor, }; } diff --git a/tests/test_cobblestone.py b/tests/test_cobblestone.py index e407509f6161..38fd6ad218c7 100644 --- a/tests/test_cobblestone.py +++ b/tests/test_cobblestone.py @@ -3,15 +3,23 @@ # for complete details. +from __future__ import annotations + +import mmap import os import pytest +from cryptography import cobblestone as cobblestone_module from cryptography.cobblestone import ( + BufferReader, Cobblestone128Decryptor, Cobblestone128Encryptor, + Cobblestone128RangeDecryptor, Cobblestone256Decryptor, Cobblestone256Encryptor, + Cobblestone256RangeDecryptor, + FileReader, ) from cryptography.exceptions import AlreadyFinalized, InvalidTag @@ -393,6 +401,378 @@ def test_accepts_buffers(self, variant): assert dec.update(bytearray(ciphertext)) + dec.finalize() == plaintext +MAX_CHUNK_COUNT = 1 << 38 + +RANGE_VARIANTS = [ + pytest.param( + (Cobblestone128Encryptor, Cobblestone128RangeDecryptor), + id="cobblestone128", + ), + pytest.param( + (Cobblestone256Encryptor, Cobblestone256RangeDecryptor), + id="cobblestone256", + ), +] + + +class _CountingReader: + """A minimal RangeReader, standing in for remote range fetches. + + ``max_read`` caps how much a single ``read_at`` returns, so that callers + which need more must issue further reads. + """ + + def __init__(self, data: bytes, max_read: int | None = None): + self._data = bytes(data) + self._max_read = max_read + self.reads = 0 + + def read_at(self, offset, length): + self.reads += 1 + if self._max_read is not None: + length = min(length, self._max_read) + return self._data[offset : offset + length] + + +RANGES = [ + (0, 10), # start of the first chunk + (100, 50), # within the first chunk + (CHUNK_SIZE - 10, 20), # spanning the 0/1 chunk boundary + (CHUNK_SIZE, CHUNK_SIZE), # exactly one full interior chunk + (CHUNK_SIZE + 5, 2 * CHUNK_SIZE), # spanning several chunks + (50000, 100), # within the final (short) chunk + (3 * CHUNK_SIZE, 5000), # the whole final chunk + (3 * CHUNK_SIZE + 4990, 10), # the very end of the message +] + + +@pytest.mark.parametrize("variant", RANGE_VARIANTS) +class TestCobblestoneRangeDecryptor: + def _encrypt(self, variant, context, plaintext): + encryptor_cls, _ = variant + key = encryptor_cls.generate_key() + return key, _encrypt_all(encryptor_cls, key, context, plaintext) + + @pytest.mark.parametrize("offset,length", RANGES) + def test_decrypt_range_matches_plaintext(self, variant, offset, length): + _, range_cls = variant + context = b"range context" + plaintext = os.urandom(3 * CHUNK_SIZE + 5000) + key, ciphertext = self._encrypt(variant, context, plaintext) + expected = plaintext[offset : offset + length] + + readers = [ + BufferReader(ciphertext), + BufferReader(bytearray(ciphertext)), + BufferReader(memoryview(ciphertext)), + _CountingReader(ciphertext), + ] + for reader in readers: + dec = range_cls(key, context) + assert dec.decrypt_range(reader, offset, length) == expected + + def test_decrypt_range_whole_message(self, variant): + _, range_cls = variant + plaintext = os.urandom(2 * CHUNK_SIZE + 1234) + key, ciphertext = self._encrypt(variant, b"", plaintext) + dec = range_cls(key, b"") + result = dec.decrypt_range(BufferReader(ciphertext), 0, len(plaintext)) + assert result == plaintext + + def test_decrypt_range_on_exact_chunk_boundary_message(self, variant): + # A message whose length is a multiple of CHUNK_SIZE has an empty + # final chunk; ranges not touching it still decrypt. + _, range_cls = variant + plaintext = os.urandom(2 * CHUNK_SIZE) + key, ciphertext = self._encrypt(variant, b"", plaintext) + dec = range_cls(key, b"") + reader = BufferReader(ciphertext) + assert dec.decrypt_range(reader, 0, 2 * CHUNK_SIZE) == plaintext + assert ( + dec.decrypt_range(reader, 2 * CHUNK_SIZE - 5, 5) == plaintext[-5:] + ) + + def test_buffer_reader_over_mmap(self, variant): + _, range_cls = variant + plaintext = os.urandom(2 * CHUNK_SIZE + 777) + key, ciphertext = self._encrypt(variant, b"ctx", plaintext) + mm = mmap.mmap(-1, len(ciphertext)) + try: + mm[:] = ciphertext + dec = range_cls(key, b"ctx") + result = dec.decrypt_range(BufferReader(mm), CHUNK_SIZE - 3, 10) + assert result == plaintext[CHUNK_SIZE - 3 : CHUNK_SIZE + 7] + finally: + mm.close() + + def test_file_reader(self, variant, tmp_path): + _, range_cls = variant + plaintext = os.urandom(2 * CHUNK_SIZE + 999) + key, ciphertext = self._encrypt(variant, b"", plaintext) + path = tmp_path / "message.bin" + path.write_bytes(ciphertext) + + with path.open("rb") as f: + dec = range_cls(key, b"") + reader = FileReader(f) + result = dec.decrypt_range(reader, CHUNK_SIZE - 4, 12) + assert result == plaintext[CHUNK_SIZE - 4 : CHUNK_SIZE + 8] + assert dec.decrypt_range(reader, 0, 20) == plaintext[:20] + + def test_file_reader_without_pread(self, variant, tmp_path, monkeypatch): + # Platforms without os.pread fall back to a locked seek/read. + monkeypatch.setattr(cobblestone_module, "_HAS_PREAD", False) + _, range_cls = variant + plaintext = os.urandom(CHUNK_SIZE + 50) + key, ciphertext = self._encrypt(variant, b"", plaintext) + path = tmp_path / "message.bin" + path.write_bytes(ciphertext) + + with path.open("rb") as f: + dec = range_cls(key, b"") + result = dec.decrypt_range(FileReader(f), CHUNK_SIZE, 50) + assert result == plaintext[CHUNK_SIZE:] + + def test_reader_returning_short_reads(self, variant): + # A reader that returns less than asked is read from repeatedly. + _, range_cls = variant + plaintext = os.urandom(2 * CHUNK_SIZE) + key, ciphertext = self._encrypt(variant, b"", plaintext) + reader = _CountingReader(ciphertext, max_read=1000) + dec = range_cls(key, b"") + assert ( + dec.decrypt_range(reader, 0, CHUNK_SIZE) == plaintext[:CHUNK_SIZE] + ) + + def test_zero_length_returns_empty(self, variant): + _, range_cls = variant + key, ciphertext = self._encrypt(variant, b"", b"some data") + dec = range_cls(key, b"") + reader = _CountingReader(ciphertext) + assert dec.decrypt_range(reader, 0, 0) == b"" + assert dec.decrypt_range(reader, 3, 0) == b"" + # A zero-length range needs no ciphertext at all. + assert reader.reads == 0 + + def test_instance_is_reusable_for_many_ranges(self, variant): + _, range_cls = variant + plaintext = os.urandom(3 * CHUNK_SIZE) + key, ciphertext = self._encrypt(variant, b"", plaintext) + dec = range_cls(key, b"") + reader = BufferReader(ciphertext) + assert dec.decrypt_range(reader, 10, 20) == plaintext[10:30] + assert ( + dec.decrypt_range(reader, 2 * CHUNK_SIZE, 100) + == plaintext[2 * CHUNK_SIZE : 2 * CHUNK_SIZE + 100] + ) + + def test_instance_reused_across_different_messages(self, variant): + # Each call re-reads the header, so one decryptor may serve readers + # for different messages (which have different salts). + _, range_cls = variant + first = os.urandom(CHUNK_SIZE + 10) + second = os.urandom(CHUNK_SIZE + 10) + encryptor_cls, _ = variant + key = encryptor_cls.generate_key() + ct1 = _encrypt_all(encryptor_cls, key, b"ctx", first) + ct2 = _encrypt_all(encryptor_cls, key, b"ctx", second) + + dec = range_cls(key, b"ctx") + assert dec.decrypt_range(BufferReader(ct1), 0, 100) == first[:100] + assert dec.decrypt_range(BufferReader(ct2), 0, 100) == second[:100] + assert dec.decrypt_range(BufferReader(ct1), 0, 100) == first[:100] + + def test_multi_chunk_range_is_a_single_body_read(self, variant): + _, range_cls = variant + plaintext = os.urandom(4 * CHUNK_SIZE) + key, ciphertext = self._encrypt(variant, b"", plaintext) + reader = _CountingReader(ciphertext) + dec = range_cls(key, b"") + dec.decrypt_range(reader, 100, 2 * CHUNK_SIZE) + # One read for the header, one contiguous read for the covering chunks. + assert reader.reads == 2 + + def test_wrong_key_rejected(self, variant): + encryptor_cls, range_cls = variant + _, ciphertext = self._encrypt(variant, b"", os.urandom(CHUNK_SIZE + 1)) + dec = range_cls(encryptor_cls.generate_key(), b"") + with pytest.raises(InvalidTag): + dec.decrypt_range(BufferReader(ciphertext), 0, 10) + + def test_wrong_context_rejected(self, variant): + _, range_cls = variant + key, ciphertext = self._encrypt( + variant, b"context a", os.urandom(CHUNK_SIZE + 1) + ) + dec = range_cls(key, b"context b") + with pytest.raises(InvalidTag): + dec.decrypt_range(BufferReader(ciphertext), 0, 10) + + @pytest.mark.parametrize( + "position", + [ + 0, # salt + SALT_LEN, # commitment + HEADER_LEN, # first chunk ciphertext + HEADER_LEN + WIRE_CHUNK_SIZE - 1, # first chunk tag + ], + ) + def test_tampering_in_range_detected(self, variant, position): + _, range_cls = variant + key, ciphertext = self._encrypt( + variant, b"", os.urandom(CHUNK_SIZE + 100) + ) + ciphertext = bytearray(ciphertext) + ciphertext[position] ^= 1 + dec = range_cls(key, b"") + with pytest.raises(InvalidTag): + dec.decrypt_range(BufferReader(bytes(ciphertext)), 0, 100) + + def test_reading_past_end_rejected(self, variant): + _, range_cls = variant + plaintext = os.urandom(CHUNK_SIZE + 500) + key, ciphertext = self._encrypt(variant, b"", plaintext) + dec = range_cls(key, b"") + reader = BufferReader(ciphertext) + # Starting exactly at the end, and overshooting the end, both fail + # rather than returning short: past the authenticated end there are no + # bytes to return. + with pytest.raises(InvalidTag): + dec.decrypt_range(reader, len(plaintext), 1) + with pytest.raises(InvalidTag): + dec.decrypt_range(reader, len(plaintext) - 2, 5) + + def test_truncated_source_in_range_rejected(self, variant): + _, range_cls = variant + plaintext = os.urandom(2 * CHUNK_SIZE + 10) + key, ciphertext = self._encrypt(variant, b"", plaintext) + # Drop everything after the first chunk. + reader = BufferReader(ciphertext[: HEADER_LEN + WIRE_CHUNK_SIZE]) + + dec = range_cls(key, b"") + # A range served entirely by the surviving first chunk still succeeds: + # a range read cannot detect truncation beyond the requested range. + assert dec.decrypt_range(reader, 0, 100) == plaintext[:100] + # A range needing a dropped chunk is rejected. + with pytest.raises(InvalidTag): + dec.decrypt_range(reader, CHUNK_SIZE, 10) + + def test_short_header_rejected(self, variant): + # A source too short to even hold the 56-byte header is truncated. + _, range_cls = variant + key, _ = self._encrypt(variant, b"", b"data") + dec = range_cls(key, b"") + with pytest.raises(InvalidTag): + dec.decrypt_range(BufferReader(b"\x00" * (HEADER_LEN - 1)), 0, 1) + + def test_final_chunk_shorter_than_tag_rejected(self, variant): + # A final wire chunk shorter than the tag cannot be authenticated. + _, range_cls = variant + key, ciphertext = self._encrypt(variant, b"", os.urandom(100)) + # Keep the full header, but leave only a few (< TAG_LEN) body bytes. + reader = BufferReader(ciphertext[: HEADER_LEN + TAG_LEN - 1]) + dec = range_cls(key, b"") + with pytest.raises(InvalidTag): + dec.decrypt_range(reader, 0, 1) + + def test_too_large_offset_rejected(self, variant): + _, range_cls = variant + key, ciphertext = self._encrypt(variant, b"", b"data") + dec = range_cls(key, b"") + with pytest.raises(ValueError): + dec.decrypt_range( + BufferReader(ciphertext), MAX_CHUNK_COUNT * CHUNK_SIZE, 1 + ) + + def test_negative_arguments_rejected(self, variant): + _, range_cls = variant + key, ciphertext = self._encrypt(variant, b"", b"data") + dec = range_cls(key, b"") + reader = BufferReader(ciphertext) + with pytest.raises((ValueError, OverflowError)): + dec.decrypt_range(reader, -1, 1) + with pytest.raises((ValueError, OverflowError)): + dec.decrypt_range(reader, 0, -1) + + def test_reader_without_read_at_rejected(self, variant): + _, range_cls = variant + key, ciphertext = self._encrypt(variant, b"", b"data") + dec = range_cls(key, b"") + # A bytes-like is not a reader: it must be wrapped in a BufferReader. + with pytest.raises(AttributeError): + dec.decrypt_range(ciphertext, 0, 10) + + def test_reader_returning_non_buffer_rejected(self, variant): + _, range_cls = variant + key, _ = self._encrypt(variant, b"", b"data") + + class BadReader: + def read_at(self, offset, length): + return "not bytes" + + dec = range_cls(key, b"") + with pytest.raises(TypeError): + dec.decrypt_range(BadReader(), 0, 10) + + def test_invalid_key_size(self, variant): + encryptor_cls, range_cls = variant + key_len = len(encryptor_cls.generate_key()) + for length in [0, key_len - 1, key_len + 1, 64]: + with pytest.raises(ValueError): + range_cls(b"\x00" * length, b"") + + def test_streaming_decryptor_has_no_decrypt_range(self, variant): + # Random access lives on its own class: the streaming decryptor is + # unchanged, so the two modes cannot be interleaved on one object. + _, range_cls = variant + assert not hasattr(Cobblestone128Decryptor, "decrypt_range") + assert not hasattr(Cobblestone256Decryptor, "decrypt_range") + assert not hasattr(range_cls, "update") + assert not hasattr(range_cls, "finalize") + + +class TestRangeReaders: + def test_buffer_reader_read_at(self): + reader = BufferReader(b"0123456789") + assert bytes(reader.read_at(0, 4)) == b"0123" + assert bytes(reader.read_at(4, 3)) == b"456" + # Reads past the end are short rather than an error. + assert bytes(reader.read_at(8, 10)) == b"89" + assert bytes(reader.read_at(20, 4)) == b"" + + def test_file_reader_read_at(self, tmp_path): + path = tmp_path / "data.bin" + path.write_bytes(b"0123456789") + with path.open("rb") as f: + reader = FileReader(f) + assert bytes(reader.read_at(0, 4)) == b"0123" + assert bytes(reader.read_at(4, 3)) == b"456" + assert bytes(reader.read_at(20, 4)) == b"" + + def test_file_reader_read_at_without_pread(self, tmp_path, monkeypatch): + monkeypatch.setattr(cobblestone_module, "_HAS_PREAD", False) + path = tmp_path / "data.bin" + path.write_bytes(b"0123456789") + with path.open("rb") as f: + reader = FileReader(f) + assert bytes(reader.read_at(0, 4)) == b"0123" + assert bytes(reader.read_at(4, 3)) == b"456" + + @pytest.mark.skipif( + not cobblestone_module._HAS_PREAD, reason="platform has no os.pread" + ) + def test_file_reader_does_not_disturb_cursor(self, tmp_path): + # With pread the file position is untouched, which is what allows one + # reader to serve concurrent ranges. + path = tmp_path / "data.bin" + path.write_bytes(b"0123456789") + with path.open("rb") as f: + f.seek(2) + reader = FileReader(f) + assert bytes(reader.read_at(6, 2)) == b"67" + assert f.tell() == 2 + + class TestVariantsAreDistinct: def test_key_sizes_differ(self): assert len(Cobblestone128Encryptor.generate_key()) == 16