From 4ba4c287db7071f3ad1ac3cc0f024442ce55d17e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 09:29:58 -0600 Subject: [PATCH 1/4] PYTHON-6285 Add hostile-input and boundary tests for the BSON and message C extensions --- .evergreen/scripts/setup_tests.py | 10 +- test/test_bson.py | 167 +++++++++++++++++++++++++++++- test/test_message.py | 88 +++++++++++++++- 3 files changed, 259 insertions(+), 6 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index c12599cbc0..ed98064d80 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -57,13 +57,15 @@ # Map the test name to test group. GROUP_MAP = dict(fuzz="fuzz", mockupdb="mockupdb", perf="perf") -# The test files run under ASan/UBSan (PYTHON-6048). They exercise the -# buffer-protocol acquisition path in bson/_cbsonmodule.c -# (_get_buffer/_prepare_input_buffer) and the wire-message builders in +# The test files run under ASan/UBSan (PYTHON-6048, PYTHON-6285). They +# exercise the buffer-protocol acquisition path in bson/_cbsonmodule.c +# (_get_buffer/_prepare_input_buffer), the wire-message builders in # pymongo/_cmessagemodule.c (_op_msg, _batched_op_msg, -# _encode_batched_write_command). +# _encode_batched_write_command), and the hostile-input and boundary tests +# driving the decoder and builders directly. SANITIZER_FILES = [ "test/test_bson.py", + "test/test_message.py", "test/test_raw_bson.py", "test/test_raw_bson_shared.py", "test/test_client.py", diff --git a/test/test_bson.py b/test/test_bson.py index 0380d1f22e..c933af267d 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -31,6 +31,7 @@ import uuid from collections import OrderedDict, abc from io import BytesIO +from typing import Any sys.path[0:0] = [""] @@ -63,7 +64,8 @@ from bson.codec_options import CodecOptions, DatetimeConversion from bson.datetime_ms import _DATETIME_ERROR_SUGGESTION from bson.dbref import DBRef -from bson.errors import InvalidBSON, InvalidDocument +from bson.decimal128 import Decimal128 +from bson.errors import BSONError, InvalidBSON, InvalidDocument from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey @@ -2081,5 +2083,168 @@ def test_long_long_to_string(self): print("_cbson was not imported. Check compilation logs.") +def _nested_document(depth: int) -> dict[str, Any]: + doc: dict[str, Any] = {"leaf": 1} + for _ in range(depth): + doc = {"d": doc} + return doc + + +def _typed_documents() -> list[dict[str, Any]]: + """One document per BSON type, at value extremes where applicable.""" + return [ + {}, + {"int32": 1}, + {"int64 max": Int64(2**63 - 1), "int64 min": Int64(-(2**63))}, + {"double": 1.5, "double min": sys.float_info.min, "double max": sys.float_info.max}, + {"string": "hello é", "string empty": ""}, + {"string big": "x" * 70000}, + { + "binary": Binary(b"\x00\xffbin", 0), + "binary old": Binary(b"old", 2), + "binary empty": Binary(b"", 0), + }, + { + "datetime zero": DatetimeMS(0), + "datetime min": DatetimeMS(-(2**63)), + "datetime max": DatetimeMS(2**63 - 1), + }, + {"timestamp": Timestamp(0, 1), "timestamp max": Timestamp(0xFFFFFFFF, 0xFFFFFFFF)}, + { + "decimal128": Decimal128("1.25"), + "decimal128 nan": Decimal128("NaN"), + "decimal128 inf": Decimal128("Infinity"), + "decimal128 neg0": Decimal128("-0"), + }, + {"regex": Regex("a*b", "im"), "regex empty": Regex("", "")}, + {"code": Code("function() { return 1; }"), "code scope": Code("f", {"x": [1, 2]})}, + { + "dbref": DBRef("coll", "id"), + "objectid": ObjectId(b"0123456789ab"), + "minkey": MinKey(), + "maxkey": MaxKey(), + "null": None, + "bool": True, + }, + {"array": [1, "two", None, {"nested": True}], "subdoc": {"b": {"c": 1}}}, + {"bytes": b"raw bytes"}, + {"key" * 63 + "e": 1}, + {"deep": _nested_document(50)}, + ] + + +_DATETIME_MS_CODEC = CodecOptions(datetime_conversion=DatetimeConversion.DATETIME_MS) + + +class TestBSONEncodeDecodeBoundaries(unittest.TestCase): + """Encode every BSON type at value extremes and re-encode byte-stably. + + Drives buffer growth transitions and every _downcast_and_check path + under sanitizer builds. + """ + + def _round_trip(self, doc): + encoded = bson.encode(doc, codec_options=_DATETIME_MS_CODEC) + decoded = bson.decode(encoded, codec_options=_DATETIME_MS_CODEC) + # Compare byte-level stability rather than decoded == doc so that + # NaN payloads (which are not equal to themselves) round-trip too. + self.assertEqual(bson.encode(decoded, codec_options=_DATETIME_MS_CODEC), encoded) + + def test_round_trip_value_extremes(self): + for doc in _typed_documents(): + self._round_trip(doc) + + def test_round_trip_size_boundaries(self): + # String sizes that cross buffer doubling boundaries. + for size in (127, 128, 255, 256, 2**16, 2**16 + 1): + self._round_trip({"a": "x" * size}) + + +class TestBSONHostileLengths(unittest.TestCase): + """Decode must reject malformed length fields with a BSONError. + + Exercises the C decoder's int-boundary validation without allocating + multi-GiB buffers. + """ + + def _assert_bson_error(self, data, msg=None): + with self.assertRaises(BSONError, msg=msg): + bson.decode(data) + + def test_declared_document_lengths(self): + base = bson.encode({"a": "b" * 20, "c": [1, 2]}) + for length in (0, -1, 0x7F7F7F7F, 0x7FFFFFFF, len(base) + 1): + data = bytearray(base) + data[0:4] = struct.pack(" Date: Tue, 17 Nov 2026 10:49:26 -0600 Subject: [PATCH 2/4] Fix buffer-boundary sizes in BSON round-trip test --- test/test_bson.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/test_bson.py b/test/test_bson.py index c933af267d..53a63bcf07 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -2155,9 +2155,13 @@ def test_round_trip_value_extremes(self): self._round_trip(doc) def test_round_trip_size_boundaries(self): - # String sizes that cross buffer doubling boundaries. - for size in (127, 128, 255, 256, 2**16, 2**16 + 1): - self._round_trip({"a": "x" * size}) + # Encoded sizes that exactly fill the C encoder's buffer at each + # doubling boundary (no realloc), and one byte more (forces a + # realloc). Encoding {"a": "x" * n} adds 13 bytes of BSON + # overhead, so use n = capacity - 13 and capacity - 12. + for capacity in (256, 512, 1024, 2**16, 2**17): + for size in (capacity - 13, capacity - 12): + self._round_trip({"a": "x" * size}) class TestBSONHostileLengths(unittest.TestCase): From 006a2658124eca14fb78ae88c0a2f02cefc56a1b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 13:25:15 -0600 Subject: [PATCH 3/4] Address review feedback on C-extension boundary tests --- test/test_bson.py | 79 ++++++++++++++++++++++++------------------ test/test_message.py | 81 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 111 insertions(+), 49 deletions(-) diff --git a/test/test_bson.py b/test/test_bson.py index 53a63bcf07..ef2958b7ce 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -2090,7 +2090,7 @@ def _nested_document(depth: int) -> dict[str, Any]: return doc -def _typed_documents() -> list[dict[str, Any]]: +def _extreme_typed_documents() -> list[dict[str, Any]]: """One document per BSON type, at value extremes where applicable.""" return [ {}, @@ -2133,14 +2133,19 @@ def _typed_documents() -> list[dict[str, Any]]: ] +# Maximum int32 value; the largest length field a malformed document can claim. +_MAX_INT32 = 0x7FFFFFFF + _DATETIME_MS_CODEC = CodecOptions(datetime_conversion=DatetimeConversion.DATETIME_MS) class TestBSONEncodeDecodeBoundaries(unittest.TestCase): """Encode every BSON type at value extremes and re-encode byte-stably. - Drives buffer growth transitions and every _downcast_and_check path - under sanitizer builds. + The boundary-sized documents walk the C encoder's buffer through its + doubling thresholds (bson/buffer.c), and the string, binary, and regex + values exercise the C encoder's _downcast_and_check length validations, + all under sanitizer builds. """ def _round_trip(self, doc): @@ -2151,17 +2156,19 @@ def _round_trip(self, doc): self.assertEqual(bson.encode(decoded, codec_options=_DATETIME_MS_CODEC), encoded) def test_round_trip_value_extremes(self): - for doc in _typed_documents(): - self._round_trip(doc) + for doc in _extreme_typed_documents(): + with self.subTest(next(iter(doc), "empty")): + self._round_trip(doc) def test_round_trip_size_boundaries(self): # Encoded sizes that exactly fill the C encoder's buffer at each # doubling boundary (no realloc), and one byte more (forces a - # realloc). Encoding {"a": "x" * n} adds 13 bytes of BSON + # realloc). Encoding {"a": "x" * n} adds 13 bytes of BSON # overhead, so use n = capacity - 13 and capacity - 12. for capacity in (256, 512, 1024, 2**16, 2**17): for size in (capacity - 13, capacity - 12): - self._round_trip({"a": "x" * size}) + with self.subTest(capacity=capacity, size=size): + self._round_trip({"a": "x" * size}) class TestBSONHostileLengths(unittest.TestCase): @@ -2177,39 +2184,44 @@ def _assert_bson_error(self, data, msg=None): def test_declared_document_lengths(self): base = bson.encode({"a": "b" * 20, "c": [1, 2]}) - for length in (0, -1, 0x7F7F7F7F, 0x7FFFFFFF, len(base) + 1): - data = bytearray(base) - data[0:4] = struct.pack(" 8192. The + # later documents are small, so only the first drives the growth. docs = [{"s": "x" * n} for n in (5000, 50, 5)] _, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", docs, _OPTS) - self.assertEqual(struct.unpack(" Date: Tue, 17 Nov 2026 15:55:34 -0600 Subject: [PATCH 4/4] address review --- test/test_bson.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/test_bson.py b/test/test_bson.py index ef2958b7ce..8f1e549edd 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -2164,7 +2164,10 @@ def test_round_trip_size_boundaries(self): # Encoded sizes that exactly fill the C encoder's buffer at each # doubling boundary (no realloc), and one byte more (forces a # realloc). Encoding {"a": "x" * n} adds 13 bytes of BSON - # overhead, so use n = capacity - 13 and capacity - 12. + # overhead, so use n = capacity - 13 and capacity - 12. A capacity + # of _MAX_INT32 (the maximum BSON document size) is deliberately + # excluded: exercising buffer_grow's > INT_MAX/2 branch would + # require allocating a multi-GiB document. for capacity in (256, 512, 1024, 2**16, 2**17): for size in (capacity - 13, capacity - 12): with self.subTest(capacity=capacity, size=size): @@ -2237,30 +2240,33 @@ def _mutated(self, data): for value in self.MUTATION_VALUES: flipped = bytearray(data) flipped[offset] = value - yield bytes(flipped) + yield f"mutated offset {offset} = {value:#04x}", bytes(flipped) inverted = bytearray(data) inverted[offset] ^= 0xFF - yield bytes(inverted) + yield f"mutated offset {offset} = inverted", bytes(inverted) def _truncated(self, data): stride = max(1, len(data) // 64) for i in range(4, len(data), stride): - yield data[:i] + yield f"truncated at {i}", data[:i] def _assert_decodes_or_errors(self, data): - """Decode must either succeed or raise; never crash.""" + """Decode must either succeed or raise a BSONError; never crash.""" try: bson.decode(data, codec_options=_DATETIME_MS_CODEC) - except (BSONError, ValueError): + except BSONError: pass def test_mutations_decode_cleanly(self): for doc in _extreme_typed_documents(): + label = next(iter(doc), "empty") data = bson.encode(doc, codec_options=_DATETIME_MS_CODEC) - for corrupted in self._mutated(data): - self._assert_decodes_or_errors(corrupted) - for truncated in self._truncated(data): - self._assert_decodes_or_errors(truncated) + for detail, corrupted in self._mutated(data): + with self.subTest(f"{label} {detail}"): + self._assert_decodes_or_errors(corrupted) + for detail, truncated in self._truncated(data): + with self.subTest(f"{label} {detail}"): + self._assert_decodes_or_errors(truncated) if __name__ == "__main__":