Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .evergreen/scripts/setup_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,15 @@
# Map the test name to test group.
GROUP_MAP = dict(fuzz="fuzz", mockupdb="mockupdb", perf="perf")

# The test files run under ASan/UBSan (PYTHON-6048). They exercise the
# buffer-protocol acquisition path in bson/_cbsonmodule.c
# (_get_buffer/_prepare_input_buffer) and the wire-message builders in
# The test files run under ASan/UBSan (PYTHON-6048, PYTHON-6285). They
# exercise the buffer-protocol acquisition path in bson/_cbsonmodule.c
# (_get_buffer/_prepare_input_buffer), the wire-message builders in
# pymongo/_cmessagemodule.c (_op_msg, _batched_op_msg,
# _encode_batched_write_command).
# _encode_batched_write_command), and the hostile-input and boundary tests
# driving the decoder and builders directly.
SANITIZER_FILES = [
"test/test_bson.py",
"test/test_message.py",
"test/test_raw_bson.py",
"test/test_raw_bson_shared.py",
"test/test_client.py",
Expand Down
190 changes: 189 additions & 1 deletion test/test_bson.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import uuid
from collections import OrderedDict, abc
from io import BytesIO
from typing import Any

sys.path[0:0] = [""]

Expand Down Expand Up @@ -63,7 +64,8 @@
from bson.codec_options import CodecOptions, DatetimeConversion
from bson.datetime_ms import _DATETIME_ERROR_SUGGESTION
from bson.dbref import DBRef
from bson.errors import InvalidBSON, InvalidDocument
from bson.decimal128 import Decimal128
from bson.errors import BSONError, InvalidBSON, InvalidDocument
from bson.int64 import Int64
from bson.max_key import MaxKey
from bson.min_key import MinKey
Expand Down Expand Up @@ -2081,5 +2083,191 @@ def test_long_long_to_string(self):
print("_cbson was not imported. Check compilation logs.")


def _nested_document(depth: int) -> dict[str, Any]:
doc: dict[str, Any] = {"leaf": 1}
for _ in range(depth):
doc = {"d": doc}
return doc


def _extreme_typed_documents() -> list[dict[str, Any]]:
"""One document per BSON type, at value extremes where applicable."""
return [
{},
{"int32": 1},
{"int64 max": Int64(2**63 - 1), "int64 min": Int64(-(2**63))},
{"double": 1.5, "double min": sys.float_info.min, "double max": sys.float_info.max},
{"string": "hello é", "string empty": ""},
{"string big": "x" * 70000},
{
"binary": Binary(b"\x00\xffbin", 0),
"binary old": Binary(b"old", 2),
"binary empty": Binary(b"", 0),
},
{
"datetime zero": DatetimeMS(0),
"datetime min": DatetimeMS(-(2**63)),
"datetime max": DatetimeMS(2**63 - 1),
},
{"timestamp": Timestamp(0, 1), "timestamp max": Timestamp(0xFFFFFFFF, 0xFFFFFFFF)},
{
"decimal128": Decimal128("1.25"),
"decimal128 nan": Decimal128("NaN"),
"decimal128 inf": Decimal128("Infinity"),
"decimal128 neg0": Decimal128("-0"),
},
{"regex": Regex("a*b", "im"), "regex empty": Regex("", "")},
{"code": Code("function() { return 1; }"), "code scope": Code("f", {"x": [1, 2]})},
{
"dbref": DBRef("coll", "id"),
"objectid": ObjectId(b"0123456789ab"),
"minkey": MinKey(),
"maxkey": MaxKey(),
"null": None,
"bool": True,
},
{"array": [1, "two", None, {"nested": True}], "subdoc": {"b": {"c": 1}}},
{"bytes": b"raw bytes"},
{"key" * 63 + "e": 1},
{"deep": _nested_document(50)},
]


# Maximum int32 value; the largest length field a malformed document can claim.
_MAX_INT32 = 0x7FFFFFFF

_DATETIME_MS_CODEC = CodecOptions(datetime_conversion=DatetimeConversion.DATETIME_MS)


class TestBSONEncodeDecodeBoundaries(unittest.TestCase):
"""Encode every BSON type at value extremes and re-encode byte-stably.

The boundary-sized documents walk the C encoder's buffer through its
doubling thresholds (bson/buffer.c), and the string, binary, and regex
values exercise the C encoder's _downcast_and_check length validations,
all under sanitizer builds.
"""

def _round_trip(self, doc):
encoded = bson.encode(doc, codec_options=_DATETIME_MS_CODEC)
decoded = bson.decode(encoded, codec_options=_DATETIME_MS_CODEC)
# Compare byte-level stability rather than decoded == doc so that
# NaN payloads (which are not equal to themselves) round-trip too.
self.assertEqual(bson.encode(decoded, codec_options=_DATETIME_MS_CODEC), encoded)

def test_round_trip_value_extremes(self):
for doc in _extreme_typed_documents():
with self.subTest(next(iter(doc), "empty")):
self._round_trip(doc)

def test_round_trip_size_boundaries(self):
# Encoded sizes that exactly fill the C encoder's buffer at each
# doubling boundary (no realloc), and one byte more (forces a
# realloc). Encoding {"a": "x" * n} adds 13 bytes of BSON
# overhead, so use n = capacity - 13 and capacity - 12. A capacity
# of _MAX_INT32 (the maximum BSON document size) is deliberately
# excluded: exercising buffer_grow's > INT_MAX/2 branch would
# require allocating a multi-GiB document.
for capacity in (256, 512, 1024, 2**16, 2**17):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also have a case that tests against _MAX_INT32?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not, it would require too much memory, added a comment.

for size in (capacity - 13, capacity - 12):
with self.subTest(capacity=capacity, size=size):
self._round_trip({"a": "x" * size})

Comment thread
NoahStapp marked this conversation as resolved.
Resolved

class TestBSONHostileLengths(unittest.TestCase):
"""Decode must reject malformed length fields with a BSONError.

Exercises the C decoder's int-boundary validation without allocating
multi-GiB buffers.
"""

def _assert_bson_error(self, data, msg=None):
with self.assertRaises(BSONError, msg=msg):
bson.decode(data)

def test_declared_document_lengths(self):
base = bson.encode({"a": "b" * 20, "c": [1, 2]})
for length in (0, -1, 0x7F7F7F7F, _MAX_INT32, len(base) + 1):
with self.subTest(length=length):
data = bytearray(base)
data[0:4] = struct.pack("<i", length)
self._assert_bson_error(bytes(data))

def test_element_string_lengths(self):
base = bson.encode({"a": "x" * 20})
# The string length field starts at offset 7.
for length in (-1, _MAX_INT32):
with self.subTest(length=length):
data = bytearray(base)
data[7:11] = struct.pack("<i", length)
self._assert_bson_error(bytes(data))

def test_element_subdocument_lengths(self):
base = bson.encode({"a": {"b": 1}})
# The subdocument length field starts at offset 7.
for length in (0, -1, _MAX_INT32):
with self.subTest(length=length):
data = bytearray(base)
data[7:11] = struct.pack("<i", length)
self._assert_bson_error(bytes(data))

def test_element_binary_lengths(self):
base = bson.encode({"a": Binary(b"1234567890", 0)})
# The binary length field starts at offset 7.
for length in (-1, _MAX_INT32):
with self.subTest(length=length):
data = bytearray(base)
data[7:11] = struct.pack("<i", length)
self._assert_bson_error(bytes(data))

def test_truncated_documents(self):
base = bson.encode({"a": "b" * 20, "c": [1, 2]})
for i in range(4, len(base)):
with self.subTest(length=i):
self._assert_bson_error(base[:i])


class TestBSONMutations(unittest.TestCase):
"""Every byte mutation must decode successfully or raise a BSONError,
never crash or read out of bounds.
"""

MUTATION_VALUES = (0x00, 0x7F, 0xFF)

def _mutated(self, data):
stride = max(1, len(data) // 64)
for offset in range(0, len(data), stride):
for value in self.MUTATION_VALUES:
flipped = bytearray(data)
flipped[offset] = value
yield f"mutated offset {offset} = {value:#04x}", bytes(flipped)
inverted = bytearray(data)
inverted[offset] ^= 0xFF
yield f"mutated offset {offset} = inverted", bytes(inverted)

def _truncated(self, data):
stride = max(1, len(data) // 64)
for i in range(4, len(data), stride):
yield f"truncated at {i}", data[:i]

def _assert_decodes_or_errors(self, data):
"""Decode must either succeed or raise a BSONError; never crash."""
try:
bson.decode(data, codec_options=_DATETIME_MS_CODEC)
except BSONError:
pass

def test_mutations_decode_cleanly(self):
for doc in _extreme_typed_documents():
label = next(iter(doc), "empty")
data = bson.encode(doc, codec_options=_DATETIME_MS_CODEC)
for detail, corrupted in self._mutated(data):
with self.subTest(f"{label} {detail}"):
self._assert_decodes_or_errors(corrupted)
for detail, truncated in self._truncated(data):
with self.subTest(f"{label} {detail}"):
self._assert_decodes_or_errors(truncated)


if __name__ == "__main__":
unittest.main()
137 changes: 136 additions & 1 deletion test/test_message.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

sys.path[0:0] = [""]

from bson import CodecOptions, encode
from bson import CodecOptions, decode, encode
from bson.objectid import ObjectId
from pymongo.common import MIN_SUPPORTED_WIRE_VERSION, MONGOS_EXHAUST_WIRE_VERSION
from pymongo.compression_support import ZlibContext, _have_zlib
Expand All @@ -46,6 +46,15 @@
from pymongo.server_type import SERVER_TYPE
from test import unittest

try:
from pymongo import _cmessage
except ImportError:
_cmessage = None

# OP_MSG header: 4 message length + 4 request id + 4 responseTo + 4 opCode
# + 4 flags + 1 payload-type byte, preceding the type-0 command document.
_OP_MSG_HEADER_SIZE = 21

_OPTS = CodecOptions()


Expand Down Expand Up @@ -467,5 +476,131 @@ def test_get_more_with_comment(self):
self.assertEqual(cmd["comment"], "my comment")


class _FakeBulkWriteContext:
"""Stub exposing the max sizes read by _cbson_batched_op_msg."""

max_bson_size = 16 * 1024 * 1024

def __init__(self, max_write_batch_size=100000, max_message_size=48 * 1024 * 1024):
self.max_write_batch_size = max_write_batch_size
self.max_message_size = max_message_size


class TestOpMsgBuilders(unittest.TestCase):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this also have a case that forces a batch split due to the number of documents exceeding max_write_batch_size?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

"""Directly drive the C wire-message builders.

Exercises buffer growth and length back-patching in _cbson_op_msg and
_cbson_batched_op_msg under sanitizer builds.
"""

def setUp(self):
if _cmessage is None:
self.skipTest("_cmessage not built")

def _assert_total_length(self, msg):
"""The first 4 bytes of an OP_MSG are its back-patched total length."""
self.assertEqual(struct.unpack("<i", msg[:4])[0], len(msg))

def test_op_msg_no_identifier(self):
_, msg, total_size, max_doc_size = _cmessage._op_msg(0, {"ismaster": 1}, "", None, _OPTS)
self.assertIsInstance(msg, bytes)
self._assert_total_length(msg)
# The header precedes the command document.
self.assertEqual(total_size, len(msg) - _OP_MSG_HEADER_SIZE)
self.assertEqual(max_doc_size, 0)
self.assertEqual(decode(msg[_OP_MSG_HEADER_SIZE:]), {"ismaster": 1})

def test_op_msg_with_identifier(self):
docs: list[dict[str, Any]] = [{"b": 1}, {"c": "two"}]
_, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", docs, _OPTS)
self._assert_total_length(msg)
self.assertEqual(max_doc_size, max(len(encode(doc)) for doc in docs))
command_size = struct.unpack("<i", msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + 4])[0]
self.assertEqual(
decode(msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + command_size]),
{"insert": "coll"},
)
# Type-1 payload: type byte, 4-byte length, cstring identifier.
payload_start = _OP_MSG_HEADER_SIZE + command_size
self.assertEqual(msg[payload_start], 0x01)
payload_length = struct.unpack("<i", msg[payload_start + 1 : payload_start + 5])[0]
self.assertEqual(
payload_length,
4 + len(b"documents\x00") + sum(len(encode(doc)) for doc in docs),
)
offset = payload_start + 5 + len(b"documents\x00")
for doc in docs:
size = len(encode(doc))
self.assertEqual(decode(msg[offset : offset + size]), doc)
offset += size
self.assertEqual(offset, len(msg))

def test_op_msg_empty_docs(self):
_, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", [], _OPTS)
self._assert_total_length(msg)
self.assertEqual(max_doc_size, 0)

def test_op_msg_multiple_buffer_doublings(self):
# The C encoder starts with a 256-byte buffer (INITIAL_BUFFER_SIZE in
# bson/buffer.c) and repeatedly doubles it until the data fits. The
# 5013-byte first document (plus the 21-byte header, command document,
# and payload prefix) forces 5 consecutive doublings: 256 -> 8192. The
# later documents are small, so only the first drives the growth.
docs = [{"s": "x" * n} for n in (5000, 50, 5)]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this have a comment explaining how/why these sizes cause the buffer to double multiple times? It's not obvious from the code.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

_, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", docs, _OPTS)
self._assert_total_length(msg)
self.assertEqual(max_doc_size, max(len(encode(doc)) for doc in docs))

def test_batched_op_msg_insert(self):
docs = [{"_id": i, "s": "x" * 10} for i in range(3)]
_, msg, to_publish = _cmessage._batched_op_msg(
0, {"insert": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext()
)
self.assertIsInstance(to_publish, list)
self._assert_total_length(msg)
command_size = struct.unpack("<i", msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + 4])[0]
self.assertEqual(
decode(msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + command_size]),
{"insert": "coll"},
)
self.assertEqual(len(to_publish), len(docs))

def test_batched_op_msg_update(self):
docs = [{"q": {}, "u": {"$set": {"a": 1}}}]
_, msg, to_publish = _cmessage._batched_op_msg(
1, {"update": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext()
)
self._assert_total_length(msg)
self.assertIsInstance(to_publish, list)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also check the length of to_publish, similar to the test above?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

self.assertEqual(len(to_publish), len(docs))

def test_batched_op_msg_split_on_batch_size(self):
# Exceeding max_write_batch_size ends the batch after exactly that
# many documents; the remaining documents are left for the caller to
# batch separately.
docs = [{"_id": i, "s": "x" * 10} for i in range(5)]
_, msg, to_publish = _cmessage._batched_op_msg(
0, {"insert": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext(max_write_batch_size=2)
)
self.assertEqual(len(to_publish), 2)
self._assert_total_length(msg)
command_size = struct.unpack("<i", msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + 4])[0]
self.assertEqual(
decode(msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + command_size]),
{"insert": "coll"},
)

def test_batched_op_msg_split_on_message_size(self):
# A document that would push the message past max_message_size is
# rolled back out of the buffer and left for the caller to batch
# separately.
docs = [{"_id": i, "s": "x" * 100} for i in range(3)]
_, msg, to_publish = _cmessage._batched_op_msg(
0, {"insert": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext(max_message_size=350)
)
self.assertEqual(len(to_publish), 2)
self._assert_total_length(msg)


if __name__ == "__main__":
unittest.main()
Loading