Repository navigation
PYTHON-6285 Add hostile-input and boundary tests for the BSON and message C extensions #3233
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
4ba4c28
e3b15dc
006a265
497e1f7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,7 +23,7 @@ | |
|
|
||
| sys.path[0:0] = [""] | ||
|
|
||
| from bson import CodecOptions, encode | ||
| from bson import CodecOptions, decode, encode | ||
| from bson.objectid import ObjectId | ||
| from pymongo.common import MIN_SUPPORTED_WIRE_VERSION, MONGOS_EXHAUST_WIRE_VERSION | ||
| from pymongo.compression_support import ZlibContext, _have_zlib | ||
|
|
@@ -46,6 +46,15 @@ | |
| from pymongo.server_type import SERVER_TYPE | ||
| from test import unittest | ||
|
|
||
| try: | ||
| from pymongo import _cmessage | ||
| except ImportError: | ||
| _cmessage = None | ||
|
|
||
| # OP_MSG header: 4 message length + 4 request id + 4 responseTo + 4 opCode | ||
| # + 4 flags + 1 payload-type byte, preceding the type-0 command document. | ||
| _OP_MSG_HEADER_SIZE = 21 | ||
|
|
||
| _OPTS = CodecOptions() | ||
|
|
||
|
|
||
|
|
@@ -467,5 +476,131 @@ def test_get_more_with_comment(self): | |
| self.assertEqual(cmd["comment"], "my comment") | ||
|
|
||
|
|
||
| class _FakeBulkWriteContext: | ||
| """Stub exposing the max sizes read by _cbson_batched_op_msg.""" | ||
|
|
||
| max_bson_size = 16 * 1024 * 1024 | ||
|
|
||
| def __init__(self, max_write_batch_size=100000, max_message_size=48 * 1024 * 1024): | ||
| self.max_write_batch_size = max_write_batch_size | ||
| self.max_message_size = max_message_size | ||
|
|
||
|
|
||
| class TestOpMsgBuilders(unittest.TestCase): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this also have a case that forces a batch split due to the number of documents exceeding
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
| """Directly drive the C wire-message builders. | ||
|
|
||
| Exercises buffer growth and length back-patching in _cbson_op_msg and | ||
| _cbson_batched_op_msg under sanitizer builds. | ||
| """ | ||
|
|
||
| def setUp(self): | ||
| if _cmessage is None: | ||
| self.skipTest("_cmessage not built") | ||
|
|
||
| def _assert_total_length(self, msg): | ||
| """The first 4 bytes of an OP_MSG are its back-patched total length.""" | ||
| self.assertEqual(struct.unpack("<i", msg[:4])[0], len(msg)) | ||
|
|
||
| def test_op_msg_no_identifier(self): | ||
| _, msg, total_size, max_doc_size = _cmessage._op_msg(0, {"ismaster": 1}, "", None, _OPTS) | ||
| self.assertIsInstance(msg, bytes) | ||
| self._assert_total_length(msg) | ||
| # The header precedes the command document. | ||
| self.assertEqual(total_size, len(msg) - _OP_MSG_HEADER_SIZE) | ||
| self.assertEqual(max_doc_size, 0) | ||
| self.assertEqual(decode(msg[_OP_MSG_HEADER_SIZE:]), {"ismaster": 1}) | ||
|
|
||
| def test_op_msg_with_identifier(self): | ||
| docs: list[dict[str, Any]] = [{"b": 1}, {"c": "two"}] | ||
| _, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", docs, _OPTS) | ||
| self._assert_total_length(msg) | ||
| self.assertEqual(max_doc_size, max(len(encode(doc)) for doc in docs)) | ||
| command_size = struct.unpack("<i", msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + 4])[0] | ||
| self.assertEqual( | ||
| decode(msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + command_size]), | ||
| {"insert": "coll"}, | ||
| ) | ||
| # Type-1 payload: type byte, 4-byte length, cstring identifier. | ||
| payload_start = _OP_MSG_HEADER_SIZE + command_size | ||
| self.assertEqual(msg[payload_start], 0x01) | ||
| payload_length = struct.unpack("<i", msg[payload_start + 1 : payload_start + 5])[0] | ||
| self.assertEqual( | ||
| payload_length, | ||
| 4 + len(b"documents\x00") + sum(len(encode(doc)) for doc in docs), | ||
| ) | ||
| offset = payload_start + 5 + len(b"documents\x00") | ||
| for doc in docs: | ||
| size = len(encode(doc)) | ||
| self.assertEqual(decode(msg[offset : offset + size]), doc) | ||
| offset += size | ||
| self.assertEqual(offset, len(msg)) | ||
|
|
||
| def test_op_msg_empty_docs(self): | ||
| _, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", [], _OPTS) | ||
| self._assert_total_length(msg) | ||
| self.assertEqual(max_doc_size, 0) | ||
|
|
||
| def test_op_msg_multiple_buffer_doublings(self): | ||
| # The C encoder starts with a 256-byte buffer (INITIAL_BUFFER_SIZE in | ||
| # bson/buffer.c) and repeatedly doubles it until the data fits. The | ||
| # 5013-byte first document (plus the 21-byte header, command document, | ||
| # and payload prefix) forces 5 consecutive doublings: 256 -> 8192. The | ||
| # later documents are small, so only the first drives the growth. | ||
| docs = [{"s": "x" * n} for n in (5000, 50, 5)] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can this have a comment explaining how/why these sizes cause the buffer to double multiple times? It's not obvious from the code.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
| _, msg, _, max_doc_size = _cmessage._op_msg(0, {"insert": "coll"}, "documents", docs, _OPTS) | ||
| self._assert_total_length(msg) | ||
| self.assertEqual(max_doc_size, max(len(encode(doc)) for doc in docs)) | ||
|
|
||
| def test_batched_op_msg_insert(self): | ||
| docs = [{"_id": i, "s": "x" * 10} for i in range(3)] | ||
| _, msg, to_publish = _cmessage._batched_op_msg( | ||
| 0, {"insert": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext() | ||
| ) | ||
| self.assertIsInstance(to_publish, list) | ||
| self._assert_total_length(msg) | ||
| command_size = struct.unpack("<i", msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + 4])[0] | ||
| self.assertEqual( | ||
| decode(msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + command_size]), | ||
| {"insert": "coll"}, | ||
| ) | ||
| self.assertEqual(len(to_publish), len(docs)) | ||
|
|
||
| def test_batched_op_msg_update(self): | ||
| docs = [{"q": {}, "u": {"$set": {"a": 1}}}] | ||
| _, msg, to_publish = _cmessage._batched_op_msg( | ||
| 1, {"update": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext() | ||
| ) | ||
| self._assert_total_length(msg) | ||
| self.assertIsInstance(to_publish, list) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we also check the length of to_publish, similar to the test above?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
| self.assertEqual(len(to_publish), len(docs)) | ||
|
|
||
| def test_batched_op_msg_split_on_batch_size(self): | ||
| # Exceeding max_write_batch_size ends the batch after exactly that | ||
| # many documents; the remaining documents are left for the caller to | ||
| # batch separately. | ||
| docs = [{"_id": i, "s": "x" * 10} for i in range(5)] | ||
| _, msg, to_publish = _cmessage._batched_op_msg( | ||
| 0, {"insert": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext(max_write_batch_size=2) | ||
| ) | ||
| self.assertEqual(len(to_publish), 2) | ||
| self._assert_total_length(msg) | ||
| command_size = struct.unpack("<i", msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + 4])[0] | ||
| self.assertEqual( | ||
| decode(msg[_OP_MSG_HEADER_SIZE : _OP_MSG_HEADER_SIZE + command_size]), | ||
| {"insert": "coll"}, | ||
| ) | ||
|
|
||
| def test_batched_op_msg_split_on_message_size(self): | ||
| # A document that would push the message past max_message_size is | ||
| # rolled back out of the buffer and left for the caller to batch | ||
| # separately. | ||
| docs = [{"_id": i, "s": "x" * 100} for i in range(3)] | ||
| _, msg, to_publish = _cmessage._batched_op_msg( | ||
| 0, {"insert": "coll"}, docs, True, _OPTS, _FakeBulkWriteContext(max_message_size=350) | ||
| ) | ||
| self.assertEqual(len(to_publish), 2) | ||
| self._assert_total_length(msg) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| unittest.main() | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we also have a case that tests against
_MAX_INT32?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not, it would require too much memory, added a comment.