From 1b21b6dde3da06ac5b72e76a7591b683c3b37a1c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 30 Oct 2026 22:05:52 -0500 Subject: [PATCH 1/8] PYTHON-6207 back bson encode and decode with Rust and run the full BSON performance suite --- .evergreen/generated_configs/functions.yml | 72 ++ .evergreen/generated_configs/tasks.yml | 22 + .evergreen/generated_configs/variants.yml | 10 +- .evergreen/scripts/bson_codec_compare.sh | 206 ++++ .evergreen/scripts/generate_config.py | 78 ++ .evergreen/scripts/generate_config_utils.py | 8 +- .evergreen/scripts/test_generate_config.py | 86 ++ .gitignore | 11 + .pre-commit-config.yaml | 3 +- AGENTS.md | 1 + rust/bson/Cargo.lock | 140 +++ rust/bson/Cargo.toml | 21 + rust/bson/build.rs | 6 + rust/bson/pyproject.toml | 15 + rust/bson/src/buffer.rs | 105 ++ rust/bson/src/datetime_conv.rs | 126 ++ rust/bson/src/decode.rs | 1048 ++++++++++++++++ rust/bson/src/encode.rs | 1066 +++++++++++++++++ rust/bson/src/errors.rs | 48 + rust/bson/src/lib.rs | 449 +++++++ rust/bson/src/options.rs | 200 ++++ rust/bson/src/types.rs | 109 ++ rust/bson/tests/test_rbson.py | 976 +++++++++++++++ test/performance/bson_impl_select.py | 76 ++ test/performance/conftest.py | 6 + .../data/extended_bson/deep_bson.json | 1 + .../data/extended_bson/flat_bson.json | 1 + .../data/extended_bson/full_bson.json | 1 + tools/compare_bson_perf.py | 58 + 29 files changed, 4946 insertions(+), 3 deletions(-) create mode 100755 .evergreen/scripts/bson_codec_compare.sh create mode 100644 rust/bson/Cargo.lock create mode 100644 rust/bson/Cargo.toml create mode 100644 rust/bson/build.rs create mode 100644 rust/bson/pyproject.toml create mode 100644 rust/bson/src/buffer.rs create mode 100644 rust/bson/src/datetime_conv.rs create mode 100644 rust/bson/src/decode.rs create mode 100644 rust/bson/src/encode.rs create mode 100644 rust/bson/src/errors.rs create mode 100644 rust/bson/src/lib.rs create mode 100644 rust/bson/src/options.rs create mode 100644 rust/bson/src/types.rs create mode 100644 rust/bson/tests/test_rbson.py create mode 100644 test/performance/bson_impl_select.py create mode 100644 test/performance/conftest.py create mode 100644 test/performance/data/extended_bson/deep_bson.json create mode 100644 test/performance/data/extended_bson/flat_bson.json create mode 100644 test/performance/data/extended_bson/full_bson.json create mode 100644 tools/compare_bson_perf.py diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index cca9166af5..59f36578fb 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -257,6 +257,78 @@ functions: - COVERAGE type: test + # Upload bson compare artifacts + upload bson compare artifacts: + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} + type: setup + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/bson-compare/${task_id}-${execution}-results_c.json + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/results_c.json + permissions: public-read + content_type: application/json + display_name: BSON C results + optional: "true" + type: setup + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/bson-compare/${task_id}-${execution}-results_rust.json + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/results_rust.json + permissions: public-read + content_type: application/json + display_name: BSON Rust results + optional: "true" + type: setup + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/bson-compare/${task_id}-${execution}-bson_comparison.md + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/bson_comparison.md + permissions: public-read + content_type: text/markdown + display_name: BSON codec comparison + optional: "true" + type: setup + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/bson-compare/${task_id}-${execution}-perf_deepdec_c.txt + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/perf_deepdec_c.txt + permissions: public-read + content_type: text/plain + display_name: BSON perf profile (C, DeepDecoding) + optional: "true" + type: setup + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/bson-compare/${task_id}-${execution}-perf_deepdec_rust.txt + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/perf_deepdec_rust.txt + permissions: public-read + content_type: text/plain + display_name: BSON perf profile (Rust, DeepDecoding) + optional: "true" + type: setup + # Upload coverage codecov upload codecov: - command: subprocess.exec diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 8e2214c680..a76828bef3 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -190,6 +190,28 @@ tasks: - AWS_SESSION_TOKEN type: test + # Bson codec compare tests + - name: bson-codec-compare + commands: + - func: run server + vars: + VERSION: v8.0-perf + SSL: nossl + UV_PYTHON: 3.11.9 + UV_PYTHON_PREFERENCE: only-managed + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/bson_codec_compare.sh + working_dir: src + include_expansions_in_env: + - UV_PYTHON + - UV_PYTHON_PREFERENCE + type: test + - func: upload bson compare artifacts + tags: [bson-codec-compare] + # Coverage report tests - name: coverage-report commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 67d069f438..d1d1051540 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -115,6 +115,14 @@ buildvariants: run_on: - rhel8.10-small + # Bson codec compare tests + - name: bson-codec-compare + tasks: + - name: bson-codec-compare + display_name: BSON Codec Compare + run_on: + - ubuntu2204-small + # Compression tests - name: compression-snappy-rhel8 tasks: @@ -490,7 +498,7 @@ buildvariants: - name: .perf display_name: Performance Benchmarks run_on: - - rhel90-dbx-perf-large + - ubuntu2204-small batchtime: 1440 # Pyopenssl tests diff --git a/.evergreen/scripts/bson_codec_compare.sh b/.evergreen/scripts/bson_codec_compare.sh new file mode 100755 index 0000000000..c9b5cff77a --- /dev/null +++ b/.evergreen/scripts/bson_codec_compare.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# Compare BSON codec throughput: the C extension vs the Rust-backed codec. +# +# Provisions the perf test environment (venv, benchmark data), builds the +# Rust codec from rust/bson, runs the BSON micro-benchmarks once per codec, +# and prints a side-by-side comparison. +# +# Requires the MongoDB server started (the perf tests' setUpClass connects); +# the Evergreen task runs `run server` first. Locally: +# MONGODB_VERSION=8.0 just run-server +set -euo pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +SCRIPT_DIR="$(cd "$SCRIPT_DIR" > /dev/null 2>&1 && pwd)" +ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" +cd "$ROOT_DIR" + +# Source (not execute) so the exports apply here: this puts the pinned uv +# (required-version in pyproject.toml) first on PATH and ensures it is +# installed — the host image's uv may be older and would abort every uv +# invocation. Sources env.sh/test-env.sh and runs install-dependencies.sh +# (idempotent; 'setup system' already ran it). +. "$SCRIPT_DIR/setup-dev-env.sh" + +# Provision the perf environment: venv, benchmark data, test-env.sh. +bash "$SCRIPT_DIR/setup-tests.sh" perf sync + +# Source the env written by setup_tests.py (TEST_PATH, UV_ARGS, ...). +if [ -f "$SCRIPT_DIR/test-env.sh" ]; then + . "$SCRIPT_DIR/test-env.sh" +fi + +# Ensure a Rust toolchain. Evergreen sets CARGO_HOME to +# $DRIVERS_TOOLS/.cargo (via env.sh from configure-env.sh); rustup defaults +# to $HOME/.cargo elsewhere — source the env from wherever it installed so +# cargo is on PATH. +if ! command -v cargo > /dev/null 2>&1; then + echo "Installing Rust toolchain..." + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal +fi +_cargo_env="${CARGO_HOME:-$HOME/.cargo}/env" +if [ -f "$_cargo_env" ]; then + # shellcheck disable=SC1090 # the env file's location depends on CARGO_HOME + . "$_cargo_env" +fi +if ! command -v cargo > /dev/null 2>&1; then + echo "cargo is not on PATH after the Rust toolchain setup" >&2 + exit 1 +fi + +# Sync the venv with the perf environment (pytest via the test extra, +# simplejson via the perf group). This must run BEFORE the codec build: +# uv sync prunes locally-installed packages that are not in uv.lock. +uv sync ${UV_ARGS:-} + +# Build the Rust codec into the venv (after the sync — see above). Two uv +# pitfalls to avoid here: `uv run --with maturin` would run maturin inside +# a throwaway overlay environment (the codec lands in the overlay, not the +# venv), and uv run's implicit sync would prune the freshly built codec. +uv pip install maturin +# Target the building machine's CPU: this task builds with `maturin develop` +# on the same host that benchmarks (never cross-compiled or distributed), and +# the pip-built C extension only gets baseline codegen. Set via RUSTFLAGS — +# cargo only discovers a crate-local .cargo/config.toml from the invocation +# directory upward, which maturin (run from the checkout root) would miss. +export RUSTFLAGS="${RUSTFLAGS:-}-C target-cpu=native" +.venv/bin/maturin develop --release -m rust/bson/Cargo.toml + +# Run the benchmarks with the venv's interpreter directly: no implicit +# sync, so the locally-installed codec survives both runs. -m perf is +# required: the repo's addopts (-m "default or default_async") would +# otherwise deselect the unmarked perf suite. The selector skips the JSON +# benchmarks (json_util is pure Python and identical for both codecs). +TEST_SELECTOR='(Flat or Deep or Full) and not Json' + +# Run each codec's pass twice in swapped order (C→Rust, then Rust→C) and +# merge to the per-benchmark mean: machine state (turbo/thermal drift) moves +# over the ~6-minute window, so a single back-to-back C/Rust pair is +# systematically biased in favor of whichever codec runs first. The swap +# cancels that bias; the merged files are what gets compared and uploaded. +BSON_IMPL=c OUTPUT_FILE=results_c_a.json \ + .venv/bin/python -m pytest \ + test/performance/perf_test.py -m perf -k "$TEST_SELECTOR" -q + +BSON_IMPL=rust OUTPUT_FILE=results_rust_a.json \ + .venv/bin/python -m pytest \ + test/performance/perf_test.py -m perf -k "$TEST_SELECTOR" -q + +BSON_IMPL=rust OUTPUT_FILE=results_rust_b.json \ + .venv/bin/python -m pytest \ + test/performance/perf_test.py -m perf -k "$TEST_SELECTOR" -q + +BSON_IMPL=c OUTPUT_FILE=results_c_b.json \ + .venv/bin/python -m pytest \ + test/performance/perf_test.py -m perf -k "$TEST_SELECTOR" -q + +# Merge the swapped-order pairs into the two files the comparison reads. +.venv/bin/python - <<'PY' +import copy +import json + + +def merge(paths, out): + values = {} + for path in paths: + for result in json.load(open(path)): + name = result["info"]["test_name"] + values.setdefault(name, []).append(result) + merged = [] + for name, results in sorted(values.items()): + # Carry the first run's entry through unchanged (info, args, + # improvement_direction, measurement_unit — perf_test.py's schema) + # and replace only the throughput value with the mean of the runs. + # "type" keeps the per-run aggregation label for schema compatibility. + entry = copy.deepcopy(results[0]) + idx = next( + i for i, m in enumerate(entry["metrics"]) + if m["name"] == "megabytes_per_sec" + ) + entry["metrics"][idx]["value"] = sum( + r["metrics"][idx]["value"] for r in results + ) / len(results) + merged.append(entry) + with open(out, "w") as fid: + fid.write(json.dumps(merged, indent=4)) + + +merge(["results_c_a.json", "results_c_b.json"], "results_c.json") +merge(["results_rust_a.json", "results_rust_b.json"], "results_rust.json") +PY +rm results_c_a.json results_c_b.json results_rust_a.json results_rust_b.json + +# Print the comparison (also uploaded as a task artifact). +.venv/bin/python tools/compare_bson_perf.py results_c.json results_rust.json | tee bson_comparison.md + +# Profile DeepDecoding — the one benchmark still below the parity bar on the +# perf host (~0.90x Rust/C, stable across runs) — once per codec, for +# hotspot data to close the remaining gap. Entirely non-fatal: profiling is +# best-effort and the benchmark numbers never depend on it. +# The rhel90 perf image does not ship perf; try the platform package manager +# with passwordless sudo (the established pattern on Evergreen hosts, e.g. +# setup-system.sh) before skipping. +if ! command -v perf > /dev/null 2>&1; then + if command -v dnf > /dev/null 2>&1; then + sudo dnf install -y perf || sudo yum install -y perf || true + elif command -v apt-get > /dev/null 2>&1; then + sudo apt-get install -y -qq perf || true + fi +fi +if command -v perf > /dev/null 2>&1; then + # Lower the paranoid floor so user-space sampling works on hardened + # images (kernel samples are dropped at >=2; user-space only is enough). + sudo sysctl -w kernel.perf_event_paranoid=1 > /dev/null 2>&1 || true + for impl in c rust; do + echo "Profiling DeepDecoding (BSON_IMPL=$impl)..." + perf record -F 199 --call-graph dwarf,16384 \ + -o "perf_deepdec_${impl}.data" \ + -- env BSON_IMPL="$impl" OUTPUT_FILE="results_profile_${impl}.json" \ + .venv/bin/python -m pytest test/performance/perf_test.py -m perf \ + -k 'DeepDecoding and not Json' -q \ + > /dev/null || echo "perf record failed (BSON_IMPL=$impl);" \ + "its profile may be missing or partial" >&2 + if [ -f "perf_deepdec_${impl}.data" ]; then + # --no-children: self time is what distinguishes the codecs; the raw + # perf.data is discarded after the report (binary and large). + perf report --stdio --no-children --percent-limit 0.5 \ + -i "perf_deepdec_${impl}.data" > "perf_deepdec_${impl}.txt" || true + rm -f "perf_deepdec_${impl}.data" + # A failed report leaves an empty/partial file; don't upload one. + [ -s "perf_deepdec_${impl}.txt" ] || rm -f "perf_deepdec_${impl}.txt" + fi + rm -f "results_profile_${impl}.json" + done +elif command -v valgrind > /dev/null 2>&1; then + # No perf (e.g. the rhel90 image and it cannot be installed there): fall + # back to callgrind, which the ubuntu2204 image ships preinstalled. + # Callgrind's instruction counts are exact and deterministic, so hotspot + # attribution is valid at any run length — FASTBENCH keeps the ~30x + # slowdown bounded, and pytest/interpreter startup churn is discountable + # in the annotation (it lands in importlib/PyImport frames). + # The native build emits AVX-512 (EVEX) instructions via memchr's runtime + # dispatch, and valgrind 3.18 cannot decode EVEX — it SIGILLs mid-run. + # Rebuild for the profiled run on x86-64-v3 (AVX2, VEX-encoded, which + # valgrind handles — the C extension is baseline x86-64 and profiles + # fine). Nothing after this block uses the codec. + echo "Rebuilding the Rust codec without AVX-512 for valgrind..." + RUSTFLAGS="-C target-cpu=x86-64-v3" .venv/bin/maturin develop --release \ + -m rust/bson/Cargo.toml + for impl in c rust; do + echo "Profiling DeepDecoding with callgrind (BSON_IMPL=$impl)..." + FASTBENCH=1 BSON_IMPL="$impl" OUTPUT_FILE="results_profile_${impl}.json" \ + valgrind --tool=callgrind --callgrind-out-file="callgrind_${impl}.out" \ + .venv/bin/python -m pytest test/performance/perf_test.py -m perf \ + -k 'DeepDecoding and not Json' -q \ + || echo "callgrind run failed (BSON_IMPL=$impl); skipping its profile" >&2 + if [ -f "callgrind_${impl}.out" ]; then + callgrind_annotate "callgrind_${impl}.out" > "perf_deepdec_${impl}.txt" || true + rm -f "callgrind_${impl}.out" + # A failed annotation leaves an empty/partial file; don't upload one. + [ -s "perf_deepdec_${impl}.txt" ] || rm -f "perf_deepdec_${impl}.txt" + fi + rm -f "results_profile_${impl}.json" + done +else + echo "perf could not be installed and valgrind is missing; skipping the DeepDecoding profiles" >&2 +fi diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 3589a81c86..90c8580aa1 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -580,6 +580,13 @@ def create_perf_variants(): return [create_variant([".perf"], "Performance Benchmarks", host=host, batchtime=BATCHTIME_DAY)] +def create_bson_codec_compare_variants(): + # Patch-only variant (no batchtime): compares the C and Rust-backed + # BSON codecs on the perf host. + host = HOSTS["perf"] + return [create_variant(["bson-codec-compare"], "BSON Codec Compare", host=host)] + + def create_aws_auth_variants(): variants = [] @@ -1174,6 +1181,34 @@ def create_search_index_tasks(): return [EvgTask(name=task_name, tags=tags, commands=commands)] +def create_bson_codec_compare_tasks(): + """Compare BSON codec throughput: C extension vs the Rust-backed codec.""" + vars = dict( + VERSION="v8.0-perf", + SSL="nossl", + UV_PYTHON=PERF_PYTHON_VERSION, + UV_PYTHON_PREFERENCE="only-managed", + ) + server_func = FunctionCall(func="run server", vars=vars) + # The script drives uv itself (setup-tests.sh + the benchmark runs), so + # the pinned-uv variables must be in its environment (the script sources + # setup-dev-env.sh to put the pinned uv first on PATH). + compare_func = get_subprocess_exec( + include_expansions_in_env=["UV_PYTHON", "UV_PYTHON_PREFERENCE"], + args=[".evergreen/scripts/bson_codec_compare.sh"], + ) + # Upload the result files with s3.put: attach.artifacts would instead + # parse each listed file as a JSON manifest of {name, link} entries, + # which the benchmark data and markdown comparison are not. + upload_func = FunctionCall(func="upload bson compare artifacts") + task = EvgTask( + name="bson-codec-compare", + tags=["bson-codec-compare"], + commands=[server_func, compare_func, upload_func], + ) + return [task] + + def create_perf_tasks(): tasks = [] for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async"]): @@ -1352,6 +1387,49 @@ def create_download_and_merge_coverage_func(): return "download and merge coverage", cmds +def create_upload_bson_compare_artifacts_func(): + """Upload the BSON codec benchmark results and comparison as task files.""" + remote_prefix = ( + "${build_variant}/${revision}/${version_id}/${build_id}" + "/bson-compare/${task_id}-${execution}" + ) + puts = [ + get_s3_put( + local_file="src/results_c.json", + remote_file=f"{remote_prefix}-results_c.json", + display_name="BSON C results", + content_type="application/json", + ), + get_s3_put( + local_file="src/results_rust.json", + remote_file=f"{remote_prefix}-results_rust.json", + display_name="BSON Rust results", + content_type="application/json", + ), + get_s3_put( + local_file="src/bson_comparison.md", + remote_file=f"{remote_prefix}-bson_comparison.md", + display_name="BSON codec comparison", + content_type="text/markdown", + ), + # Optional perf profiles of DeepDecoding (emitted by the script when + # perf is available); optional=true skips a missing file without error. + get_s3_put( + local_file="src/perf_deepdec_c.txt", + remote_file=f"{remote_prefix}-perf_deepdec_c.txt", + display_name="BSON perf profile (C, DeepDecoding)", + content_type="text/plain", + ), + get_s3_put( + local_file="src/perf_deepdec_rust.txt", + remote_file=f"{remote_prefix}-perf_deepdec_rust.txt", + display_name="BSON perf profile (Rust, DeepDecoding)", + content_type="text/plain", + ), + ] + return "upload bson compare artifacts", [get_assume_role(), *puts] + + def create_upload_mo_artifacts_func(): include = ["./**.core", "./**.mdmp"] # Windows: minidumps archive_cmd = archive_targz_pack(target="mongo-coredumps.tgz", source_dir="./", include=include) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 0ba291ab5b..cd2a677a63 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -70,7 +70,13 @@ class Host: HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64", dict()) HOSTS["ubuntu22"] = Host("ubuntu22", "ubuntu2204-small", "Ubuntu-22", dict()) HOSTS["ubuntu24"] = Host("ubuntu24", "ubuntu2404-small", "Ubuntu-24", dict()) -HOSTS["perf"] = Host("perf", "rhel90-dbx-perf-large", "", dict()) +# TEMPORARY (debug): flipped to ubuntu2204-small for the DeepDecoding +# callgrind profile run — rhel90-dbx-perf-large cannot install perf and has +# no valgrind; the ubuntu2204 image ships valgrind preinstalled. REVERT this +# line (back to "rhel90-dbx-perf-large") after the profile run: the timing +# bar belongs on the perf host. The patch build selects only +# bson-codec-compare, so no other task is affected in the patch. +HOSTS["perf"] = Host("perf", "ubuntu2204-small", "", dict()) HOSTS["debian11"] = Host("debian11", "debian11-small", "Debian11", dict()) DEFAULT_HOST = HOSTS["rhel8"] diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index f078f2f305..cd0f724ee0 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -265,6 +265,92 @@ def test_rotated_tasks_keep_nightly_coverage(): assert not offenders, f"nightly 9.0 OCSP tasks not on min-deps Python: {offenders}" +def test_bson_codec_compare_uploads_artifacts_as_files(): + """The BSON codec comparison task uploads its result files with s3.put: + attach.artifacts instead parses each listed file as a JSON manifest of + {name, link} entries, so benchmark data or markdown crash the task (the + markdown is "invalid character '|'" JSON) or attach empty entries.""" + task = next(task for task in all_tasks() if task.name == "bson-codec-compare") + offenders = [ + command + for command in task.commands + if getattr(command, "command", None) == "attach.artifacts" + ] + assert not offenders, f"bson-codec-compare still uses attach.artifacts: {offenders}" + funcs = [getattr(command, "func", None) for command in task.commands] + assert "upload bson compare artifacts" in funcs + + func_name, cmds = generate_config.create_upload_bson_compare_artifacts_func() + assert func_name == "upload bson compare artifacts" + assert [cmd.command for cmd in cmds] == [ + "ec2.assume_role", + "s3.put", + "s3.put", + "s3.put", + "s3.put", + "s3.put", + ] + puts = [cmd for cmd in cmds if cmd.command == "s3.put"] + by_local = {cmd.params["local_file"]: cmd.params for cmd in puts} + assert set(by_local) == { + "src/results_c.json", + "src/results_rust.json", + "src/bson_comparison.md", + "src/perf_deepdec_c.txt", + "src/perf_deepdec_rust.txt", + } + assert by_local["src/results_c.json"]["content_type"] == "application/json" + assert by_local["src/results_rust.json"]["content_type"] == "application/json" + assert by_local["src/bson_comparison.md"]["content_type"] == "text/markdown" + assert by_local["src/perf_deepdec_c.txt"]["content_type"] == "text/plain" + assert by_local["src/perf_deepdec_rust.txt"]["content_type"] == "text/plain" + assert all(params.get("display_name") for params in by_local.values()) + remotes = [cmd.params["remote_file"] for cmd in puts] + assert len(set(remotes)) == len(remotes), f"colliding remote paths: {remotes}" + assert all("${task_id}-${execution}" in remote for remote in remotes), remotes + + +def test_bson_codec_compare_profiles_deep_decoding(): + """The task profiles DeepDecoding (the one benchmark still below the + parity bar on the host) under perf, once per codec. Drift tripwire: the + known report files must stay written by the script AND uploaded; a new + report file needs a matching upload added here.""" + script = (Path(__file__).parent / "bson_codec_compare.sh").read_text() + assert "perf record" in script, "no perf record in bson_codec_compare.sh" + assert "'DeepDecoding and not Json'" in script, ( + "the profile runs must select DeepDecoding and exclude the Json benchmarks" + ) + assert "for impl in c rust" in script, "the profile must run once per codec" + assert "dnf install -y perf" in script, ( + "the script must try to install perf (best-effort) before skipping the profiles" + ) + assert "apt-get install" in script, ( + "the script must try to install perf (best-effort) before skipping the profiles" + ) + # rhel90 has no perf and cannot install it; the ubuntu2204 image ships + # valgrind — the fallback must keep the same DeepDecoding contract. + assert "callgrind_annotate" in script, "no callgrind fallback for hosts without perf (rhel90)" + assert "--tool=callgrind" in script, ( + "the callgrind fallback must profile DeepDecoding in fast mode" + ) + assert "FASTBENCH=1" in script, "the callgrind fallback must profile DeepDecoding in fast mode" + # valgrind 3.18 cannot decode AVX-512 (EVEX) instructions, which + # target-cpu=native emits (memchr's runtime dispatch); the profiled + # build must stay on AVX2 (x86-64-v3, VEX-encoded) to avoid SIGILL. + assert "target-cpu=x86-64-v3" in script, ( + "the callgrind fallback must rebuild the codec without AVX-512" + ) + emitted = {"perf_deepdec_c.txt", "perf_deepdec_rust.txt"} + assert '"perf_deepdec_${impl}.txt"' in script, "the script never writes the profile reports" + _, cmds = generate_config.create_upload_bson_compare_artifacts_func() + uploaded = { + cmd.params["local_file"].removeprefix("src/") + for cmd in cmds + if cmd.command == "s3.put" and cmd.params["local_file"].startswith("src/perf_deepdec_") + } + assert uploaded == emitted, f"uploaded profiles {uploaded} != emitted {emitted}" + + def test_non_standard_cov_tasks_have_no_cov_companions(): """Each non-standard coverage task also emits a no-cov companion, so the macOS/Win64 encryption variants (which filter .cov) have a task to diff --git a/.gitignore b/.gitignore index c113d3ec21..30386fdee5 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,14 @@ error_log # Fuzzing (PYTHON-4232) /fuzz-scratch/ + +# Rust build artifacts +rust/**/target/ + +# BSON codec comparison outputs (bson_codec_compare.sh, untracked) +/results_c*.json +/results_rust*.json +/results_profile_*.json +/bson_comparison.md +/perf_deepdec_c.txt +/perf_deepdec_rust.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55c1ed6add..3daa16eda8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -116,7 +116,8 @@ repos: # - test/test_bson.py:267: isnt ==> isn't # - test/versioned-api/crud-api-version-1-strict.json:514: nin ==> inn, min, bin, nine # - test/test_client.py:188: te ==> the, be, we, to - args: ["-L", "fle,fo,infinit,isnt,nin,te,aks"] + # - rust/bson/src/*.rs: crate ==> create ("crate" is a Rust compilation unit) + args: ["-L", "fle,fo,infinit,isnt,nin,te,aks,crate,statics"] exclude: ^pymongo/public_suffix_list\.dat$ - repo: local diff --git a/AGENTS.md b/AGENTS.md index 43a2a11d70..6691069358 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,7 @@ Official Python driver for MongoDB. - `test/` — sync test suite (generated from `test/asynchronous/` by `just synchro`) and shared test suite - `test/unified_format/` — cross-driver spec tests (Unified Test Format) - `fuzz/` — BSON fuzz tests (not pytest; see `fuzz/README.md`) +- `rust/bson/` — experimental Rust-backed BSON codec (`_rbson`, pyo3/maturin) with differential tests in `rust/bson/tests/`; build with `maturin develop --release -m rust/bson/Cargo.toml`, benchmark with `BSON_IMPL=rust` (see `test/performance/bson_impl_select.py` and `.evergreen/scripts/bson_codec_compare.sh`) Do not edit files in `pymongo/synchronous/` or mirrored files in `test/` directly: they are generated by `just synchro` from `pymongo/asynchronous/` and `test/asynchronous/`. A file in `test/` is mirrored if a file of the same name exists in `test/asynchronous/`. diff --git a/rust/bson/Cargo.lock b/rust/bson/Cargo.lock new file mode 100644 index 0000000000..318195bc10 --- /dev/null +++ b/rust/bson/Cargo.lock @@ -0,0 +1,140 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pymongo-rbson" +version = "0.1.0" +dependencies = [ + "memchr", + "pyo3", + "pyo3-build-config", +] + +[[package]] +name = "pyo3" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" diff --git a/rust/bson/Cargo.toml b/rust/bson/Cargo.toml new file mode 100644 index 0000000000..799b9dbf81 --- /dev/null +++ b/rust/bson/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "pymongo-rbson" +version = "0.1.0" +edition = "2021" +description = "Rust-backed BSON codec for PyMongo (drop-in alternative to bson._cbson)" + +[lib] +name = "_rbson" +crate-type = ["cdylib"] + +[dependencies] +memchr = "2.8.3" +pyo3 = { version = "0.29", features = ["extension-module"] } + +[build-dependencies] +pyo3-build-config = { version = "0.29", features = ["resolve-config"] } + +[profile.release] +opt-level = 3 +lto = "fat" +codegen-units = 1 diff --git a/rust/bson/build.rs b/rust/bson/build.rs new file mode 100644 index 0000000000..5c8b5a7560 --- /dev/null +++ b/rust/bson/build.rs @@ -0,0 +1,6 @@ +fn main() { + // Emits pyo3's version cfgs (Py_3_7, Py_3_8, ...) both as active + // `rustc-cfg` flags and as `rustc-check-cfg` declarations for rustc's + // unexpected_cfgs lint. + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/rust/bson/pyproject.toml b/rust/bson/pyproject.toml new file mode 100644 index 0000000000..2be261f708 --- /dev/null +++ b/rust/bson/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["maturin>=1.15,<2.0"] +build-backend = "maturin" + +[project] +name = "pymongo-rbson" +version = "0.1.0" +description = "Rust-backed BSON codec for PyMongo (drop-in alternative to bson._cbson)" +requires-python = ">=3.11" +dependencies = [] + +[tool.maturin] +# The extension module is the top-level `_rbson` (like `bson._cbson`), not a +# package: `import _rbson` yields the native module directly. +module-name = "_rbson" diff --git a/rust/bson/src/buffer.rs b/rust/bson/src/buffer.rs new file mode 100644 index 0000000000..a102495488 --- /dev/null +++ b/rust/bson/src/buffer.rs @@ -0,0 +1,105 @@ +//! Growable little-endian byte buffer for BSON encoding. +//! +//! Mirrors the semantics of `bson/buffer.c`: append-only writes with the +//! ability to backpatch previously written 4-byte length fields. + +/// Growable byte buffer with BSON-oriented write helpers. +pub struct Buffer { + buf: Vec, +} + +impl Buffer { + #[inline] + pub fn new() -> Self { + // BSON minimum size is 5 bytes; most documents are larger. + Buffer { + buf: Vec::with_capacity(256), + } + } + + #[inline] + pub fn len(&self) -> usize { + self.buf.len() + } + + #[inline] + pub fn write_u8(&mut self, value: u8) { + self.buf.push(value); + } + + #[inline] + pub fn write_u32_le(&mut self, value: u32) { + self.buf.extend_from_slice(&value.to_le_bytes()); + } + + #[inline] + pub fn write_i32_le(&mut self, value: i32) { + self.buf.extend_from_slice(&value.to_le_bytes()); + } + + #[inline] + pub fn write_i64_le(&mut self, value: i64) { + self.buf.extend_from_slice(&value.to_le_bytes()); + } + + #[inline] + pub fn write_f64_le(&mut self, value: f64) { + self.buf.extend_from_slice(&value.to_bits().to_le_bytes()); + } + + /// Writes raw bytes without any framing. + #[inline] + pub fn write_bytes(&mut self, bytes: &[u8]) { + self.buf.extend_from_slice(bytes); + } + + /// Returns the current offset, for later `backpatch_u32` calls. + #[inline] + pub fn position(&self) -> usize { + self.buf.len() + } + + /// Overwrites 4 bytes at `at` with `value` (little-endian). + #[inline] + pub fn backpatch_u32(&mut self, at: usize, value: u32) { + self.buf[at..at + 4].copy_from_slice(&value.to_le_bytes()); + } + + /// Consumes the buffer, returning the written bytes. + #[inline] + pub fn into_vec(self) -> Vec { + self.buf + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn length_backpatching() { + let mut buf = Buffer::new(); + buf.write_u32_le(0); // placeholder + buf.write_bytes(b"a\x00"); + buf.write_u8(0); + let total = buf.len() as u32; + buf.backpatch_u32(0, total); + let mut expected = total.to_le_bytes().to_vec(); + expected.extend_from_slice(b"a\x00\x00"); + assert_eq!(buf.into_vec(), expected); + } + + #[test] + fn scalar_writes_are_little_endian() { + let mut buf = Buffer::new(); + buf.write_i32_le(-1); + buf.write_f64_le(1.0); + assert_eq!( + buf.into_vec(), + [ + 0xff, 0xff, 0xff, 0xff, // -1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, // 1.0f64 + ] + ); + } +} diff --git a/rust/bson/src/datetime_conv.rs b/rust/bson/src/datetime_conv.rs new file mode 100644 index 0000000000..bd7c74669a --- /dev/null +++ b/rust/bson/src/datetime_conv.rs @@ -0,0 +1,126 @@ +//! datetime ↔ epoch-millis conversion. +//! +//! Mirrors `millis_from_datetime`/`decode_datetime` in `_cbsonmodule.c` and +//! the helpers in `bson/datetime_ms.py`: naive datetime fields are treated +//! as UTC (the `timegm` interpretation). + +/// Days since 1970-01-01 for a civil date (Howard Hinnant's algorithm). +fn days_from_civil(y: i64, m: u32, d: u32) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; // [0, 399] + let mp = (m as i64 + 9) % 12; // [0, 11] + let doy = (153 * mp + 2) / 5 + d as i64 - 1; // [0, 365] + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +/// Inverse of `days_from_civil`: days since 1970-01-01 → (year, month, day). +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; // [0, 146096] + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399] + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] + let mp = (5 * doy + 2) / 153; // [0, 11] + let d = doy - (153 * mp + 2) / 5 + 1; // [1, 31] + let m = if mp < 10 { mp + 3 } else { mp - 9 }; // [1, 12] + (y + if m <= 2 { 1 } else { 0 }, m as u32, d as u32) +} + +/// Computes epoch millis for naive datetime fields interpreted as UTC. +/// +/// Mirrors `millis_from_datetime`: `timegm(fields) * 1000 + microsecond / 1000`. +pub fn millis_from_fields( + year: i32, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + microsecond: u32, +) -> i64 { + let days = days_from_civil(year as i64, month as u32, day as u32); + let secs = days * 86_400 + hour as i64 * 3600 + minute as i64 * 60 + second as i64; + secs * 1000 + (microsecond / 1000) as i64 +} + +/// Decomposes epoch millis into UTC datetime fields. +/// +/// Microseconds are the millisecond remainder times 1000 (BSON datetimes +/// have millisecond resolution). +pub fn millis_to_fields(millis: i64) -> (i32, u8, u8, u8, u8, u8, u32) { + let days = millis.div_euclid(86_400_000); + let rem = millis.rem_euclid(86_400_000); + let (y, m, d) = civil_from_days(days); + let hour = (rem / 3_600_000) as u8; + let minute = (rem % 3_600_000 / 60_000) as u8; + let second = (rem % 60_000 / 1000) as u8; + let microsecond = (rem % 1000 * 1000) as u32; + ( + y as i32, + m as u8, + d as u8, + hour, + minute, + second, + microsecond, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn epoch() { + assert_eq!(millis_from_fields(1970, 1, 1, 0, 0, 0, 0), 0); + assert_eq!(millis_to_fields(0), (1970, 1, 1, 0, 0, 0, 0)); + } + + #[test] + fn known_moments() { + // 2026-10-20T14:30:00Z + assert_eq!( + millis_from_fields(2026, 10, 20, 14, 30, 0, 0), + 1_792_506_600_000 + ); + assert_eq!( + millis_to_fields(1_792_506_600_000), + (2026, 10, 20, 14, 30, 0, 0) + ); + // One millisecond before the epoch. + assert_eq!(millis_from_fields(1969, 12, 31, 23, 59, 59, 999_000), -1); + assert_eq!(millis_to_fields(-1), (1969, 12, 31, 23, 59, 59, 999_000)); + // Leap years. + assert_eq!( + millis_from_fields(2024, 2, 29, 0, 0, 0, 0), + 1_709_164_800_000 + ); + assert_eq!( + millis_to_fields(1_709_164_800_000), + (2024, 2, 29, 0, 0, 0, 0) + ); + } + + #[test] + fn round_trips() { + for millis in [ + -2_082_481_920_000_i64, // year 1 + -1, + 0, + 1, + 999, + 1000, + 1_792_494_600_123, + 2_534_023_008_799_999, // 9999-12-31T23:59:59.999Z + ] { + let fields = millis_to_fields(millis); + let round_tripped = millis_from_fields( + fields.0, fields.1, fields.2, fields.3, fields.4, fields.5, fields.6, + ); + assert_eq!(round_tripped, millis, "millis={millis}"); + } + } +} diff --git a/rust/bson/src/decode.rs b/rust/bson/src/decode.rs new file mode 100644 index 0000000000..285decb1ee --- /dev/null +++ b/rust/bson/src/decode.rs @@ -0,0 +1,1048 @@ +//! Decoding BSON bytes to Python objects. +//! +//! A direct port of the C extension's decoder (`_cbsonmodule.c`: +//! `get_value` / `elements_to_dict`). The C extension is the behavioral +//! reference. + +use std::sync::Mutex; + +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyBytes, PyDateTime, PyDeltaAccess, PyDict, PyList, PyString, PyTzInfo}; + +use crate::datetime_conv::millis_to_fields; +use crate::errors::{bson_error, invalid_bson}; +use crate::options::{CodecOptionsData, DATETIME_AUTO, DATETIME_CLAMP, DATETIME_MS}; +use crate::types::Types; + +// BSON type bytes. +const BSONNUM: u8 = 0x01; +const BSONSTR: u8 = 0x02; +const BSONOBJ: u8 = 0x03; +const BSONARR: u8 = 0x04; +const BSONBIN: u8 = 0x05; +const BSONUND: u8 = 0x06; +const BSONOID: u8 = 0x07; +const BSONBOO: u8 = 0x08; +const BSONDAT: u8 = 0x09; +const BSONNUL: u8 = 0x0A; +const BSONRGX: u8 = 0x0B; +const BSONREF: u8 = 0x0C; +const BSONCOD: u8 = 0x0D; +const BSONSYM: u8 = 0x0E; +const BSONCWS: u8 = 0x0F; +const BSONINT: u8 = 0x10; +const BSONTIM: u8 = 0x11; +const BSONLON: u8 = 0x12; +const BSONDEC: u8 = 0x13; +const BSONMAX: u8 = 0x7F; +const BSONMIN: u8 = 0xFF; + +/// The smallest legal BSON document: 4-byte length + terminator. +pub const BSON_MIN_SIZE: usize = 5; + +/// Maximum BSON document size (mirrors BSON_MAX_SIZE in _cbsonmodule.c). +pub const BSON_MAX_SIZE: usize = i32::MAX as usize; + +/// Context for a decode pass: the borrowed data plus the owning Python +/// object (needed to build memoryview slices for raw document classes). +pub struct DecodeInput<'a, 'py> { + pub data: &'a [u8], + /// The original buffer object (bytes, bytearray, memoryview, ...) or + /// None when unknown. + pub owner: Option<&'a Bound<'py, PyAny>>, +} + +/// Decodes a single BSON document (`_bson_to_dict`). +/// +/// `data` must be exactly one document (the C validates +/// `size == total_size` and a NUL terminator). +pub fn bson_to_dict<'py>( + py: Python<'py>, + input: &DecodeInput<'_, 'py>, + options: &CodecOptionsData<'py>, +) -> PyResult> { + let data = input.data; + if data.len() < BSON_MIN_SIZE { + return Err(invalid_bson(py, "not enough data for a BSON document")); + } + // The C reads the size prefix as a signed int32; negative values + // (and zero) are "invalid message size". + let size = i32::from_le_bytes([data[0], data[1], data[2], data[3]]); + if size < BSON_MIN_SIZE as i32 { + return Err(invalid_bson(py, "invalid message size")); + } + let size = size as usize; + if data.len() < size || data.len() > BSON_MAX_SIZE { + return Err(invalid_bson(py, "objsize too large")); + } + if size != data.len() || data[size - 1] != 0 { + return Err(invalid_bson(py, "bad eoo")); + } + elements_to_dict(py, input, &data[..size], options).map(|(value, _)| value) +} + +/// Decodes all elements of a BSON document into a new +/// `options.document_class` instance, mirroring `elements_to_dict`. +/// +/// `data` must be exactly one valid document: a 4-byte length prefix equal +/// to `data.len()` and a NUL terminator (checked by `bson_to_dict`, +/// `read_size_at`, `read_size_prefix`, and the Code-with-scope case). +/// Returns the document and whether any field name started with `$` (see +/// `decode_elements_into`). +pub fn elements_to_dict<'py>( + py: Python<'py>, + input: &DecodeInput<'_, 'py>, + data: &[u8], + options: &CodecOptionsData<'py>, +) -> PyResult<(Py, bool)> { + let size = data.len(); + + if options.is_raw_bson { + return raw_document(py, input, data, size, options).map(|value| (value, false)); + } + + // PyDict_New() directly for the common case (mirrors the C). + let document: Bound<'py, PyAny> = if options.is_dict_class { + PyDict::new(py).into_any() + } else { + options.document_class.call0()? + }; + let has_dollar_key = { + let _guard = crate::encode::recursion_guard(py, crate::encode::GUARD_DECODE_DOCUMENT)?; + decode_elements_into(py, &document, &data[4..size - 1], options)? + }; + Ok((document.unbind(), has_dollar_key)) +} + +/// Builds a raw document class instance from the raw bytes: the whole +/// buffer is passed through when possible, larger documents become +/// memoryview slices, smaller ones are copied (`elements_to_dict`'s +/// is_raw_bson path in the C). +fn raw_document<'py>( + py: Python<'py>, + input: &DecodeInput<'_, 'py>, + data: &[u8], + size: usize, + options: &CodecOptionsData<'py>, +) -> PyResult> { + let owner = input.owner.and_then(|o| o.cast::().ok()); + // Only immutable bytes owners may be viewed or sliced; anything else + // (bytearray, foreign buffer) is always copied. + let bson_bytes: Bound<'py, PyAny> = if let Some(owner) = owner { + let owner_data = owner.as_bytes(); + // The doc slice must lie within the owner buffer (mirrors the C's + // pointer-difference offset check). + let offset = if data.as_ptr() >= owner_data.as_ptr() { + data.as_ptr().addr() - owner_data.as_ptr().addr() + } else { + return Err(invalid_bson(py, "invalid buffer offset")); + }; + if offset + size > owner_data.len() { + return Err(invalid_bson(py, "invalid buffer offset")); + } + let threshold = Types::get(py)?.raw_bson_view_threshold; + if offset == 0 && size == owner_data.len() { + // The document spans the entire buffer: pass it through. + owner.clone().into_any() + } else if size >= threshold { + // Zero-copy: a read-only slice of the immutable buffer. + let view = pyo3::types::PyMemoryView::from(owner.clone().as_any())?; + view.get_item(pyo3::types::PySlice::new( + py, + offset as isize, + (offset + size) as isize, + 1, + ))? + } else { + PyBytes::new(py, &data[..size]).into_any() + } + } else { + PyBytes::new(py, &data[..size]).into_any() + }; + let document = options + .document_class + .call1((bson_bytes, options.options_obj(py)))?; + Ok(document.unbind()) +} + +/// Decodes elements from `body` (the elements of a document, excluding the +/// length prefix and the terminator) into an existing mapping. Returns +/// whether any field name started with `$` — a necessary condition for the +/// document to be shaped like a DBRef (its keys must include "$ref" and +/// "$id"), letting `get_value` skip `_dbref_hook` in the common case. +pub fn decode_elements_into<'py>( + py: Python<'py>, + document: &Bound<'py, PyAny>, + body: &[u8], + options: &CodecOptionsData<'py>, +) -> PyResult { + let input = DecodeInput { + data: body, + owner: None, + }; + // PyDict_SetItem() is faster than PyObject_SetItem() because it avoids + // method lookup overhead (mirrors the C). + let dict_target: Option> = if options.is_dict_class { + document.cast::().ok().cloned() + } else { + None + }; + let mut position = 0; + let mut has_dollar_key = false; + while position < body.len() { + let type_byte = body[position]; + position += 1; + // Read the cstring name. Mirrors the C's strlen-then-check: a name + // that is not NUL-terminated within the element's extent raises + // "field name too large". + let name_start = position; + position += memchr::memchr(0, &body[position..]).unwrap_or(body.len() - position); + if position >= body.len() { + return Err(invalid_bson(py, "field name too large")); + } + let name = &body[name_start..position]; + position += 1; // skip the NUL + + // The name is decoded before the value (mirrors the C's + // _element_to_dict); a UnicodeDecodeError is wrapped in InvalidBSON. + let key = decode_key(py, name, options).map_err(|err| rewrap_as_invalid_bson(py, err))?; + + let (value, new_position) = + get_value(py, &input, name, type_byte, position, body.len(), options) + .map_err(|err| rewrap_as_invalid_bson(py, err))?; + position = new_position; + + if let Some(dict) = &dict_target { + dict.set_item(key, value)?; + } else { + document.set_item(key, value)?; + } + has_dollar_key |= name.first() == Some(&b'$'); + } + if position != body.len() { + return Err(invalid_bson(py, "bad object or element length")); + } + Ok(has_dollar_key) +} + +/// Decodes the value of one element starting at `position` (just after the +/// name). Returns `(value, new_position)`, mirroring the C `get_value`. +/// +/// `end` is the exclusive upper bound of the element's extent: every read +/// for this value must lie below it (the C passes `max` for the same +/// purpose). +#[inline] +pub fn get_value<'py>( + py: Python<'py>, + input: &DecodeInput<'_, 'py>, + name: &[u8], + type_byte: u8, + position: usize, + end: usize, + options: &CodecOptionsData<'py>, +) -> PyResult<(Py, usize)> { + get_value_with_raw_array(py, input, name, type_byte, position, end, options, false) +} + +/// `get_value` with the `raw_array` flag: when set, array elements (0x04) +/// are returned as raw bytes instead of decoded lists (used by +/// RawBSONDocument's `_RawArrayBSONDocument`). +#[allow(clippy::too_many_lines)] +#[allow(clippy::too_many_arguments)] +pub fn get_value_with_raw_array<'py>( + py: Python<'py>, + input: &DecodeInput<'_, 'py>, + name: &[u8], + type_byte: u8, + mut position: usize, + end: usize, + options: &CodecOptionsData<'py>, + raw_array: bool, +) -> PyResult<(Py, usize)> { + let data = input.data; + // The catch-all for any bounds violation inside get_value mirrors the + // C's `invalid:` label: "invalid length or type code". + let invalid = || invalid_bson(py, "invalid length or type code"); + let value: Py = match type_byte { + BSONNUM => { + if position + 8 > end { + return Err(invalid()); + } + let v = f64::from_le_bytes(data[position..position + 8].try_into().unwrap()); + position += 8; + v.into_pyobject(py)?.unbind().into_any() + } + BSONSTR | BSONSYM => { + // Strings and deprecated symbols decode to str. + let length = read_string_length(py, data, position, end)?; + let string_start = position + 4; + if string_start + length > end { + return Err(invalid()); + } + if data[string_start + length - 1] != 0 { + return Err(invalid()); + } + let slice = &data[string_start..string_start + length - 1]; + position = string_start + length; + let string = decode_utf8(py, slice, options)?; + string.unbind() + } + BSONOBJ => { + let size = read_size_at(py, data, position, end)?; + let (value, has_dollar_key) = + elements_to_dict(py, input, &data[position..position + size], options) + .map_err(|err| rewrap_as_invalid_bson(py, err))?; + position += size; + // Hook for DBRefs ($ref/$id shaped docs decode to DBRef); + // skipped when no key could be "$ref"/"$id" (every DBRef key + // starts with "$") or for raw document classes (mirrors the C). + if options.is_raw_bson || !has_dollar_key { + value + } else { + dbref_hook(py, value, options)? + } + } + BSONARR => { + let size = read_size_at(py, data, position, end)?; + if raw_array { + // Treat the whole array element as a binary buffer. + let raw = PyBytes::new(py, &data[position..position + size]); + position += size; + raw.unbind().into_any() + } else { + // Arrays decode to lists; the numeric keys are skipped + // (they are in order). + let arr_end = position + size - 1; + if data[arr_end] != 0 { + return Err(invalid()); + } + let list = PyList::empty(py); + let mut pos = position + 4; + let _guard = crate::encode::recursion_guard(py, crate::encode::GUARD_DECODE_LIST)?; + while pos < arr_end { + let element_type = data[pos]; + pos += 1; // past the type byte + pos += memchr::memchr(0, &data[pos..arr_end]).ok_or_else(invalid)?; + pos += 1; // Skip the NUL: keys are in order. + let (item, new_pos) = + get_value(py, input, name, element_type, pos, arr_end, options) + .map_err(|err| rewrap_as_invalid_bson(py, err))?; + pos = new_pos; + if unsafe { pyo3::ffi::PyList_Append(list.as_ptr(), item.as_ptr()) } < 0 { + return Err(PyErr::fetch(py)); + } + } + if pos != arr_end { + return Err(invalid()); + } + position = arr_end + 1; // past the terminator + list.into_any().unbind() + } + } + BSONBIN => { + if position + 5 > end { + return Err(invalid()); + } + let length = read_u32(py, data, position)? as usize; + if length > end - position - 5 { + return Err(invalid()); + } + let subtype = data[position + 4]; + position += 5; + let data_start = position; + let types = Types::get(py)?; + if subtype == 2 { + if length < 4 { + return Err(invalid()); + } + let length2 = read_u32(py, data, position)? as usize; + if length2 != length - 4 { + return Err(invalid()); + } + } + if subtype == 0 { + // Plain bytes. + let slice = &data[data_start..data_start + length]; + position += length; + PyBytes::new(py, slice).unbind().into_any() + } else if subtype == 3 || subtype == 4 { + // UUID handling. + let raw = if subtype == 2 { + &data[data_start + 4..data_start + length] + } else { + &data[data_start..data_start + length] + }; + let payload = PyBytes::new(py, raw); + position += length; + let binary = types.binary.bind(py).call1((payload, subtype as i32))?; + let uuid_rep = options.uuid_representation; + if uuid_rep == 0 + || (subtype == 4 && uuid_rep != 4) + || (subtype == 3 && uuid_rep == 4) + { + binary.unbind() + } else { + binary + .call_method1(pyo3::intern!(py, "as_uuid"), (uuid_rep,))? + .unbind() + } + } else { + let raw = if subtype == 2 { + &data[data_start + 4..data_start + length] + } else { + &data[data_start..data_start + length] + }; + let payload = PyBytes::new(py, raw); + position += length; + types + .binary + .bind(py) + .call1((payload, subtype as i32))? + .unbind() + } + } + BSONUND | BSONNUL => py.None(), + BSONOID => { + if position + 12 > end { + return Err(invalid()); + } + let oid_bytes = PyBytes::new(py, &data[position..position + 12]); + position += 12; + let types = Types::get(py)?; + vectorcall(py, &types.object_id, &[oid_bytes.as_ptr()])?.unbind() + } + BSONBOO => { + if position + 1 > end { + return Err(invalid()); + } + let byte = data[position]; + if byte != 0 && byte != 1 { + return Err(invalid_bson( + py, + &format!("invalid boolean value: {byte:x}"), + )); + } + position += 1; + (byte == 1) + .into_pyobject(py)? + .to_owned() + .into_any() + .unbind() + } + BSONDAT => { + if position + 8 > end { + return Err(invalid()); + } + let millis = i64::from_le_bytes(data[position..position + 8].try_into().unwrap()); + position += 8; + decode_datetime(py, millis, options)? + } + BSONRGX => { + // pattern cstr + flags cstr, both bounded by the element + // extent (PYTHON-6110/6111). + let pattern_end = find_nul(py, data, position, end)?; + let pattern = decode_utf8(py, &data[position..pattern_end], options)?; + let flags_start = pattern_end + 1; + let flags_end = find_nul(py, data, flags_start, end)?; + let mut flags: i64 = 0; + for &byte in &data[flags_start..flags_end] { + match byte { + b'i' => flags |= 2, + b'l' => flags |= 4, + b'm' => flags |= 8, + b's' => flags |= 16, + b'u' => flags |= 32, + b'x' => flags |= 64, + _ => {} + } + } + position = flags_end + 1; + let types = Types::get(py)?; + types.regex.bind(py).call1((pattern, flags))?.unbind() + } + BSONREF => { + // Deprecated DBPointer decodes to DBRef(collection, ObjectId). + let length = read_string_length(py, data, position, end)?; + let string_start = position + 4; + // Encoded string length + string + 12 byte ObjectId. + if string_start + length + 12 > end { + return Err(invalid()); + } + if data[string_start + length - 1] != 0 { + return Err(invalid()); + } + let collection = + decode_utf8(py, &data[string_start..string_start + length - 1], options)?; + let oid_start = string_start + length; + let oid_bytes = PyBytes::new(py, &data[oid_start..oid_start + 12]); + position = oid_start + 12; + let types = Types::get(py)?; + let oid = types.object_id.bind(py).call1((oid_bytes,))?; + types.dbref.bind(py).call1((collection, oid))?.unbind() + } + BSONCOD => { + let (code, next) = get_string_payload(py, data, position, end, options)?; + position = next; + let types = Types::get(py)?; + vectorcall(py, &types.code, &[code.as_ptr()])?.unbind() + } + BSONCWS => { + if position + 8 > end { + return Err(invalid()); + } + let cws_size = read_u32(py, data, position)? as usize; + if cws_size > end - position { + return Err(invalid()); + } + let code_size = read_u32(py, data, position + 4)? as usize; + // cws length + code length + code + scope length + let min_len = 4 + 4 + code_size + 4; + if code_size == 0 + || code_size > end - position + || min_len > end - position + || min_len < code_size + { + return Err(invalid()); + } + let code_start = position + 8; + if data[code_start + code_size - 1] != 0 { + return Err(invalid()); + } + let code = decode_utf8(py, &data[code_start..code_start + code_size - 1], options)?; + let scope_start = code_start + code_size; + let scope_size = read_u32(py, data, scope_start)? as usize; + // code length + code + scope length + scope + let total = 4 + 4 + code_size + scope_size; + if scope_size < BSON_MIN_SIZE || total != cws_size || total < scope_size { + return Err(invalid()); + } + if data[scope_start + scope_size - 1] != 0 { + return Err(invalid()); + } + let scope = elements_to_dict( + py, + input, + &data[scope_start..scope_start + scope_size], + options, + ) + .map_err(|err| rewrap_as_invalid_bson(py, err)) + .map(|(value, _)| value)?; + position = scope_start + scope_size; + let types = Types::get(py)?; + types.code.bind(py).call1((code, scope))?.unbind() + } + BSONINT => { + if position + 4 > end { + return Err(invalid()); + } + let v = i32::from_le_bytes(data[position..position + 4].try_into().unwrap()); + position += 4; + v.into_pyobject(py)?.unbind().into_any() + } + BSONTIM => { + if position + 8 > end { + return Err(invalid()); + } + let inc = u32::from_le_bytes(data[position..position + 4].try_into().unwrap()); + let time = u32::from_le_bytes(data[position + 4..position + 8].try_into().unwrap()); + position += 8; + let time_obj = unsafe { pyo3::ffi::PyLong_FromUnsignedLong(time as _) }; + let inc_obj = unsafe { pyo3::ffi::PyLong_FromUnsignedLong(inc as _) }; + let time_obj = unsafe { Bound::from_owned_ptr_or_err(py, time_obj)? }; + let inc_obj = unsafe { Bound::from_owned_ptr_or_err(py, inc_obj)? }; + let types = Types::get(py)?; + vectorcall(py, &types.timestamp, &[time_obj.as_ptr(), inc_obj.as_ptr()])?.unbind() + } + BSONLON => { + if position + 8 > end { + return Err(invalid()); + } + let v = i64::from_le_bytes(data[position..position + 8].try_into().unwrap()); + position += 8; + // int64 decodes to bson.int64.Int64, not plain int. + let long = unsafe { pyo3::ffi::PyLong_FromLongLong(v) }; + let long = unsafe { Bound::from_owned_ptr_or_err(py, long)? }; + let types = Types::get(py)?; + vectorcall(py, &types.int64, &[long.as_ptr()])?.unbind() + } + BSONDEC => { + if position + 16 > end { + return Err(invalid()); + } + let bid = PyBytes::new(py, &data[position..position + 16]); + position += 16; + let from_bid = pyo3::intern!(py, "from_bid"); + let types = Types::get(py)?; + vectorcall_method1(py, &types.decimal128, from_bid.as_any(), &bid.into_any())?.unbind() + } + BSONMIN => { + let types = Types::get(py)?; + vectorcall(py, &types.min_key, &[])?.unbind() + } + BSONMAX => { + let types = Types::get(py)?; + vectorcall(py, &types.max_key, &[])?.unbind() + } + _ => { + let type_repr = PyBytes::new(py, &[type_byte]) + .repr()? + .to_cow()? + .into_owned(); + return Err(invalid_bson( + py, + &format!( + "Detected unknown BSON type {} for fieldname '{}'. \ + Are you using the latest driver version?", + type_repr, + String::from_utf8_lossy(name), + ), + )); + } + }; + + // Apply a custom decoder for this value's exact type, if registered. + if let Some(decoder_map) = &options.decoder_map_dict { + let value_bound = value.bind(py); + if let Some(converter) = decoder_map.get_item(value_bound.get_type())? { + return Ok((converter.call1((value_bound,))?.unbind(), position)); + } + } + Ok((value, position)) +} + +/// Constructs a `DatetimeMS` for the millis value (vectorcall, no tuple). +fn datetime_ms_from_millis<'py>( + py: Python<'py>, + millis: i64, + types: &Types, +) -> PyResult> { + let long = unsafe { pyo3::ffi::PyLong_FromLongLong(millis) }; + let long = unsafe { Bound::from_owned_ptr_or_err(py, long)? }; + Ok(vectorcall(py, &types.datetime_ms, &[long.as_ptr()])?.unbind()) +} + +/// Converts a decoded `$ref`/`$id`-shaped document into a DBRef +/// (`_dbref_hook` in the C). The DBRef keys are removed from the document +/// and the remaining keys become `_extra`. +/// +/// Only called for documents whose field names included a `$`-prefixed key +/// (a necessary condition for "$ref" and "$id" to be present). +fn dbref_hook<'py>( + py: Python<'py>, + value: Py, + _options: &CodecOptionsData<'py>, +) -> PyResult> { + let doc = value.bind(py); + let lookup = |key: &str| -> PyResult>> { + // Dict fast path: PyDict_GetItemWithError raises no exceptions for + // missing keys, and one lookup serves both the presence check and + // the value (PyMapping_HasKey + PyObject_GetItem in the C); other + // mappings fall back to get_item with a caught KeyError. + if let Ok(dict) = doc.cast::() { + let key = pyo3::types::PyString::intern(py, key); + let item = unsafe { pyo3::ffi::PyDict_GetItemWithError(dict.as_ptr(), key.as_ptr()) }; + if item.is_null() { + return if unsafe { pyo3::ffi::PyErr_Occurred() }.is_null() { + Ok(None) + } else { + Err(PyErr::fetch(py)) + }; + } + return unsafe { Ok(Some(Bound::from_borrowed_ptr(py, item))) }; + } + match doc.get_item(key) { + Ok(item) => Ok(Some(item)), + Err(err) if err.is_instance_of::(py) => Ok(None), + Err(err) => Err(err), + } + }; + let Some(reference) = lookup("$ref")? else { + return Ok(value); + }; + let Some(id) = lookup("$id")? else { + return Ok(value); + }; + let database = lookup("$db")?; + // Type checks: $ref must be str, $db must be str (or absent). + if reference.cast::().is_err() { + return Ok(value); + } + if let Some(db) = &database { + if db.cast::().is_err() { + return Ok(value); + } + } + let types = Types::get(py)?; + // Remove the DBRef keys from the document (the remainder becomes + // DBRef's _extra), matching the C. + doc.del_item(pyo3::intern!(py, "$ref"))?; + doc.del_item(pyo3::intern!(py, "$id"))?; + let database = match database { + Some(db) => { + doc.del_item(pyo3::intern!(py, "$db"))?; + db + } + None => py.None().bind(py).clone(), + }; + let dbref = types.dbref.bind(py).call1((reference, id, database, doc))?; + Ok(dbref.unbind()) +} + +/// Finds the next NUL byte at or after `position`, bounded by `end` +/// (bounded scans, mirroring PYTHON-6110/6111). +#[inline(always)] +fn find_nul(py: Python<'_>, data: &[u8], position: usize, end: usize) -> PyResult { + let stop = end.min(data.len()); + memchr::memchr(0, &data[position..stop]) + .map(|i| position + i) + .ok_or_else(|| invalid_bson(py, "invalid length or type code")) +} + +/// Reads a length-prefixed string payload, returning `(str, new_position)`. +#[inline(always)] +fn get_string_payload<'py>( + py: Python<'py>, + data: &[u8], + position: usize, + end: usize, + options: &CodecOptionsData<'py>, +) -> PyResult<(Bound<'py, PyAny>, usize)> { + let length = read_string_length(py, data, position, end)?; + let string_start = position + 4; + if string_start + length > end { + return Err(invalid_bson(py, "invalid length or type code")); + } + if data[string_start + length - 1] != 0 { + return Err(invalid_bson(py, "invalid length or type code")); + } + let slice = &data[string_start..string_start + length - 1]; + let string = decode_utf8(py, slice, options)?; + Ok((string, string_start + length)) +} + +/// Datetime decoding with `datetime_conversion` mode support +/// (`decode_datetime` in the C). +fn decode_datetime<'py>( + py: Python<'py>, + mut millis: i64, + options: &CodecOptionsData<'py>, +) -> PyResult> { + let types = Types::get(py)?; + + if options.datetime_conversion == DATETIME_MS { + return datetime_ms_from_millis(py, millis, types); + } + + if options.datetime_conversion == DATETIME_CLAMP || options.datetime_conversion == DATETIME_AUTO + { + // The representable range shrinks/grows with the target timezone. + let (mut min_millis, mut max_millis) = (types.min_millis, types.max_millis); + if options.tz_aware { + if let Some(tzinfo) = &options.tzinfo { + let min_offset = offset_millis(py, tzinfo, types.min_datetime.bind(py))?; + let max_offset = offset_millis(py, tzinfo, types.max_datetime.bind(py))?; + if min_offset < 0 { + min_millis -= min_offset; + } + if max_offset > 0 { + max_millis -= max_offset; + } + } + } + if options.datetime_conversion == DATETIME_CLAMP { + millis = millis.clamp(min_millis, max_millis); + } else if !(min_millis..=max_millis).contains(&millis) { + return datetime_ms_from_millis(py, millis, types); + } + } + + let (year, month, day, hour, minute, second, microsecond) = millis_to_fields(millis); + let tzinfo_arg = if options.tz_aware { + Some(types.utc.bind(py).cast::()?.clone()) + } else { + None + }; + let naive = match PyDateTime::new( + py, + year, + month, + day, + hour, + minute, + second, + microsecond, + tzinfo_arg.as_ref(), + ) { + Ok(dt) => dt, + // Out-of-range ValueError gets the DATETIME_AUTO suggestion + // appended (mirrors datetime_from_millis in the C). + Err(err) => return Err(append_datetime_suggestion(py, err)), + }; + let value: Bound<'py, PyAny> = naive.into_any(); + if options.tz_aware { + if let Some(target) = &options.tzinfo { + return Ok(value + .call_method1(pyo3::intern!(py, "astimezone"), (target,))? + .unbind()); + } + } + Ok(value.unbind()) +} + +/// Appends the DATETIME_AUTO suggestion to out-of-range datetime +/// ValueErrors (mirrors `datetime_from_millis` in the C). +fn append_datetime_suggestion(py: Python<'_>, err: PyErr) -> PyErr { + if !err.is_instance_of::(py) { + return err; + } + let message = err + .value(py) + .str() + .and_then(|m| m.to_cow().map(|c| c.into_owned())) + .unwrap_or_default(); + pyo3::exceptions::PyValueError::new_err(format!( + "{message} (Consider Using CodecOptions(datetime_conversion=DATETIME_AUTO) \ + or MongoClient(datetime_conversion='DATETIME_AUTO')). \ + See: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes" + )) +} + +/// Returns the utc offset of `at_datetime` under `tzinfo` in millis +/// (`tzinfo.utcoffset(at_datetime)`). +fn offset_millis<'py>( + py: Python<'py>, + tzinfo: &Bound<'py, PyAny>, + at_datetime: &Bound<'py, PyAny>, +) -> PyResult { + let offset = tzinfo.call_method1(pyo3::intern!(py, "utcoffset"), (at_datetime,))?; + if offset.is_none() { + return Ok(0); + } + let delta = offset + .cast::() + .map_err(|_| { + bson_error( + py, + "BSONError", + "tzinfo.utcoffset() did not return a datetime.timedelta", + ) + })? + .to_owned(); + let days = delta.get_days() as i64; + let seconds = delta.get_seconds() as i64; + let micros = delta.get_microseconds() as i64; + Ok((days * 86_400 + seconds) * 1000 + micros / 1000) +} + +/// Reads and validates a nested document size at `position`, bounded by +/// `end`. Mirrors the checks in the C get_value's document case: every +/// failure lands on the shared "invalid length or type code" message. +#[inline(always)] +fn read_size_at(py: Python<'_>, data: &[u8], position: usize, end: usize) -> PyResult { + if position + 4 > end { + return Err(invalid_bson(py, "invalid length or type code")); + } + let size = u32::from_le_bytes(data[position..position + 4].try_into().unwrap()) as usize; + if size < BSON_MIN_SIZE || size > end - position { + return Err(invalid_bson(py, "invalid length or type code")); + } + if data[position + size - 1] != 0 { + return Err(invalid_bson(py, "invalid length or type code")); + } + Ok(size) +} + +/// Reads a length-prefixed string's length (which includes the trailing +/// NUL) and validates it, guarding against negative lengths. All failures +/// land on the shared "invalid length or type code" message (the C's +/// `goto invalid`). +#[inline(always)] +fn read_string_length(py: Python<'_>, data: &[u8], position: usize, end: usize) -> PyResult { + if position + 4 > end { + return Err(invalid_bson(py, "invalid length or type code")); + } + let length = i32::from_le_bytes(data[position..position + 4].try_into().unwrap()); + if length < 1 { + return Err(invalid_bson(py, "invalid length or type code")); + } + Ok(length as usize) +} + +#[inline(always)] +fn read_u32(py: Python<'_>, data: &[u8], position: usize) -> PyResult { + let bytes = data + .get(position..position + 4) + .ok_or_else(|| invalid_bson(py, "invalid length or type code"))?; + Ok(u32::from_le_bytes(bytes.try_into().unwrap())) +} + +/// Calls `callee` with positional arguments through `PyObject_Vectorcall` +/// (a C argument array — no argument tuple allocation, mirroring the C +/// extension's constructor call sites). +#[inline(always)] +fn vectorcall<'py>( + py: Python<'py>, + callee: &Py, + args: &[*mut pyo3::ffi::PyObject], +) -> PyResult> { + unsafe { + Bound::from_owned_ptr_or_err( + py, + pyo3::ffi::PyObject_Vectorcall( + callee.as_ptr(), + args.as_ptr(), + args.len(), + std::ptr::null_mut(), + ), + ) + } +} + +/// Calls `obj.(arg)` through `PyObject_VectorcallMethod`. +#[inline(always)] +fn vectorcall_method1<'py>( + py: Python<'py>, + obj: &Py, + name: &Bound<'py, PyAny>, + arg: &Bound<'py, PyAny>, +) -> PyResult> { + let args = [obj.as_ptr(), arg.as_ptr()]; + unsafe { + Bound::from_owned_ptr_or_err( + py, + pyo3::ffi::PyObject_VectorcallMethod( + name.as_ptr(), + args.as_ptr(), + 2, + std::ptr::null_mut(), + ), + ) + } +} + +/// Decodes UTF-8 bytes with the codec options' error handler. +/// +/// Fast path: strict (the default) passes a NULL handler. +#[inline(always)] +pub fn decode_utf8<'py>( + py: Python<'py>, + bytes: &[u8], + options: &CodecOptionsData<'py>, +) -> PyResult> { + let ptr = unsafe { + pyo3::ffi::PyUnicode_DecodeUTF8( + bytes.as_ptr() as *const _, + bytes.len() as _, + options + .utf8_handler + .as_deref() + .map(|h| h.as_ptr()) + .unwrap_or(std::ptr::null()), + ) + }; + let decoded = unsafe { Bound::from_owned_ptr_or_err(py, ptr)? }; + Ok(decoded.into_any()) +} + +/// Field names repeat heavily within and across documents; decoding each +/// occurrence with PyUnicode_DecodeUTF8 (allocation + decode) and hashing the +/// fresh string for every dict insertion is a large share of the decode loop. +/// A bounded direct-mapped cache of raw key bytes to decoded strings removes +/// both on a hit: the cached string's hash is computed once by CPython and +/// reused for every subsequent PyDict_SetItem. (Unlike the C extension, +/// repeat decodes return the *same* str object for a repeated name — benign, +/// strings are immutable.) Only enabled for the strict UTF-8 handler, where +/// decoding is a pure function of the bytes. +/// +/// Thread safety: the module requires the GIL (`gil_used = true` in +/// lib.rs), so every entry point holds it; the Mutex additionally guards +/// the Vec against the (theoretical) free-threaded case, and statics are +/// never dropped in Rust, so cached strings are only ever released by +/// eviction, under the GIL. The cache — like `Types` — is process-wide: +/// importing the module into more than one interpreter is not supported. +const KEY_CACHE_SLOTS: usize = 1 << 13; +/// Keys longer than this skip the cache: the copy of the raw bytes needed to +/// cache them would rival the decode cost being avoided. +const KEY_CACHE_MAX_KEY_LEN: usize = 64; + +/// A cache slot: the raw key bytes and the decoded string. +type KeyCacheEntry = (Box<[u8]>, Py); + +struct KeyCache { + slots: Vec>, +} + +impl KeyCache { + const fn new() -> Self { + Self { slots: Vec::new() } + } +} + +static KEY_CACHE: Mutex = Mutex::new(KeyCache::new()); + +/// FNV-1a: cheap and adequate for a direct-mapped cache. +#[inline] +fn key_hash(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for &byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +#[inline] +fn decode_key_cached<'py>( + py: Python<'py>, + bytes: &[u8], + options: &CodecOptionsData<'py>, +) -> PyResult> { + let mut cache = KEY_CACHE.lock().unwrap_or_else(|err| err.into_inner()); + if cache.slots.len() != KEY_CACHE_SLOTS { + cache.slots.resize_with(KEY_CACHE_SLOTS, || None); + } + let index = (key_hash(bytes) as usize) & (KEY_CACHE_SLOTS - 1); + let slot = &mut cache.slots[index]; + if let Some((cached, string)) = slot { + if cached.as_ref() == bytes { + return Ok(string.bind(py).clone()); + } + } + let decoded = decode_utf8(py, bytes, options)?; + *slot = Some((Box::from(bytes), decoded.clone().unbind())); + Ok(decoded) +} + +/// Decodes a field-name key, using the intern cache when the strict handler +/// is in effect (any other handler makes decoding depend on mutable options +/// state, so the cache is bypassed). +#[inline] +pub fn decode_key<'py>( + py: Python<'py>, + bytes: &[u8], + options: &CodecOptionsData<'py>, +) -> PyResult> { + if options.utf8_handler.is_none() && bytes.len() <= KEY_CACHE_MAX_KEY_LEN { + return decode_key_cached(py, bytes, options); + } + decode_utf8(py, bytes, options) +} + +/// Wraps unexpected errors during value decoding in InvalidBSON, mirroring +/// `_rewrap_as_invalid_bson` in the C extension. +#[cold] +pub fn rewrap_as_invalid_bson(py: Python<'_>, err: PyErr) -> PyErr { + let is_invalid_bson = Types::get(py) + .ok() + .and_then(|types| err.value(py).is_instance(types.invalid_bson.bind(py)).ok()) + .unwrap_or(false); + if is_invalid_bson { + return err; + } + let message = err + .value(py) + .str() + .and_then(|m| m.to_cow().map(|c| c.into_owned())) + .unwrap_or_default(); + bson_error(py, "InvalidBSON", &message) +} diff --git a/rust/bson/src/encode.rs b/rust/bson/src/encode.rs new file mode 100644 index 0000000000..f3556cf996 --- /dev/null +++ b/rust/bson/src/encode.rs @@ -0,0 +1,1066 @@ +//! Encoding Python objects to BSON bytes. +//! +//! A direct port of the C extension's encoder (`_cbsonmodule.c`: +//! `write_dict` / `write_element_to_buffer`), including the `_type_marker` +//! fast-dispatch protocol. The C extension is the behavioral reference. + +use std::ffi::CStr; + +use pyo3::buffer::PyBuffer; +use pyo3::prelude::*; +use pyo3::types::{ + PyAny, PyBool, PyBytes, PyDateAccess, PyDateTime, PyDict, PyFloat, PyInt, PyList, PyString, + PyTimeAccess, PyTuple, +}; +use pyo3::PyTypeInfo; + +use crate::buffer::Buffer; +use crate::datetime_conv::millis_from_fields; +use crate::errors::{invalid_document, invalid_string_data}; +use crate::options::{read_type_marker, CodecOptionsData}; +use crate::types::Types; + +// BSON type bytes. +const BSONNUM: u8 = 0x01; +const BSONSTR: u8 = 0x02; +const BSONOBJ: u8 = 0x03; +const BSONARR: u8 = 0x04; +const BSONBIN: u8 = 0x05; +const BSONOID: u8 = 0x07; +const BSONBOO: u8 = 0x08; +const BSONDAT: u8 = 0x09; +const BSONNUL: u8 = 0x0A; +const BSONRGX: u8 = 0x0B; +const BSONCOD: u8 = 0x0D; +const BSONCWS: u8 = 0x0F; +const BSONINT: u8 = 0x10; +const BSONTIM: u8 = 0x11; +const BSONLON: u8 = 0x12; +const BSONDEC: u8 = 0x13; +const BSONMAX: u8 = 0x7F; +const BSONMIN: u8 = 0xFF; + +// _type_marker value of RawBSONDocument (see bson/raw_bson.py). +const MARKER_RAW_BSON: i64 = 101; + +const I32_MIN: i64 = -2147483648; +const I32_MAX: i64 = 2147483647; +/// BSON_MAX_SIZE from _cbsonmodule.c. +const BSON_MAX_SIZE_I64: i64 = I32_MAX; + +/// True when the value's type is exactly `T` (mirrors `Py*_CheckExact`). +fn is_exact<'py, T: PyTypeInfo>(value: &Bound<'py, PyAny>) -> bool { + value.is_exact_instance_of::() +} + +/// Encodes a mapping to BSON bytes (`_dict_to_bson`). +pub fn dict_to_bson<'py>( + py: Python<'py>, + document: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, + top_level: bool, +) -> PyResult> { + let mut buf = Buffer::new(); + // The C's handle_invalid_doc_error wraps only failures of the element + // iteration loops (not the _id pre-write), so the wrapping lives in + // write_dict. + write_dict(py, &mut buf, document, check_keys, options, top_level)?; + Ok(buf.into_vec()) +} + +/// Wraps `InvalidDocument` errors with the "Invalid document: " prefix and +/// the document as second arg (mirrors the C `_copy_and_set_exception`). +fn wrap_invalid_document<'py>(py: Python<'py>, document: &Bound<'py, PyAny>, err: PyErr) -> PyErr { + let types = match Types::get(py) { + Ok(types) => types, + Err(_) => return err, + }; + let is_invalid_document = err + .value(py) + .is_instance(types.invalid_document.bind(py)) + .unwrap_or(false); + if !is_invalid_document { + return err; + } + let message = match err.value(py).str() { + Ok(m) => match m.to_cow() { + Ok(c) => c.into_owned(), + Err(_) => return err, + }, + Err(_) => return err, + }; + let class = types.invalid_document.bind(py); + match class.call1((format!("Invalid document: {message}"), document)) { + Ok(new_exc) => { + // Transfer the original traceback so the error points at the + // encode site (mirrors the C's _transfer_traceback). + let old_tb = err.value(py).getattr("__traceback__").ok(); + if let Some(tb) = old_tb { + let _ = new_exc.setattr("__traceback__", tb); + } + PyErr::from_value(new_exc) + } + Err(_) => err, + } +} + +/// True when `key` is a string equal to `expected`. +fn is_str_eq(key: &Bound<'_, PyAny>, expected: &str) -> bool { + match key.cast::() { + Ok(s) => s.to_str().map(|s| s == expected).unwrap_or(false), + Err(_) => false, + } +} + +/// Writes `length + elements + terminator` for a mapping, with the `_id` +/// element first when `top_level` — mirroring the C's `write_dict`, which +/// owns the framing. A raw document (marker 101) writes only its raw +/// bytes: the raw document's own length prefix frames it, so no extra +/// length or terminator is written. +pub fn write_dict<'py>( + py: Python<'py>, + buf: &mut Buffer, + mapping: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, + top_level: bool, +) -> PyResult<()> { + // Fast path: dict or dict subclass (e.g. SON). Non-dict mappings first + // check for a raw document passthrough, then for the Mapping protocol. + if mapping.cast::().is_err() { + if let Some(marker) = read_type_marker(mapping)? { + if marker == MARKER_RAW_BSON { + let raw = mapping.getattr(pyo3::intern!(py, "raw"))?; + let data = raw_bytes_of(py, &raw)?; + buf.write_bytes(&data); + return Ok(()); + } + } + let types = Types::get(py)?; + if !mapping.is_instance(types.mapping.bind(py))? { + return Err(pyo3::exceptions::PyTypeError::new_err(format!( + "encoder expected a mapping type but got: {}", + py_repr(mapping) + ))); + } + } + + let start = buf.position(); + buf.write_u32_le(0); // length placeholder + let _guard = recursion_guard(py, GUARD_DOCUMENT)?; + + let mut skip_id = false; + if top_level { + // _id is encoded first, regardless of its position in the mapping. + // The lookup must not trigger mapping hooks (e.g. defaultdict's + // __missing__): dict lookups use the raw storage API, other + // mappings use a has-key check first. + let id_value = if let Ok(dict) = mapping.cast::() { + dict.get_item(pyo3::intern!(py, "_id"))? + } else { + match mapping.get_item(pyo3::intern!(py, "_id")) { + Ok(value) => Some(value), + Err(err) if err.is_instance_of::(py) => None, + Err(err) => return Err(err), + } + }; + if let Some(value) = id_value { + write_value( + py, buf, b"_id\x00", &value, check_keys, options, false, false, + )?; + skip_id = true; + } + } + // Fast path: dict or dict subclass (e.g. SON). Other Mappings are + // iterated via their `items()` method. + if let Ok(dict) = mapping.cast::() { + for (key, value) in dict.iter() { + if skip_id && is_str_eq(&key, "_id") { + continue; + } + if let Err(err) = write_pair(py, buf, &key, &value, check_keys, options) { + return Err(fail_element(py, top_level, mapping, err)); + } + } + } else { + let items = mapping.getattr(pyo3::intern!(py, "items"))?.call0()?; + for item in items.try_iter()? { + let item = item?; + let pair = item.cast::().map_err(|_| { + pyo3::exceptions::PyTypeError::new_err("encoder expected mapping items") + })?; + let key = pair.get_item(0)?; + let value = pair.get_item(1)?; + if skip_id && is_str_eq(&key, "_id") { + continue; + } + if let Err(err) = write_pair(py, buf, &key, &value, check_keys, options) { + return Err(fail_element(py, top_level, mapping, err)); + } + } + } + buf.write_u8(0); + let length = (buf.len() - start) as u32; + buf.backpatch_u32(start, length); + Ok(()) +} + +/// The element-iteration failure path: at the top level, InvalidDocument +/// errors are wrapped with the document (mirrors the C's +/// `handle_invalid_doc_error` call sites, which gate on `top_level` and +/// exclude the `_id` pre-write). +fn fail_element<'py>( + py: Python<'py>, + top_level: bool, + mapping: &Bound<'py, PyAny>, + err: PyErr, +) -> PyErr { + if !top_level { + return err; + } + wrap_invalid_document(py, mapping, err) +} + +/// Validates a key and writes its element (`decode_and_write_pair`'s +/// encode-side counterpart: `write_pair` in the C extension). +fn write_pair<'py>( + py: Python<'py>, + buf: &mut Buffer, + key: &Bound<'py, PyAny>, + value: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, +) -> PyResult<()> { + // Keys must be strings. + let key_str = match key.cast::() { + Ok(s) => s, + Err(_) => { + return Err(invalid_document( + py, + &format!( + "documents must have only string keys, key was {}", + key.repr()?.to_cow()? + ), + )) + } + }; + // UnicodeEncodeError propagates, matching C. + let key_utf8 = key_str.to_cow()?; + + // Keys must not contain NUL bytes (detected via strlen in C). + if key_utf8.as_bytes().contains(&0) { + return Err(invalid_document( + py, + "Key names must not contain the NULL byte", + )); + } + + if check_keys { + if key_utf8.starts_with('$') { + return Err(invalid_document( + py, + &format!("key '{}' must not start with '$'", key_utf8), + )); + } + if key_utf8.contains('.') { + return Err(invalid_document( + py, + &format!("key '{}' must not contain '.'", key_utf8), + )); + } + } + + // The element name (key bytes + NUL) goes in a stack buffer for the + // common short-key case; the C builds it on the stack the same way. + let key_bytes = key_utf8.as_bytes(); + const STACK_NAME_LEN: usize = 96; + let mut stack_name = [0u8; STACK_NAME_LEN]; + if key_bytes.len() < STACK_NAME_LEN { + stack_name[..key_bytes.len()].copy_from_slice(key_bytes); + return write_value( + py, + buf, + &stack_name[..key_bytes.len() + 1], + value, + check_keys, + options, + false, + false, + ); + } + let mut name = Vec::with_capacity(key_utf8.len() + 1); + name.extend_from_slice(key_utf8.as_bytes()); + name.push(0); + write_value(py, buf, &name, value, check_keys, options, false, false) +} + +/// Encodes a name/value pair, dispatching on the value's type. +/// +/// Dispatch order mirrors `write_element_to_buffer` in the C extension: +/// exact builtins (no attribute lookups), then `_type_marker`, then the +/// isinstance waterfall, then the type registry, then the fallback encoder. +#[allow(clippy::too_many_arguments)] +fn write_value<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, + in_custom_call: bool, + in_fallback_call: bool, +) -> PyResult<()> { + // ---- Exact builtin fast path (no attribute lookups). ---- + // bool must be checked before int (bool is an int subclass). + if is_exact::(value) { + let v: bool = value.extract()?; + buf.write_u8(BSONBOO); + buf.write_bytes(name); + buf.write_u8(v as u8); + return Ok(()); + } + if is_exact::(value) { + return write_int_dispatch( + py, + buf, + name, + value, + check_keys, + options, + in_custom_call, + in_fallback_call, + ); + } + if is_exact::(value) { + let v: f64 = value.extract()?; + buf.write_u8(BSONNUM); + buf.write_bytes(name); + buf.write_f64_le(v); + return Ok(()); + } + if value.is_none() { + buf.write_u8(BSONNUL); + buf.write_bytes(name); + return Ok(()); + } + if is_exact::(value) { + return write_document_value(py, buf, name, value, check_keys, options); + } + if is_exact::(value) || is_exact::(value) { + return write_array_value(py, buf, name, value, check_keys, options); + } + if is_exact::(value) { + let bytes = value.cast::().map_err(PyErr::from)?; + return write_binary_subtype_zero(py, buf, name, bytes); + } + if is_exact::(value) { + let string = value.cast::().map_err(PyErr::from)?; + return write_string_value(py, buf, name, string); + } + + // ---- _type_marker dispatch (exact ints only). ---- + let marker = read_type_marker(value)?; + if let Some(marker) = marker { + if write_marker_value( + py, + buf, + name, + value, + marker, + check_keys, + options, + in_custom_call, + in_fallback_call, + )? { + return Ok(()); + } + // Unknown marker: fall through to the isinstance chain. + } + + // ---- isinstance waterfall for builtins and stdlib types. ---- + if value.is_instance_of::() { + let v: bool = value.extract()?; + buf.write_u8(BSONBOO); + buf.write_bytes(name); + buf.write_u8(v as u8); + return Ok(()); + } + if value.is_instance_of::() { + return write_int_dispatch( + py, + buf, + name, + value, + check_keys, + options, + in_custom_call, + in_fallback_call, + ); + } + if value.is_instance_of::() { + let v: f64 = value.extract()?; + buf.write_u8(BSONNUM); + buf.write_bytes(name); + buf.write_f64_le(v); + return Ok(()); + } + if value.is_none() { + buf.write_u8(BSONNUL); + buf.write_bytes(name); + return Ok(()); + } + if value.is_instance_of::() { + return write_document_value(py, buf, name, value, check_keys, options); + } + if value.is_instance_of::() || value.is_instance_of::() { + return write_array_value(py, buf, name, value, check_keys, options); + } + if value.is_instance_of::() { + let bytes = value.cast::().map_err(PyErr::from)?; + return write_binary_subtype_zero(py, buf, name, bytes); + } + if value.is_instance_of::() { + let string = value.cast::().map_err(PyErr::from)?; + return write_string_value(py, buf, name, string); + } + if value.cast::().is_ok() { + return write_datetime_value(py, buf, name, value); + } + let types = Types::get(py)?; + if value.is_instance(types.re_type.bind(py))? { + return write_regex_value(py, buf, name, value); + } + if value.is_instance(types.mapping.bind(py))? { + return write_document_value(py, buf, name, value, check_keys, options); + } + if value.is_instance(types.uuid.bind(py))? { + // Encoded as Binary.from_uuid(value, uuid_representation). + let binary = types.binary.bind(py).call_method1( + pyo3::intern!(py, "from_uuid"), + (value, options.uuid_representation), + )?; + return write_value( + py, + buf, + name, + &binary, + check_keys, + options, + in_custom_call, + in_fallback_call, + ); + } + + // ---- Custom type encoder (exact type match), then fallback. ---- + // The re-dispatch is bounded by CPython's recursion counter (the C + // relies on the Python frames of the callbacks; guarding explicitly + // keeps the Rust stack bounded the same way). + if let Some(converted) = custom_encoder_value(py, options, value, in_custom_call)? { + let _guard = recursion_guard(py, GUARD_EMPTY)?; + return write_value(py, buf, name, &converted, check_keys, options, true, false); + } + if let Some(converted) = fallback_encoder_value(py, options, value, in_fallback_call)? { + let _guard = recursion_guard(py, GUARD_EMPTY)?; + return write_value(py, buf, name, &converted, check_keys, options, false, true); + } + + // ---- Cannot encode. ---- + buf.write_bytes(name); // Keep the buffer consistent for error reporting. + Err(cannot_encode(py, value, in_fallback_call)) +} + +/// RAII guard around CPython's recursion limit, mirroring the C's +/// `Py_EnterRecursiveCall`/`Py_LeaveRecursiveCall` pairs. Cyclic structures +/// raise RecursionError instead of overflowing the stack. +pub struct RecursionGuard { + // Not Send: Py_EnterRecursiveCall/LeaveRecursiveCall touch the + // current thread's recursion counter. + _not_send: std::marker::PhantomData<*const ()>, +} + +/// Enters a CPython recursion check; returns an `Err` with the pending +/// error (typically RecursionError) when the limit is hit. +/// +/// The message is a static C string (CPython reads it when formatting the +/// RecursionError): no per-call allocation, so guards are cheap enough +/// for every document, array, and converted value. +pub fn recursion_guard(py: Python<'_>, message: &'static CStr) -> PyResult { + let ok = unsafe { pyo3::ffi::Py_EnterRecursiveCall(message.as_ptr()) }; + if ok != 0 { + return Err(PyErr::fetch(py)); + } + Ok(RecursionGuard { + _not_send: std::marker::PhantomData, + }) +} + +/// The `Py_EnterRecursiveCall` messages, matching the C extension's. +pub const GUARD_EMPTY: &CStr = c""; +pub const GUARD_DOCUMENT: &CStr = c" while encoding a BSON document"; +pub const GUARD_ARRAY: &CStr = c" while encoding a BSON array"; +pub const GUARD_DECODE_DOCUMENT: &CStr = c" while decoding a BSON document"; +pub const GUARD_DECODE_LIST: &CStr = c" while decoding a list value"; + +impl Drop for RecursionGuard { + fn drop(&mut self) { + unsafe { pyo3::ffi::Py_LeaveRecursiveCall() }; + } +} + +/// Handles `_type_marker` values (Binary=5, ObjectId=7, DatetimeMS=9, +/// Regex=11, Code=13, Timestamp=17, Int64=18, Decimal128=19, DBRef=100, +/// RawBSONDocument=101, MaxKey=127, MinKey=255). Returns `false` for +/// unrecognized markers, which fall through to the isinstance chain +/// (mirrors the C switch without a default case). +#[allow(clippy::too_many_arguments)] +fn write_marker_value<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, + marker: i64, + _check_keys: bool, + options: &CodecOptionsData<'py>, + _in_custom_call: bool, + _in_fallback_call: bool, +) -> PyResult { + match marker { + 5 => { + // Binary: read .subtype; Binary is a bytes subclass. + let subtype: i64 = value + .getattr(pyo3::intern!(py, "subtype"))? + .extract() + .map_err(|_| pyo3::exceptions::PyTypeError::new_err("invalid subtype"))?; + let data = value.cast::().map_err(PyErr::from)?.as_bytes(); + // Mirrors the C's _downcast_and_check(size, extra): the length + // plus its framing (subtype byte, plus the extra length prefix + // for subtype 2) must fit in an int32. + let extra: i64 = if subtype == 2 { 4 + 1 } else { 1 }; + if data.len() as i64 > BSON_MAX_SIZE_I64 - extra { + return Err(invalid_string_data( + py, + "String length must be <= 2147483647", + )); + } + let size = data.len() as i32; + buf.write_u8(BSONBIN); + buf.write_bytes(name); + if subtype == 2 { + // The payload carries an extra length prefix. + buf.write_i32_le(size + 4); + buf.write_u8(2); + } + buf.write_i32_le(size); + if subtype != 2 { + buf.write_u8(subtype as u8); + } + buf.write_bytes(data); + Ok(true) + } + 7 => { + // ObjectId: .binary is the 12-byte representation. + let binary = value.getattr(pyo3::intern!(py, "binary"))?; + let bytes = binary.cast::().map_err(PyErr::from)?; + buf.write_u8(BSONOID); + buf.write_bytes(name); + buf.write_bytes(bytes.as_bytes()); + Ok(true) + } + 9 => { + // DatetimeMS: int(value) is the millis value (the C uses + // PyNumber_Long, which routes through __int__ and requires the + // result to be an int; a float __int__ raises TypeError). + let as_int = value.call_method0(pyo3::intern!(py, "__int__"))?; + if !as_int.is_instance_of::() { + return Err(pyo3::exceptions::PyTypeError::new_err( + "int() argument must be an int, not a float", + )); + } + let millis: i64 = as_int.extract().map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err( + "MongoDB datetimes can only handle up to 8-byte ints", + ) + })?; + buf.write_u8(BSONDAT); + buf.write_bytes(name); + buf.write_i64_le(millis); + Ok(true) + } + 11 => { + write_regex_value(py, buf, name, value)?; + Ok(true) + } + 13 => { + // Code: the value is the code string; .scope decides between + // plain code (0x0D) and code w/ scope (0x0F). + let scope = value.getattr(pyo3::intern!(py, "scope"))?; + let code = value.cast::().map_err(PyErr::from)?; + if scope.is_none() { + buf.write_u8(BSONCOD); + buf.write_bytes(name); + write_string_payload(buf, code)?; + return Ok(true); + } + buf.write_u8(BSONCWS); + buf.write_bytes(name); + let start = buf.position(); + buf.write_u32_le(0); // total length placeholder + write_string_payload(buf, code)?; + // The scope is a full document (length + elements + term), + // written without key checking (mirrors the C). + write_dict(py, buf, &scope, false, options, false)?; + let length = (buf.len() - start) as u32; + buf.backpatch_u32(start, length); + Ok(true) + } + 17 => { + // Timestamp: i32 inc, then i32 time. Mirrors the C's + // PyLong_AsUnsignedLong: negative or >unsigned-long values + // raise OverflowError; [2³¹, 2³²) truncates to i32. + let inc: u64 = value + .getattr(pyo3::intern!(py, "inc"))? + .extract() + .map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err( + "can't convert negative value to unsigned long", + ) + })?; + let time: u64 = value + .getattr(pyo3::intern!(py, "time"))? + .extract() + .map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err( + "can't convert negative value to unsigned long", + ) + })?; + buf.write_u8(BSONTIM); + buf.write_bytes(name); + buf.write_i32_le(inc as i32); + buf.write_i32_le(time as i32); + Ok(true) + } + 18 => { + // Int64. + let v: i64 = value.extract().map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err( + "MongoDB can only handle up to 8-byte ints", + ) + })?; + buf.write_u8(BSONLON); + buf.write_bytes(name); + buf.write_i64_le(v); + Ok(true) + } + 19 => { + // Decimal128: .bid is the 16-byte representation. + let bid = value.getattr(pyo3::intern!(py, "bid"))?; + let bytes = bid.cast::().map_err(PyErr::from)?; + buf.write_u8(BSONDEC); + buf.write_bytes(name); + buf.write_bytes(bytes.as_bytes()); + Ok(true) + } + 100 => { + // DBRef: encoded as its as_doc() document. + let as_doc = value.call_method0(pyo3::intern!(py, "as_doc"))?; + buf.write_u8(BSONOBJ); + buf.write_bytes(name); + write_dict(py, buf, &as_doc, false, options, false)?; + Ok(true) + } + 101 => { + // RawBSONDocument: .raw (bytes or memoryview) written as-is. + let raw = value.getattr(pyo3::intern!(py, "raw"))?; + let data = raw_bytes_of(py, &raw)?; + buf.write_u8(BSONOBJ); + buf.write_bytes(name); + buf.write_bytes(&data); + Ok(true) + } + 255 => { + buf.write_u8(BSONMIN); + buf.write_bytes(name); + Ok(true) + } + 127 => { + buf.write_u8(BSONMAX); + buf.write_bytes(name); + Ok(true) + } + _ => Ok(false), // unknown marker: fall through to the chain + } +} + +/// Extracts bytes from a bytes or memoryview object (`write_raw_doc`). +fn raw_bytes_of<'py>(_py: Python<'py>, raw: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(bytes) = raw.cast::() { + return Ok(bytes.as_bytes().to_vec()); + } + if !raw.is_instance_of::() { + return Err(pyo3::exceptions::PyTypeError::new_err(format!( + "RawBSONDocument.raw must be bytes or memoryview, not {}", + raw.get_type().name()?.to_cow()? + ))); + } + let buffer = PyBuffer::::get(raw)?; + // Defensive checks mirroring the C's write_raw_doc/_get_buffer: a + // misbehaving exporter may hand back non-contiguous or null buffers. + if buffer.buf_ptr().is_null() || !buffer.is_c_contiguous() || buffer.item_size() != 1 { + return Err(pyo3::exceptions::PyTypeError::new_err( + "RawBSONDocument.raw must be bytes or memoryview", + )); + } + let slice = + unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; + Ok(slice.to_vec()) +} + +/// Writes the regex element shared by the `Regex` class and compiled +/// patterns: `0x0B + name + pattern cstr + flags cstr`. +fn write_regex_value<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, +) -> PyResult<()> { + let int_flags: i64 = value + .getattr(pyo3::intern!(py, "flags"))? + .extract() + .map_err(|_| pyo3::exceptions::PyTypeError::new_err("invalid regex flags"))?; + let pattern_obj = value.getattr(pyo3::intern!(py, "pattern"))?; + + let pattern_bytes: Vec = if let Ok(p) = pattern_obj.cast::() { + // UnicodeEncodeError propagates for invalid patterns, matching C. + p.to_str()?.as_bytes().to_vec() + } else if let Ok(p) = pattern_obj.cast::() { + // Bytes patterns are revalidated as UTF-8 (mirrors the C). + let data = p.as_bytes(); + match std::str::from_utf8(data) { + Ok(_) => {} + Err(_) => { + return Err(invalid_string_data( + py, + "regex patterns must be valid UTF-8", + )) + } + } + data.to_vec() + } else { + return Err(pyo3::exceptions::PyTypeError::new_err( + "a bytes-like object is required for the regex pattern", + )); + }; + if pattern_bytes.contains(&0) { + return Err(invalid_document( + py, + "regex patterns must not contain the NULL byte", + )); + } + + buf.write_u8(BSONRGX); + buf.write_bytes(name); + buf.write_bytes(&pattern_bytes); + buf.write_u8(0); + // Flag chars are written in bit order: i l m s u x. + let mut flags = Vec::new(); + if int_flags & 2 != 0 { + flags.push(b'i'); + } + if int_flags & 4 != 0 { + flags.push(b'l'); + } + if int_flags & 8 != 0 { + flags.push(b'm'); + } + if int_flags & 16 != 0 { + flags.push(b's'); + } + if int_flags & 32 != 0 { + flags.push(b'u'); + } + if int_flags & 64 != 0 { + flags.push(b'x'); + } + buf.write_bytes(&flags); + buf.write_u8(0); + Ok(()) +} + +/// Formats a value's repr as an owned String (error paths only). +fn py_repr(value: &Bound<'_, PyAny>) -> String { + value + .repr() + .and_then(|r| r.to_cow().map(|c| c.into_owned())) + .unwrap_or_default() +} + +/// The `PyLong` encode path shared by exact ints and int subclasses: +/// int32 when it fits, else int64. On overflow the error is cleared and +/// the custom/fallback encoders get a chance (mirrors the C behavior); +/// the final OverflowError comes from `cannot_encode`. +#[allow(clippy::too_many_arguments)] +fn write_int_dispatch<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, + in_custom_call: bool, + in_fallback_call: bool, +) -> PyResult<()> { + match value.extract::() { + Ok(v) => { + if (I32_MIN..=I32_MAX).contains(&v) { + buf.write_u8(BSONINT); + buf.write_bytes(name); + buf.write_i32_le(v as i32); + } else { + buf.write_u8(BSONLON); + buf.write_bytes(name); + buf.write_i64_le(v); + } + Ok(()) + } + Err(_) => { + // Overflow: give the custom/fallback encoders a chance, then + // fail in cannot_encode (which raises OverflowError for ints). + if let Some(converted) = custom_encoder_value(py, options, value, in_custom_call)? { + return write_value( + py, + buf, + name, + &converted, + check_keys, + options, + true, + in_fallback_call, + ); + } + if let Some(converted) = fallback_encoder_value(py, options, value, in_fallback_call)? { + return write_value(py, buf, name, &converted, check_keys, options, false, true); + } + Err(cannot_encode(py, value, in_fallback_call)) + } + } +} + +/// Consults `options.type_registry._encoder_map` for an exact-type encoder. +fn custom_encoder_value<'py>( + py: Python<'py>, + options: &CodecOptionsData<'py>, + value: &Bound<'py, PyAny>, + in_custom_call: bool, +) -> PyResult>> { + if in_custom_call { + return Ok(None); + } + let Some(encoder_map) = options.encoder_map(py) else { + return Ok(None); + }; + let converter = encoder_map.get_item(value.get_type())?; + let Some(converter) = converter else { + return Ok(None); + }; + Ok(Some(converter.call1((value,))?)) +} + +/// Consults `options.type_registry._fallback_encoder`. +fn fallback_encoder_value<'py>( + py: Python<'py>, + options: &CodecOptionsData<'py>, + value: &Bound<'py, PyAny>, + in_fallback_call: bool, +) -> PyResult>> { + if in_fallback_call { + return Ok(None); + } + let Some(fallback) = options.fallback_encoder(py) else { + return Ok(None); + }; + Ok(Some(fallback.call1((value,))?)) +} + +/// The final failure path, mirroring `_set_cannot_encode`: OverflowError +/// for ints that do not fit in 8 bytes, InvalidDocument otherwise. +fn cannot_encode(py: Python<'_>, value: &Bound<'_, PyAny>, _was_fallback: bool) -> PyErr { + if value.is_instance_of::() && value.extract::().is_err() { + return pyo3::exceptions::PyOverflowError::new_err( + "MongoDB can only handle up to 8-byte ints", + ); + } + invalid_document( + py, + &format!( + "cannot encode object: {}, of type: {}", + py_repr(value), + py_repr(&value.get_type().into_any()), + ), + ) +} + +/// Writes `0x03 + name + document` for dict-like values. +fn write_document_value<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, +) -> PyResult<()> { + buf.write_u8(BSONOBJ); + buf.write_bytes(name); + write_dict(py, buf, value, check_keys, options, false) +} + +/// Writes `0x04 + name + array` (arrays are documents with numeric keys). +fn write_array_value<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, + check_keys: bool, + options: &CodecOptionsData<'py>, +) -> PyResult<()> { + buf.write_u8(BSONARR); + buf.write_bytes(name); + let start = buf.position(); + buf.write_u32_le(0); // length placeholder + let _guard = recursion_guard(py, GUARD_ARRAY)?; + let len = value.len()?; + // List/tuple values (including subclasses — the cast is + // isinstance-based, matching the C's subclass-inclusive PyList_Check / + // PyTuple_Check) are read through the concrete sequence API + // (PyList_GET_ITEM / PyTuple_GET_ITEM in the C), which bypasses + // __getitem__ overrides. The mapping-protocol fallback below only + // stays for defensive symmetry with the dispatch above it. + let list = value.cast::().ok(); + let tuple = if list.is_none() { + value.cast::().ok() + } else { + None + }; + // The numeric element key ("0", "1", ... + NUL) is built in a stack + // buffer (snprintf into a local in the C): no per-element allocation. + let mut index_key = [0u8; 12]; + for i in 0..len { + let item = if let Some(list) = &list { + list.get_item(i) + } else if let Some(tuple) = &tuple { + tuple.get_item(i) + } else { + value.get_item(i) + }?; + // Array lengths are int32, so i < 2^31 has at most 10 digits: the + // write below always fits 11 bytes including the NUL. + let mut pos = index_key.len() - 1; + index_key[pos] = 0; + let mut n = i; + loop { + pos -= 1; + index_key[pos] = b'0' + (n % 10) as u8; + n /= 10; + if n == 0 { + break; + } + } + write_value( + py, + buf, + &index_key[pos..], + &item, + check_keys, + options, + false, + false, + )?; + } + buf.write_u8(0); + let length = (buf.len() - start) as u32; + buf.backpatch_u32(start, length); + Ok(()) +} + +/// Writes `bytes` values as BSON binary subtype 0. +fn write_binary_subtype_zero( + _py: Python<'_>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'_, PyBytes>, +) -> PyResult<()> { + let data = value.as_bytes(); + if data.len() > i32::MAX as usize { + return Err(invalid_string_data( + _py, + "String length must be <= 2147483647", + )); + } + buf.write_u8(BSONBIN); + buf.write_bytes(name); + buf.write_i32_le(data.len() as i32); + buf.write_u8(0); // subtype + buf.write_bytes(data); + Ok(()) +} + +/// Writes `0x02 + name + i32(len+1) + utf8 + NUL`. +fn write_string_value<'py>( + _py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyString>, +) -> PyResult<()> { + buf.write_u8(BSONSTR); + buf.write_bytes(name); + write_string_payload(buf, value) +} + +/// Writes the string payload: `i32(len+1) + utf8 + NUL`. +fn write_string_payload(buf: &mut Buffer, value: &Bound<'_, PyString>) -> PyResult<()> { + // UnicodeEncodeError propagates, matching C. + let utf8 = value.to_cow()?; + if utf8.len() >= i32::MAX as usize { + return Err(invalid_string_data( + value.py(), + "String length must be <= 2147483647", + )); + } + buf.write_i32_le(utf8.len() as i32 + 1); + buf.write_bytes(utf8.as_bytes()); + buf.write_u8(0); + Ok(()) +} + +/// Writes datetime values as `0x09 + name + i64 millis`. +/// +/// Aware datetimes are shifted by `utcoffset()` first (mirrors the C +/// encoder, which subtracts the offset and then treats the fields as UTC). +fn write_datetime_value<'py>( + py: Python<'py>, + buf: &mut Buffer, + name: &[u8], + value: &Bound<'py, PyAny>, +) -> PyResult<()> { + let dt = value.cast::().map_err(PyErr::from)?; + let offset = dt.call_method0(pyo3::intern!(py, "utcoffset"))?; + let effective: Bound<'py, PyDateTime> = if offset.is_none() { + dt.clone() + } else { + let shifted = value.sub(offset)?; + shifted.cast::().map_err(PyErr::from)?.clone() + }; + let millis = millis_from_fields( + effective.get_year(), + effective.get_month(), + effective.get_day(), + effective.get_hour(), + effective.get_minute(), + effective.get_second(), + effective.get_microsecond(), + ); + buf.write_u8(BSONDAT); + buf.write_bytes(name); + buf.write_i64_le(millis); + Ok(()) +} diff --git a/rust/bson/src/errors.rs b/rust/bson/src/errors.rs new file mode 100644 index 0000000000..09eafb2855 --- /dev/null +++ b/rust/bson/src/errors.rs @@ -0,0 +1,48 @@ +//! Raising `bson.errors` exceptions from Rust. +//! +//! Mirrors `_cbsonmodule.c`'s `_error()` helper: the exception classes are +//! looked up on the `bson.errors` module. Error paths are cold, so a +//! per-call module lookup (cached after first use) is fine. + +use pyo3::sync::PyOnceLock; + +use pyo3::prelude::*; + +static ERRORS_MODULE: PyOnceLock> = PyOnceLock::new(); + +/// Returns the `bson.errors` module, importing and caching it on first use. +fn errors_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + ERRORS_MODULE + .get_or_try_init(py, || { + let module = py.import("bson.errors")?; + Ok::<_, PyErr>(module.into_any().unbind()) + }) + .map(|m| m.bind(py)) +} + +/// Builds a `PyErr` for `bson.errors.` with `message`. +#[cold] +pub fn bson_error(py: Python<'_>, name: &str, message: &str) -> PyErr { + let error = errors_module(py) + .and_then(|m| m.getattr(name)) + .and_then(|exc| exc.call1((message,))); + match error { + Ok(value) => PyErr::from_value(value), + Err(_) => pyo3::exceptions::PyRuntimeError::new_err(message.to_string()), + } +} + +/// Shorthand for `bson.errors.InvalidBSON`. +pub fn invalid_bson(py: Python<'_>, message: &str) -> PyErr { + bson_error(py, "InvalidBSON", message) +} + +/// Shorthand for `bson.errors.InvalidDocument`. +pub fn invalid_document(py: Python<'_>, message: &str) -> PyErr { + bson_error(py, "InvalidDocument", message) +} + +/// Shorthand for `bson.errors.InvalidStringData`. +pub fn invalid_string_data(py: Python<'_>, message: &str) -> PyErr { + bson_error(py, "InvalidStringData", message) +} diff --git a/rust/bson/src/lib.rs b/rust/bson/src/lib.rs new file mode 100644 index 0000000000..c2c0a42492 --- /dev/null +++ b/rust/bson/src/lib.rs @@ -0,0 +1,449 @@ +//! `_rbson` — a Rust-backed BSON codec for PyMongo. +//! +//! Drop-in alternative to `bson._cbson` exposing the same six private +//! entry points with identical signatures and semantics (the C extension is +//! the behavioral reference): +//! +//! - `_dict_to_bson(document, check_keys, codec_options, top_level=True)` +//! - `_bson_to_dict(bson, codec_options)` +//! - `_decode_all(bson, codec_options)` +//! - `_element_to_dict(bson, position, obj_end, codec_options, raw_array)` +//! - `_raw_to_dict(bson, position, obj_end, codec_options, result, raw_array)` +//! - `_array_of_documents_to_buffer(array)` +//! +//! The GIL is held for the duration of every call (parity with the C +//! extension): the module declares `Py_mod_gil = Py_MOD_GIL_USED` +//! (`gil_used = true`), so free-threaded CPython builds re-enable the GIL +//! when importing it. Note that pyo3 does not declare +//! `Py_mod_multiple_interpreters`, and the import machinery defaults a +//! missing slot to SUPPORTED — so this module *can* be imported into a +//! shared-GIL subinterpreter even though its process-wide statics (`Types`, +//! the field-name cache) would then be shared across interpreters. +//! Importing the module into more than one interpreter is not supported. + +mod buffer; +mod datetime_conv; +mod decode; +mod encode; +mod errors; +mod options; +mod types; + +use pyo3::buffer::PyBuffer; +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyBytes, PyList, PyModule, PyTuple}; + +use buffer::Buffer; +use decode::BSON_MIN_SIZE; +use errors::invalid_bson; +use options::CodecOptionsData; +use types::Types; + +/// True when any document in the stream is at least `threshold` bytes +/// (mirrors `_contains_view_eligible_doc`). +fn contains_view_eligible_doc(data: &[u8], threshold: usize) -> bool { + let mut position = 0; + while data.len() - position >= 4 { + let size = u32::from_le_bytes(data[position..position + 4].try_into().unwrap()) as usize; + if size < BSON_MIN_SIZE || size > data.len() - position { + return false; + } + if size >= threshold { + return true; + } + position += size; + } + false +} + +/// Prepares the input buffer owner for decoding, mirroring +/// `_prepare_input_buffer`: raw-document decoding from mutable/foreign +/// buffers copies them to bytes when the stream contains a +/// view-eligible (>= threshold) document. +fn prepare_input_buffer<'py>( + py: Python<'py>, + object: &Bound<'py, PyAny>, + options: &CodecOptionsData<'py>, +) -> PyResult> { + if !options.is_raw_bson + || object.is_instance_of::() + || object.is_instance_of::() + { + return Ok(object.clone()); + } + let eligible = with_buffer_view(object, |data| { + Ok(contains_view_eligible_doc( + data, + Types::get(py)?.raw_bson_view_threshold, + )) + })?; + if eligible { + // Copy to bytes so memoryview-backed views outlive the input + // (mirrors PyBytes_FromObject). + let copy = with_buffer_view(object, |data| Ok(PyBytes::new(py, data)))?; + return Ok(copy.into_any()); + } + Ok(object.clone()) +} + +/// Calls `f` with a `&[u8]` view of any buffer-protocol object (bytes, +/// bytearray, memoryview, mmap, ...), mirroring `_get_buffer` in the C +/// extension. The view is zero-copy. +fn with_buffer_view( + object: &Bound<'_, PyAny>, + f: impl FnOnce(&[u8]) -> PyResult, +) -> PyResult { + if let Ok(bytes) = object.cast::() { + return f(bytes.as_bytes()); + } + let buffer = PyBuffer::::get(object)?; + with_get_buffer_checks(&buffer)?; + let slice = + unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; + let result = f(slice); + drop(buffer); + result +} + +/// The defensive checks of the C's `_get_buffer`: pyo3 acquires buffers +/// with PyBUF_FULL_RO (not PyBUF_SIMPLE), so a misbehaving exporter can +/// hand back non-contiguous or null buffers; mirroring the C, each is +/// rejected with a ValueError. +fn with_get_buffer_checks(buffer: &PyBuffer) -> PyResult<()> { + // `!view->buf || view->len < 0`. + if buffer.buf_ptr().is_null() { + return Err(pyo3::exceptions::PyValueError::new_err("invalid buffer")); + } + // `!PyBuffer_IsContiguous(view, 'C')`. + if !buffer.is_c_contiguous() { + return Err(pyo3::exceptions::PyValueError::new_err( + "must be a contiguous buffer", + )); + } + // `view->itemsize != 1`. + if buffer.item_size() != 1 { + return Err(pyo3::exceptions::PyValueError::new_err( + "buffer data must be ascii or utf8", + )); + } + Ok(()) +} + +/// Encodes a Python mapping to BSON bytes. +#[pyfunction] +#[pyo3( + signature = (document, check_keys, codec_options, top_level = true), + text_signature = "(document, check_keys, codec_options, top_level=True)" +)] +fn _dict_to_bson<'py>( + py: Python<'py>, + document: &Bound<'py, PyAny>, + check_keys: bool, + codec_options: &Bound<'py, PyAny>, + top_level: bool, +) -> PyResult> { + let options = CodecOptionsData::parse(codec_options)?; + + // A RawBSONDocument encodes to its raw bytes unconditionally (mirrors + // the C `_cbson_dict_to_bson`, which checks the marker regardless of + // top_level): bytes are returned as-is, anything else is converted + // with PyBytes_FromObject. + if let Some(marker) = options::read_type_marker(document)? { + if marker == options::RAW_BSON_DOCUMENT_MARKER { + let raw = document.getattr(pyo3::intern!(py, "raw"))?; + return match raw.cast::() { + Ok(bytes) => Ok(bytes.clone()), + Err(_) => { + // Copy via the buffer protocol (mirrors + // PyBytes_FromObject). + let data = with_buffer_view(&raw, |data| Ok(data.to_vec()))?; + Ok(PyBytes::new(py, &data)) + } + }; + } + } + + let bytes = encode::dict_to_bson(py, document, check_keys, &options, top_level)?; + Ok(PyBytes::new(py, &bytes)) +} + +/// Decodes a single BSON document into `codec_options.document_class`. +#[pyfunction] +fn _bson_to_dict<'py>( + py: Python<'py>, + bson: &Bound<'py, PyAny>, + codec_options: &Bound<'py, PyAny>, +) -> PyResult> { + let options = CodecOptionsData::parse(codec_options)?; + let owner = prepare_input_buffer(py, bson, &options)?; + with_buffer_view(&owner, |data| { + decode::bson_to_dict( + py, + &decode::DecodeInput { + data, + owner: Some(&owner), + }, + &options, + ) + }) +} + +/// Decodes a buffer containing one or more BSON documents into a list. +#[pyfunction] +fn _decode_all<'py>( + py: Python<'py>, + bson: &Bound<'py, PyAny>, + codec_options: &Bound<'py, PyAny>, +) -> PyResult> { + let options = CodecOptionsData::parse(codec_options)?; + let owner = prepare_input_buffer(py, bson, &options)?; + with_buffer_view(&owner, |data| { + let input = decode::DecodeInput { + data, + owner: Some(&owner), + }; + let mut position = 0; + let results = PyList::empty(py); + while position < data.len() { + let slice = &data[position..]; + let size = read_size_prefix(py, slice)?; + if size > slice.len() { + return Err(invalid_bson(py, "objsize too large")); + } + // The C's _cbson_decode_all propagates elements_to_dict + // failures untouched — the InvalidBSON wrapping happened + // deeper, in get_value. + let value = decode::elements_to_dict(py, &input, &slice[..size], &options) + .map(|(value, _)| value)?; + results.append(value)?; + position += size; + } + Ok(results.into_any()) + }) +} + +/// Reads the 4-byte size prefix of the next document and validates it. +fn read_size_prefix(py: Python<'_>, data: &[u8]) -> PyResult { + if data.len() < BSON_MIN_SIZE { + return Err(invalid_bson(py, "not enough data for a BSON document")); + } + // The C reads the size prefix as a signed int32; negative values + // (and zero) are "invalid message size". + let size = i32::from_le_bytes([data[0], data[1], data[2], data[3]]); + if size < BSON_MIN_SIZE as i32 { + return Err(invalid_bson(py, "invalid message size")); + } + let size = size as usize; + if size > data.len() { + return Err(invalid_bson(py, "objsize too large")); + } + if data[size - 1] != 0 { + return Err(invalid_bson(py, "bad eoo")); + } + Ok(size) +} + +fn rewrap(py: Python<'_>, err: PyErr) -> PyErr { + decode::rewrap_as_invalid_bson(py, err) +} + +/// Decodes a single key/value pair at `position`, returning +/// `(name, value, new_position)`. The input must be a bytes object +/// (mirrors `_cbson_element_to_dict`). +#[pyfunction] +#[pyo3(signature = (bson, position, obj_end, codec_options, raw_array=false))] +fn _element_to_dict<'py>( + py: Python<'py>, + bson: &Bound<'py, PyAny>, + position: u32, + obj_end: u32, + codec_options: &Bound<'py, PyAny>, + raw_array: bool, +) -> PyResult> { + let bytes = bson.cast::().map_err(|_| { + pyo3::exceptions::PyTypeError::new_err( + "argument to _element_to_dict must be a bytes object", + ) + })?; + let options = CodecOptionsData::parse(codec_options)?; + let data = bytes.as_bytes(); + let input = decode::DecodeInput { + data, + owner: Some(bson), + }; + let max = obj_end as usize; + let mut pos = position as usize; + if pos > max || max > data.len() { + return Err(invalid_bson(py, "bad object or element length")); + } + if pos >= max { + return Err(invalid_bson(py, "field name too large")); + } + let type_byte = data[pos]; + pos += 1; + let name_start = pos; + while pos < max && data[pos] != 0 { + pos += 1; + } + if pos >= max { + return Err(invalid_bson(py, "field name too large")); + } + let name_bytes = &data[name_start..pos]; + pos += 1; // skip the NUL + + // The name is decoded before the value (mirrors the C's + // _element_to_dict); a UnicodeDecodeError is wrapped in InvalidBSON. + let name = decode::decode_key(py, name_bytes, &options).map_err(|err| rewrap(py, err))?; + let (value, new_position) = decode::get_value_with_raw_array( + py, &input, name_bytes, type_byte, pos, max, &options, raw_array, + ) + .map_err(|err| rewrap(py, err))?; + let result = PyTuple::new( + py, + [ + name.unbind(), + value, + new_position.into_pyobject(py)?.into_any().unbind(), + ], + )?; + Ok(result.into_any()) +} + +/// Decodes all elements of a raw BSON document into an existing mapping +/// (`result`), used by RawBSONDocument's lazy inflation. Returns `result`. +#[pyfunction] +#[pyo3(signature = (bson, position, obj_end, codec_options, result_mapping, raw_array=false))] +fn _raw_to_dict<'py>( + py: Python<'py>, + bson: &Bound<'py, PyAny>, + position: u32, + obj_end: u32, + codec_options: &Bound<'py, PyAny>, + result_mapping: &Bound<'py, PyAny>, + raw_array: bool, +) -> PyResult> { + let options = CodecOptionsData::parse(codec_options)?; + let owner = prepare_input_buffer(py, bson, &options)?; + with_buffer_view(&owner, |data| { + let input = decode::DecodeInput { + data, + owner: Some(&owner), + }; + // obj_end must be the index of the document's eoo byte. + if obj_end as usize == 0 + || obj_end as usize >= data.len() + || position > obj_end + || data[obj_end as usize] != 0 + { + return Err(invalid_bson(py, "bad object or element length")); + } + let end = obj_end as usize - 1; + let mut pos = position as usize; + while pos < end { + let type_byte = data[pos]; + pos += 1; + let name_start = pos; + while pos < end && data[pos] != 0 { + pos += 1; + } + if pos >= end { + return Err(invalid_bson(py, "field name too large")); + } + let name_bytes = &data[name_start..pos]; + pos += 1; // skip the NUL + + // The name is decoded before the value (mirrors the C); a + // UnicodeDecodeError is wrapped in InvalidBSON. + let name = + decode::decode_key(py, name_bytes, &options).map_err(|err| rewrap(py, err))?; + let (value, new_pos) = decode::get_value_with_raw_array( + py, + &input, + name_bytes, + type_byte, + pos, + // Per-element values are bounded by the eoo index + // (the C passes obj_end as max to _element_to_dict). + obj_end as usize, + &options, + raw_array, + ) + .map_err(|err| rewrap(py, err))?; + pos = new_pos; + + result_mapping.set_item(name, value)?; + } + if pos != obj_end as usize { + return Err(invalid_bson(py, "bad object or element length")); + } + Ok(result_mapping.clone().unbind()) + }) +} + +/// Converts a BSON array-of-documents into a stream of concatenated BSON +/// documents (used for batched write commands). +#[pyfunction] +fn _array_of_documents_to_buffer<'py>( + py: Python<'py>, + array: &Bound<'py, PyAny>, +) -> PyResult> { + with_buffer_view(array, |data| { + if data.len() < BSON_MIN_SIZE { + return Err(invalid_bson(py, "not enough data for a BSON document")); + } + let size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; + if size != data.len() || size < BSON_MIN_SIZE { + return Err(invalid_bson(py, "objsize too large")); + } + if data[size - 1] != 0 { + return Err(invalid_bson(py, "bad eoo")); + } + + let mut buffer = Buffer::new(); + let mut position = 4; + while position < size - 1 { + // Verify the value is an embedded document. + let type_byte = data[position]; + if type_byte != 3 { + return Err(invalid_bson(py, "array element was not an object")); + } + position += 1; + // Just skip the key. + while position < size && data[position] != 0 { + position += 1; + } + if position >= size { + return Err(invalid_bson(py, "invalid array content")); + } + position += 1; + if position >= size || size - position < BSON_MIN_SIZE { + return Err(invalid_bson(py, "invalid array content")); + } + let value_length = + u32::from_le_bytes(data[position..position + 4].try_into().unwrap()) as usize; + if value_length < BSON_MIN_SIZE { + return Err(invalid_bson(py, "invalid message size")); + } + if value_length >= size - position { + return Err(invalid_bson(py, "invalid array content")); + } + buffer.write_bytes(&data[position..position + value_length]); + position += value_length; + } + if position != size - 1 { + return Err(invalid_bson(py, "bad object or element length")); + } + Ok(PyBytes::new(py, &buffer.into_vec()).into_any()) + }) +} + +#[pymodule(gil_used = true)] +fn _rbson(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_dict_to_bson, module)?)?; + module.add_function(wrap_pyfunction!(_bson_to_dict, module)?)?; + module.add_function(wrap_pyfunction!(_decode_all, module)?)?; + module.add_function(wrap_pyfunction!(_element_to_dict, module)?)?; + module.add_function(wrap_pyfunction!(_raw_to_dict, module)?)?; + module.add_function(wrap_pyfunction!(_array_of_documents_to_buffer, module)?)?; + Ok(()) +} diff --git a/rust/bson/src/options.rs b/rust/bson/src/options.rs new file mode 100644 index 0000000000..c1de0611f2 --- /dev/null +++ b/rust/bson/src/options.rs @@ -0,0 +1,200 @@ +//! Parsing of `bson.CodecOptions` (a `NamedTuple`) into a Rust-side struct. +//! +//! Mirrors `convert_codec_options` in `_cbsonmodule.c`: the options object is +//! consumed positionally with the field order +//! `(document_class, tz_aware, uuid_representation, +//! unicode_decode_error_handler, tzinfo, type_registry, datetime_conversion)`. + +use std::ffi::CString; + +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyDict, PyInt, PyString, PyTuple}; +use pyo3::PyTypeInfo; + +/// The BSON type marker of `RawBSONDocument` (`bson/codec_options.py`). +pub const RAW_BSON_DOCUMENT_MARKER: i64 = 101; + +// `DatetimeConversion` enum values (an int enum in bson/codec_options.py). +pub const DATETIME_MS: u8 = 3; +pub const DATETIME_CLAMP: u8 = 2; +pub const DATETIME_AUTO: u8 = 4; + +/// Parsed codec options, valid for the lifetime of the enclosing Python call. +pub struct CodecOptionsData<'py> { + /// The document class (e.g. `dict`, `SON`, `RawBSONDocument`). + pub document_class: Bound<'py, PyAny>, + pub tz_aware: bool, + pub uuid_representation: u8, + pub tzinfo: Option>, + pub datetime_conversion: u8, + /// True when `document_class` is a raw BSON document class. + pub is_raw_bson: bool, + /// True when `document_class` is exactly `dict`. + pub is_dict_class: bool, + /// Cached `type_registry._encoder_map` (a dict, possibly empty). + encoder_map: Bound<'py, PyAny>, + /// Cached `type_registry._fallback_encoder` (None when unset). + fallback_encoder: Bound<'py, PyAny>, + /// The original CodecOptions object (passed to raw document classes). + options_obj: Bound<'py, PyAny>, + /// The decoder map as a dict, resolved once (None when empty). + pub decoder_map_dict: Option>, + /// The UTF-8 decode error handler, resolved once (None = strict). + pub utf8_handler: Option, +} + +impl<'py> CodecOptionsData<'py> { + /// The original options object, passed to raw document constructors. + pub fn options_obj(&self, _py: Python<'_>) -> Bound<'py, PyAny> { + self.options_obj.clone() + } +} + +/// Reads the `_type_marker` attribute of an object. +/// +/// Mirrors `_type_marker()` in `_cbsonmodule.c`: the attribute is read +/// optionally and only accepted when it is an *exact* `int` (objects with +/// broken `__getattr__` implementations could return arbitrary types). +/// Returns `None` when the marker is absent or not an exact int; returns +/// `Err` only when the attribute read itself fails with a non-AttributeError. +pub fn read_type_marker<'py>(object: &Bound<'py, PyAny>) -> PyResult> { + let py = object.py(); + #[cfg(Py_3_13)] + let marker = { + // PyObject_GetOptionalAttr (the C's 3.13+ path) finds the attribute + // without materializing an AttributeError — the common case is a + // document_class without a marker (plain dict). + let intern = pyo3::intern!(py, "_type_marker"); + let mut raw: *mut pyo3::ffi::PyObject = std::ptr::null_mut(); + let found = unsafe { + pyo3::ffi::PyObject_GetOptionalAttr(object.as_ptr(), intern.as_ptr(), &mut raw) + }; + if found < 0 { + return Err(PyErr::fetch(py)); + } + if found == 0 { + None + } else { + Some(unsafe { Bound::from_owned_ptr_or_err(py, raw) }?) + } + }; + #[cfg(not(Py_3_13))] + let marker = match object.getattr(pyo3::intern!(py, "_type_marker")) { + Ok(marker) => Some(marker), + Err(err) => { + if err.is_instance_of::(py) { + None + } else { + return Err(err); + } + } + }; + if let Some(marker) = marker { + if marker.is_instance_of::() && marker.get_type().is(PyInt::type_object(py)) { + // Exact int (not a bool or int subclass). + if let Ok(value) = marker.extract::() { + return Ok(Some(value)); + } + } + } + Ok(None) +} + +impl<'py> CodecOptionsData<'py> { + /// Parses a `CodecOptions` (NamedTuple) instance positionally. + pub fn parse(options_obj: &Bound<'py, PyAny>) -> PyResult { + let py = options_obj.py(); + // Direct tuple access (mirrors PyArg_ParseTuple on the NamedTuple); + // an out-of-line sequence would add per-field method dispatch. + let seq = options_obj + .cast::() + .map_err(|_| pyo3::exceptions::PyTypeError::new_err("codec_options must be a tuple"))?; + if seq.len() < 7 { + return Err(pyo3::exceptions::PyIndexError::new_err( + "tuple index out of range", + )); + } + + let document_class = unsafe { seq.get_item_unchecked(0) }; + let tz_aware: bool = unsafe { seq.get_item_unchecked(1) }.extract()?; + let uuid_representation: u8 = unsafe { seq.get_item_unchecked(2) }.extract()?; + let unicode_decode_error_handler = unsafe { seq.get_item_unchecked(3) }; + let tzinfo_obj = unsafe { seq.get_item_unchecked(4) }; + let type_registry = unsafe { seq.get_item_unchecked(5) }; + let datetime_conversion: u8 = unsafe { seq.get_item_unchecked(6) }.extract()?; + + let is_raw_bson = matches!( + read_type_marker(&document_class)?, + Some(RAW_BSON_DOCUMENT_MARKER) + ); + let is_dict_class = document_class.is(PyDict::type_object(py)); + + let encoder_map = type_registry.getattr(pyo3::intern!(py, "_encoder_map"))?; + let fallback_encoder = type_registry.getattr(pyo3::intern!(py, "_fallback_encoder"))?; + let decoder_map = type_registry.getattr(pyo3::intern!(py, "_decoder_map"))?; + + let decoder_map_dict = { + let dict = decoder_map.cast::().ok().cloned(); + match dict { + Some(d) if d.is_empty() => None, + other => other, + } + }; + let utf8_handler = { + let handler = unicode_decode_error_handler + .cast::() + .ok() + .cloned(); + match handler { + Some(h) => h + .to_str() + .ok() + .filter(|s| *s != "strict") + .and_then(|s| CString::new(s.to_string()).ok()), + None => None, + } + }; + + Ok(CodecOptionsData { + document_class, + tz_aware, + uuid_representation, + tzinfo: if tzinfo_obj.is_none() { + None + } else { + Some(tzinfo_obj) + }, + datetime_conversion, + is_raw_bson, + is_dict_class, + encoder_map, + fallback_encoder, + options_obj: options_obj.clone(), + decoder_map_dict, + utf8_handler, + }) + } + + /// The encoder map as a dict-like object, or None when empty. + /// + /// Mirrors the C's `is_encoder_empty` check: an empty map is skipped + /// without a per-value lookup. + pub fn encoder_map(&self, _py: Python<'_>) -> Option> { + // TypeRegistry always builds _encoder_map as a dict. + let dict = self.encoder_map.cast::().ok()?.clone(); + if dict.is_empty() { + None + } else { + Some(dict) + } + } + + /// The fallback encoder, or None when unset. + pub fn fallback_encoder(&self, _py: Python<'_>) -> Option> { + if self.fallback_encoder.is_none() { + None + } else { + Some(self.fallback_encoder.clone()) + } + } +} diff --git a/rust/bson/src/types.rs b/rust/bson/src/types.rs new file mode 100644 index 0000000000..5ea7f17449 --- /dev/null +++ b/rust/bson/src/types.rs @@ -0,0 +1,109 @@ +//! Python type objects needed by the codec, imported once and cached. +//! +//! Mirrors the module-state caching in `_cbsonmodule.c` (which resolves +//! `bson.int64.Int64`, `re`'s pattern type, etc. at module init). + +use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; + +pub struct Types { + /// `bson.int64.Int64` + pub int64: Py, + /// `bson.objectid.ObjectId` + pub object_id: Py, + /// `bson.binary.Binary` + pub binary: Py, + /// `bson.code.Code` + pub code: Py, + /// `bson.dbref.DBRef` + pub dbref: Py, + /// `bson.regex.Regex` + pub regex: Py, + /// `bson.timestamp.Timestamp` + pub timestamp: Py, + /// `bson.decimal128.Decimal128` + pub decimal128: Py, + /// `bson.min_key.MinKey` + pub min_key: Py, + /// `bson.max_key.MaxKey` + pub max_key: Py, + /// `bson.datetime_ms.DatetimeMS` + pub datetime_ms: Py, + /// `uuid.UUID` + pub uuid: Py, + /// The type of `re.compile("")` (i.e. `re.Pattern`). + pub re_type: Py, + /// `collections.abc.Mapping` + pub mapping: Py, + /// `bson.tz_util.utc` (the canonical UTC FixedOffset). + pub utc: Py, + /// `bson.errors.InvalidDocument` + pub invalid_document: Py, + /// `bson.errors.InvalidBSON` + pub invalid_bson: Py, + /// `bson.datetime_ms._MIN_UTC_MS` + pub min_millis: i64, + /// `bson.datetime_ms._MAX_UTC_MS` + pub max_millis: i64, + /// `bson.datetime_ms._MIN_UTC` + pub min_datetime: Py, + /// `bson.datetime_ms._MAX_UTC` + pub max_datetime: Py, + /// `bson.codec_options._RAW_BSON_VIEW_THRESHOLD` + pub raw_bson_view_threshold: usize, +} + +fn import_type(py: Python<'_>, module: &str, attr: &str) -> PyResult> { + Ok(py.import(module)?.getattr(attr)?.unbind()) +} + +static TYPES: PyOnceLock = PyOnceLock::new(); + +impl Types { + /// Returns the cached type objects, importing them on first use. + pub fn get(py: Python<'_>) -> PyResult<&'static Types> { + TYPES.get_or_try_init(py, || { + let re_module = py.import("re")?; + let re_type: Py = re_module + .getattr("compile")? + .call(("", 0), None)? + .get_type() + .into_any() + .unbind(); + Ok(Types { + int64: import_type(py, "bson.int64", "Int64")?, + object_id: import_type(py, "bson.objectid", "ObjectId")?, + binary: import_type(py, "bson.binary", "Binary")?, + code: import_type(py, "bson.code", "Code")?, + dbref: import_type(py, "bson.dbref", "DBRef")?, + regex: import_type(py, "bson.regex", "Regex")?, + timestamp: import_type(py, "bson.timestamp", "Timestamp")?, + decimal128: import_type(py, "bson.decimal128", "Decimal128")?, + min_key: import_type(py, "bson.min_key", "MinKey")?, + max_key: import_type(py, "bson.max_key", "MaxKey")?, + datetime_ms: import_type(py, "bson.datetime_ms", "DatetimeMS")?, + uuid: import_type(py, "uuid", "UUID")?, + re_type, + mapping: import_type(py, "collections.abc", "Mapping")?, + utc: import_type(py, "bson.tz_util", "utc")?, + invalid_document: import_type(py, "bson.errors", "InvalidDocument")?, + invalid_bson: import_type(py, "bson.errors", "InvalidBSON")?, + min_millis: import_type(py, "bson.datetime_ms", "_MIN_UTC_MS")? + .bind(py) + .extract()?, + max_millis: import_type(py, "bson.datetime_ms", "_MAX_UTC_MS")? + .bind(py) + .extract()?, + min_datetime: import_type(py, "bson.datetime_ms", "_MIN_UTC")?, + max_datetime: import_type(py, "bson.datetime_ms", "_MAX_UTC")?, + raw_bson_view_threshold: import_type( + py, + "bson.codec_options", + "_RAW_BSON_VIEW_THRESHOLD", + )? + .bind(py) + .extract()?, + }) + }) + } +} diff --git a/rust/bson/tests/test_rbson.py b/rust/bson/tests/test_rbson.py new file mode 100644 index 0000000000..5930e037a2 --- /dev/null +++ b/rust/bson/tests/test_rbson.py @@ -0,0 +1,976 @@ +"""Differential tests for the Rust-backed BSON codec (_rbson). + +The C extension (``bson._cbson``) is the behavioral reference: wherever the +Rust implementation handles a value, its output must match the C extension's +byte-for-byte on encode and value-for-value (including exact types) on decode. + +These tests need no MongoDB server. Run from the repo root:: + + .venv/bin/python -m pytest rust/bson/tests -v + +after building the crate (from rust/bson: ``maturin develop --release``). +""" + +from __future__ import annotations + +import datetime +import os +import re +import sys +from pathlib import Path + +import pytest + +sys.path[0:0] = [str(Path(__file__).parent.parent.parent.parent)] + +import _rbson # noqa: E402 + +import bson as bson_module # noqa: E402 +from bson import ( # noqa: E402 + CodecOptions, + _cbson, +) +from bson.errors import InvalidBSON, InvalidDocument # noqa: E402 + +pytestmark = pytest.mark.default + +OPTIONS = CodecOptions() + + +def encode_c(doc, check_keys=False, opts=OPTIONS, top_level=True): + return _cbson._dict_to_bson(doc, check_keys, opts, top_level) + + +def encode_rust(doc, check_keys=False, opts=OPTIONS, top_level=True): + return _rbson._dict_to_bson(doc, check_keys, opts, top_level) + + +def decode_c(data, opts=OPTIONS): + return _cbson._bson_to_dict(data, opts) + + +def decode_rust(data, opts=OPTIONS): + return _rbson._bson_to_dict(data, opts) + + +def assert_same_encode(doc, check_keys=False, opts=OPTIONS): + expected = encode_c(doc, check_keys=check_keys, opts=opts) + actual = encode_rust(doc, check_keys=check_keys, opts=opts) + assert actual == expected, f"\nexpected: {expected!r}\nactual: {actual!r}" + + +def assert_same_decode(data, opts=OPTIONS): + expected = decode_c(data, opts) + actual = decode_rust(data, opts) + assert actual == expected, f"\nexpected: {expected!r}\nactual: {actual!r}" + for key in expected: + assert type(actual[key]) is type(expected[key]), ( + f"value under {key!r}: {type(actual[key])!r} != {type(expected[key])!r}" + ) + return actual + + +def assert_same_round_trip(doc, check_keys=False, opts=OPTIONS): + assert_same_encode(doc, check_keys=check_keys, opts=opts) + result = assert_same_decode(encode_rust(doc, check_keys=check_keys, opts=opts), opts) + assert result == doc + return result + + +# --- Scaffold: empty document ------------------------------------------------ + + +def test_empty_document_round_trip(): + encoded = encode_rust({}) + assert encoded == b"\x05\x00\x00\x00\x00" + assert decode_rust(encoded) == {} + assert_same_round_trip({}) + + +# --- Builtin scalars --------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", + [ + "hello", + "", + "áéíóú ☃", + True, + False, + None, + 0.0, + -0.5, + 1.5, + 1e300, + -2.2250738585072014e-308, + float("inf"), + float("-inf"), + b"\x00\x01\xff", + b"", + ], +) +def test_scalar_encode_matches_c(value): + assert_same_round_trip({"a": value}) + + +def test_nan_encode_matches_c(): + # NaN compares unequal to itself, so compare bytes. + assert_same_encode({"a": float("nan")}) + + +@pytest.mark.parametrize( + "value", + [0, 1, -1, 42, -(2**31), 2**31 - 1, -(2**31) - 1, 2**31, 2**63 - 1, -(2**63)], +) +def test_int_encode_matches_c(value): + assert_same_round_trip({"a": value}) + + +def test_int64_boundary_overflows(): + for value in (2**63, -(2**63) - 1): + for encoder in (encode_c, encode_rust): + with pytest.raises(OverflowError): + encoder({"a": value}) + + +# --- Containers -------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", + [ + [], + [1, 2, 3], + [1, True, 3.8, "world"], + [[1, 2], [3, [4, 5]]], + [[]], + [{"nested": [1, {"deep": "doc"}]}], + ], +) +def test_array_encode_matches_c(value): + assert_same_round_trip({"a": value}) + + +@pytest.mark.parametrize( + "value", + [ + ({},), + (1, "a", None), + ], +) +def test_tuple_encodes_like_c_and_decodes_as_list(value): + # Tuples encode as arrays; they decode back as lists (like the C ext). + assert_same_encode({"a": value}) + decoded = assert_same_decode(encode_rust({"a": value})) + assert decoded == {"a": list(value)} + + +@pytest.mark.parametrize( + "value", + [ + {}, + {"b": 1, "c": "two"}, + {"nested": {"deep": {"deeper": [1, 2, None]}}}, + {"unicode_key_☃": "áéíóú"}, + {"empty": {}}, + ], +) +def test_document_encode_matches_c(value): + assert_same_round_trip({"a": value}) + + +def test_subdocuments_and_arrays_are_containers(): + # Arrays are encoded as documents with numeric string keys. + encoded = encode_rust({"a": [1]}) + expected = b"\x14\x00\x00\x00\x04a\x00\x0c\x00\x00\x00\x100\x00\x01\x00\x00\x00\x00\x00" + assert encoded == expected + assert encoded == encode_c({"a": [1]}) + assert_same_round_trip({"a": [1]}) + + +# --- Top-level _id ordering -------------------------------------------------- + + +def test_top_level_id_is_encoded_first(): + assert_same_round_trip({"z": 1, "_id": 2}) + encoded = encode_rust({"z": 1, "_id": 2}) + assert encoded.index(b"_id") < encoded.index(b"z") + assert encoded == encode_c({"z": 1, "_id": 2}) + + +def test_non_top_level_id_keeps_document_order(): + encoded = encode_rust({"a": {"z": 1, "_id": 2}}) + assert encoded.index(b"z") < encoded.index(b"_id") + assert encoded == encode_c({"a": {"z": 1, "_id": 2}}) + + +def test_top_level_false_preserves_order(): + encoded = encode_rust({"z": 1, "_id": 2}, top_level=False) + assert encoded.index(b"z") < encoded.index(b"_id") + assert encoded == encode_c({"z": 1, "_id": 2}, top_level=False) + + +# --- Key validation ---------------------------------------------------------- + + +@pytest.mark.parametrize("key", ["$a", "a.b"]) +def test_check_keys_rejects_dollar_and_dot(key): + for encoder in (encode_c, encode_rust): + with pytest.raises(InvalidDocument): + encoder({key: 1}, check_keys=True) + # Without check_keys the same keys encode. + assert_same_encode({key: 1}) + + +def test_dollar_and_dot_allowed_without_check_keys(): + assert_same_round_trip({"$a": 1, "a.b": 2}) + + +def test_non_string_key_rejected(): + for encoder in (encode_c, encode_rust): + with pytest.raises(InvalidDocument): + encoder({1: "a"}) + + +def test_nul_byte_key_rejected(): + for encoder in (encode_c, encode_rust): + with pytest.raises(InvalidDocument, match="NULL"): + encoder({"a\x00b": 1}) + + +def test_non_mapping_top_level_rejected(): + for encoder in (encode_c, encode_rust): + with pytest.raises(TypeError): + encoder(42) # type: ignore[arg-type] + + +# --- Builtin subclasses ------------------------------------------------------ + + +class StrSubclass(str): + pass + + +class IntSubclass(int): + pass + + +class FloatSubclass(float): + pass + + +class BytesSubclass(bytes): + pass + + +class ListSubclass(list): + pass + + +class TupleSubclass(tuple): + pass + + +class DictSubclass(dict): + pass + + +@pytest.mark.parametrize( + "value", + [ + StrSubclass("sub"), + IntSubclass(42), + FloatSubclass(1.5), + BytesSubclass(b"bin"), + ListSubclass([1, 2]), + TupleSubclass((1, 2)), + DictSubclass({"a": 1}), + True, # exact bool stays bool + ], +) +def test_builtin_subclasses_encode_like_c(value): + assert_same_encode({"a": value}) + + +def test_str_subclass_with_lone_surrogate_raises(): + class BadStr(str): + pass + + for encoder in (encode_c, encode_rust): + with pytest.raises(UnicodeEncodeError): + encoder({"a": BadStr("\ud800")}) + + +# --- datetime ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", + [ + datetime.datetime(2026, 10, 20, 14, 30, 0), + datetime.datetime(1970, 1, 1), + datetime.datetime(1969, 12, 31, 23, 59, 59, 999000), + datetime.datetime(1, 1, 2), + datetime.datetime(9999, 12, 31, 23, 59, 59, 999000), + datetime.datetime(2026, 10, 20, 14, 30, 0, 123000), + datetime.datetime(2026, 10, 20, 14, 30, 0, 999999), + ], +) +def test_naive_datetime_encode_matches_c(value): + assert_same_encode({"a": value}) + + +def test_utc_aware_datetime_encode_matches_c(): + from datetime import timezone + + value = datetime.datetime(2026, 10, 20, 14, 30, 0, tzinfo=timezone.utc) + assert_same_encode({"a": value}) + + +# --- _type_marker classes (encode) -------------------------------------------- + +import uuid as uuid_module # noqa: E402 + +from bson import ( # noqa: E402 + Binary, + Code, + DatetimeMS, + DBRef, + Decimal128, + Int64, + MaxKey, + MinKey, + ObjectId, + Regex, + Timestamp, +) +from bson.binary import BinaryVectorDtype # noqa: E402 +from bson.codec_options import TypeRegistry # noqa: E402 +from bson.raw_bson import RawBSONDocument # noqa: E402 + + +def test_binary_subtypes(): + assert_same_encode({"a": Binary(b"ab", 0)}) + assert_same_encode({"a": Binary(b"ab", 2)}) + assert_same_encode({"a": Binary(b"ab", 128)}) + assert_same_encode({"a": Binary(b"ab", 254)}) + # Vectors (subtype 9) write their header bytes unchanged. + vector = Binary.from_vector([1, 2, 3], BinaryVectorDtype.INT8) + assert_same_encode({"a": vector}) + + +def test_objectid(): + assert_same_encode({"a": ObjectId(b"123456789012")}) + assert_same_encode({"a": ObjectId("67e0e10a3f2b1c4d5e6f7a8b")}) + + +def test_int64_marker(): + assert_same_encode({"a": Int64(7)}) + assert_same_encode({"a": Int64(-(2**31) - 1)}) + + # An Int64 subclass uses the marker too. + class MyInt64(Int64): + pass + + assert_same_encode({"a": MyInt64(5)}) + + +def test_code(): + assert_same_encode({"a": Code("x")}) + assert_same_encode({"a": Code("x", None)}) + assert_same_encode({"a": Code("x", {})}) + assert_same_encode({"a": Code("x", {"s": 1})}) + assert_same_encode({"a": Code("áé", {"nested": {"a": [1, "b"]}})}) + + +def test_dbref(): + assert_same_encode({"a": DBRef("coll", 5)}) + assert_same_encode({"a": DBRef("coll", 5, "db")}) + assert_same_encode({"a": DBRef("coll", 5, "db", extra=1)}) + + +def test_regex(): + assert_same_encode({"a": Regex("a*", "i")}) + assert_same_encode({"a": Regex(b"a*", 0)}) + assert_same_encode({"a": re.compile("a*", re.I)}) + assert_same_encode({"a": re.compile("")}) + + +def test_timestamp(): + assert_same_encode({"a": Timestamp(4, 13)}) + assert_same_encode({"a": Timestamp(0, 0)}) + + +def test_min_max_key_and_marker_only_class(): + assert_same_encode({"a": MinKey()}) + assert_same_encode({"a": MaxKey()}) + # A bare class with only the marker attribute encodes like MaxKey. + + class MyMaxKey: + _type_marker = 127 + + assert_same_encode({"a": MyMaxKey()}) + + +def test_datetime_ms(): + assert_same_encode({"a": DatetimeMS(12345)}) + assert_same_encode({"a": DatetimeMS(-1)}) + + +def test_decimal128(): + assert_same_encode({"a": Decimal128("1.5")}) + assert_same_encode({"a": Decimal128("-0.000")}) + + +def test_raw_bson_document_nested(): + raw = encode_c({"inner": "doc"}) + raw_doc = RawBSONDocument(raw) + assert_same_encode({"a": raw_doc}) + assert_same_encode({"outer": {"nested": raw_doc}}) + + +def test_raw_bson_document_top_level(): + raw = encode_c({"inner": "doc"}) + raw_doc = RawBSONDocument(raw) + # Top level: the raw bytes are returned as-is. + assert encode_rust(raw_doc) == raw + assert encode_c(raw_doc) == raw + + +def test_uuid_encodings(): + value = uuid_module.UUID("3fbee3e7-9c2e-4f47-a1aa-fc3b7b3d5c1e") + standard = CodecOptions(uuid_representation=4) + legacy = CodecOptions(uuid_representation=3) + java = CodecOptions(uuid_representation=5) + csharp = CodecOptions(uuid_representation=6) + assert_same_encode({"a": value}, opts=standard) + assert_same_encode({"a": value}, opts=legacy) + assert_same_encode({"a": value}, opts=java) + assert_same_encode({"a": value}, opts=csharp) + # UNSPECIFIED raises for bare UUIDs. + for encoder in (encode_c, encode_rust): + with pytest.raises(ValueError, match="cannot encode native"): + encoder({"a": value}) + + +def test_non_utc_aware_datetime(): + from datetime import timedelta, timezone + + value = datetime.datetime( + 2026, 10, 20, 14, 30, 0, tzinfo=timezone(timedelta(hours=5, minutes=30)) + ) + assert_same_encode({"a": value}) + + +def test_custom_type_registry(): + from bson.codec_options import TypeEncoder + + class Wrapped: + def __init__(self, n): + self.n = n + + # The encoder's type_codecs match by exact type. + class WrappedEncoder(TypeEncoder): + type_name = "wrapped" + + python_type = Wrapped + + def transform_python(self, value): + return {"n": value.n} + + opts = CodecOptions(type_registry=TypeRegistry([WrappedEncoder()])) + assert_same_encode({"a": Wrapped(3)}, opts=opts) + + +def test_fallback_encoder(): + def fallback(value): + return str(value) + + opts = CodecOptions(type_registry=TypeRegistry(fallback_encoder=fallback)) + + class Unknown: + def __str__(self): + return "unknown!" + + assert_same_encode({"a": Unknown()}, opts=opts) + # The fallback also rescues oversized ints. + assert_same_encode({"a": 2**64}, opts=opts) + + +def test_cannot_encode_unknown_type(): + class Unknown: + pass + + for encoder in (encode_c, encode_rust): + with pytest.raises(InvalidDocument, match="cannot encode object"): + encoder({"a": Unknown()}) + + +# --- decode of BSON classes and codec options (Task 4) ------------------------ + + +def test_decode_objectid(): + result = assert_same_round_trip({"a": ObjectId(b"123456789012")}) + assert isinstance(result["a"], ObjectId) + + +def test_decode_binary(): + for subtype in (1, 2, 128, 254): + result = assert_same_round_trip({"a": Binary(b"ab", subtype)}) + assert type(result["a"]) is Binary + assert result["a"].subtype == subtype + # Subtype 0 decodes to plain bytes. + result = assert_same_round_trip({"a": b"ab"}) + assert type(result["a"]) is bytes + # Vectors round trip as Binary subtype 9. + vector = Binary.from_vector([1, 2, 3], BinaryVectorDtype.INT8) + result = assert_same_round_trip({"a": vector}) + assert type(result["a"]) is Binary + assert result["a"].subtype == 9 + + +def test_decode_binary_subtype2_strips_prefix(): + data = encode_c({"a": Binary(b"ab", 2)}) + result = decode_rust(data) + assert type(result["a"]) is Binary + assert result["a"] == Binary(b"ab", 2) + + +def test_decode_uuid(): + value = uuid_module.UUID("3fbee3e7-9c2e-4f47-a1aa-fc3b7b3d5c1e") + # STANDARD representation decodes to UUID. + result = assert_same_round_trip({"a": value}, opts=CodecOptions(uuid_representation=4)) + assert isinstance(result["a"], uuid_module.UUID) + # PYTHON_LEGACY round trips to UUID too (wire subtype 3). + result = assert_same_round_trip({"a": value}, opts=CodecOptions(uuid_representation=3)) + assert isinstance(result["a"], uuid_module.UUID) + # UNSPECIFIED keeps Binary. + result = assert_same_round_trip({"a": Binary.from_uuid(value)}, opts=CodecOptions()) + assert type(result["a"]) is Binary + # Subtype/representation mismatches keep Binary: + # - wire subtype 4 with a non-standard representation configured; + doc_sub4 = b"\x19\x00\x00\x00\x05a\x00\x0c\x00\x00\x00\x04" + value.bytes[:12] + b"\x00" + result = assert_same_decode(doc_sub4, CodecOptions(uuid_representation=3)) + assert type(result["a"]) is Binary + assert result["a"].subtype == 4 + # - wire subtype 3 with the standard representation configured. + doc_sub3 = b"\x19\x00\x00\x00\x05a\x00\x0c\x00\x00\x00\x03" + value.bytes[:12] + b"\x00" + result = assert_same_decode(doc_sub3, CodecOptions(uuid_representation=4)) + assert type(result["a"]) is Binary + assert result["a"].subtype == 3 + + +def test_decode_int64_is_int64(): + result = assert_same_round_trip({"a": Int64(7)}) + assert type(result["a"]) is Int64 + + +def test_decode_code_and_scope(): + result = assert_same_round_trip({"a": Code("x")}) + assert type(result["a"]) is Code + result = assert_same_round_trip({"a": Code("x", {"s": 1})}) + assert type(result["a"]) is Code + assert result["a"].scope == {"s": 1} + + +def test_decode_dbref_hook(): + result = assert_same_decode(encode_c({"a": {"$ref": "c", "$id": 5}})) + assert isinstance(result["a"], DBRef) + assert result["a"].collection == "c" + assert result["a"].id == 5 + result = assert_same_decode(encode_c({"a": {"$ref": "c", "$id": 5, "$db": "d"}})) + assert isinstance(result["a"], DBRef) + assert result["a"].database == "d" + # Extra keys become kwargs. + result = assert_same_decode(encode_c({"a": {"$ref": "c", "$id": 5, "x": 1}})) + assert isinstance(result["a"], DBRef) + assert result["a"].x == 1 + # Non-DBRef-shaped docs stay dicts. + result = assert_same_round_trip({"a": {"$ref": 1, "$id": 5}}) + assert isinstance(result["a"], dict) + result = assert_same_round_trip({"a": {"$ref": "c"}}) + assert isinstance(result["a"], dict) + + +def test_decode_regex(): + result = assert_same_round_trip({"a": Regex("a*", "i")}) + assert type(result["a"]) is Regex + assert result["a"].pattern == "a*" + assert result["a"].flags == 2 + # Bytes patterns decode to str patterns (round-trip lossy, like the C). + assert_same_encode({"a": Regex(b"a*", 0)}) + result = assert_same_decode(encode_rust({"a": Regex(b"a*", 0)})) + assert type(result["a"]) is Regex + assert result["a"].pattern == "a*" + + +def test_decode_timestamp(): + result = assert_same_round_trip({"a": Timestamp(4, 13)}) + assert type(result["a"]) is Timestamp + assert result["a"].time == 4 + assert result["a"].inc == 13 + + +def test_decode_decimal128(): + result = assert_same_round_trip({"a": Decimal128("1.5")}) + assert type(result["a"]) is Decimal128 + assert result["a"] == Decimal128("1.5") + + +def test_decode_min_max_key(): + result = assert_same_round_trip({"a": MinKey()}) + assert type(result["a"]) is MinKey + result = assert_same_round_trip({"a": MaxKey()}) + assert type(result["a"]) is MaxKey + + +def test_decode_datetime_ms_mode(): + opts = CodecOptions(datetime_conversion=2) # DATETIME_CLAMP + result = assert_same_round_trip({"a": datetime.datetime(2026, 10, 20, 14, 30, 0)}, opts=opts) + assert isinstance(result["a"], datetime.datetime) + # Out-of-range values clamp to datetime.min/max. + result = decode_c(encode_c({"a": DatetimeMS(-(2**62))}, opts=opts), opts) + assert result["a"] == datetime.datetime.min + + +def test_decode_datetime_ms_type(): + opts = CodecOptions(datetime_conversion=3) # DATETIME_MS + result = assert_same_round_trip({"a": DatetimeMS(12345)}, opts=opts) + assert type(result["a"]) is DatetimeMS + + +def test_decode_datetime_auto_mode(): + opts = CodecOptions(datetime_conversion=4) # DATETIME_AUTO + result = assert_same_round_trip({"a": datetime.datetime(2026, 10, 20, 14, 30, 0)}, opts=opts) + assert isinstance(result["a"], datetime.datetime) + result = assert_same_round_trip({"a": DatetimeMS(-(2**62))}, opts=opts) + assert type(result["a"]) is DatetimeMS + + +def test_decode_tz_aware(): + from datetime import timedelta, timezone + + opts = CodecOptions(tz_aware=True) + data = encode_c({"a": datetime.datetime(2026, 10, 20, 14, 30, 0)}) + result = assert_same_decode(data, opts) + assert result["a"].utcoffset() == timedelta(0) + # With a tzinfo, times are converted. + tz = timezone(timedelta(hours=-5)) + opts = CodecOptions(tz_aware=True, tzinfo=tz) + result = assert_same_decode(data, opts) + assert result["a"].utcoffset() == timedelta(hours=-5) + + +def test_decode_unicode_error_handler(): + # Encode an invalid surrogate-free 4-byte sequence manually: decode + # invalid UTF-8 bytes with the strict handler raises InvalidBSON, + # "replace" substitutes U+FFFD. + import struct + + def make_string_doc(raw): + payload = struct.pack("= 4096 bytes decode from memoryview slices. + inner = encode_c({"pad": "x" * 5000}) + outer = encode_c({"nested": RawBSONDocument(inner)}) + opts = CodecOptions(document_class=RawBSONDocument) + result = assert_same_decode(outer, opts) + assert isinstance(result, RawBSONDocument) + nested = result["nested"] + assert isinstance(nested, RawBSONDocument) + assert isinstance(nested.raw, memoryview) + assert bytes(nested.raw) == inner + # Top-level documents pass their bytes buffer through unchanged. + result = assert_same_decode(inner, opts) + assert isinstance(result.raw, bytes) + + +def test_document_class_son(): + from bson.son import SON + + opts = CodecOptions(document_class=SON) + data = encode_c({"b": 1, "a": 2}) + result = assert_same_decode(data, opts) + assert type(result) is SON + assert list(result) == ["b", "a"] + + +def test_decode_invalid_fails(): + for bad in (b"", b"\x03", b"\x04\x00\x00\x00\x00", b"\x0e\x00\x00\x00\x02a\x00"): + for decoder in (decode_c, decode_rust): + with pytest.raises(InvalidBSON): + decoder(bad) + + +# --- recursion safety --------------------------------------------------------- + + +def test_cyclic_documents_raise_recursion_error(): + # Mirrors test_bson.py::test_bad_encode: the C survives cycles via + # CPython's recursion guard; the Rust codec must too. + evil_list: dict = {"a": []} + evil_list["a"].append(evil_list) + evil_dict: dict = {} + evil_dict["a"] = evil_dict + for evil_data in (evil_dict, evil_list): + for encoder in (encode_c, encode_rust): + with pytest.raises(RecursionError): + encoder(evil_data) + + +# --- review findings: raw-document framing and boundary parity ----------------- + + +def test_raw_bson_document_top_level_false(): + # I-1: a RawBSONDocument with top_level=False must pass the raw bytes + # through unchanged (the C checks the marker unconditionally). + raw = encode_c({"inner": "doc"}) + raw_doc = RawBSONDocument(raw) + assert encode_rust(raw_doc, top_level=False) == raw + assert encode_rust(raw_doc, top_level=False) == encode_c(raw_doc, top_level=False) + + +def test_code_with_raw_bson_document_scope(): + # I-2: a RawBSONDocument scope encodes as the raw bytes with no extra + # framing; the result must decode back to the same document. + raw = encode_c({"inner": "doc"}) + doc = {"a": Code("x", RawBSONDocument(raw))} + assert_same_encode(doc) + decoded = decode_c(encode_rust(doc)) + assert decoded["a"].scope == {"inner": "doc"} + + +def test_element_to_dict_position_boundary(): + # M-1: position at the end of the buffer raises InvalidBSON. + data = encode_c({}) + for position, obj_end in ((5, 5), (6, 5)): + for decoder in (_cbson._element_to_dict, _rbson._element_to_dict): + with pytest.raises(InvalidBSON): + decoder(data, position, obj_end, OPTIONS, False) + + +@pytest.mark.parametrize("bad", [b"\xff\xff\xff\xff\x00", b"\x00\x00\x00\x00\x00"]) +def test_decode_invalid_size_prefix_message(bad): + # M-2: negative/zero size prefixes raise "invalid message size". + for decoder in (decode_c, decode_rust): + with pytest.raises(InvalidBSON, match="invalid message size"): + decoder(bad) + + +def test_select_impl_rebinds(): + # M-8: the harness's own contract — select_impl() rebinds the entry + # points (a no-op rebind would be invisible to the C-parity suites). + from test.performance.bson_impl_select import select_impl + + if os.environ.get("BSON_IMPL", "c") == "rust": + assert select_impl() == "rust" + assert bson_module._dict_to_bson is _rbson._dict_to_bson + + assert bson_module.raw_bson._raw_to_dict is _rbson._raw_to_dict + + +def test_invalid_id_not_wrapped(): + # MIN-1: a failure while writing the _id element is not wrapped with + # the "Invalid document: " prefix (the C's handle_invalid_doc_error + # covers only the iteration loops). + for encoder in (encode_c, encode_rust): + with pytest.raises(InvalidDocument) as exc_info: + encoder({"_id": object()}) + assert "Invalid document:" not in str(exc_info.value) + # Iteration-loop failures ARE wrapped at the top level. + for encoder in (encode_c, encode_rust): + with pytest.raises(InvalidDocument) as exc_info: + encoder({"a": object()}) + assert "Invalid document:" in str(exc_info.value) + + +def test_non_contiguous_buffer_rejected(): + # MIN-4: a non-contiguous buffer exporter raises ValueError instead of + # silently decoding wrong bytes. + values = bytearray(b"\x0d\x00\x00\x00\x02a\x00\x01\x00\x00\x00x\x00\x00") + view = memoryview(values)[::-1] # reversed: not C-contiguous + with pytest.raises(ValueError, match="contiguous"): + _rbson._bson_to_dict(view, OPTIONS) + + +def test_invalid_utf8_field_name_wrapped(): + # MIN-2: an invalid-UTF-8 field name raises InvalidBSON (the C wraps + # the UnicodeDecodeError), for every entry point. + import struct + + def make(raw_name): + payload = raw_name + b"\x00" + b"\x10\x01\x00\x00\x00" + length = 4 + len(payload) + 1 + return struct.pack("= 2)[:2] + first, second = first.decode(), second.decode() + + data_one = encode_c({first: 1}) + data_two = encode_c({second: 2}) + data_both = encode_c({first: 1, second: 2}) + for _ in range(5): + # Each decode evicts the other name (and, for data_both, the doc's + # own first name is evicted by its second). + assert assert_same_decode(data_one) == {first: 1} + assert assert_same_decode(data_two) == {second: 2} + assert assert_same_decode(data_both) == {first: 1, second: 2} + + +def test_field_name_cache_long_key(): + # Keys over KEY_CACHE_MAX_KEY_LEN bypass the cache and still decode. + doc = {"l" * 65: 1, "short": 2} + data = encode_c(doc) + for _ in range(2): + assert assert_same_decode(data) == doc + + +def test_field_name_cache_not_shared_across_handlers(): + # The cache is only enabled for the strict handler: a successful decode + # under a non-strict handler must not poison later strict decodes of + # the same bytes (which must still raise InvalidBSON). + import struct + + def make(raw_name): + element = b"\x10" + raw_name + b"\x00" + b"\x01\x00\x00\x00" # int32 + length = 4 + len(element) + 1 + return struct.pack("`` at import time; those attributes are rebound as well. + +This module must be imported before the benchmarks execute (the +``test/performance/conftest.py`` imports it during pytest startup), but it +may be imported at any point before the first encode/decode call. +""" + +from __future__ import annotations + +import os + +_IMPLS = ("c", "rust") + +# The private entry points exposed by both _cbson and _rbson. +# Note: _element_to_dict/_raw_to_dict have the 5-argument C-style signature +# (the C extension's); bson/__init__.py's Python wrappers (which add a +# `view` argument) are dead code when the C extension is importable, so +# rebinding the module attribute to the C-style signature is safe. +_RBSON_ENTRY_POINTS = ( + "_dict_to_bson", + "_bson_to_dict", + "_decode_all", + "_element_to_dict", + "_raw_to_dict", + "_array_of_documents_to_buffer", +) + +# Modules that captured a private entry point with `from bson import ...` +# and hold a direct reference; (module, attribute) pairs to rebind. +_CAPTURING_MODULES = ( + ("bson.raw_bson", "_raw_to_dict"), + ("pymongo.message", "_dict_to_bson"), + ("pymongo.synchronous.change_stream", "_bson_to_dict"), + ("pymongo.asynchronous.change_stream", "_bson_to_dict"), + ("pymongo.synchronous.encryption", "_dict_to_bson"), + ("pymongo.asynchronous.encryption", "_dict_to_bson"), +) + + +def select_impl() -> str: + """Applies the BSON_IMPL selection and returns the active impl name.""" + impl = os.environ.get("BSON_IMPL", "c").strip().lower() + if impl not in _IMPLS: + raise ValueError(f"BSON_IMPL must be one of {_IMPLS}, got {impl!r}") + + if impl == "rust": + import _rbson + + import bson + + for name in _RBSON_ENTRY_POINTS: + setattr(bson, name, getattr(_rbson, name)) + for module_name, attr in _CAPTURING_MODULES: + try: + module = __import__(module_name, fromlist=[attr]) + except ImportError: + continue # optional dependency, not loaded + if getattr(module, attr, None) is not None: + setattr(module, attr, getattr(_rbson, attr)) + return impl + + +ACTIVE_IMPL = select_impl() diff --git a/test/performance/conftest.py b/test/performance/conftest.py new file mode 100644 index 0000000000..0907855894 --- /dev/null +++ b/test/performance/conftest.py @@ -0,0 +1,6 @@ +# Selects the BSON codec implementation (BSON_IMPL=c|rust) before the perf +# test modules run. See bson_impl_select.py. +from __future__ import annotations + +# Importing triggers codec selection (side effect; see bson_impl_select.py). +from test.performance import bson_impl_select diff --git a/test/performance/data/extended_bson/deep_bson.json b/test/performance/data/extended_bson/deep_bson.json new file mode 100644 index 0000000000..3cdf4ad4a9 --- /dev/null +++ b/test/performance/data/extended_bson/deep_bson.json @@ -0,0 +1 @@ +{"right": {"right": {"right": {"right": {"right": {"rightValue": "EIXQykWD", "leftValue": "VRVcZnIk"}, "left": {"rightValue": "oRbShlgw", "leftValue": "ojDbdtPA"}}, "left": {"right": {"rightValue": "CzHfouDM", "leftValue": "GQXwTsrM"}, "left": {"rightValue": "JEQOwZLQ", "leftValue": "LgWNfbhn"}}}, "left": {"right": {"right": {"rightValue": "khozCROb", "leftValue": "rTMhLQtj"}, "left": {"rightValue": "iJBmstFH", "leftValue": "LvtNlLgv"}}, "left": {"right": {"rightValue": "UYcmaFMh", "leftValue": "tuanWIAx"}, "left": {"rightValue": "WwptcESl", "leftValue": "ixOtlORA"}}}}, "left": {"right": {"right": {"right": {"rightValue": "dicfQcdX", "leftValue": "xYMiAHwv"}, "left": {"rightValue": "WbMXdChI", "leftValue": "oWLWuHVi"}}, "left": {"right": {"rightValue": "wRodFaZo", "leftValue": "aTcaLCnp"}, "left": {"rightValue": "ijcnduCw", "leftValue": "KFbXjDfX"}}}, "left": {"right": {"right": {"rightValue": "zzEcxhSZ", "leftValue": "bWSAvFzm"}, "left": {"rightValue": "mQjffcog", "leftValue": "hDbObPsH"}}, "left": {"right": {"rightValue": "LOxUOaEy", "leftValue": "DzfmhHAL"}, "left": {"rightValue": "oGVrSHkQ", "leftValue": "GGTRCnke"}}}}}, "left": {"right": {"right": {"right": {"right": {"rightValue": "PHHXnUgq", "leftValue": "xkZzQsMR"}, "left": {"rightValue": "bSpBOtXU", "leftValue": "lmRMwRFf"}}, "left": {"right": {"rightValue": "ReWinBoZ", "leftValue": "TOlxDiMX"}, "left": {"rightValue": "KuopgxIC", "leftValue": "iWKMSGCV"}}}, "left": {"right": {"right": {"rightValue": "pydwIHaq", "leftValue": "KMJNfqEW"}, "left": {"rightValue": "JtXAjgEq", "leftValue": "YmMYBRkX"}}, "left": {"right": {"rightValue": "wmdxiSSk", "leftValue": "sABNQDEX"}, "left": {"rightValue": "YnINlnHU", "leftValue": "qMdmLtPq"}}}}, "left": {"right": {"right": {"right": {"rightValue": "rHcvRSjR", "leftValue": "uNIIxBLx"}, "left": {"rightValue": "atKAMwgJ", "leftValue": "XWFVhrTV"}}, "left": {"right": {"rightValue": "HIQHjrLx", "leftValue": "YughZJOY"}, "left": {"rightValue": "qYeyvcvK", "leftValue": "mzHqtLHA"}}}, "left": {"right": {"right": {"rightValue": "BlifVvAC", "leftValue": "BvZTPsYB"}, "left": {"rightValue": "XPHqYTOu", "leftValue": "qUNMEFgS"}}, "left": {"right": {"rightValue": "zkxbAuKr", "leftValue": "YpAkENEL"}, "left": {"rightValue": "wuBwgsDI", "leftValue": "ONIZsGFD"}}}}}} diff --git a/test/performance/data/extended_bson/flat_bson.json b/test/performance/data/extended_bson/flat_bson.json new file mode 100644 index 0000000000..6b0fe3fc44 --- /dev/null +++ b/test/performance/data/extended_bson/flat_bson.json @@ -0,0 +1 @@ +{"pfZSRHnn":{"$numberDouble":"4837384839313709000"},"XeRkAyCq":{"$numberInt":"12"},"oRWMNJTE":"pbJrsZrkbKNGqDNwPPrxpeWVBRWgREWtfaZZCrLkBUmdAXdyaPcrMwhDzBXCpZtXjloYMgNgfIfEyrvq","hnVgYIQi":"RwMsHQcqgXvkmNpPIndeDusPOwRfluOOvXnTxwpiZvzkMWQaEQXjyfjoSMcqpeGmULGQlhWvHxmxFNqe","cxOHMeDJ":"RTMMHBNebriQZeGrExTKSJsMRbeKcIzaEqpMwOlVfgNreVIHDkLFBjDWJMubctZtjlozdPqZWyhillEw","MeUYSkPS":"sunCadRBarhKilkQEMRPVYQULkReALfyzmotkXUeDMcEiRvxgqyBiTQhmuoFXQxMotjJFjYTMrEyZFEM","IsorvnMR":true,"vvUeXASH":true,"zEgGhhZf":false,"SUWXijHT":{"$numberInt":"13"},"HicJbMpj":true,"ijwXMKqI":{"$numberInt":"30"},"RemSsnnR":false,"McpOBmaR":"fKGDZnXeXYTKnMITrDZrwVjNojiGCkNHtnWSxYwAiOtmIaVzDzQfifoorkYKjESnrauTyfClKjyczlPG","ddVenEkK":{"$numberInt":"41"},"CqCssWxW":"iXTkPsEpuEDVFxDIJsZVLuzUHCNfIiCQXYFjFBNHabJBENLnGhIsjUaLMWUaUqUiUiHRcPycHhHfrNdm","taoNnQYY":{"$numberDouble":"6412916066386615000"},"MXMxLVBk":{"$numberInt":"-83"},"yeTUgNrU":false,"vkEDWgmN":"HMobufHXBFQvVXzYiYqKpWMTqviPqPJtGEEVsebUiwVSbvYExShHJSVrwBEcovhlVhhAJKERrsmaipwn","TgSwBbgp":{"$numberDouble":"1912191031177068500"},"FDYGeSiR":{"$numberLong":"487277598556628711"},"xrzGnsEK":"UgqXkyWUceZzifQuDdMLiOtOwnPoolsdcjfsWkbQfWfOzgPpXEbefKjgRCxjCLgDRqpHypvyclTHDoxD","VVvwKVRG":{"$numberInt":"-81"},"TmUnYUrv":{"$numberLong":"7726451032369080086"},"OCsIhHxq":"VxBUZZSjnrXwRplwxRSJGwkFBFVpGvLFpWTFrmSJNJcgQQUFmlEYUcFnXOdJrOFQCaIlmBMzoihXdofI","pKjOghFa":true,"VCSKFCoE":{"$numberLong":"434237187180930726"},"yKfZnGKG":{"$numberDouble":"6813471041107657000"},"UKwbAKGw":{"$numberInt":"-32"},"iFFGfTXc":"hfiYFaVdPKSVRQlUccqrBdSxbXlRNDDaUCQsFmuXkUSJLPshjiFbjnmWlfVsiXXflAioxjJUFQhBWxTk","RwAVVKHM":"mgrXmDmkEyXOcafRFdcgQWPnDJERlFruGXatARowzYEoXCjeGzUmNFqnFAOxlTPRIuAEEEIFluDgqPHz","AtWNZJXa":{"$numberLong":"3508212275855267880"},"XxvXmHiQ":{"$numberDouble":"7674526623550603000"},"wmDLUkXt":{"$numberLong":"6862757006749435415"},"zSYvADVf":{"$numberLong":"4604914032121925983"},"VcCSqSmp":true,"jmglLvAS":"VRgSVSHCZypXPQVtNMQhBVncwgFlzwuDUGqtlEiPZekhblDroihrDBKGomJreRVvxEsbEntNMXMZWogD","ZmtEJFSO":"OSFQgOFQgWjYqkdWfrCsigaqKtrlvhMXRPzZkIMUoNQTyzksczZAxCUpugucyVtOsAnCZOXPWOsRrxiK","MqfkBZJF":{"$numberInt":"-74"},"doshbrpF":{"$numberLong":"6122240408803684263"},"jWaFvVAz":{"$numberDouble":"4715021990214510000"},"LVNIFCYm":false,"VplFgewF":{"$numberInt":"34"},"eRTIdIJR":"XbGmdGmGWEGFksccUGkSQDfTCpsgwkFncUsKPMGDoAnqsYcOmKhhFpNEbCcoGxcRvgMLXPXPtGyWTizz","zswQbWEI":{"$numberLong":"6244091767194659669"},"obcwwqWZ":{"$numberDouble":"1896750106775623700"},"vlSZaxCV":"OtIbYgSoPwVCZLkuJTaYGycvxNQLQISxykLwSPAfuqutkkOsBaIfZkCWBrFphwtREOcbguXIfyfolWFC","QobifTeZ":"DYnMUSYAFqdovehITtzUczCWFCsGIZohIDsXlcwXRUWRBdsbUZUhbYmRhQlNMOWpIaXBehDXJFbGcdOM","XXKbyIXG":{"$numberLong":"2072366708468029955"},"dNSuxlSU":"QnxFGQAgRGNTNDoexinDfHbjtzzBgHOuDgdjrtzGEIaqPsgyNrmiWYyKAUyXVYQGLkKEQDhYDwgrIFDQ","HrUPbFHD":{"$numberDouble":"325761556852724740"},"XEBqaXkB":{"$numberDouble":"4019504911325976600"},"xWpeGNjl":{"$numberLong":"7449003994039447076"},"aicoMxZq":false,"LngvlnTV":{"$numberDouble":"1970843135463072800"},"WYJdGJLu":"phaJTVYjUODRlTcpQmsfpOmPvbCeGxLvJiAUQFUMHoYxPzvbLxTyqvmXtRDHjZtYmGzxihwoWDhVoeZY","TRpgnInA":false,"JhImQOkw":true,"KMKBtlov":{"$numberLong":"860986396812465750"},"gySFZeAE":{"$numberDouble":"3015363760891914000"},"RPsQhgRD":"lMIstKuqhARucceUyKpUkdSXwIAOeOhfJJyJaifJYnQRHWauhGovMmgwCqkKwphGEQnHishVfNZprBJl","rmzUAgmk":"srpWYNgASRjyQzUTVMpyTqQdAwGrIduuOzolvbmKNcNRDksZAoJKKDwhyqSSkGvSmpMOgaeslHWzaoce","Ibrdrtgg":"RqbUIwdCfDmzWikDAgKfWuQfilTKSeEyPIxyBaIqZzjbTTxSPkbLbstWFhGcWSnAumROlwADbnmUCRjN","WmMOvgFc":false,"KnhgtAOJ":"IyuKFgZwvnAGBNCoemNXfwQjMfUlvWOYyuwmSOWMNxxVSgraCIKQPiRNuAiOkuErAHgDwIHEOhSxPWlf","qrJASGzU":false,"egxZaSsw":false,"cVjWCrlu":{"$numberLong":"2748531972151590948"},"OfTmCvDx":"IoaWOFbeGbFAgmTDdgRVQZBocrhtcEbUkPuIGpRpewYLHwhzcWtAVcqmgQcxcjTUjNUQLYqDyzdGWIOv","MNuWZMLP":true,"JrJzKiIx":"GkDzDJPJFYxRGoyzkQNPIpNPkwIXdaWQhIyTpYROuBxIZUZsImnCWfNmoTgYOuYODuxAeOqrKNVMgoTV","JzgaUWVG":{"$numberDouble":"5703784872945009000"},"WHSQVLKG":true,"SYtZkQbC":"atHWUNSMsYTkOhIVdOXfTfunoQQIuRWzdxPQDOIOagxKUOewsHOWHPZWnvtIWGtJYkbkYclyPGHjhjqo","nBKWWUWk":{"$numberInt":"41"},"Rbxpznea":{"$numberInt":"34"},"tIJEYSYM":"jGphtvhjtsZDtmacwidBiDdGLohKmbCYHanoFpxyqhIDaUGhxLdZwjyXSxmXClUycMWCNBVmCHNQFPjI","dHsYhRbV":"MpSSMTdVdDdUljTFtgNEQLzCUWneoIDMyZzOvlAczEdVhFNPokYsjnKPwTjcSPBJhoqZEiyElvTKoBZK","iwfbMdcv":{"$numberDouble":"8139611435673170000"},"dCLfYqqM":{"$numberInt":"23"},"ahFCBmqT":{"$numberLong":"7114997365939674931"},"TkXMwZlU":{"$numberLong":"3854513878611629146"},"LUPqMOHS":"SKNaboMrDhvPGbymhONwDrWPbhIgKmFmIAqaNTiSEHGxqCdyWPXClLhOmIxzJXyJwpduAvdCrebmyWVv","hwHOTmmW":{"$numberDouble":"5160167887703515000"},"kfvcFmKw":false,"bkuaZWRT":{"$numberLong":"977724985626757127"},"nKhiSITP":{"$numberInt":"59"},"CYhSCkWB":"uQZqBkRYVaYFubIssZEPDMoJwZNUXoHWdutFeuwnbzwlwskOxqJEhulXzctrrhuXAduwqpxYRXVzvEyF","vSLTtfDF":"XgPsrVgaBzFFzulnuoYLRQDLjFOfsOCbAHiajSHeGSYdMbxlWWIFaPxOsnNqYbLRLVEeWOCEKFYIlupp","CEtYKsdd":"ULtshGrugvoWEbLeIuwpnMjYAQfxthdAAMbXQLNKlkqTZlrczjoKrdFFTVWVsFiNsPLfrAOOMrjBKUpt","dVkWIafN":{"$numberDouble":"6092406401066639000"},"HQiykral":{"$numberDouble":"6997084378132750000"},"YDHWnEXV":"glMkrFRyjzyKXLrYKXVuCzbjygQgzGfPnZaGrYPJZJczkDtRKyQFiIxIMVFpfiegBAgWylCPKUnXHNyt","pacTBmxE":{"$numberDouble":"5976360937339736000"},"ddPdLgGg":{"$numberInt":"-11"},"xWUlYggc":"AUOduJdhWBfjzZAjbtISwIbBJDekDcGkFKpkuPuSEKvyJovAOsbnBUSkgJPCozLVFtqKWlcbkbLMDAUe","dpbwfSRb":"LYdHhOzEWLioOBePsqKpSTanmJsqGmJGmBhAGmgaUZWOUfppmzGbzBjYBnyqNhAszSAyKVIFspeBvvFF","AgYYbYPr":{"$numberInt":"72"},"wjfyueDC":true,"fEheUtop":"QXACxLaREdGLHJLPLMfCEwHgEDoMiwbTBWMsKXsPgtAQGUVHNnshCipscgmQDEFeCZijffBNQYBDflvF","pOMEwSod":{"$numberInt":"-96"},"BwTXiovJ":{"$numberInt":"-23"},"TDUzNJiH":"RZZTBkdVIIydZfwHaVrFJGsIUMIObjlUHiyCIZFbjRHrhgyCIXJQNROinoekjzrStNhmVONXpFTuhZCu","KyxOoCqS":{"$numberInt":"-81"},"zMCFzcWY":"cptNXXHxnUyOVkDCHgaFBvzTrXkyngxRpUhorRpZTwetLlbemqxvYDaIkzPVPdQhLsyokFFrsCpsIcQF","FpduyhQP":true,"JXMyYkfb":{"$numberDouble":"3513302241369264000"},"yXSBbPeT":"lYVEceJTwIsxYgeHfyhPyUHilJBbFbYbbmptwtzKvXhITbtiicRyRyaxtaLmXqeHTgDsIHHNCZsDewWM","wjAWaOog":"oVhUaqzlBGrSveZHuRAGDENonbZFPQWWcCMQVnIOnBTBsYfetnRUfIkgqrMTQaCtVRMHFeLvjyoeAojc","cepcgozk":"HWWMoqFPswvQGTohBvAOnpfsjOgPmvuWfVODrpySXWabfjLQcAJfScmKBZeQgGNDhhHIQcWDYlFdTPOu","DQBQcQFj":{"$numberLong":"8199992792049175759"},"sYtnozSc":{"$numberLong":"7484353617351081136"},"BBqZInWV":"AzsVuMwTaJDIRJPdOoCtSchUgUSwaCIiDpsXVpbGxxHeYpNsCSKtiHJlWAAVFlFEAhVpJeWXpiRXlhhd","dtywOLeD":"LyYVrOxlWLyBfldYXiwvEKzkDxnxVoeKGZrvDybrJkuEyUAfmaxIjLENDWMyZlDrsWFEOGrriEtDQCLg","WoFGfdvb":"QKphEuwokwrAKuiBoyUQaMRRYHETLBbVIvIjYudcCiFwwwngoDmjqGuHAnQGFNuPPmzNKsuRcpMkKDYR","zDzSGNnW":true,"ReOZakjB":{"$numberDouble":"66287689751350270"},"uMDWqLMf":"ThdZGzzcvZOfAAZuYxcxhmbyOELCwOWjzaRPMIfkxsEtmQKHLAnBNgNouVFzcmMnxSqWKvHftNUdFGry","GiAHzFII":{"$numberInt":"69"},"UtbwOKLt":{"$numberLong":"2280370138792319183"},"DJsnHZIC":{"$numberInt":"-40"},"SqNvlUZF":{"$numberLong":"5994784905659127787"},"PvfnpsMV":{"$numberInt":"-65"},"UpdMADoN":true,"jbUymqiB":"RstgOATQyfadznEncFuQxWiXZDnBlNSVTnZyyDukKwUtfwFAIslMLeZMReWmOupCOShgfpsLjvdQsTrH","qHzOMXeT":{"$numberInt":"-56"},"XGxlHrXf":true,"sGWJTAcT":false,"VtzeOlCT":"YCXdGpHeDrPjQllqOZbXdGDzIjFCDBIbqUwoVAUkcdfuPvMLsgYFJCQfDVHxxIbToFRJANYYfZGCOpni","mlfZVfVT":{"$numberDouble":"1362707589677182000"},"HVHyetUM":{"$numberLong":"2679099034338649685"},"gErhgZTh":"hbAkaEVuyVtmEMjRmvnKugRdszFExBLKpFeSczaRQBvqAWLqBNKRPisGtlCbgtbcpFMrvIkQwycqzQGj","omnwvBbA":{"$numberInt":"-44"},"AgSNVyBb":{"$numberLong":"5022992695992337622"},"CDIGOuIZ":{"$numberDouble":"4277641814126091300"},"CFujXoob":{"$numberInt":"-92"},"reiKnuza":{"$numberLong":"6584276617939036140"},"PjKiuWnQ":{"$numberDouble":"5348924571675368000"},"pQyCJaEd":{"$numberDouble":"4291392859207246000"},"xZOksssj":{"$numberLong":"6571311739487813793"},"AMQrGQmu":{"$numberLong":"8087840999310375327"},"HQeCoswW":{"$numberDouble":"6409675101451264000"},"pPtPsgRl":"XVgpEfMdrVZdLjSeJbYCLQaXfYDqkGpjUrEwVLNyRvDLIxonippcArEpnTMdooUauoGmguuGNhxisdFq","_id":{"$oid":"568176370279243c4c57a495"}} \ No newline at end of file diff --git a/test/performance/data/extended_bson/full_bson.json b/test/performance/data/extended_bson/full_bson.json new file mode 100644 index 0000000000..42ba9e0f6a --- /dev/null +++ b/test/performance/data/extended_bson/full_bson.json @@ -0,0 +1 @@ +{"KpnXZaDQ":{"$numberInt":"94"},"BOQAeydE":{"$binary":{"base64":"RWNVVkhUUmVsWEhzcnhxV25WdnVQTWFERUJMRFRrYmlOVUFGYmh3QWZzRGtTRkVHT1lrTWFKR2twUUFIZVVGWkJ1Y1RlYlpNTGF2VG51Vk8=","subType":"00"}},"kVDldkCH":{"$numberLong":"7791511468172299508"},"olUSRZtj":{},"ayrwiTMT":{"$maxKey":1},"nyYyaFrV":{"$numberInt":"8"},"zlVZQePF":true,"iRVlyXVm":{"$numberLong":"2167305701202279400"},"lyWwkZGg":{"$code":"ynoqoVfRjpUAZooFnnxIjIwFuYZtGrYKmntfyvBQOPsLMGcJGsaFeTvcTweCJIGGtFnFOnYfezldSuSy","$scope":{}},"DfzEYsYW":false,"VFKRjaPW":{"$minKey":1},"pDeYcUIu":{"$date":{"$numberLong":"776822400000"}},"kqKTGXUm":{"$binary":{"base64":"c2VhUHJCUEZwZ3lSekhDVWhMQ1VSdExiTENReGx4SnBzaXhPanlreFluWnhjQXhGYWlFS0xhaFJZVGZhaktVdnREeUFxZ2hIZlpDZWpoTk0=","subType":"00"}},"iNbJUZuj":{"$code":"QEUjFSxUAHTQGjJRGuawHllxxbITZGkBplrnbHyjiXVhZEnFmauehFkYOaIaYPbtyAsBMdRyonmwrflg","$scope":{"fKyfBXJY":{"$numberInt":"-39"},"UflrpcEy":{"$numberInt":"-99"},"MFzjzCYI":{"$numberInt":"-72"},"eTohHXcM":{"$numberInt":"56"},"LcgVuFFV":{"$numberInt":"-68"}}},"buEAkIge":{"$date":{"$numberLong":"1449360000000"}},"NAKOhrML":{"$numberInt":"-30"},"CAWdqfmI":{"$numberLong":"5403474030514800337"},"hAcOFtfN":{"$binary":{"base64":"Y1RHZ01abE9oaXd2ZFJSZUxLaFliS2t6Q2xsVFNucHBXb1hTSFZSSVJOSUd6amtSTlNmb05jRnZrWWxoQ2pVa0lDWVZ6ZWh5ZmZqTFNXYlc=","subType":"00"}},"qPDeTZWq":{"$numberLong":"9055278099777449039"},"ARdpUnvr":{"$code":"UubGxpjznChawiBpCJjWgjYaHAjdqHFbOBQyDkpBtMMtyZEIetHzpZshLLNChAQObxpMHdqsLkNoyFtl","$scope":{"oGlfmjDq":{"$numberInt":"79"},"SJqmyxSJ":{"$numberInt":"90"},"KGCCxetX":{"$numberInt":"-84"},"OMDIwkuc":{"$numberInt":"-81"},"mJtyLYxY":{"$numberInt":"66"},"mYPcOozX":{"$numberInt":"44"},"dsIvQCvi":{"$numberInt":"52"}}},"geaenijV":{"$timestamp":{"t":1,"i":1027123200}},"EuKkhmHw":{"$minKey":1},"kcVXzXHn":{"$code":"rcFbGNwyjskyGgevDrFrsDOstYxGvADYxnEuRepZGlnGUHtyPBoDsvyEHQzfwNLCSiaElUWgQmBRUWyE","$scope":{}},"dSMWIFLD":{"$numberLong":"7306035949033000157"},"UuDsAOGk":{"$code":"umFRKcVcQPMdvBGLMEhgNIiDjlxBzzeqlGJeJHhTUPCDeCxnnCMGkxbKnSxstvURPRlRoGMYMejQNULY","$scope":{}},"KUVCyPFv":"oQqKPHpzjHknSfdFdEymIJOZxYOQotZKAFlanBTLUBUFCPCpgXEdqXoNVVReXxJkzCpwLQqNVKvPGJJW","kFaSNVoS":{"$maxKey":1},"SQIAGqNE":{"$numberInt":"-58"},"MsFvjFIM":{"$code":"jYbFSnAjtwHgLOKGmENJthmsPIKxbQIulkOVicMfFRnxmkIkIPOcZyPyhMFrweORAcDwwJpWuSwxYqFv","$scope":{}},"ZZbaMgQp":{"$regularExpression":{"pattern":"uWxyRVYi","options":""}},"GqcHZeLf":{"$numberLong":"8240785954293476870"},"xCNVQDhK":{"$timestamp":{"t":1,"i":1438819200}},"jMeFYftg":{"$numberLong":"455743854322593800"},"yltfrVvf":{"$code":"WWoPrkbAWYFjTUsIgIpaCkPcymjMGotwFvITafFXXzvrAYSoGcfDkdlIIGYKqejFJwPJElNrShMdgJOZ","$scope":{"TKLtwaMQ":{"$numberInt":"-41"},"GniljOvp":{"$numberInt":"-74"},"PMKRGIPz":{"$numberInt":"-98"}}},"NzsNfcyY":[{"$numberInt":"5"},{"$numberInt":"2"},{"$numberInt":"5"},{"$numberInt":"10"},{"$numberInt":"6"},{"$numberInt":"8"},{"$numberInt":"3"}],"hhVNsfZZ":{},"OagyDZLm":true,"VIJGlGfX":[{"$numberInt":"8"},{"$numberInt":"2"},{"$numberInt":"9"},{"$numberInt":"8"}],"RBrgnptQ":true,"ebfJeJwS":"RCFBSqorwgFNcBwCWsgxjnSvWQQTUeUcTiDliSmuEldTlhzBFuLyysIKwaiSwZCdCnqRyiRfFNnKXVXE","LQygaTou":{"$code":"pVShLZkivMkARMdSFjHSaBcZPreJuYBsSpxIzQSVteWGQSXfFesEmCYzZyHhjpSLuuBEcEefRdyphvWw","$scope":{"Pnilqgzk":{"$numberInt":"43"}}},"uNvbuffj":{"$date":{"$numberLong":"1229817600000"}},"SHvLWKjg":{"$binary":{"base64":"SE51VlZGWXdBYkpqaVhIRlBJR3hDd2JUb2plVnBrQWZIZUN1U2Rab2ZudVZPZmpKR01QaFpxTUdPWHdiRnpibGx1c0ZnY0NscFpZRmdqb0Q=","subType":"00"}},"wKCwNLSh":{"$minKey":1},"giVmVwzU":{"$code":"JSgaICRscEgxNZjwPeEocTpfhdMNhAqbmOzwtuRMdhyTxwAKwLdplIJjWXzZYxIeLlofqqQPfTxJmoZX","$scope":{}},"BGAqpNkE":{"$minKey":1},"ZmCYgzpS":{"$timestamp":{"t":923788800,"i":1}},"PGiZAWYN":{"$date":{"$numberLong":"1270339200000"}},"wShiBppY":true,"tuTaYuPG":"aLfBnCaptjatvIvMUHkijCexLohaUYmmugMsQpyedSBfAFgvSktBNYTGBmmLUIUGsmhQCCSerSEHRMpZ","mUXXRWFX":false,"xoLWHvAD":{"$code":"VcWVKVTUfFgthzjgmctPeijKmNDrlOWddmBcMGulYjhiWyRpBjlUjJjTBXCbWAyYNZUGiqSlMjrZShPZ","$scope":{}},"WqNFzMgH":false,"JQbxXYUE":"NtsOrOgfmcvAXehXybWnzADxfildzokfBvdPWmvEShmKsNGuZkZmdUbuUmGLWuRwvkIksIGAVJtHeWaK","AZbAtRCC":{"$code":"AUKJNhEvuziRiApNRrScrKlADiLovaxVLOxyGodLKWuQJRwMMwOnqLwTLVDglnKBYOLyOvoYyRwEJCKH","$scope":{"IJpzzYpT":{"$numberInt":"52"},"CtVYmtxF":{"$numberInt":"-75"},"ELcEqqir":{"$numberInt":"-83"},"zMJPPwpk":{"$numberInt":"-30"}}},"JlhVrQmD":"TdJBbKVLawqzBqmqzuRpVVetrdgAbyHNMCGWuCKMUPJJQxDVpBuBjLhkRjilTxYmdkYNkflsBRdCBGHp","vSvdWAnJ":{"$binary":{"base64":"R1BaektHUlhZZ3J4S1BYUWVFTGd4Y2lkQVNNblRzeHNObmJ4Rm1hbExVRUNGd2NydVR6VVJCelpNdllIRHhUUk1NVEhWd21wZndOcHZza0w=","subType":"00"}},"JEVgGziE":{"$numberLong":"2044627925402213400"},"LCHLPrcd":{"$maxKey":1},"fWBoraHq":{"$timestamp":{"t":1,"i":984268800}},"bxrxTsho":[{"$numberInt":"8"},{"$numberInt":"10"},{"$numberInt":"10"},{"$numberInt":"2"},{"$numberInt":"9"},{"$numberInt":"5"},{"$numberInt":"2"},{"$numberInt":"1"},{"$numberInt":"7"},{"$numberInt":"7"}],"uhmyCSEv":{"$maxKey":1},"rclBQefx":{"$numberLong":"5827556087171336561"},"iEoENLRz":{},"yPKwWWxb":false,"fvLXcTQB":false,"DQuSYbZR":{"$binary":{"base64":"anhGWHB5d05uT0JLbmhEc05ncWdNaVJMTVlCSmJSbXZ0a3RWZnh1cnZLY2JaS3NjQlB2RHV6WVhFQnJ3eWZ0eEpIdGF0eXduYUtJbWlhc3U=","subType":"00"}},"uSbYKmdX":{"$maxKey":1},"nLsRKLJQ":"HPzjoSuhGkNSwsULyFwEIlRZlGEkxutczLMBOeeOvaWCAZRKZoTUFpcLNHQBGCVAskBBJGRvBLyRkRXn","kQhDjXIO":false,"duzmrLJI":{"$code":"tyYTwBCQgIUaLlAFjjJsrDdCczsyCJXvMDWdfnArhrEQnrBxxXpmBIxukADpKExQuXKpPSnBdWmEgXoV","$scope":{"EXKjeucO":{"$numberInt":"49"},"lsZZCuLP":{"$numberInt":"-43"},"giussUAr":{"$numberInt":"11"},"asWnYFAn":{"$numberInt":"-5"},"KsNFIwWL":{"$numberInt":"-94"},"avUNuoZK":{"$numberInt":"-68"},"ursJhljA":{"$numberInt":"-18"}}},"zqpnKCOL":{"$minKey":1},"SQOAGVaT":{"$date":{"$numberLong":"905817600000"}},"WboSdRaB":true,"bRaWfHwf":{"$numberInt":"-70"},"QdkSZoFx":{"$minKey":1},"EAYSerbF":{"$numberLong":"941789166999923700"},"IsYFQBkP":{"$maxKey":1},"BZogAEUM":{"$regularExpression":{"pattern":"luJBIsSO","options":""}},"duTaTKGF":{"$timestamp":{"t":1,"i":1252972800}},"IJOIvDcw":{"$date":{"$numberLong":"775958400000"}},"TtYjxGJH":{"$numberLong":"8268697854246446000"},"caPGDEGj":{},"dNVHpBIF":[{"$numberInt":"7"},{"$numberInt":"6"},{"$numberInt":"2"},{"$numberInt":"7"}],"hUhWCZbY":[{"$numberInt":"5"},{"$numberInt":"6"},{"$numberInt":"8"},{"$numberInt":"5"},{"$numberInt":"9"},{"$numberInt":"1"},{"$numberInt":"1"}],"hjSPiBiC":true,"UwZlNrHv":[{"$numberInt":"7"},{"$numberInt":"3"},{"$numberInt":"3"},{"$numberInt":"5"},{"$numberInt":"9"},{"$numberInt":"7"},{"$numberInt":"5"},{"$numberInt":"4"}],"juiYRtal":{"$numberLong":"6906562979624261000"},"RFhPJzgh":{"$numberInt":"-7"},"_id":{"$oid":"568176370279243c4c57a496"},"LNpbbRfA":{"$timestamp":{"t":808444800,"i":1}}} diff --git a/tools/compare_bson_perf.py b/tools/compare_bson_perf.py new file mode 100644 index 0000000000..601cc18ddb --- /dev/null +++ b/tools/compare_bson_perf.py @@ -0,0 +1,58 @@ +"""Merges two benchmark results.json files into a side-by-side comparison. + +Usage:: + + python tools/compare_bson_perf.py results_c.json results_rust.json + +Each input is the OUTPUT_FILE produced by ``test/performance/perf_test.py`` +(a list of ``{"info": {"test_name": ...}, "metrics": [...]}`` entries). +The comparison is printed as a table and written to stdout in markdown. +""" + +from __future__ import annotations + +import json +import sys + + +def load(path): + with open(path) as fid: + return json.load(fid) + + +def metrics_by_name(results): + """Maps test name → megabytes_per_sec (one entry per test today).""" + metrics = {} + for entry in results: + name = entry["info"]["test_name"] + for metric in entry["metrics"]: + if metric["name"] == "megabytes_per_sec": + metrics[name] = metric["value"] + return metrics + + +def main() -> int: + if len(sys.argv) != 3: + print(__doc__) + return 2 + baseline = metrics_by_name(load(sys.argv[1])) + challenger = metrics_by_name(load(sys.argv[2])) + + print(f"| {'Benchmark':<24} | {'C MB/s':>10} | {'Rust MB/s':>10} | {'Rust/C':>7} |") + print(f"| {'-' * 24} | {'-' * 10} | {'-' * 10} | {'-' * 7} |") + for name in sorted(baseline): + base = baseline[name] + challenger_value = challenger.get(name) + if challenger_value is None: + print(f"| {name:<24} | {base:>10.1f} | {'n/a':>10} | {'n/a':>7} |") + continue + ratio = "n/a" if base == 0 else f"{challenger_value / base:.2f}x" + print(f"| {name:<24} | {base:>10.1f} | {challenger_value:>10.1f} | {ratio:>7} |") + for name in sorted(challenger): + if name not in baseline: + print(f"| {name:<24} | {'n/a':>10} | {challenger[name]:>10.1f} | {'n/a':>7} |") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 2d51b27289e736b07b17ca3d5e1129486cf76f47 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 2 Nov 2026 08:58:45 -0600 Subject: [PATCH 2/8] update comment --- .evergreen/scripts/bson_codec_compare.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.evergreen/scripts/bson_codec_compare.sh b/.evergreen/scripts/bson_codec_compare.sh index c9b5cff77a..741ccaf056 100755 --- a/.evergreen/scripts/bson_codec_compare.sh +++ b/.evergreen/scripts/bson_codec_compare.sh @@ -133,10 +133,9 @@ rm results_c_a.json results_c_b.json results_rust_a.json results_rust_b.json # Print the comparison (also uploaded as a task artifact). .venv/bin/python tools/compare_bson_perf.py results_c.json results_rust.json | tee bson_comparison.md -# Profile DeepDecoding — the one benchmark still below the parity bar on the -# perf host (~0.90x Rust/C, stable across runs) — once per codec, for -# hotspot data to close the remaining gap. Entirely non-fatal: profiling is -# best-effort and the benchmark numbers never depend on it. +# Profile DeepDecoding once per codec for decode hotspot data. Entirely +# non-fatal: profiling is best-effort and the benchmark numbers never +# depend on it. # The rhel90 perf image does not ship perf; try the platform package manager # with passwordless sudo (the established pattern on Evergreen hosts, e.g. # setup-system.sh) before skipping. From cf89930604eb16ad1a3269f36b66cb83b24107c0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 2 Nov 2026 10:17:48 -0600 Subject: [PATCH 3/8] address review: buffer size cap, buffer soundness, C parity, CI fixes --- .evergreen/generated_configs/variants.yml | 2 +- .evergreen/scripts/bson_codec_compare.sh | 13 +- .evergreen/scripts/generate_config.py | 8 +- .evergreen/scripts/generate_config_utils.py | 8 +- rust/bson/Cargo.toml | 9 +- rust/bson/src/buffer.rs | 122 +++++++++-- rust/bson/src/decode.rs | 6 +- rust/bson/src/encode.rs | 224 +++++++++++--------- rust/bson/src/lib.rs | 55 +++-- rust/bson/tests/test_rbson.py | 111 ++++++++++ 10 files changed, 406 insertions(+), 152 deletions(-) diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index d1d1051540..26ff886bd9 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -498,7 +498,7 @@ buildvariants: - name: .perf display_name: Performance Benchmarks run_on: - - ubuntu2204-small + - rhel90-dbx-perf-large batchtime: 1440 # Pyopenssl tests diff --git a/.evergreen/scripts/bson_codec_compare.sh b/.evergreen/scripts/bson_codec_compare.sh index 741ccaf056..729b63b205 100755 --- a/.evergreen/scripts/bson_codec_compare.sh +++ b/.evergreen/scripts/bson_codec_compare.sh @@ -63,7 +63,7 @@ uv pip install maturin # the pip-built C extension only gets baseline codegen. Set via RUSTFLAGS — # cargo only discovers a crate-local .cargo/config.toml from the invocation # directory upward, which maturin (run from the checkout root) would miss. -export RUSTFLAGS="${RUSTFLAGS:-}-C target-cpu=native" +export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C target-cpu=native" .venv/bin/maturin develop --release -m rust/bson/Cargo.toml # Run the benchmarks with the venv's interpreter directly: no implicit @@ -182,9 +182,16 @@ elif command -v valgrind > /dev/null 2>&1; then # Rebuild for the profiled run on x86-64-v3 (AVX2, VEX-encoded, which # valgrind handles — the C extension is baseline x86-64 and profiles # fine). Nothing after this block uses the codec. + # The rebuild is part of the best-effort profiling contract: under + # set -e a failure here must not fail the task (all comparison results + # are already written) — the task still needs to reach its + # artifact-upload command, so treat it as a skipped profile. echo "Rebuilding the Rust codec without AVX-512 for valgrind..." - RUSTFLAGS="-C target-cpu=x86-64-v3" .venv/bin/maturin develop --release \ - -m rust/bson/Cargo.toml + if ! RUSTFLAGS="-C target-cpu=x86-64-v3" .venv/bin/maturin develop --release \ + -m rust/bson/Cargo.toml; then + echo "codec rebuild for valgrind failed; skipping the DeepDecoding profiles" >&2 + exit 0 + fi for impl in c rust; do echo "Profiling DeepDecoding with callgrind (BSON_IMPL=$impl)..." FASTBENCH=1 BSON_IMPL="$impl" OUTPUT_FILE="results_profile_${impl}.json" \ diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 90c8580aa1..4b08e23274 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -582,8 +582,12 @@ def create_perf_variants(): def create_bson_codec_compare_variants(): # Patch-only variant (no batchtime): compares the C and Rust-backed - # BSON codecs on the perf host. - host = HOSTS["perf"] + # BSON codecs. It is a self-contained C-vs-Rust comparison, so it does + # not need the perf host's historical comparability — unlike the .perf + # suite, whose shared HOSTS["perf"] assignment must not move. Ubuntu is + # selected here for the profiling runs: the image ships valgrind, and + # perf installs there (the perf host can do neither). + host = HOSTS["ubuntu22"] return [create_variant(["bson-codec-compare"], "BSON Codec Compare", host=host)] diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index cd2a677a63..0ba291ab5b 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -70,13 +70,7 @@ class Host: HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64", dict()) HOSTS["ubuntu22"] = Host("ubuntu22", "ubuntu2204-small", "Ubuntu-22", dict()) HOSTS["ubuntu24"] = Host("ubuntu24", "ubuntu2404-small", "Ubuntu-24", dict()) -# TEMPORARY (debug): flipped to ubuntu2204-small for the DeepDecoding -# callgrind profile run — rhel90-dbx-perf-large cannot install perf and has -# no valgrind; the ubuntu2204 image ships valgrind preinstalled. REVERT this -# line (back to "rhel90-dbx-perf-large") after the profile run: the timing -# bar belongs on the perf host. The patch build selects only -# bson-codec-compare, so no other task is affected in the patch. -HOSTS["perf"] = Host("perf", "ubuntu2204-small", "", dict()) +HOSTS["perf"] = Host("perf", "rhel90-dbx-perf-large", "", dict()) HOSTS["debian11"] = Host("debian11", "debian11-small", "Debian11", dict()) DEFAULT_HOST = HOSTS["rhel8"] diff --git a/rust/bson/Cargo.toml b/rust/bson/Cargo.toml index 799b9dbf81..546c864eb8 100644 --- a/rust/bson/Cargo.toml +++ b/rust/bson/Cargo.toml @@ -10,7 +10,14 @@ crate-type = ["cdylib"] [dependencies] memchr = "2.8.3" -pyo3 = { version = "0.29", features = ["extension-module"] } +pyo3 = { version = "0.29" } + +[features] +# Extension-module is a default feature so maturin's builds keep it; test +# binaries build with --no-default-features to link libpython and run the +# unit tests (an extension module cannot link a test harness). +default = ["extension-module"] +extension-module = ["pyo3/extension-module"] [build-dependencies] pyo3-build-config = { version = "0.29", features = ["resolve-config"] } diff --git a/rust/bson/src/buffer.rs b/rust/bson/src/buffer.rs index a102495488..5842f43ae3 100644 --- a/rust/bson/src/buffer.rs +++ b/rust/bson/src/buffer.rs @@ -1,7 +1,48 @@ //! Growable little-endian byte buffer for BSON encoding. //! //! Mirrors the semantics of `bson/buffer.c`: append-only writes with the -//! ability to backpatch previously written 4-byte length fields. +//! ability to backpatch previously written 4-byte length fields. Like the +//! C's `buffer_assure_space`, cumulative growth beyond `INT_MAX` (the BSON +//! document size limit) fails instead of wrapping the length fields the +//! buffer's own consumers backpatch and decode, and allocation failure +//! raises instead of aborting (the C's MemoryError). + +use pyo3::exceptions::{PyMemoryError, PyValueError}; +use pyo3::prelude::*; + +/// A write could not complete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReserveError { + /// The cumulative write would exceed the signed-int32 BSON document + /// size limit (mirrors the C's `buffer_assure_space` ValueError). + Overflow, + /// The allocation failed (mirrors the C's MemoryError). + Allocation, +} + +impl std::fmt::Display for ReserveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ReserveError::Overflow => f.write_str("Document would overflow BSON size limit"), + ReserveError::Allocation => f.write_str("could not allocate memory for the document"), + } + } +} + +impl std::error::Error for ReserveError {} + +impl From for PyErr { + fn from(err: ReserveError) -> PyErr { + match err { + ReserveError::Overflow => { + // Mirrors the C's buffer_assure_space: a plain ValueError. + PyValueError::new_err("Document would overflow BSON size limit") + } + // Mirrors the C's buffer_grow: a plain MemoryError. + ReserveError::Allocation => PyMemoryError::new_err(()), + } + } +} /// Growable byte buffer with BSON-oriented write helpers. pub struct Buffer { @@ -22,35 +63,62 @@ impl Buffer { self.buf.len() } + /// Reserves `additional` bytes, failing when the cumulative size would + /// exceed `i32::MAX` (checked before any allocation, mirroring the C's + /// `buffer_assure_space`) or when the allocation fails (the C's + /// MemoryError; `try_reserve` raises instead of aborting). #[inline] - pub fn write_u8(&mut self, value: u8) { + fn reserve(&mut self, additional: usize) -> Result<(), ReserveError> { + match self.buf.len().checked_add(additional) { + Some(total) if total <= i32::MAX as usize => {} + _ => return Err(ReserveError::Overflow), + } + self.buf + .try_reserve(additional) + .map_err(|_| ReserveError::Allocation) + } + + #[inline] + pub fn write_u8(&mut self, value: u8) -> Result<(), ReserveError> { + self.reserve(1)?; self.buf.push(value); + Ok(()) } #[inline] - pub fn write_u32_le(&mut self, value: u32) { + pub fn write_u32_le(&mut self, value: u32) -> Result<(), ReserveError> { + self.reserve(4)?; self.buf.extend_from_slice(&value.to_le_bytes()); + Ok(()) } #[inline] - pub fn write_i32_le(&mut self, value: i32) { + pub fn write_i32_le(&mut self, value: i32) -> Result<(), ReserveError> { + self.reserve(4)?; self.buf.extend_from_slice(&value.to_le_bytes()); + Ok(()) } #[inline] - pub fn write_i64_le(&mut self, value: i64) { + pub fn write_i64_le(&mut self, value: i64) -> Result<(), ReserveError> { + self.reserve(8)?; self.buf.extend_from_slice(&value.to_le_bytes()); + Ok(()) } #[inline] - pub fn write_f64_le(&mut self, value: f64) { + pub fn write_f64_le(&mut self, value: f64) -> Result<(), ReserveError> { + self.reserve(8)?; self.buf.extend_from_slice(&value.to_bits().to_le_bytes()); + Ok(()) } /// Writes raw bytes without any framing. #[inline] - pub fn write_bytes(&mut self, bytes: &[u8]) { + pub fn write_bytes(&mut self, bytes: &[u8]) -> Result<(), ReserveError> { + self.reserve(bytes.len())?; self.buf.extend_from_slice(bytes); + Ok(()) } /// Returns the current offset, for later `backpatch_u32` calls. @@ -76,12 +144,42 @@ impl Buffer { mod tests { use super::*; + #[test] + fn writes_past_the_bson_size_limit_fail() { + let mut buf = Buffer::new(); + // Huge additions fail the cap check before any allocation: the + // check is pure math, so even usize::MAX cannot abort or allocate. + assert_eq!(buf.reserve(usize::MAX), Err(ReserveError::Overflow)); + assert_eq!( + buf.reserve((i32::MAX as usize) + 1), + Err(ReserveError::Overflow) + ); + assert_eq!(buf.len(), 0); + // A failed write leaves nothing behind; later writes still succeed. + buf.write_u8(1).unwrap(); + assert_eq!(buf.len(), 1); + // Exactly INT_MAX total bytes is the largest legal document. Grow + // in 512 MiB chunks from one reused source so the test peaks at + // ~2.5 GiB (the capped buffer plus one chunk) rather than twice + // that; the last write lands exactly on the boundary, and one more + // byte overflows. + let chunk = vec![0u8; 1 << 29]; + for _ in 0..3 { + buf.write_bytes(&chunk).unwrap(); + } + buf.write_bytes(&chunk[..((i32::MAX as usize) - 1) % chunk.len()]) + .unwrap(); + assert_eq!(buf.len(), i32::MAX as usize); + assert_eq!(buf.write_u8(0), Err(ReserveError::Overflow)); + assert_eq!(buf.len(), i32::MAX as usize); + } + #[test] fn length_backpatching() { let mut buf = Buffer::new(); - buf.write_u32_le(0); // placeholder - buf.write_bytes(b"a\x00"); - buf.write_u8(0); + buf.write_u32_le(0).unwrap(); // placeholder + buf.write_bytes(b"a\x00").unwrap(); + buf.write_u8(0).unwrap(); let total = buf.len() as u32; buf.backpatch_u32(0, total); let mut expected = total.to_le_bytes().to_vec(); @@ -92,8 +190,8 @@ mod tests { #[test] fn scalar_writes_are_little_endian() { let mut buf = Buffer::new(); - buf.write_i32_le(-1); - buf.write_f64_le(1.0); + buf.write_i32_le(-1).unwrap(); + buf.write_f64_le(1.0).unwrap(); assert_eq!( buf.into_vec(), [ diff --git a/rust/bson/src/decode.rs b/rust/bson/src/decode.rs index 285decb1ee..7e3c660c87 100644 --- a/rust/bson/src/decode.rs +++ b/rust/bson/src/decode.rs @@ -663,12 +663,14 @@ fn dbref_hook<'py>( return Ok(value); }; let database = lookup("$db")?; - // Type checks: $ref must be str, $db must be str (or absent). + // Type checks: $ref must be str; $db must be str, None (the key may be + // present with a null value), or absent — matching the C's + // `database == Py_None || PyUnicode_Check(database)`. if reference.cast::().is_err() { return Ok(value); } if let Some(db) = &database { - if db.cast::().is_err() { + if !db.is_none() && db.cast::().is_err() { return Ok(value); } } diff --git a/rust/bson/src/encode.rs b/rust/bson/src/encode.rs index f3556cf996..5ae7fb6796 100644 --- a/rust/bson/src/encode.rs +++ b/rust/bson/src/encode.rs @@ -133,7 +133,7 @@ pub fn write_dict<'py>( if marker == MARKER_RAW_BSON { let raw = mapping.getattr(pyo3::intern!(py, "raw"))?; let data = raw_bytes_of(py, &raw)?; - buf.write_bytes(&data); + buf.write_bytes(&data)?; return Ok(()); } } @@ -147,7 +147,7 @@ pub fn write_dict<'py>( } let start = buf.position(); - buf.write_u32_le(0); // length placeholder + buf.write_u32_le(0)?; // length placeholder let _guard = recursion_guard(py, GUARD_DOCUMENT)?; let mut skip_id = false; @@ -155,15 +155,13 @@ pub fn write_dict<'py>( // _id is encoded first, regardless of its position in the mapping. // The lookup must not trigger mapping hooks (e.g. defaultdict's // __missing__): dict lookups use the raw storage API, other - // mappings use a has-key check first. + // mappings probe with PyMapping_HasKey first — which swallows all + // lookup errors and treats the key as missing. let id_value = if let Ok(dict) = mapping.cast::() { dict.get_item(pyo3::intern!(py, "_id"))? } else { - match mapping.get_item(pyo3::intern!(py, "_id")) { - Ok(value) => Some(value), - Err(err) if err.is_instance_of::(py) => None, - Err(err) => return Err(err), - } + // Swallows all lookup errors (PyMapping_HasKey parity). + mapping.get_item(pyo3::intern!(py, "_id")).ok() }; if let Some(value) = id_value { write_value( @@ -173,7 +171,9 @@ pub fn write_dict<'py>( } } // Fast path: dict or dict subclass (e.g. SON). Other Mappings are - // iterated via their `items()` method. + // iterated by key, looking each value up — mirroring the C, which + // does not require an items() method (a Mapping registered via + // Mapping.register may only provide __iter__/__getitem__/__len__). if let Ok(dict) = mapping.cast::() { for (key, value) in dict.iter() { if skip_id && is_str_eq(&key, "_id") { @@ -184,14 +184,15 @@ pub fn write_dict<'py>( } } } else { - let items = mapping.getattr(pyo3::intern!(py, "items"))?.call0()?; - for item in items.try_iter()? { - let item = item?; - let pair = item.cast::().map_err(|_| { - pyo3::exceptions::PyTypeError::new_err("encoder expected mapping items") - })?; - let key = pair.get_item(0)?; - let value = pair.get_item(1)?; + for key in mapping.try_iter()? { + let key = key?; + let value = match mapping.get_item(&key) { + Ok(value) => value, + // The C replaces any lookup failure with KeyError(key). + Err(_) => { + return Err(pyo3::exceptions::PyKeyError::new_err(key.unbind())); + } + }; if skip_id && is_str_eq(&key, "_id") { continue; } @@ -200,7 +201,7 @@ pub fn write_dict<'py>( } } } - buf.write_u8(0); + buf.write_u8(0)?; let length = (buf.len() - start) as u32; buf.backpatch_u32(start, length); Ok(()) @@ -315,9 +316,9 @@ fn write_value<'py>( // bool must be checked before int (bool is an int subclass). if is_exact::(value) { let v: bool = value.extract()?; - buf.write_u8(BSONBOO); - buf.write_bytes(name); - buf.write_u8(v as u8); + buf.write_u8(BSONBOO)?; + buf.write_bytes(name)?; + buf.write_u8(v as u8)?; return Ok(()); } if is_exact::(value) { @@ -334,14 +335,14 @@ fn write_value<'py>( } if is_exact::(value) { let v: f64 = value.extract()?; - buf.write_u8(BSONNUM); - buf.write_bytes(name); - buf.write_f64_le(v); + buf.write_u8(BSONNUM)?; + buf.write_bytes(name)?; + buf.write_f64_le(v)?; return Ok(()); } if value.is_none() { - buf.write_u8(BSONNUL); - buf.write_bytes(name); + buf.write_u8(BSONNUL)?; + buf.write_bytes(name)?; return Ok(()); } if is_exact::(value) { @@ -381,9 +382,9 @@ fn write_value<'py>( // ---- isinstance waterfall for builtins and stdlib types. ---- if value.is_instance_of::() { let v: bool = value.extract()?; - buf.write_u8(BSONBOO); - buf.write_bytes(name); - buf.write_u8(v as u8); + buf.write_u8(BSONBOO)?; + buf.write_bytes(name)?; + buf.write_u8(v as u8)?; return Ok(()); } if value.is_instance_of::() { @@ -400,14 +401,14 @@ fn write_value<'py>( } if value.is_instance_of::() { let v: f64 = value.extract()?; - buf.write_u8(BSONNUM); - buf.write_bytes(name); - buf.write_f64_le(v); + buf.write_u8(BSONNUM)?; + buf.write_bytes(name)?; + buf.write_f64_le(v)?; return Ok(()); } if value.is_none() { - buf.write_u8(BSONNUL); - buf.write_bytes(name); + buf.write_u8(BSONNUL)?; + buf.write_bytes(name)?; return Ok(()); } if value.is_instance_of::() { @@ -466,7 +467,9 @@ fn write_value<'py>( } // ---- Cannot encode. ---- - buf.write_bytes(name); // Keep the buffer consistent for error reporting. + // An overflow here (the buffer is at the size limit) must not mask the + // cannot-encode error that follows. + let _ = buf.write_bytes(name); // Keep the buffer consistent for error reporting. Err(cannot_encode(py, value, in_fallback_call)) } @@ -544,27 +547,27 @@ fn write_marker_value<'py>( )); } let size = data.len() as i32; - buf.write_u8(BSONBIN); - buf.write_bytes(name); + buf.write_u8(BSONBIN)?; + buf.write_bytes(name)?; if subtype == 2 { // The payload carries an extra length prefix. - buf.write_i32_le(size + 4); - buf.write_u8(2); + buf.write_i32_le(size + 4)?; + buf.write_u8(2)?; } - buf.write_i32_le(size); + buf.write_i32_le(size)?; if subtype != 2 { - buf.write_u8(subtype as u8); + buf.write_u8(subtype as u8)?; } - buf.write_bytes(data); + buf.write_bytes(data)?; Ok(true) } 7 => { // ObjectId: .binary is the 12-byte representation. let binary = value.getattr(pyo3::intern!(py, "binary"))?; let bytes = binary.cast::().map_err(PyErr::from)?; - buf.write_u8(BSONOID); - buf.write_bytes(name); - buf.write_bytes(bytes.as_bytes()); + buf.write_u8(BSONOID)?; + buf.write_bytes(name)?; + buf.write_bytes(bytes.as_bytes())?; Ok(true) } 9 => { @@ -582,9 +585,9 @@ fn write_marker_value<'py>( "MongoDB datetimes can only handle up to 8-byte ints", ) })?; - buf.write_u8(BSONDAT); - buf.write_bytes(name); - buf.write_i64_le(millis); + buf.write_u8(BSONDAT)?; + buf.write_bytes(name)?; + buf.write_i64_le(millis)?; Ok(true) } 11 => { @@ -597,15 +600,15 @@ fn write_marker_value<'py>( let scope = value.getattr(pyo3::intern!(py, "scope"))?; let code = value.cast::().map_err(PyErr::from)?; if scope.is_none() { - buf.write_u8(BSONCOD); - buf.write_bytes(name); + buf.write_u8(BSONCOD)?; + buf.write_bytes(name)?; write_string_payload(buf, code)?; return Ok(true); } - buf.write_u8(BSONCWS); - buf.write_bytes(name); + buf.write_u8(BSONCWS)?; + buf.write_bytes(name)?; let start = buf.position(); - buf.write_u32_le(0); // total length placeholder + buf.write_u32_le(0)?; // total length placeholder write_string_payload(buf, code)?; // The scope is a full document (length + elements + term), // written without key checking (mirrors the C). @@ -634,10 +637,10 @@ fn write_marker_value<'py>( "can't convert negative value to unsigned long", ) })?; - buf.write_u8(BSONTIM); - buf.write_bytes(name); - buf.write_i32_le(inc as i32); - buf.write_i32_le(time as i32); + buf.write_u8(BSONTIM)?; + buf.write_bytes(name)?; + buf.write_i32_le(inc as i32)?; + buf.write_i32_le(time as i32)?; Ok(true) } 18 => { @@ -647,25 +650,25 @@ fn write_marker_value<'py>( "MongoDB can only handle up to 8-byte ints", ) })?; - buf.write_u8(BSONLON); - buf.write_bytes(name); - buf.write_i64_le(v); + buf.write_u8(BSONLON)?; + buf.write_bytes(name)?; + buf.write_i64_le(v)?; Ok(true) } 19 => { // Decimal128: .bid is the 16-byte representation. let bid = value.getattr(pyo3::intern!(py, "bid"))?; let bytes = bid.cast::().map_err(PyErr::from)?; - buf.write_u8(BSONDEC); - buf.write_bytes(name); - buf.write_bytes(bytes.as_bytes()); + buf.write_u8(BSONDEC)?; + buf.write_bytes(name)?; + buf.write_bytes(bytes.as_bytes())?; Ok(true) } 100 => { // DBRef: encoded as its as_doc() document. let as_doc = value.call_method0(pyo3::intern!(py, "as_doc"))?; - buf.write_u8(BSONOBJ); - buf.write_bytes(name); + buf.write_u8(BSONOBJ)?; + buf.write_bytes(name)?; write_dict(py, buf, &as_doc, false, options, false)?; Ok(true) } @@ -673,19 +676,19 @@ fn write_marker_value<'py>( // RawBSONDocument: .raw (bytes or memoryview) written as-is. let raw = value.getattr(pyo3::intern!(py, "raw"))?; let data = raw_bytes_of(py, &raw)?; - buf.write_u8(BSONOBJ); - buf.write_bytes(name); - buf.write_bytes(&data); + buf.write_u8(BSONOBJ)?; + buf.write_bytes(name)?; + buf.write_bytes(&data)?; Ok(true) } 255 => { - buf.write_u8(BSONMIN); - buf.write_bytes(name); + buf.write_u8(BSONMIN)?; + buf.write_bytes(name)?; Ok(true) } 127 => { - buf.write_u8(BSONMAX); - buf.write_bytes(name); + buf.write_u8(BSONMAX)?; + buf.write_bytes(name)?; Ok(true) } _ => Ok(false), // unknown marker: fall through to the chain @@ -758,10 +761,10 @@ fn write_regex_value<'py>( )); } - buf.write_u8(BSONRGX); - buf.write_bytes(name); - buf.write_bytes(&pattern_bytes); - buf.write_u8(0); + buf.write_u8(BSONRGX)?; + buf.write_bytes(name)?; + buf.write_bytes(&pattern_bytes)?; + buf.write_u8(0)?; // Flag chars are written in bit order: i l m s u x. let mut flags = Vec::new(); if int_flags & 2 != 0 { @@ -782,8 +785,8 @@ fn write_regex_value<'py>( if int_flags & 64 != 0 { flags.push(b'x'); } - buf.write_bytes(&flags); - buf.write_u8(0); + buf.write_bytes(&flags)?; + buf.write_u8(0)?; Ok(()) } @@ -813,13 +816,13 @@ fn write_int_dispatch<'py>( match value.extract::() { Ok(v) => { if (I32_MIN..=I32_MAX).contains(&v) { - buf.write_u8(BSONINT); - buf.write_bytes(name); - buf.write_i32_le(v as i32); + buf.write_u8(BSONINT)?; + buf.write_bytes(name)?; + buf.write_i32_le(v as i32)?; } else { - buf.write_u8(BSONLON); - buf.write_bytes(name); - buf.write_i64_le(v); + buf.write_u8(BSONLON)?; + buf.write_bytes(name)?; + buf.write_i64_le(v)?; } Ok(()) } @@ -909,8 +912,8 @@ fn write_document_value<'py>( check_keys: bool, options: &CodecOptionsData<'py>, ) -> PyResult<()> { - buf.write_u8(BSONOBJ); - buf.write_bytes(name); + buf.write_u8(BSONOBJ)?; + buf.write_bytes(name)?; write_dict(py, buf, value, check_keys, options, false) } @@ -923,24 +926,33 @@ fn write_array_value<'py>( check_keys: bool, options: &CodecOptionsData<'py>, ) -> PyResult<()> { - buf.write_u8(BSONARR); - buf.write_bytes(name); + buf.write_u8(BSONARR)?; + buf.write_bytes(name)?; let start = buf.position(); - buf.write_u32_le(0); // length placeholder + buf.write_u32_le(0)?; // length placeholder let _guard = recursion_guard(py, GUARD_ARRAY)?; - let len = value.len()?; // List/tuple values (including subclasses — the cast is // isinstance-based, matching the C's subclass-inclusive PyList_Check / // PyTuple_Check) are read through the concrete sequence API // (PyList_GET_ITEM / PyTuple_GET_ITEM in the C), which bypasses - // __getitem__ overrides. The mapping-protocol fallback below only - // stays for defensive symmetry with the dispatch above it. + // __getitem__ overrides. The size likewise comes from the concrete + // PyList_Size/PyTuple_Size storage, not a subclass's __len__ — + // value.len() would dispatch to it and mis-size the array. The + // mapping-protocol fallback below only stays for defensive symmetry + // with the dispatch above it. let list = value.cast::().ok(); let tuple = if list.is_none() { value.cast::().ok() } else { None }; + let len = if let Some(list) = &list { + list.len() + } else if let Some(tuple) = &tuple { + tuple.len() + } else { + value.len()? + }; // The numeric element key ("0", "1", ... + NUL) is built in a stack // buffer (snprintf into a local in the C): no per-element allocation. let mut index_key = [0u8; 12]; @@ -976,7 +988,7 @@ fn write_array_value<'py>( false, )?; } - buf.write_u8(0); + buf.write_u8(0)?; let length = (buf.len() - start) as u32; buf.backpatch_u32(start, length); Ok(()) @@ -996,11 +1008,11 @@ fn write_binary_subtype_zero( "String length must be <= 2147483647", )); } - buf.write_u8(BSONBIN); - buf.write_bytes(name); - buf.write_i32_le(data.len() as i32); - buf.write_u8(0); // subtype - buf.write_bytes(data); + buf.write_u8(BSONBIN)?; + buf.write_bytes(name)?; + buf.write_i32_le(data.len() as i32)?; + buf.write_u8(0)?; // subtype + buf.write_bytes(data)?; Ok(()) } @@ -1011,8 +1023,8 @@ fn write_string_value<'py>( name: &[u8], value: &Bound<'py, PyString>, ) -> PyResult<()> { - buf.write_u8(BSONSTR); - buf.write_bytes(name); + buf.write_u8(BSONSTR)?; + buf.write_bytes(name)?; write_string_payload(buf, value) } @@ -1026,9 +1038,9 @@ fn write_string_payload(buf: &mut Buffer, value: &Bound<'_, PyString>) -> PyResu "String length must be <= 2147483647", )); } - buf.write_i32_le(utf8.len() as i32 + 1); - buf.write_bytes(utf8.as_bytes()); - buf.write_u8(0); + buf.write_i32_le(utf8.len() as i32 + 1)?; + buf.write_bytes(utf8.as_bytes())?; + buf.write_u8(0)?; Ok(()) } @@ -1059,8 +1071,8 @@ fn write_datetime_value<'py>( effective.get_second(), effective.get_microsecond(), ); - buf.write_u8(BSONDAT); - buf.write_bytes(name); - buf.write_i64_le(millis); + buf.write_u8(BSONDAT)?; + buf.write_bytes(name)?; + buf.write_i64_le(millis)?; Ok(()) } diff --git a/rust/bson/src/lib.rs b/rust/bson/src/lib.rs index c2c0a42492..0b19345a12 100644 --- a/rust/bson/src/lib.rs +++ b/rust/bson/src/lib.rs @@ -60,6 +60,11 @@ fn contains_view_eligible_doc(data: &[u8], threshold: usize) -> bool { /// `_prepare_input_buffer`: raw-document decoding from mutable/foreign /// buffers copies them to bytes when the stream contains a /// view-eligible (>= threshold) document. +/// +/// The scan and copy read the exporter's memory through a transient borrow +/// that spans no Python callbacks (the eligibility scan is pure Rust), so +/// the snapshot of `with_buffer_view` is not required here; at most one +/// copy is made, straight into the returned bytes object. fn prepare_input_buffer<'py>( py: Python<'py>, object: &Bound<'py, PyAny>, @@ -71,24 +76,29 @@ fn prepare_input_buffer<'py>( { return Ok(object.clone()); } - let eligible = with_buffer_view(object, |data| { - Ok(contains_view_eligible_doc( - data, - Types::get(py)?.raw_bson_view_threshold, - )) - })?; - if eligible { + let buffer = PyBuffer::::get(object)?; + with_get_buffer_checks(&buffer)?; + let data = + unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; + if contains_view_eligible_doc(data, Types::get(py)?.raw_bson_view_threshold) { // Copy to bytes so memoryview-backed views outlive the input // (mirrors PyBytes_FromObject). - let copy = with_buffer_view(object, |data| Ok(PyBytes::new(py, data)))?; - return Ok(copy.into_any()); + let copy = PyBytes::new(py, data); + drop(buffer); + Ok(copy.into_any()) + } else { + drop(buffer); + Ok(object.clone()) } - Ok(object.clone()) } /// Calls `f` with a `&[u8]` view of any buffer-protocol object (bytes, /// bytearray, memoryview, mmap, ...), mirroring `_get_buffer` in the C -/// extension. The view is zero-copy. +/// extension. Immutable bytes are borrowed zero-copy; every other exporter +/// is snapshotted to an owned copy first — its memory is writable through +/// other paths (a buffer export prevents resizing, not writes), and `f` +/// invokes decoder callbacks that can run Python code, so an immutable +/// Rust slice over that memory would be unsound. fn with_buffer_view( object: &Bound<'_, PyAny>, f: impl FnOnce(&[u8]) -> PyResult, @@ -98,11 +108,14 @@ fn with_buffer_view( } let buffer = PyBuffer::::get(object)?; with_get_buffer_checks(&buffer)?; - let slice = - unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; - let result = f(slice); + // Copy while the GIL is held and before any callback runs: nothing can + // write through the exporter during this window, and the view is + // dropped before `f` executes. + let data = + unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) } + .to_vec(); drop(buffer); - result + f(&data) } /// The defensive checks of the C's `_get_buffer`: pyo3 acquires buffers @@ -342,11 +355,17 @@ fn _raw_to_dict<'py>( while pos < end { let type_byte = data[pos]; pos += 1; + // The name scan is bounded by obj_end (the eoo index), not by + // end: a no-payload final value (None, MinKey, MaxKey) puts the + // name's NUL at obj_end - 1, the last byte before the + // terminator (the C's _element_to_dict scans with max = + // obj_end). The eoo check above guarantees the scan cannot run + // past the buffer. let name_start = pos; - while pos < end && data[pos] != 0 { + while pos < obj_end as usize && data[pos] != 0 { pos += 1; } - if pos >= end { + if pos >= obj_end as usize { return Err(invalid_bson(py, "field name too large")); } let name_bytes = &data[name_start..pos]; @@ -427,7 +446,7 @@ fn _array_of_documents_to_buffer<'py>( if value_length >= size - position { return Err(invalid_bson(py, "invalid array content")); } - buffer.write_bytes(&data[position..position + value_length]); + buffer.write_bytes(&data[position..position + value_length])?; position += value_length; } if position != size - 1 { diff --git a/rust/bson/tests/test_rbson.py b/rust/bson/tests/test_rbson.py index 5930e037a2..0980f59b87 100644 --- a/rust/bson/tests/test_rbson.py +++ b/rust/bson/tests/test_rbson.py @@ -974,3 +974,114 @@ def make(raw_name): decode_rust(bad) with pytest.raises(InvalidBSON): decode_c(bad) + + +# --- Copilot review findings -------------------------------------------------- + + +def test_dbref_none_db_value_matches_c(): + # A present $db whose value is None must still produce a DBRef (the C's + # _dbref_hook accepts `database == Py_None`), not a plain dict. + result = assert_same_decode(encode_c({"a": {"$ref": "c", "$id": 5, "$db": None}})) + assert isinstance(result["a"], DBRef) + assert result["a"].database is None + + +def test_encode_registered_mapping_without_items(): + # A Mapping registered with Mapping.register (not subclassed) need not + # inherit the ABC's items() mixin: the C iterates keys and looks each + # value up, so the Rust must not require an items() method. + import collections.abc + + class RegisteredMapping: + def __init__(self, data): + self._data = dict(data) + + def __iter__(self): + return iter(self._data) + + def __getitem__(self, key): + return self._data[key] + + def __len__(self): + return len(self._data) + + collections.abc.Mapping.register(RegisteredMapping) + doc = {"b": RegisteredMapping({"x": 1}), "a": RegisteredMapping({"y": 2})} + assert_same_encode(doc) + + # Missing keys during iteration raise KeyError, like the C. + class MissingValue(RegisteredMapping): + def __getitem__(self, key): + if key == "gone": + raise KeyError(key) + return super().__getitem__(key) + + collections.abc.Mapping.register(MissingValue) + with pytest.raises(KeyError): + encode_rust({"a": MissingValue({"gone": 1})}) + with pytest.raises(KeyError): + encode_c({"a": MissingValue({"gone": 1})}) + + +def test_encode_list_subclass_len_override(): + # A list/tuple subclass's __len__ must not change the encoded element + # count: the C sizes arrays with PyList_Size/PyTuple_Size. + class LyingList(list): + def __len__(self): + return 0 + + class LyingTuple(tuple): + def __len__(self): + return 0 + + assert_same_encode({"a": LyingList([1, 2])}) + assert_same_encode({"a": LyingTuple((1, 2))}) + assert decode_rust(encode_rust({"a": LyingList([1, 2])})) == {"a": [1, 2]} + + +def test_raw_to_dict_no_payload_last_value(): + # The last element of a document may have no payload (None/MinKey/MaxKey): + # its name's NUL is the byte before the terminator, and inflating such a + # raw document must not raise "field name too large". + from bson import MaxKey, MinKey + + for doc in ( + {"a": None}, + {"x": 1, "a": None}, + {"a": MinKey()}, + {"a": MaxKey()}, + {"x": MinKey(), "y": MaxKey()}, + ): + raw = encode_c(doc) + expected = _cbson._raw_to_dict(raw, 4, len(raw) - 1, OPTIONS, {}, False) + result = {} + actual = _rbson._raw_to_dict(raw, 4, len(raw) - 1, OPTIONS, result, False) + assert actual is result + assert result == expected == doc + # RawBSONDocument's lazy inflation runs the same entry point. + assert_same_decode(raw, CodecOptions(document_class=RawBSONDocument)) + + +def test_decode_bytearray_mutated_by_decoder(): + # Decoding borrows the exporter's memory only through an owned snapshot: + # a decoder callback may mutate (even empty) the source buffer mid-decode + # without corrupting the values decoded from the snapshot. + from bson.codec_options import TypeDecoder, TypeRegistry + + doc = {"a": "one", "b": "two", "c": "three"} + buf = bytearray(encode_c(doc)) + + class StrDecoder(TypeDecoder): + bson_type = str + + def transform_bson(self, value): + buf.clear() + return value + + opts = CodecOptions(type_registry=TypeRegistry([StrDecoder()])) + result = decode_rust(bytes(buf), opts) + assert result == doc + # bytearray and memoryview inputs decode exactly like bytes. + assert decode_rust(bytearray(encode_c(doc)), opts) == decode_c(encode_c(doc), opts) + assert decode_rust(memoryview(encode_c(doc)), opts) == decode_c(encode_c(doc), opts) From 394510bf3efdcfd8e0b35d3d74674c4c4d868dac Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Nov 2026 06:06:27 -0600 Subject: [PATCH 4/8] address review: threshold init before buffer borrow, decoded-key DBRef flag, fixed-size OID/Decimal128 payloads, handler conversion errors, parity suite in compare task --- .evergreen/scripts/bson_codec_compare.sh | 7 ++ rust/bson/src/decode.rs | 21 +++++- rust/bson/src/encode.rs | 35 ++++++++-- rust/bson/src/lib.rs | 11 +-- rust/bson/src/options.rs | 24 +++++-- rust/bson/tests/test_rbson.py | 89 +++++++++++++++++++++++- 6 files changed, 171 insertions(+), 16 deletions(-) diff --git a/.evergreen/scripts/bson_codec_compare.sh b/.evergreen/scripts/bson_codec_compare.sh index 729b63b205..1bd2d061ec 100755 --- a/.evergreen/scripts/bson_codec_compare.sh +++ b/.evergreen/scripts/bson_codec_compare.sh @@ -66,6 +66,13 @@ uv pip install maturin export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C target-cpu=native" .venv/bin/maturin develop --release -m rust/bson/Cargo.toml +# Run the differential codec suite before benchmarking: it compares the +# Rust codec against the C extension (the behavioral reference) value for +# value, so a fast but incorrect codec fails the task here instead of +# producing a successful comparison. The suite lives outside the repo's +# configured testpaths and no other CI command invokes it. +.venv/bin/python -m pytest rust/bson/tests/test_rbson.py -q + # Run the benchmarks with the venv's interpreter directly: no implicit # sync, so the locally-installed codec survives both runs. -m perf is # required: the repo's addopts (-m "default or default_async") would diff --git a/rust/bson/src/decode.rs b/rust/bson/src/decode.rs index 7e3c660c87..117215caaf 100644 --- a/rust/bson/src/decode.rs +++ b/rust/bson/src/decode.rs @@ -212,12 +212,31 @@ pub fn decode_elements_into<'py>( .map_err(|err| rewrap_as_invalid_bson(py, err))?; position = new_position; + // The flag must track the *decoded* key, because a non-strict + // handler can mangle the raw bytes: with "ignore", b'\xff$ref' + // decodes to '$ref', and the C's _dbref_hook (which sees only the + // decoded mapping) then returns a DBRef where a raw-byte check + // would return a plain dict. With the strict handler the decoded + // key preserves the raw bytes exactly ('$' is ASCII, so the first + // byte decides), keeping the check on the borrowed slice. A key + // that cannot be re-borrowed as UTF-8 (surrogates under + // "surrogateescape") is treated as not starting with '$': it + // cannot equal the "$ref"/"$id" the hook requires, and any real + // "$ref"/"$id" keys decode borrowably and are seen. + has_dollar_key |= if options.utf8_handler.is_none() { + name.first() == Some(&b'$') + } else { + key.cast::() + .ok() + .and_then(|key| key.to_str().ok()) + .is_some_and(|key| key.starts_with('$')) + }; + if let Some(dict) = &dict_target { dict.set_item(key, value)?; } else { document.set_item(key, value)?; } - has_dollar_key |= name.first() == Some(&b'$'); } if position != body.len() { return Err(invalid_bson(py, "bad object or element length")); diff --git a/rust/bson/src/encode.rs b/rust/bson/src/encode.rs index 5ae7fb6796..cc85f0ed04 100644 --- a/rust/bson/src/encode.rs +++ b/rust/bson/src/encode.rs @@ -562,12 +562,26 @@ fn write_marker_value<'py>( Ok(true) } 7 => { - // ObjectId: .binary is the 12-byte representation. + // ObjectId: .binary is the 12-byte representation. The C + // writes exactly 12 bytes (bson/_cbsonmodule.c), truncating a + // longer payload from a marker-bearing object or subclass; a + // shorter one would overread the buffer, which the Rust + // rejects instead. let binary = value.getattr(pyo3::intern!(py, "binary"))?; let bytes = binary.cast::().map_err(PyErr::from)?; + let payload = bytes.as_bytes(); + if payload.len() < 12 { + return Err(invalid_document( + py, + &format!( + "Expected ObjectId binary to be 12 bytes, got {}", + payload.len() + ), + )); + } buf.write_u8(BSONOID)?; buf.write_bytes(name)?; - buf.write_bytes(bytes.as_bytes())?; + buf.write_bytes(&payload[..12])?; Ok(true) } 9 => { @@ -656,12 +670,25 @@ fn write_marker_value<'py>( Ok(true) } 19 => { - // Decimal128: .bid is the 16-byte representation. + // Decimal128: .bid is the 16-byte representation. As with + // ObjectId, the C writes exactly 16 bytes, truncating a + // longer payload; a shorter one is rejected rather than + // overread. let bid = value.getattr(pyo3::intern!(py, "bid"))?; let bytes = bid.cast::().map_err(PyErr::from)?; + let payload = bytes.as_bytes(); + if payload.len() < 16 { + return Err(invalid_document( + py, + &format!( + "Expected Decimal128 bid to be 16 bytes, got {}", + payload.len() + ), + )); + } buf.write_u8(BSONDEC)?; buf.write_bytes(name)?; - buf.write_bytes(bytes.as_bytes())?; + buf.write_bytes(&payload[..16])?; Ok(true) } 100 => { diff --git a/rust/bson/src/lib.rs b/rust/bson/src/lib.rs index 0b19345a12..ed536a0ff0 100644 --- a/rust/bson/src/lib.rs +++ b/rust/bson/src/lib.rs @@ -62,9 +62,11 @@ fn contains_view_eligible_doc(data: &[u8], threshold: usize) -> bool { /// view-eligible (>= threshold) document. /// /// The scan and copy read the exporter's memory through a transient borrow -/// that spans no Python callbacks (the eligibility scan is pure Rust), so -/// the snapshot of `with_buffer_view` is not required here; at most one -/// copy is made, straight into the returned bytes object. +/// that spans no Python code: `Types::get` is resolved *before* the buffer +/// is acquired (its first use imports modules and compiles a regex, and +/// that initialization could run Python that mutates the exporter), and +/// the eligibility scan itself is pure Rust. At most one copy is made, +/// straight into the returned bytes object. fn prepare_input_buffer<'py>( py: Python<'py>, object: &Bound<'py, PyAny>, @@ -76,11 +78,12 @@ fn prepare_input_buffer<'py>( { return Ok(object.clone()); } + let threshold = Types::get(py)?.raw_bson_view_threshold; let buffer = PyBuffer::::get(object)?; with_get_buffer_checks(&buffer)?; let data = unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; - if contains_view_eligible_doc(data, Types::get(py)?.raw_bson_view_threshold) { + if contains_view_eligible_doc(data, threshold) { // Copy to bytes so memoryview-backed views outlive the input // (mirrors PyBytes_FromObject). let copy = PyBytes::new(py, data); diff --git a/rust/bson/src/options.rs b/rust/bson/src/options.rs index c1de0611f2..6658c8f83c 100644 --- a/rust/bson/src/options.rs +++ b/rust/bson/src/options.rs @@ -146,11 +146,25 @@ impl<'py> CodecOptionsData<'py> { .ok() .cloned(); match handler { - Some(h) => h - .to_str() - .ok() - .filter(|s| *s != "strict") - .and_then(|s| CString::new(s.to_string()).ok()), + Some(h) => { + // Mirrors the C's PyArg_ParseTuple "z" format, which + // fails the options conversion when the handler cannot + // be converted: an unencodable (surrogate) handler + // raises UnicodeEncodeError from to_str, and an + // embedded NUL is a ValueError (matching CPython's + // "embedded null character"). Python's CodecOptions + // constructor only checks isinstance(str), so both + // are reachable; silently treating either as strict + // would change decoding behavior. + let handler = h.to_str()?; + if handler == "strict" { + None + } else { + Some(CString::new(handler.to_string()).map_err(|_| { + pyo3::exceptions::PyValueError::new_err("embedded null character") + })?) + } + } None => None, } }; diff --git a/rust/bson/tests/test_rbson.py b/rust/bson/tests/test_rbson.py index 0980f59b87..46f9212aa8 100644 --- a/rust/bson/tests/test_rbson.py +++ b/rust/bson/tests/test_rbson.py @@ -1076,12 +1076,97 @@ class StrDecoder(TypeDecoder): bson_type = str def transform_bson(self, value): - buf.clear() + # In-place overwrite: clear() would deallocate the buffer + # instead of writing through it. + buf[:] = bytes(len(buf)) return value opts = CodecOptions(type_registry=TypeRegistry([StrDecoder()])) - result = decode_rust(bytes(buf), opts) + # Pass the bytearray itself: the callback must mutate the exporter + # being decoded, or the test cannot detect a missing snapshot (a + # bytes() copy would be immune to the mutation by construction). + result = decode_rust(buf, opts) assert result == doc # bytearray and memoryview inputs decode exactly like bytes. assert decode_rust(bytearray(encode_c(doc)), opts) == decode_c(encode_c(doc), opts) assert decode_rust(memoryview(encode_c(doc)), opts) == decode_c(encode_c(doc), opts) + + +def test_dbref_dollar_keys_after_ignore_decode(): + # Field names whose leading invalid bytes drop out under "ignore" + # decode to "$ref"/"$id"; the C's _dbref_hook sees the decoded + # mapping and returns a DBRef. The Rust dollar-key flag must + # therefore track the decoded key, not the raw bytes. + import struct + + inner = ( + struct.pack(" Date: Tue, 3 Nov 2026 06:25:24 -0600 Subject: [PATCH 5/8] fix typing --- tools/compare_bson_perf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/compare_bson_perf.py b/tools/compare_bson_perf.py index 601cc18ddb..d8ca6da8d6 100644 --- a/tools/compare_bson_perf.py +++ b/tools/compare_bson_perf.py @@ -13,16 +13,17 @@ import json import sys +from typing import Any -def load(path): +def load(path: str) -> list[dict[str, Any]]: with open(path) as fid: return json.load(fid) -def metrics_by_name(results): +def metrics_by_name(results: list[dict[str, Any]]) -> dict[str, float]: """Maps test name → megabytes_per_sec (one entry per test today).""" - metrics = {} + metrics: dict[str, float] = {} for entry in results: name = entry["info"]["test_name"] for metric in entry["metrics"]: From 69270b1887fa4fee0e0b741b9fbd34ba9df7a55c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Nov 2026 06:32:40 -0600 Subject: [PATCH 6/8] trim comment and docstring prose across the Rust codec --- .evergreen/scripts/bson_codec_compare.sh | 59 ++++++--------- rust/bson/src/buffer.rs | 26 +++---- rust/bson/src/decode.rs | 66 ++++++----------- rust/bson/src/encode.rs | 80 +++++++------------- rust/bson/src/errors.rs | 8 +- rust/bson/src/lib.rs | 94 +++++++++--------------- rust/bson/src/options.rs | 38 ++++------ rust/bson/tests/test_rbson.py | 23 +++--- test/performance/bson_impl_select.py | 29 ++------ 9 files changed, 150 insertions(+), 273 deletions(-) diff --git a/.evergreen/scripts/bson_codec_compare.sh b/.evergreen/scripts/bson_codec_compare.sh index 1bd2d061ec..fcb58aa942 100755 --- a/.evergreen/scripts/bson_codec_compare.sh +++ b/.evergreen/scripts/bson_codec_compare.sh @@ -15,10 +15,9 @@ SCRIPT_DIR="$(cd "$SCRIPT_DIR" > /dev/null 2>&1 && pwd)" ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" cd "$ROOT_DIR" -# Source (not execute) so the exports apply here: this puts the pinned uv +# Source (not execute) so the exports apply: this puts the pinned uv # (required-version in pyproject.toml) first on PATH and ensures it is -# installed — the host image's uv may be older and would abort every uv -# invocation. Sources env.sh/test-env.sh and runs install-dependencies.sh +# installed. Sources env.sh/test-env.sh and runs install-dependencies.sh # (idempotent; 'setup system' already ran it). . "$SCRIPT_DIR/setup-dev-env.sh" @@ -53,16 +52,14 @@ fi # uv sync prunes locally-installed packages that are not in uv.lock. uv sync ${UV_ARGS:-} -# Build the Rust codec into the venv (after the sync — see above). Two uv -# pitfalls to avoid here: `uv run --with maturin` would run maturin inside -# a throwaway overlay environment (the codec lands in the overlay, not the -# venv), and uv run's implicit sync would prune the freshly built codec. +# Build the Rust codec into the venv. `uv run --with maturin` would install +# the codec into a throwaway overlay environment, and uv run's implicit +# sync would prune it again, so install maturin into the venv directly. +# Target the building machine's CPU (this task builds with `maturin +# develop` on the host that benchmarks): cargo only discovers a +# crate-local .cargo/config.toml from the invocation directory upward, +# which maturin (run from the checkout root) would miss — set RUSTFLAGS. uv pip install maturin -# Target the building machine's CPU: this task builds with `maturin develop` -# on the same host that benchmarks (never cross-compiled or distributed), and -# the pip-built C extension only gets baseline codegen. Set via RUSTFLAGS — -# cargo only discovers a crate-local .cargo/config.toml from the invocation -# directory upward, which maturin (run from the checkout root) would miss. export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C target-cpu=native" .venv/bin/maturin develop --release -m rust/bson/Cargo.toml @@ -81,10 +78,9 @@ export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C target-cpu=native" TEST_SELECTOR='(Flat or Deep or Full) and not Json' # Run each codec's pass twice in swapped order (C→Rust, then Rust→C) and -# merge to the per-benchmark mean: machine state (turbo/thermal drift) moves -# over the ~6-minute window, so a single back-to-back C/Rust pair is -# systematically biased in favor of whichever codec runs first. The swap -# cancels that bias; the merged files are what gets compared and uploaded. +# merge to the per-benchmark mean: machine state (turbo/thermal drift) +# moves over the ~6-minute window and would bias a single back-to-back +# pair in favor of whichever codec runs first. BSON_IMPL=c OUTPUT_FILE=results_c_a.json \ .venv/bin/python -m pytest \ test/performance/perf_test.py -m perf -k "$TEST_SELECTOR" -q @@ -142,10 +138,8 @@ rm results_c_a.json results_c_b.json results_rust_a.json results_rust_b.json # Profile DeepDecoding once per codec for decode hotspot data. Entirely # non-fatal: profiling is best-effort and the benchmark numbers never -# depend on it. -# The rhel90 perf image does not ship perf; try the platform package manager -# with passwordless sudo (the established pattern on Evergreen hosts, e.g. -# setup-system.sh) before skipping. +# depend on it. The rhel90 perf image does not ship perf; try the platform +# package manager with passwordless sudo before skipping. if ! command -v perf > /dev/null 2>&1; then if command -v dnf > /dev/null 2>&1; then sudo dnf install -y perf || sudo yum install -y perf || true @@ -178,21 +172,16 @@ if command -v perf > /dev/null 2>&1; then rm -f "results_profile_${impl}.json" done elif command -v valgrind > /dev/null 2>&1; then - # No perf (e.g. the rhel90 image and it cannot be installed there): fall - # back to callgrind, which the ubuntu2204 image ships preinstalled. - # Callgrind's instruction counts are exact and deterministic, so hotspot - # attribution is valid at any run length — FASTBENCH keeps the ~30x - # slowdown bounded, and pytest/interpreter startup churn is discountable - # in the annotation (it lands in importlib/PyImport frames). - # The native build emits AVX-512 (EVEX) instructions via memchr's runtime - # dispatch, and valgrind 3.18 cannot decode EVEX — it SIGILLs mid-run. - # Rebuild for the profiled run on x86-64-v3 (AVX2, VEX-encoded, which - # valgrind handles — the C extension is baseline x86-64 and profiles - # fine). Nothing after this block uses the codec. - # The rebuild is part of the best-effort profiling contract: under - # set -e a failure here must not fail the task (all comparison results - # are already written) — the task still needs to reach its - # artifact-upload command, so treat it as a skipped profile. + # No perf (e.g. the rhel90 image): fall back to callgrind, which the + # ubuntu2204 image ships preinstalled. Callgrind's instruction counts are + # exact and deterministic, so hotspot attribution is valid at any run + # length — FASTBENCH keeps the ~30x slowdown bounded, and pytest/interpreter + # startup churn is discountable in the annotation (it lands in importlib + # frames). The native build emits AVX-512 (EVEX) via memchr's runtime + # dispatch and valgrind 3.18 cannot decode EVEX — rebuild for x86-64-v3 + # (AVX2), which valgrind handles. Nothing after this block uses the codec, + # and under set -e a rebuild failure must not fail the task (all + # comparison results are already written). echo "Rebuilding the Rust codec without AVX-512 for valgrind..." if ! RUSTFLAGS="-C target-cpu=x86-64-v3" .venv/bin/maturin develop --release \ -m rust/bson/Cargo.toml; then diff --git a/rust/bson/src/buffer.rs b/rust/bson/src/buffer.rs index 5842f43ae3..c1c6f86fc3 100644 --- a/rust/bson/src/buffer.rs +++ b/rust/bson/src/buffer.rs @@ -1,11 +1,9 @@ //! Growable little-endian byte buffer for BSON encoding. //! -//! Mirrors the semantics of `bson/buffer.c`: append-only writes with the -//! ability to backpatch previously written 4-byte length fields. Like the -//! C's `buffer_assure_space`, cumulative growth beyond `INT_MAX` (the BSON -//! document size limit) fails instead of wrapping the length fields the -//! buffer's own consumers backpatch and decode, and allocation failure -//! raises instead of aborting (the C's MemoryError). +//! Mirrors `bson/buffer.c`: append-only writes with backpatchable 4-byte +//! length fields. Cumulative growth beyond `INT_MAX` fails instead of +//! wrapping the length fields, and allocation failure raises instead of +//! aborting. use pyo3::exceptions::{PyMemoryError, PyValueError}; use pyo3::prelude::*; @@ -35,10 +33,8 @@ impl From for PyErr { fn from(err: ReserveError) -> PyErr { match err { ReserveError::Overflow => { - // Mirrors the C's buffer_assure_space: a plain ValueError. PyValueError::new_err("Document would overflow BSON size limit") } - // Mirrors the C's buffer_grow: a plain MemoryError. ReserveError::Allocation => PyMemoryError::new_err(()), } } @@ -63,10 +59,8 @@ impl Buffer { self.buf.len() } - /// Reserves `additional` bytes, failing when the cumulative size would - /// exceed `i32::MAX` (checked before any allocation, mirroring the C's - /// `buffer_assure_space`) or when the allocation fails (the C's - /// MemoryError; `try_reserve` raises instead of aborting). + /// Reserves `additional` bytes, failing before any allocation when the + /// cumulative size would exceed `i32::MAX`. #[inline] fn reserve(&mut self, additional: usize) -> Result<(), ReserveError> { match self.buf.len().checked_add(additional) { @@ -158,11 +152,9 @@ mod tests { // A failed write leaves nothing behind; later writes still succeed. buf.write_u8(1).unwrap(); assert_eq!(buf.len(), 1); - // Exactly INT_MAX total bytes is the largest legal document. Grow - // in 512 MiB chunks from one reused source so the test peaks at - // ~2.5 GiB (the capped buffer plus one chunk) rather than twice - // that; the last write lands exactly on the boundary, and one more - // byte overflows. + // Grow in 512 MiB chunks so the test peaks at ~2.5 GiB rather than + // twice that; the last write lands exactly on the boundary, and + // one more byte overflows. let chunk = vec![0u8; 1 << 29]; for _ in 0..3 { buf.write_bytes(&chunk).unwrap(); diff --git a/rust/bson/src/decode.rs b/rust/bson/src/decode.rs index 117215caaf..2184a7aa94 100644 --- a/rust/bson/src/decode.rs +++ b/rust/bson/src/decode.rs @@ -165,11 +165,10 @@ fn raw_document<'py>( Ok(document.unbind()) } -/// Decodes elements from `body` (the elements of a document, excluding the -/// length prefix and the terminator) into an existing mapping. Returns -/// whether any field name started with `$` — a necessary condition for the -/// document to be shaped like a DBRef (its keys must include "$ref" and -/// "$id"), letting `get_value` skip `_dbref_hook` in the common case. +/// Decodes elements from `body` (a document's elements, excluding the +/// length prefix and terminator) into an existing mapping. Returns whether +/// any field name started with `$` — a necessary condition for a DBRef +/// shape, letting `get_value` skip the hook in the common case. pub fn decode_elements_into<'py>( py: Python<'py>, document: &Bound<'py, PyAny>, @@ -212,17 +211,12 @@ pub fn decode_elements_into<'py>( .map_err(|err| rewrap_as_invalid_bson(py, err))?; position = new_position; - // The flag must track the *decoded* key, because a non-strict - // handler can mangle the raw bytes: with "ignore", b'\xff$ref' - // decodes to '$ref', and the C's _dbref_hook (which sees only the - // decoded mapping) then returns a DBRef where a raw-byte check - // would return a plain dict. With the strict handler the decoded - // key preserves the raw bytes exactly ('$' is ASCII, so the first - // byte decides), keeping the check on the borrowed slice. A key - // that cannot be re-borrowed as UTF-8 (surrogates under - // "surrogateescape") is treated as not starting with '$': it - // cannot equal the "$ref"/"$id" the hook requires, and any real - // "$ref"/"$id" keys decode borrowably and are seen. + // Track the decoded key, not the raw bytes: a non-strict handler + // can mangle b'\xff$ref' into '$ref', and the C's _dbref_hook + // sees only the decoded mapping. Strict decodes preserve raw + // bytes, so the first byte decides; a key that cannot be + // re-borrowed as UTF-8 (surrogates under "surrogateescape") + // cannot equal "$ref"/"$id" either. has_dollar_key |= if options.utf8_handler.is_none() { name.first() == Some(&b'$') } else { @@ -796,7 +790,7 @@ fn decode_datetime<'py>( ) { Ok(dt) => dt, // Out-of-range ValueError gets the DATETIME_AUTO suggestion - // appended (mirrors datetime_from_millis in the C). + // appended. Err(err) => return Err(append_datetime_suggestion(py, err)), }; let value: Bound<'py, PyAny> = naive.into_any(); @@ -856,8 +850,7 @@ fn offset_millis<'py>( } /// Reads and validates a nested document size at `position`, bounded by -/// `end`. Mirrors the checks in the C get_value's document case: every -/// failure lands on the shared "invalid length or type code" message. +/// `end` (mirrors the C get_value's document case). #[inline(always)] fn read_size_at(py: Python<'_>, data: &[u8], position: usize, end: usize) -> PyResult { if position + 4 > end { @@ -874,9 +867,7 @@ fn read_size_at(py: Python<'_>, data: &[u8], position: usize, end: usize) -> PyR } /// Reads a length-prefixed string's length (which includes the trailing -/// NUL) and validates it, guarding against negative lengths. All failures -/// land on the shared "invalid length or type code" message (the C's -/// `goto invalid`). +/// NUL), guarding against negative lengths (the C's `goto invalid`). #[inline(always)] fn read_string_length(py: Python<'_>, data: &[u8], position: usize, end: usize) -> PyResult { if position + 4 > end { @@ -898,8 +889,7 @@ fn read_u32(py: Python<'_>, data: &[u8], position: usize) -> PyResult { } /// Calls `callee` with positional arguments through `PyObject_Vectorcall` -/// (a C argument array — no argument tuple allocation, mirroring the C -/// extension's constructor call sites). +/// (no argument tuple allocation). #[inline(always)] fn vectorcall<'py>( py: Python<'py>, @@ -965,25 +955,17 @@ pub fn decode_utf8<'py>( Ok(decoded.into_any()) } -/// Field names repeat heavily within and across documents; decoding each -/// occurrence with PyUnicode_DecodeUTF8 (allocation + decode) and hashing the -/// fresh string for every dict insertion is a large share of the decode loop. -/// A bounded direct-mapped cache of raw key bytes to decoded strings removes -/// both on a hit: the cached string's hash is computed once by CPython and -/// reused for every subsequent PyDict_SetItem. (Unlike the C extension, -/// repeat decodes return the *same* str object for a repeated name — benign, -/// strings are immutable.) Only enabled for the strict UTF-8 handler, where -/// decoding is a pure function of the bytes. -/// -/// Thread safety: the module requires the GIL (`gil_used = true` in -/// lib.rs), so every entry point holds it; the Mutex additionally guards -/// the Vec against the (theoretical) free-threaded case, and statics are -/// never dropped in Rust, so cached strings are only ever released by -/// eviction, under the GIL. The cache — like `Types` — is process-wide: -/// importing the module into more than one interpreter is not supported. +/// A direct-mapped cache of raw key bytes to decoded strings: decoding and +/// hashing fresh strings for every dict insertion is a large share of the +/// decode loop, and a hit reuses the cached string's CPython hash. Repeat +/// decodes now return the same str object for a repeated name (benign; +/// strings are immutable). Enabled only for the strict handler, where +/// decoding is a pure function of the bytes. Like `Types`, the cache is +/// process-wide: importing the module into more than one interpreter is +/// not supported. const KEY_CACHE_SLOTS: usize = 1 << 13; -/// Keys longer than this skip the cache: the copy of the raw bytes needed to -/// cache them would rival the decode cost being avoided. +/// Keys longer than this skip the cache: caching them would cost as much +/// as the decode it avoids. const KEY_CACHE_MAX_KEY_LEN: usize = 64; /// A cache slot: the raw key bytes and the decoded string. diff --git a/rust/bson/src/encode.rs b/rust/bson/src/encode.rs index cc85f0ed04..f3ad08a8fa 100644 --- a/rust/bson/src/encode.rs +++ b/rust/bson/src/encode.rs @@ -62,9 +62,8 @@ pub fn dict_to_bson<'py>( top_level: bool, ) -> PyResult> { let mut buf = Buffer::new(); - // The C's handle_invalid_doc_error wraps only failures of the element - // iteration loops (not the _id pre-write), so the wrapping lives in - // write_dict. + // The C wraps only element-iteration failures (not the _id pre-write), + // so the wrapping lives in write_dict. write_dict(py, &mut buf, document, check_keys, options, top_level)?; Ok(buf.into_vec()) } @@ -114,10 +113,8 @@ fn is_str_eq(key: &Bound<'_, PyAny>, expected: &str) -> bool { } /// Writes `length + elements + terminator` for a mapping, with the `_id` -/// element first when `top_level` — mirroring the C's `write_dict`, which -/// owns the framing. A raw document (marker 101) writes only its raw -/// bytes: the raw document's own length prefix frames it, so no extra -/// length or terminator is written. +/// element first when `top_level`. A raw document (marker 101) writes only +/// its raw bytes, already framed by its own length prefix. pub fn write_dict<'py>( py: Python<'py>, buf: &mut Buffer, @@ -126,8 +123,6 @@ pub fn write_dict<'py>( options: &CodecOptionsData<'py>, top_level: bool, ) -> PyResult<()> { - // Fast path: dict or dict subclass (e.g. SON). Non-dict mappings first - // check for a raw document passthrough, then for the Mapping protocol. if mapping.cast::().is_err() { if let Some(marker) = read_type_marker(mapping)? { if marker == MARKER_RAW_BSON { @@ -152,15 +147,12 @@ pub fn write_dict<'py>( let mut skip_id = false; if top_level { - // _id is encoded first, regardless of its position in the mapping. - // The lookup must not trigger mapping hooks (e.g. defaultdict's - // __missing__): dict lookups use the raw storage API, other - // mappings probe with PyMapping_HasKey first — which swallows all - // lookup errors and treats the key as missing. + // _id encodes first regardless of its position. Lookups must not + // trigger mapping hooks: dicts use the raw storage API; other + // mappings swallow lookup errors (PyMapping_HasKey parity). let id_value = if let Ok(dict) = mapping.cast::() { dict.get_item(pyo3::intern!(py, "_id"))? } else { - // Swallows all lookup errors (PyMapping_HasKey parity). mapping.get_item(pyo3::intern!(py, "_id")).ok() }; if let Some(value) = id_value { @@ -170,10 +162,8 @@ pub fn write_dict<'py>( skip_id = true; } } - // Fast path: dict or dict subclass (e.g. SON). Other Mappings are - // iterated by key, looking each value up — mirroring the C, which - // does not require an items() method (a Mapping registered via - // Mapping.register may only provide __iter__/__getitem__/__len__). + // Mappings are iterated by key with per-key lookup: no items() method + // is required (Mapping.register'd classes may not provide one). if let Ok(dict) = mapping.cast::() { for (key, value) in dict.iter() { if skip_id && is_str_eq(&key, "_id") { @@ -207,10 +197,8 @@ pub fn write_dict<'py>( Ok(()) } -/// The element-iteration failure path: at the top level, InvalidDocument -/// errors are wrapped with the document (mirrors the C's -/// `handle_invalid_doc_error` call sites, which gate on `top_level` and -/// exclude the `_id` pre-write). +/// At the top level, InvalidDocument errors are wrapped with the document +/// (mirrors the C's `handle_invalid_doc_error` call sites). fn fail_element<'py>( py: Python<'py>, top_level: bool, @@ -223,8 +211,7 @@ fn fail_element<'py>( wrap_invalid_document(py, mapping, err) } -/// Validates a key and writes its element (`decode_and_write_pair`'s -/// encode-side counterpart: `write_pair` in the C extension). +/// Validates a key and writes its element (the C's `write_pair`). fn write_pair<'py>( py: Python<'py>, buf: &mut Buffer, @@ -272,8 +259,7 @@ fn write_pair<'py>( } } - // The element name (key bytes + NUL) goes in a stack buffer for the - // common short-key case; the C builds it on the stack the same way. + // Short keys write from a stack buffer (the C does the same). let key_bytes = key_utf8.as_bytes(); const STACK_NAME_LEN: usize = 96; let mut stack_name = [0u8; STACK_NAME_LEN]; @@ -454,9 +440,8 @@ fn write_value<'py>( } // ---- Custom type encoder (exact type match), then fallback. ---- - // The re-dispatch is bounded by CPython's recursion counter (the C - // relies on the Python frames of the callbacks; guarding explicitly - // keeps the Rust stack bounded the same way). + // The re-dispatch is recursion-guarded: the C relies on the Python + // frames of the callbacks to bound the stack. if let Some(converted) = custom_encoder_value(py, options, value, in_custom_call)? { let _guard = recursion_guard(py, GUARD_EMPTY)?; return write_value(py, buf, name, &converted, check_keys, options, true, false); @@ -482,12 +467,9 @@ pub struct RecursionGuard { _not_send: std::marker::PhantomData<*const ()>, } -/// Enters a CPython recursion check; returns an `Err` with the pending -/// error (typically RecursionError) when the limit is hit. -/// -/// The message is a static C string (CPython reads it when formatting the -/// RecursionError): no per-call allocation, so guards are cheap enough -/// for every document, array, and converted value. +/// Enters a CPython recursion check; returns the pending error (typically +/// RecursionError) when the limit is hit. The message is a static C string, +/// read by CPython when formatting the error. pub fn recursion_guard(py: Python<'_>, message: &'static CStr) -> PyResult { let ok = unsafe { pyo3::ffi::Py_EnterRecursiveCall(message.as_ptr()) }; if ok != 0 { @@ -562,11 +544,8 @@ fn write_marker_value<'py>( Ok(true) } 7 => { - // ObjectId: .binary is the 12-byte representation. The C - // writes exactly 12 bytes (bson/_cbsonmodule.c), truncating a - // longer payload from a marker-bearing object or subclass; a - // shorter one would overread the buffer, which the Rust - // rejects instead. + // The C writes exactly 12 bytes of .binary, truncating a + // longer payload; a shorter one would overread, so reject it. let binary = value.getattr(pyo3::intern!(py, "binary"))?; let bytes = binary.cast::().map_err(PyErr::from)?; let payload = bytes.as_bytes(); @@ -585,9 +564,8 @@ fn write_marker_value<'py>( Ok(true) } 9 => { - // DatetimeMS: int(value) is the millis value (the C uses - // PyNumber_Long, which routes through __int__ and requires the - // result to be an int; a float __int__ raises TypeError). + // int(value) is the millis value; a float __int__ raises + // TypeError (PyNumber_Long parity). let as_int = value.call_method0(pyo3::intern!(py, "__int__"))?; if !as_int.is_instance_of::() { return Err(pyo3::exceptions::PyTypeError::new_err( @@ -632,9 +610,8 @@ fn write_marker_value<'py>( Ok(true) } 17 => { - // Timestamp: i32 inc, then i32 time. Mirrors the C's - // PyLong_AsUnsignedLong: negative or >unsigned-long values - // raise OverflowError; [2³¹, 2³²) truncates to i32. + // PyLong_AsUnsignedLong parity: negative or >unsigned-long + // raises OverflowError; [2³¹, 2³²) truncates to i32. let inc: u64 = value .getattr(pyo3::intern!(py, "inc"))? .extract() @@ -670,10 +647,8 @@ fn write_marker_value<'py>( Ok(true) } 19 => { - // Decimal128: .bid is the 16-byte representation. As with - // ObjectId, the C writes exactly 16 bytes, truncating a - // longer payload; a shorter one is rejected rather than - // overread. + // As with ObjectId: exactly 16 bytes of .bid; a shorter + // payload is rejected rather than overread. let bid = value.getattr(pyo3::intern!(py, "bid"))?; let bytes = bid.cast::().map_err(PyErr::from)?; let payload = bytes.as_bytes(); @@ -734,8 +709,7 @@ fn raw_bytes_of<'py>(_py: Python<'py>, raw: &Bound<'py, PyAny>) -> PyResult::get(raw)?; - // Defensive checks mirroring the C's write_raw_doc/_get_buffer: a - // misbehaving exporter may hand back non-contiguous or null buffers. + // A misbehaving exporter may hand back non-contiguous or null buffers. if buffer.buf_ptr().is_null() || !buffer.is_c_contiguous() || buffer.item_size() != 1 { return Err(pyo3::exceptions::PyTypeError::new_err( "RawBSONDocument.raw must be bytes or memoryview", diff --git a/rust/bson/src/errors.rs b/rust/bson/src/errors.rs index 09eafb2855..a806747c88 100644 --- a/rust/bson/src/errors.rs +++ b/rust/bson/src/errors.rs @@ -1,8 +1,6 @@ -//! Raising `bson.errors` exceptions from Rust. -//! -//! Mirrors `_cbsonmodule.c`'s `_error()` helper: the exception classes are -//! looked up on the `bson.errors` module. Error paths are cold, so a -//! per-call module lookup (cached after first use) is fine. +//! Raising `bson.errors` exceptions from Rust, mirroring the C's +//! `_error()` helper. Error paths are cold, so a cached module lookup is +//! fine. use pyo3::sync::PyOnceLock; diff --git a/rust/bson/src/lib.rs b/rust/bson/src/lib.rs index ed536a0ff0..9cfd452c30 100644 --- a/rust/bson/src/lib.rs +++ b/rust/bson/src/lib.rs @@ -11,15 +11,9 @@ //! - `_raw_to_dict(bson, position, obj_end, codec_options, result, raw_array)` //! - `_array_of_documents_to_buffer(array)` //! -//! The GIL is held for the duration of every call (parity with the C -//! extension): the module declares `Py_mod_gil = Py_MOD_GIL_USED` -//! (`gil_used = true`), so free-threaded CPython builds re-enable the GIL -//! when importing it. Note that pyo3 does not declare -//! `Py_mod_multiple_interpreters`, and the import machinery defaults a -//! missing slot to SUPPORTED — so this module *can* be imported into a -//! shared-GIL subinterpreter even though its process-wide statics (`Types`, -//! the field-name cache) would then be shared across interpreters. -//! Importing the module into more than one interpreter is not supported. +//! Every call holds the GIL (`gil_used = true`). Importing the module into +//! more than one interpreter is unsupported: its process-wide statics would +//! be shared across them. mod buffer; mod datetime_conv; @@ -56,17 +50,13 @@ fn contains_view_eligible_doc(data: &[u8], threshold: usize) -> bool { false } -/// Prepares the input buffer owner for decoding, mirroring -/// `_prepare_input_buffer`: raw-document decoding from mutable/foreign -/// buffers copies them to bytes when the stream contains a -/// view-eligible (>= threshold) document. +/// Mirrors `_prepare_input_buffer`: for raw-document decoding, a mutable +/// or foreign buffer whose stream holds a view-eligible document is +/// copied to bytes. /// -/// The scan and copy read the exporter's memory through a transient borrow -/// that spans no Python code: `Types::get` is resolved *before* the buffer -/// is acquired (its first use imports modules and compiles a regex, and -/// that initialization could run Python that mutates the exporter), and -/// the eligibility scan itself is pure Rust. At most one copy is made, -/// straight into the returned bytes object. +/// `Types` is resolved before the buffer is acquired: its first-use +/// initialization runs Python, which must not happen while the exporter's +/// memory is borrowed as an immutable slice. fn prepare_input_buffer<'py>( py: Python<'py>, object: &Bound<'py, PyAny>, @@ -84,8 +74,7 @@ fn prepare_input_buffer<'py>( let data = unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; if contains_view_eligible_doc(data, threshold) { - // Copy to bytes so memoryview-backed views outlive the input - // (mirrors PyBytes_FromObject). + // Copy so memoryview-backed views outlive the input. let copy = PyBytes::new(py, data); drop(buffer); Ok(copy.into_any()) @@ -95,13 +84,9 @@ fn prepare_input_buffer<'py>( } } -/// Calls `f` with a `&[u8]` view of any buffer-protocol object (bytes, -/// bytearray, memoryview, mmap, ...), mirroring `_get_buffer` in the C -/// extension. Immutable bytes are borrowed zero-copy; every other exporter -/// is snapshotted to an owned copy first — its memory is writable through -/// other paths (a buffer export prevents resizing, not writes), and `f` -/// invokes decoder callbacks that can run Python code, so an immutable -/// Rust slice over that memory would be unsound. +/// Mirrors the C's `_get_buffer`: bytes are borrowed zero-copy; every +/// other buffer is copied first. Its memory is writable through other +/// paths, and `f` can run Python, so a shared slice would be unsound. fn with_buffer_view( object: &Bound<'_, PyAny>, f: impl FnOnce(&[u8]) -> PyResult, @@ -111,9 +96,7 @@ fn with_buffer_view( } let buffer = PyBuffer::::get(object)?; with_get_buffer_checks(&buffer)?; - // Copy while the GIL is held and before any callback runs: nothing can - // write through the exporter during this window, and the view is - // dropped before `f` executes. + // Copy while the GIL is held and before any callback runs. let data = unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) } .to_vec(); @@ -121,10 +104,9 @@ fn with_buffer_view( f(&data) } -/// The defensive checks of the C's `_get_buffer`: pyo3 acquires buffers -/// with PyBUF_FULL_RO (not PyBUF_SIMPLE), so a misbehaving exporter can -/// hand back non-contiguous or null buffers; mirroring the C, each is -/// rejected with a ValueError. +/// Mirrors the C's `_get_buffer` checks: pyo3 acquires buffers with +/// PyBUF_FULL_RO, so a misbehaving exporter can hand back null or +/// non-contiguous buffers. fn with_get_buffer_checks(buffer: &PyBuffer) -> PyResult<()> { // `!view->buf || view->len < 0`. if buffer.buf_ptr().is_null() { @@ -160,18 +142,15 @@ fn _dict_to_bson<'py>( ) -> PyResult> { let options = CodecOptionsData::parse(codec_options)?; - // A RawBSONDocument encodes to its raw bytes unconditionally (mirrors - // the C `_cbson_dict_to_bson`, which checks the marker regardless of - // top_level): bytes are returned as-is, anything else is converted - // with PyBytes_FromObject. + // A RawBSONDocument encodes to its raw bytes regardless of top_level + // (mirrors the C). if let Some(marker) = options::read_type_marker(document)? { if marker == options::RAW_BSON_DOCUMENT_MARKER { let raw = document.getattr(pyo3::intern!(py, "raw"))?; return match raw.cast::() { Ok(bytes) => Ok(bytes.clone()), Err(_) => { - // Copy via the buffer protocol (mirrors - // PyBytes_FromObject). + // Copy via the buffer protocol. let data = with_buffer_view(&raw, |data| Ok(data.to_vec()))?; Ok(PyBytes::new(py, &data)) } @@ -226,9 +205,8 @@ fn _decode_all<'py>( if size > slice.len() { return Err(invalid_bson(py, "objsize too large")); } - // The C's _cbson_decode_all propagates elements_to_dict - // failures untouched — the InvalidBSON wrapping happened - // deeper, in get_value. + // Failures propagate unwrapped: the C wraps them deeper, in + // get_value. let value = decode::elements_to_dict(py, &input, &slice[..size], &options) .map(|(value, _)| value)?; results.append(value)?; @@ -243,8 +221,7 @@ fn read_size_prefix(py: Python<'_>, data: &[u8]) -> PyResult { if data.len() < BSON_MIN_SIZE { return Err(invalid_bson(py, "not enough data for a BSON document")); } - // The C reads the size prefix as a signed int32; negative values - // (and zero) are "invalid message size". + // The size prefix is a signed int32; negative (or zero) is invalid. let size = i32::from_le_bytes([data[0], data[1], data[2], data[3]]); if size < BSON_MIN_SIZE as i32 { return Err(invalid_bson(py, "invalid message size")); @@ -264,8 +241,7 @@ fn rewrap(py: Python<'_>, err: PyErr) -> PyErr { } /// Decodes a single key/value pair at `position`, returning -/// `(name, value, new_position)`. The input must be a bytes object -/// (mirrors `_cbson_element_to_dict`). +/// `(name, value, new_position)`. The input must be bytes. #[pyfunction] #[pyo3(signature = (bson, position, obj_end, codec_options, raw_array=false))] fn _element_to_dict<'py>( @@ -307,8 +283,8 @@ fn _element_to_dict<'py>( let name_bytes = &data[name_start..pos]; pos += 1; // skip the NUL - // The name is decoded before the value (mirrors the C's - // _element_to_dict); a UnicodeDecodeError is wrapped in InvalidBSON. + // The name decodes before the value (mirrors the C); a + // UnicodeDecodeError is wrapped in InvalidBSON. let name = decode::decode_key(py, name_bytes, &options).map_err(|err| rewrap(py, err))?; let (value, new_position) = decode::get_value_with_raw_array( py, &input, name_bytes, type_byte, pos, max, &options, raw_array, @@ -358,12 +334,9 @@ fn _raw_to_dict<'py>( while pos < end { let type_byte = data[pos]; pos += 1; - // The name scan is bounded by obj_end (the eoo index), not by - // end: a no-payload final value (None, MinKey, MaxKey) puts the - // name's NUL at obj_end - 1, the last byte before the - // terminator (the C's _element_to_dict scans with max = - // obj_end). The eoo check above guarantees the scan cannot run - // past the buffer. + // The name scan is bounded by obj_end (the eoo index), not + // end: a no-payload final value (None, MinKey, MaxKey) puts + // its NUL at obj_end - 1. let name_start = pos; while pos < obj_end as usize && data[pos] != 0 { pos += 1; @@ -374,8 +347,7 @@ fn _raw_to_dict<'py>( let name_bytes = &data[name_start..pos]; pos += 1; // skip the NUL - // The name is decoded before the value (mirrors the C); a - // UnicodeDecodeError is wrapped in InvalidBSON. + // The name decodes before the value (mirrors the C). let name = decode::decode_key(py, name_bytes, &options).map_err(|err| rewrap(py, err))?; let (value, new_pos) = decode::get_value_with_raw_array( @@ -384,8 +356,8 @@ fn _raw_to_dict<'py>( name_bytes, type_byte, pos, - // Per-element values are bounded by the eoo index - // (the C passes obj_end as max to _element_to_dict). + // Values are bounded by the eoo index (the C passes + // obj_end as max). obj_end as usize, &options, raw_array, @@ -430,7 +402,7 @@ fn _array_of_documents_to_buffer<'py>( return Err(invalid_bson(py, "array element was not an object")); } position += 1; - // Just skip the key. + // Skip the key. while position < size && data[position] != 0 { position += 1; } diff --git a/rust/bson/src/options.rs b/rust/bson/src/options.rs index 6658c8f83c..16ed98648a 100644 --- a/rust/bson/src/options.rs +++ b/rust/bson/src/options.rs @@ -1,9 +1,6 @@ -//! Parsing of `bson.CodecOptions` (a `NamedTuple`) into a Rust-side struct. -//! -//! Mirrors `convert_codec_options` in `_cbsonmodule.c`: the options object is -//! consumed positionally with the field order -//! `(document_class, tz_aware, uuid_representation, -//! unicode_decode_error_handler, tzinfo, type_registry, datetime_conversion)`. +//! Parsing of `bson.CodecOptions` (a `NamedTuple`) into a Rust-side struct, +//! mirroring `convert_codec_options` in `_cbsonmodule.c`: the options object +//! is consumed positionally. use std::ffi::CString; @@ -50,20 +47,16 @@ impl<'py> CodecOptionsData<'py> { } } -/// Reads the `_type_marker` attribute of an object. -/// -/// Mirrors `_type_marker()` in `_cbsonmodule.c`: the attribute is read -/// optionally and only accepted when it is an *exact* `int` (objects with -/// broken `__getattr__` implementations could return arbitrary types). -/// Returns `None` when the marker is absent or not an exact int; returns -/// `Err` only when the attribute read itself fails with a non-AttributeError. +/// Reads the `_type_marker` attribute of an object, accepting only an +/// *exact* `int` (objects with broken `__getattr__` could return arbitrary +/// types). Returns `None` when absent or not an exact int; returns `Err` +/// only when the attribute read itself fails with a non-AttributeError. pub fn read_type_marker<'py>(object: &Bound<'py, PyAny>) -> PyResult> { let py = object.py(); #[cfg(Py_3_13)] let marker = { - // PyObject_GetOptionalAttr (the C's 3.13+ path) finds the attribute - // without materializing an AttributeError — the common case is a - // document_class without a marker (plain dict). + // Avoids materializing an AttributeError for the common + // no-marker case (a plain dict). let intern = pyo3::intern!(py, "_type_marker"); let mut raw: *mut pyo3::ffi::PyObject = std::ptr::null_mut(); let found = unsafe { @@ -147,15 +140,10 @@ impl<'py> CodecOptionsData<'py> { .cloned(); match handler { Some(h) => { - // Mirrors the C's PyArg_ParseTuple "z" format, which - // fails the options conversion when the handler cannot - // be converted: an unencodable (surrogate) handler - // raises UnicodeEncodeError from to_str, and an - // embedded NUL is a ValueError (matching CPython's - // "embedded null character"). Python's CodecOptions - // constructor only checks isinstance(str), so both - // are reachable; silently treating either as strict - // would change decoding behavior. + // The C's "z" parse fails the conversion for + // unencodable (surrogate) handlers and embedded NULs; + // the Python constructor only isinstance-checks, so + // both are reachable. Propagating matches the C. let handler = h.to_str()?; if handler == "strict" { None diff --git a/rust/bson/tests/test_rbson.py b/rust/bson/tests/test_rbson.py index 46f9212aa8..aa297aeaa8 100644 --- a/rust/bson/tests/test_rbson.py +++ b/rust/bson/tests/test_rbson.py @@ -1064,9 +1064,8 @@ def test_raw_to_dict_no_payload_last_value(): def test_decode_bytearray_mutated_by_decoder(): - # Decoding borrows the exporter's memory only through an owned snapshot: - # a decoder callback may mutate (even empty) the source buffer mid-decode - # without corrupting the values decoded from the snapshot. + # A decoder callback may mutate the source buffer mid-decode; values + # must come from an owned snapshot, not the exporter's memory. from bson.codec_options import TypeDecoder, TypeRegistry doc = {"a": "one", "b": "two", "c": "three"} @@ -1082,9 +1081,8 @@ def transform_bson(self, value): return value opts = CodecOptions(type_registry=TypeRegistry([StrDecoder()])) - # Pass the bytearray itself: the callback must mutate the exporter - # being decoded, or the test cannot detect a missing snapshot (a - # bytes() copy would be immune to the mutation by construction). + # Pass the bytearray itself: a bytes() copy would be immune to the + # callback's mutation, and the test could not detect a missing snapshot. result = decode_rust(buf, opts) assert result == doc # bytearray and memoryview inputs decode exactly like bytes. @@ -1093,10 +1091,9 @@ def transform_bson(self, value): def test_dbref_dollar_keys_after_ignore_decode(): - # Field names whose leading invalid bytes drop out under "ignore" - # decode to "$ref"/"$id"; the C's _dbref_hook sees the decoded - # mapping and returns a DBRef. The Rust dollar-key flag must - # therefore track the decoded key, not the raw bytes. + # With "ignore", b'\xff$ref' decodes to '$ref'; the C's _dbref_hook + # sees the decoded mapping and returns a DBRef. The dollar-key flag + # must track the decoded key, not the raw bytes. import struct inner = ( @@ -1130,10 +1127,8 @@ def test_error_handler_embedded_nul_rejected(): def test_objectid_marker_fixed_payload(): - # The C writes exactly 12 bytes of .binary (buffer_write_bytes(data, - # 12)), truncating a longer payload; a marker-bearing object with a - # longer .binary must not produce malformed BSON. A shorter payload - # would make the C read past the buffer, so the Rust rejects it. + # The C writes exactly 12 bytes of .binary, truncating longer + # payloads; a shorter payload would overread, so the Rust rejects it. class LongObjectId: _type_marker = 7 diff --git a/test/performance/bson_impl_select.py b/test/performance/bson_impl_select.py index f1b739e670..1fac8e6bd3 100644 --- a/test/performance/bson_impl_select.py +++ b/test/performance/bson_impl_select.py @@ -1,21 +1,10 @@ """Selects the BSON codec implementation for the performance benchmarks. -Set ``BSON_IMPL`` to ``c`` (the default) or ``rust``: - -- ``c``: the C extension (``bson._cbson``) — PyMongo's production codec. -- ``rust``: the Rust-backed codec (``_rbson``, built from ``rust/bson``). - -For ``rust``, the private entry points that ``bson/__init__.py`` rebinds -from ``_cbson`` at import time are rebound to the Rust module instead. The -public ``bson.encode``/``decode``/``decode_all`` functions look these -names up on the ``bson`` module at call time, so rebinding is transparent. - -Some driver modules capture the private functions with ``from bson import -`` at import time; those attributes are rebound as well. - -This module must be imported before the benchmarks execute (the -``test/performance/conftest.py`` imports it during pytest startup), but it -may be imported at any point before the first encode/decode call. +Set ``BSON_IMPL`` to ``c`` (the default) or ``rust``. For ``rust``, the +private entry points that ``bson/__init__.py`` binds from ``_cbson``, and +the driver modules that captured them with ``from bson import ...``, are +rebound to the Rust module. The public ``bson.encode``/``decode`` look +these names up at call time, so rebinding is transparent. """ from __future__ import annotations @@ -24,11 +13,9 @@ _IMPLS = ("c", "rust") -# The private entry points exposed by both _cbson and _rbson. -# Note: _element_to_dict/_raw_to_dict have the 5-argument C-style signature -# (the C extension's); bson/__init__.py's Python wrappers (which add a -# `view` argument) are dead code when the C extension is importable, so -# rebinding the module attribute to the C-style signature is safe. +# The private entry points exposed by both _cbson and _rbson. They have +# the C-style signature: bson/__init__.py's Python wrappers (which add a +# `view` argument) are dead code when the C extension is importable. _RBSON_ENTRY_POINTS = ( "_dict_to_bson", "_bson_to_dict", From 9ecfbcc2d0b4d7994a821f51285754f5ecb6c9ad Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Nov 2026 06:56:08 -0600 Subject: [PATCH 7/8] address review: key-cache deadlock, DBRef hook parity, decoder exception boundary, signed-byte buffers, patch-only compare variant --- .evergreen/generated_configs/variants.yml | 1 + .evergreen/scripts/generate_config.py | 4 +- .evergreen/scripts/generate_config_utils.py | 13 ++ .evergreen/scripts/test_generate_config.py | 14 ++ rust/bson/src/decode.rs | 79 ++++++-- rust/bson/src/encode.rs | 17 +- rust/bson/src/lib.rs | 99 +++++++--- rust/bson/tests/test_rbson.py | 209 +++++++++++++++++++- 8 files changed, 369 insertions(+), 67 deletions(-) diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 26ff886bd9..f203a62f4b 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -117,6 +117,7 @@ buildvariants: # Bson codec compare tests - name: bson-codec-compare + patch_only: true tasks: - name: bson-codec-compare display_name: BSON Codec Compare diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 4b08e23274..6930464dd7 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -588,7 +588,9 @@ def create_bson_codec_compare_variants(): # selected here for the profiling runs: the image ships valgrind, and # perf installs there (the perf host can do neither). host = HOSTS["ubuntu22"] - return [create_variant(["bson-codec-compare"], "BSON Codec Compare", host=host)] + return [ + create_variant(["bson-codec-compare"], "BSON Codec Compare", host=host, patch_only=True) + ] def create_aws_auth_variants(): diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 0ba291ab5b..6f04995222 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -93,6 +93,12 @@ class Host: ############## +# Variants that must only run on patches. shrub's BuildVariant model has no +# patch_only field, so create_variant_generic records the name and +# write_variants_to_file injects the field into the generated YAML. +PATCH_ONLY_VARIANTS: set[str] = set() + + def create_variant_generic( tasks: list[str | EvgTaskRef], display_name: str, @@ -103,6 +109,7 @@ def create_variant_generic( **kwargs: Any, ) -> BuildVariant: """Create a build variant for the given inputs.""" + patch_only = kwargs.pop("patch_only", False) task_refs = [] if isinstance(host, str): host = HOSTS[host] @@ -123,6 +130,8 @@ def create_variant_generic( if isinstance(run_on, str): run_on = [run_on] name = display_name.replace(" ", "-").replace("*-", "").lower() + if patch_only: + PATCH_ONLY_VARIANTS.add(name) return BuildVariant( name=name, display_name=display_name, @@ -285,6 +294,10 @@ def write_variants_to_file(mod): fid.write(f" # {title} tests\n") for line in out[1:]: fid.write(f"{line}\n") + # Variant entries sit at two-space indent; task refs inside + # a variant render deeper and must not match. + if line.startswith(" - name: ") and line[10:].strip() in PATCH_ONLY_VARIANTS: + fid.write(" patch_only: true\n") fid.write("\n") # Remove extra trailing newline: diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index cd0f724ee0..120e8e64b2 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -310,6 +310,20 @@ def test_bson_codec_compare_uploads_artifacts_as_files(): assert all("${task_id}-${execution}" in remote for remote in remotes), remotes +def test_bson_codec_compare_is_patch_only(): + """The BSON codec comparison variant is patch-only: omitting batchtime + does not stop Evergreen from running it on mainline commits, and + shrub's BuildVariant model cannot express the field, so the writer + injects it into the generated YAML. Drift tripwire for that injection. + """ + import yaml + + here = Path(__file__).parent + config = yaml.safe_load((here.parent / "generated_configs" / "variants.yml").read_text()) + variant = next(v for v in config["buildvariants"] if v["name"] == "bson-codec-compare") + assert variant.get("patch_only") is True + + def test_bson_codec_compare_profiles_deep_decoding(): """The task profiles DeepDecoding (the one benchmark still below the parity bar on the host) under perf, once per codec. Drift tripwire: the diff --git a/rust/bson/src/decode.rs b/rust/bson/src/decode.rs index 2184a7aa94..d28bae416e 100644 --- a/rust/bson/src/decode.rs +++ b/rust/bson/src/decode.rs @@ -210,6 +210,9 @@ pub fn decode_elements_into<'py>( get_value(py, &input, name, type_byte, position, body.len(), options) .map_err(|err| rewrap_as_invalid_bson(py, err))?; position = new_position; + // Document-level converter failures propagate unwrapped (the C's + // _element_to_dict does not rewrap get_value failures). + let value = apply_decoder(py, value, options)?; // Track the decoded key, not the raw bytes: a non-strict handler // can mangle b'\xff$ref' into '$ref', and the C's _dbref_hook @@ -306,10 +309,12 @@ pub fn get_value_with_raw_array<'py>( elements_to_dict(py, input, &data[position..position + size], options) .map_err(|err| rewrap_as_invalid_bson(py, err))?; position += size; - // Hook for DBRefs ($ref/$id shaped docs decode to DBRef); - // skipped when no key could be "$ref"/"$id" (every DBRef key - // starts with "$") or for raw document classes (mirrors the C). - if options.is_raw_bson || !has_dollar_key { + // Hook for DBRefs ($ref/$id shaped docs decode to DBRef). The + // C runs it unconditionally; the skip is only sound for exact + // dicts, which store keys as decoded. A custom class may add + // or rewrite keys in its constructor or __setitem__, so its + // result is always checked. + if options.is_raw_bson || (options.is_dict_class && !has_dollar_key) { value } else { dbref_hook(py, value, options)? @@ -340,6 +345,10 @@ pub fn get_value_with_raw_array<'py>( let (item, new_pos) = get_value(py, input, name, element_type, pos, arr_end, options) .map_err(|err| rewrap_as_invalid_bson(py, err))?; + // The C's array loop wraps item failures (goto + // invalid), converter errors included. + let item = apply_decoder(py, item, options) + .map_err(|err| rewrap_as_invalid_bson(py, err))?; pos = new_pos; if unsafe { pyo3::ffi::PyList_Append(list.as_ptr(), item.as_ptr()) } < 0 { return Err(PyErr::fetch(py)); @@ -613,14 +622,26 @@ pub fn get_value_with_raw_array<'py>( } }; - // Apply a custom decoder for this value's exact type, if registered. + Ok((value, position)) +} + +/// Applies the type registry's decoder for the value's exact type, if +/// registered. Mirrors the C's decoder-map tail in get_value: the converter +/// result is returned directly, so its failures are not wrapped here — +/// whether they are wrapped depends on the caller (document-level callers +/// propagate; the array loop wraps, like the C's `goto invalid`). +pub fn apply_decoder<'py>( + py: Python<'py>, + value: Py, + options: &CodecOptionsData<'py>, +) -> PyResult> { if let Some(decoder_map) = &options.decoder_map_dict { let value_bound = value.bind(py); if let Some(converter) = decoder_map.get_item(value_bound.get_type())? { - return Ok((converter.call1((value_bound,))?.unbind(), position)); + return Ok(converter.call1((value_bound,))?.unbind()); } } - Ok((value, position)) + Ok(value) } /// Constructs a `DatetimeMS` for the millis value (vectorcall, no tuple). @@ -647,11 +668,14 @@ fn dbref_hook<'py>( ) -> PyResult> { let doc = value.bind(py); let lookup = |key: &str| -> PyResult>> { - // Dict fast path: PyDict_GetItemWithError raises no exceptions for - // missing keys, and one lookup serves both the presence check and - // the value (PyMapping_HasKey + PyObject_GetItem in the C); other - // mappings fall back to get_item with a caught KeyError. - if let Ok(dict) = doc.cast::() { + // Exact-dict fast path: PyDict_GetItemWithError raises no + // exceptions for missing keys, and one lookup serves both the + // presence check and the value (PyMapping_HasKey + + // PyObject_GetItem in the C). Subclasses are excluded — their + // __getitem__ override must win — and other mappings fall back to + // get_item with a caught KeyError. + if doc.is_exact_instance_of::() { + let dict = doc.cast::().unwrap(); let key = pyo3::types::PyString::intern(py, key); let item = unsafe { pyo3::ffi::PyDict_GetItemWithError(dict.as_ptr(), key.as_ptr()) }; if item.is_null() { @@ -959,10 +983,10 @@ pub fn decode_utf8<'py>( /// hashing fresh strings for every dict insertion is a large share of the /// decode loop, and a hit reuses the cached string's CPython hash. Repeat /// decodes now return the same str object for a repeated name (benign; -/// strings are immutable). Enabled only for the strict handler, where -/// decoding is a pure function of the bytes. Like `Types`, the cache is -/// process-wide: importing the module into more than one interpreter is -/// not supported. +/// strings are immutable). Enabled only for valid-UTF-8 keys under the +/// strict handler with no registered override, where decoding is a pure +/// function of the bytes. Like `Types`, the cache is process-wide: +/// importing the module into more than one interpreter is not supported. const KEY_CACHE_SLOTS: usize = 1 << 13; /// Keys longer than this skip the cache: caching them would cost as much /// as the decode it avoids. @@ -1016,25 +1040,38 @@ fn decode_key_cached<'py>( Ok(decoded) } -/// Decodes a field-name key, using the intern cache when the strict handler -/// is in effect (any other handler makes decoding depend on mutable options -/// state, so the cache is bypassed). +/// Decodes a field-name key, using the intern cache for valid-UTF-8 keys +/// under the strict handler. Any other handler makes decoding depend on +/// mutable options state, and invalid UTF-8 can invoke a registered +/// "strict" handler (which may re-enter decoding or make decoding impure), +/// so both bypass the cache. #[inline] pub fn decode_key<'py>( py: Python<'py>, bytes: &[u8], options: &CodecOptionsData<'py>, ) -> PyResult> { - if options.utf8_handler.is_none() && bytes.len() <= KEY_CACHE_MAX_KEY_LEN { + if options.utf8_handler.is_none() + && bytes.len() <= KEY_CACHE_MAX_KEY_LEN + && std::str::from_utf8(bytes).is_ok() + { return decode_key_cached(py, bytes, options); } decode_utf8(py, bytes, options) } /// Wraps unexpected errors during value decoding in InvalidBSON, mirroring -/// `_rewrap_as_invalid_bson` in the C extension. +/// `_rewrap_as_invalid_bson` in the C extension: only `Exception` subclasses +/// are wrapped — `KeyboardInterrupt` and `SystemExit` pass through — and +/// an `InvalidBSON` is returned as-is. #[cold] pub fn rewrap_as_invalid_bson(py: Python<'_>, err: PyErr) -> PyErr { + if !err + .value(py) + .is_instance_of::() + { + return err; + } let is_invalid_bson = Types::get(py) .ok() .and_then(|types| err.value(py).is_instance(types.invalid_bson.bind(py)).ok()) diff --git a/rust/bson/src/encode.rs b/rust/bson/src/encode.rs index f3ad08a8fa..86404ca1de 100644 --- a/rust/bson/src/encode.rs +++ b/rust/bson/src/encode.rs @@ -6,7 +6,6 @@ use std::ffi::CStr; -use pyo3::buffer::PyBuffer; use pyo3::prelude::*; use pyo3::types::{ PyAny, PyBool, PyBytes, PyDateAccess, PyDateTime, PyDict, PyFloat, PyInt, PyList, PyString, @@ -708,16 +707,12 @@ fn raw_bytes_of<'py>(_py: Python<'py>, raw: &Bound<'py, PyAny>) -> PyResult::get(raw)?; - // A misbehaving exporter may hand back non-contiguous or null buffers. - if buffer.buf_ptr().is_null() || !buffer.is_c_contiguous() || buffer.item_size() != 1 { - return Err(pyo3::exceptions::PyTypeError::new_err( - "RawBSONDocument.raw must be bytes or memoryview", - )); - } - let slice = - unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; - Ok(slice.to_vec()) + let buffer = crate::RawBuffer::get(raw)?; + // The C's write_raw_doc goes through _get_buffer: a misbehaving + // exporter may hand back non-contiguous, oversize-item, or null + // buffers. + crate::with_get_buffer_checks(&buffer)?; + Ok(buffer.as_slice().to_vec()) } /// Writes the regex element shared by the `Regex` class and compiled diff --git a/rust/bson/src/lib.rs b/rust/bson/src/lib.rs index 9cfd452c30..336163956c 100644 --- a/rust/bson/src/lib.rs +++ b/rust/bson/src/lib.rs @@ -23,7 +23,6 @@ mod errors; mod options; mod types; -use pyo3::buffer::PyBuffer; use pyo3::prelude::*; use pyo3::types::{PyAny, PyBytes, PyList, PyModule, PyTuple}; @@ -33,6 +32,65 @@ use errors::invalid_bson; use options::CodecOptionsData; use types::Types; +/// A format-independent buffer view (`PyBUF_SIMPLE`), mirroring the C's +/// `PyObject_GetBuffer` call sites: any contiguous single-byte exporter is +/// accepted, including signed-byte buffers (`memoryview.cast("b")`, +/// `array.array("b")`) that the typed `PyBuffer` rejects. +pub(crate) struct RawBuffer { + view: pyo3::ffi::Py_buffer, +} + +impl RawBuffer { + pub(crate) fn get(object: &Bound<'_, PyAny>) -> PyResult { + // SAFETY: zero-initialized view; released in Drop on failure. + let mut view: pyo3::ffi::Py_buffer = unsafe { std::mem::zeroed() }; + // SAFETY: the view is released before returning the error. + if unsafe { + pyo3::ffi::PyObject_GetBuffer(object.as_ptr(), &mut view, pyo3::ffi::PyBUF_SIMPLE) + } != 0 + { + return Err(PyErr::fetch(object.py())); + } + Ok(Self { view }) + } + + pub(crate) fn as_slice(&self) -> &[u8] { + // SAFETY: the view is live and the caller has applied the + // `_get_buffer` checks (null, contiguity, itemsize). + unsafe { std::slice::from_raw_parts(self.view.buf as *const u8, self.view.len as usize) } + } +} + +impl Drop for RawBuffer { + fn drop(&mut self) { + unsafe { pyo3::ffi::PyBuffer_Release(&mut self.view) }; + } +} + +/// The C's `_get_buffer` checks: pyo3 acquires buffers with PyBUF_FULL_RO +/// elsewhere, so a misbehaving exporter can hand back null or +/// non-contiguous buffers. +pub(crate) fn with_get_buffer_checks(view: &RawBuffer) -> PyResult<()> { + // `!view->buf || view->len < 0`. + if view.view.buf.is_null() || view.view.len < 0 { + return Err(pyo3::exceptions::PyValueError::new_err("invalid buffer")); + } + // `!PyBuffer_IsContiguous(view, 'C')`. + // SAFETY: the view is live and validly initialized. + if unsafe { pyo3::ffi::PyBuffer_IsContiguous(&view.view, b'C' as _) } == 0 { + return Err(pyo3::exceptions::PyValueError::new_err( + "must be a contiguous buffer", + )); + } + // `view->itemsize != 1`. + if view.view.itemsize != 1 { + return Err(pyo3::exceptions::PyValueError::new_err( + "buffer data must be ascii or utf8", + )); + } + Ok(()) +} + /// True when any document in the stream is at least `threshold` bytes /// (mirrors `_contains_view_eligible_doc`). fn contains_view_eligible_doc(data: &[u8], threshold: usize) -> bool { @@ -69,10 +127,9 @@ fn prepare_input_buffer<'py>( return Ok(object.clone()); } let threshold = Types::get(py)?.raw_bson_view_threshold; - let buffer = PyBuffer::::get(object)?; + let buffer = RawBuffer::get(object)?; with_get_buffer_checks(&buffer)?; - let data = - unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) }; + let data = buffer.as_slice(); if contains_view_eligible_doc(data, threshold) { // Copy so memoryview-backed views outlive the input. let copy = PyBytes::new(py, data); @@ -94,39 +151,14 @@ fn with_buffer_view( if let Ok(bytes) = object.cast::() { return f(bytes.as_bytes()); } - let buffer = PyBuffer::::get(object)?; + let buffer = RawBuffer::get(object)?; with_get_buffer_checks(&buffer)?; // Copy while the GIL is held and before any callback runs. - let data = - unsafe { std::slice::from_raw_parts(buffer.buf_ptr() as *const u8, buffer.len_bytes()) } - .to_vec(); + let data = buffer.as_slice().to_vec(); drop(buffer); f(&data) } -/// Mirrors the C's `_get_buffer` checks: pyo3 acquires buffers with -/// PyBUF_FULL_RO, so a misbehaving exporter can hand back null or -/// non-contiguous buffers. -fn with_get_buffer_checks(buffer: &PyBuffer) -> PyResult<()> { - // `!view->buf || view->len < 0`. - if buffer.buf_ptr().is_null() { - return Err(pyo3::exceptions::PyValueError::new_err("invalid buffer")); - } - // `!PyBuffer_IsContiguous(view, 'C')`. - if !buffer.is_c_contiguous() { - return Err(pyo3::exceptions::PyValueError::new_err( - "must be a contiguous buffer", - )); - } - // `view->itemsize != 1`. - if buffer.item_size() != 1 { - return Err(pyo3::exceptions::PyValueError::new_err( - "buffer data must be ascii or utf8", - )); - } - Ok(()) -} - /// Encodes a Python mapping to BSON bytes. #[pyfunction] #[pyo3( @@ -290,6 +322,8 @@ fn _element_to_dict<'py>( py, &input, name_bytes, type_byte, pos, max, &options, raw_array, ) .map_err(|err| rewrap(py, err))?; + // Converter failures propagate unwrapped (the C's _element_to_dict). + let value = decode::apply_decoder(py, value, &options)?; let result = PyTuple::new( py, [ @@ -364,6 +398,9 @@ fn _raw_to_dict<'py>( ) .map_err(|err| rewrap(py, err))?; pos = new_pos; + // Converter failures propagate unwrapped (the C's + // _element_to_dict). + let value = decode::apply_decoder(py, value, &options)?; result_mapping.set_item(name, value)?; } diff --git a/rust/bson/tests/test_rbson.py b/rust/bson/tests/test_rbson.py index aa297aeaa8..7be91b9846 100644 --- a/rust/bson/tests/test_rbson.py +++ b/rust/bson/tests/test_rbson.py @@ -866,11 +866,14 @@ def test_invalid_id_not_wrapped(): def test_non_contiguous_buffer_rejected(): - # MIN-4: a non-contiguous buffer exporter raises ValueError instead of - # silently decoding wrong bytes. + # MIN-4: a non-contiguous buffer exporter fails to decode instead of + # silently decoding wrong bytes. Both codecs get BufferError from the + # exporter (PyBUF_SIMPLE): memoryview refuses the request itself. values = bytearray(b"\x0d\x00\x00\x00\x02a\x00\x01\x00\x00\x00x\x00\x00") view = memoryview(values)[::-1] # reversed: not C-contiguous - with pytest.raises(ValueError, match="contiguous"): + with pytest.raises((ValueError, BufferError), match="contiguous"): + decode_c(view, OPTIONS) + with pytest.raises((ValueError, BufferError), match="contiguous"): _rbson._bson_to_dict(view, OPTIONS) @@ -1165,3 +1168,203 @@ def __init__(self): with pytest.raises(InvalidDocument): encode_rust({"a": ShortDecimal128()}) + + +def _invalid_key_doc(): + # {"a": "x"}: the 0xff byte invokes the error handler + # during key decode. + import struct + + element = b"\x02\xff\x61\x00" + struct.pack(" rejects. + import array + + encoded = encode_c({"a": "x"}) + for exporter in ( + memoryview(encoded).cast("b"), + array.array("b", encoded), + ): + assert decode_rust(exporter) == {"a": "x"} + assert decode_c(exporter) == {"a": "x"} + + +def test_raw_bson_document_signed_byte_view(): + # RawBSONDocument.raw may be a signed-byte memoryview: the encode + # passthrough accepts it like the C. + raw = encode_c({"a": 1}) + view = memoryview(raw).cast("b") + assert_same_encode({"a": RawBSONDocument(view)}) From d7273c8fd6fc2d33ea147140c0b59beeab25c535 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Nov 2026 08:06:03 -0600 Subject: [PATCH 8/8] address review: dollar-key shortcut only for valid UTF-8 names --- rust/bson/src/decode.rs | 49 +++++++++++++++-------------------- rust/bson/src/lib.rs | 12 ++++----- rust/bson/tests/test_rbson.py | 49 ++++++++++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 42 deletions(-) diff --git a/rust/bson/src/decode.rs b/rust/bson/src/decode.rs index d28bae416e..27e17a5400 100644 --- a/rust/bson/src/decode.rs +++ b/rust/bson/src/decode.rs @@ -214,13 +214,11 @@ pub fn decode_elements_into<'py>( // _element_to_dict does not rewrap get_value failures). let value = apply_decoder(py, value, options)?; - // Track the decoded key, not the raw bytes: a non-strict handler - // can mangle b'\xff$ref' into '$ref', and the C's _dbref_hook - // sees only the decoded mapping. Strict decodes preserve raw - // bytes, so the first byte decides; a key that cannot be - // re-borrowed as UTF-8 (surrogates under "surrogateescape") - // cannot equal "$ref"/"$id" either. - has_dollar_key |= if options.utf8_handler.is_none() { + // The raw-byte shortcut needs valid UTF-8 plus an unmodified + // strict handler: a registered "strict" handler can still turn + // b'\xffref' into '$ref' (the C's _dbref_hook sees only the + // decoded mapping). Surrogate keys cannot equal "$ref"/"$id". + has_dollar_key |= if options.utf8_handler.is_none() && std::str::from_utf8(name).is_ok() { name.first() == Some(&b'$') } else { key.cast::() @@ -309,11 +307,10 @@ pub fn get_value_with_raw_array<'py>( elements_to_dict(py, input, &data[position..position + size], options) .map_err(|err| rewrap_as_invalid_bson(py, err))?; position += size; - // Hook for DBRefs ($ref/$id shaped docs decode to DBRef). The - // C runs it unconditionally; the skip is only sound for exact - // dicts, which store keys as decoded. A custom class may add - // or rewrite keys in its constructor or __setitem__, so its - // result is always checked. + // The C runs the DBRef hook unconditionally; the skip is only + // sound for exact dicts, which store keys as decoded. A custom + // class may add or rewrite keys, so its result is always + // checked. if options.is_raw_bson || (options.is_dict_class && !has_dollar_key) { value } else { @@ -626,10 +623,9 @@ pub fn get_value_with_raw_array<'py>( } /// Applies the type registry's decoder for the value's exact type, if -/// registered. Mirrors the C's decoder-map tail in get_value: the converter -/// result is returned directly, so its failures are not wrapped here — -/// whether they are wrapped depends on the caller (document-level callers -/// propagate; the array loop wraps, like the C's `goto invalid`). +/// registered. Mirrors the C's decoder-map tail in get_value: its failures +/// are not wrapped here — document-level callers propagate them; the array +/// loop wraps, like the C's `goto invalid`. pub fn apply_decoder<'py>( py: Python<'py>, value: Py, @@ -668,12 +664,11 @@ fn dbref_hook<'py>( ) -> PyResult> { let doc = value.bind(py); let lookup = |key: &str| -> PyResult>> { - // Exact-dict fast path: PyDict_GetItemWithError raises no - // exceptions for missing keys, and one lookup serves both the - // presence check and the value (PyMapping_HasKey + - // PyObject_GetItem in the C). Subclasses are excluded — their - // __getitem__ override must win — and other mappings fall back to - // get_item with a caught KeyError. + // Exact-dict fast path: one raw-storage lookup serves both the + // presence check and the value (PyMapping_HasKey + GetItem in the + // C). Subclasses are excluded — their __getitem__ override must + // win — and other mappings fall back to get_item with a caught + // KeyError. if doc.is_exact_instance_of::() { let dict = doc.cast::().unwrap(); let key = pyo3::types::PyString::intern(py, key); @@ -1041,10 +1036,9 @@ fn decode_key_cached<'py>( } /// Decodes a field-name key, using the intern cache for valid-UTF-8 keys -/// under the strict handler. Any other handler makes decoding depend on -/// mutable options state, and invalid UTF-8 can invoke a registered +/// under the strict handler. Anything else can invoke a registered /// "strict" handler (which may re-enter decoding or make decoding impure), -/// so both bypass the cache. +/// so it bypasses the cache. #[inline] pub fn decode_key<'py>( py: Python<'py>, @@ -1061,9 +1055,8 @@ pub fn decode_key<'py>( } /// Wraps unexpected errors during value decoding in InvalidBSON, mirroring -/// `_rewrap_as_invalid_bson` in the C extension: only `Exception` subclasses -/// are wrapped — `KeyboardInterrupt` and `SystemExit` pass through — and -/// an `InvalidBSON` is returned as-is. +/// `_rewrap_as_invalid_bson` in the C: only `Exception` subclasses are +/// wrapped, and an `InvalidBSON` is returned as-is. #[cold] pub fn rewrap_as_invalid_bson(py: Python<'_>, err: PyErr) -> PyErr { if !err diff --git a/rust/bson/src/lib.rs b/rust/bson/src/lib.rs index 336163956c..5a96fe08c5 100644 --- a/rust/bson/src/lib.rs +++ b/rust/bson/src/lib.rs @@ -32,10 +32,9 @@ use errors::invalid_bson; use options::CodecOptionsData; use types::Types; -/// A format-independent buffer view (`PyBUF_SIMPLE`), mirroring the C's -/// `PyObject_GetBuffer` call sites: any contiguous single-byte exporter is -/// accepted, including signed-byte buffers (`memoryview.cast("b")`, -/// `array.array("b")`) that the typed `PyBuffer` rejects. +/// A format-independent buffer view (`PyBUF_SIMPLE`): accepts any +/// contiguous single-byte exporter, including signed-byte buffers +/// (`memoryview.cast("b")`) that the typed `PyBuffer` rejects. pub(crate) struct RawBuffer { view: pyo3::ffi::Py_buffer, } @@ -67,9 +66,8 @@ impl Drop for RawBuffer { } } -/// The C's `_get_buffer` checks: pyo3 acquires buffers with PyBUF_FULL_RO -/// elsewhere, so a misbehaving exporter can hand back null or -/// non-contiguous buffers. +/// The C's `_get_buffer` checks: a misbehaving exporter can hand back +/// null, non-contiguous, or oversize-item buffers. pub(crate) fn with_get_buffer_checks(view: &RawBuffer) -> PyResult<()> { // `!view->buf || view->len < 0`. if view.view.buf.is_null() || view.view.len < 0 { diff --git a/rust/bson/tests/test_rbson.py b/rust/bson/tests/test_rbson.py index 7be91b9846..8e76f05c41 100644 --- a/rust/bson/tests/test_rbson.py +++ b/rust/bson/tests/test_rbson.py @@ -1181,10 +1181,8 @@ def _invalid_key_doc(): def test_registered_strict_handler_bypasses_cache(): - # A handler registered under the name "strict" runs inside - # PyUnicode_DecodeUTF8 even for a NULL errors argument, and its output - # may differ per call: invalid-UTF-8 keys must bypass the key cache - # rather than caching handler output as a strict decode. + # A handler registered as "strict" runs on invalid UTF-8 keys, and its + # output may differ per call: those keys must bypass the key cache. import codecs data = _invalid_key_doc() @@ -1251,6 +1249,44 @@ def run(): assert outcome["result"] == {"ka": "x"} +def test_dbref_dollar_keys_from_registered_strict_handler(): + # A handler registered under the name "strict" runs on invalid bytes + # even with a NULL errors argument, so b'\xffref'/b'\xffid' decode to + # "$ref"/"$id" while utf8_handler stays None. The C's _dbref_hook sees + # the decoded mapping and returns a DBRef; the raw-byte shortcut must + # only apply to valid-UTF-8 names. + import codecs + import struct + + inner = ( + struct.pack(" rejects. + # The C acquires buffers with PyBUF_SIMPLE: any contiguous + # single-byte exporter decodes, including signed-byte (format "b"). import array encoded = encode_c({"a": "x"})