PYTHON-6048 Add sanitizer variants to Evergreen

.evergreen/generated_configs/functions.yml

@@ -85,6 +85,20 @@ functions:
       params:
         directory: src
 
+  # Run sanitizer tests
+  run sanitizer tests:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - run-sanitizer-tests
+        working_dir: src
+        include_expansions_in_env:
+          - SANITIZER
+          - UV_PYTHON
+      type: test
+
   # Run server
   run server:
     - command: subprocess.exec

.evergreen/generated_configs/tasks.yml

@@ -2831,6 +2831,33 @@ tasks:
       - func: send dashboard data
     tags: [perf]
 
+  # Sanitizer tests
+  - name: test-sanitizer-asan
+    commands:
+      - func: run server
+        vars:
+          VERSION: latest
+          TOPOLOGY: standalone
+          AUTH: noauth
+          SSL: nossl
+      - func: run sanitizer tests
+        vars:
+          SANITIZER: asan
+    tags: [sanitizer, pr]
+  - name: test-sanitizer-tsan
+    commands:
+      - func: run server
+        vars:
+          VERSION: latest
+          TOPOLOGY: standalone
+          AUTH: noauth
+          SSL: nossl
+      - func: run sanitizer tests
+        vars:
+          SANITIZER: tsan
+          UV_PYTHON: 3.14t
+    tags: [sanitizer, pr, free-threaded]
+
   # Search index tests
   - name: test-search-index-helpers
     commands:

.evergreen/generated_configs/variants.yml

@@ -520,6 +520,16 @@ buildvariants:
       SUB_TEST_NAME: pyopenssl
     tags: [pr]
 
+  # Sanitizer tests
+  - name: sanitizers-ubuntu-22
+    tasks:
+      - name: test-sanitizer-asan
+      - name: test-sanitizer-tsan
+    display_name: Sanitizers Ubuntu-22
+    run_on:
+      - ubuntu2204-small
+    tags: [pr]
+
   # Search index tests
   - name: search-index-helpers-rhel8
     tasks:

.evergreen/scripts/generate_config.py

@@ -400,6 +400,13 @@ def create_no_c_ext_variants():
     return [create_variant(tasks, display_name, host=host, expansions=expansions)]
 
 
+def create_sanitizer_variants():
+    host = HOSTS["ubuntu22"]
+    tasks = ["test-sanitizer-asan", "test-sanitizer-tsan"]
+    display_name = get_variant_name("Sanitizers", host)
+    return [create_variant(tasks, display_name, host=host, tags=["pr"])]
+
+
 def create_disable_test_commands_variants():
     host = DEFAULT_HOST
     expansions = dict(AUTH="auth", SSL="ssl", DISABLE_TEST_COMMANDS="1")
@@ -718,6 +725,30 @@ def create_no_toolchain_tasks():
     return tasks
 
 
+def create_sanitizer_tasks():
+    tasks = []
+    # (sanitizer, free-threaded UV_PYTHON or None)
+    configs = [("asan", None), ("tsan", "3.14t")]
+    for sanitizer, python in configs:
+        tags = ["sanitizer", "pr"]
+        server_vars = dict(VERSION="latest", TOPOLOGY="standalone", AUTH="noauth", SSL="nossl")
+        server_func = FunctionCall(func="run server", vars=server_vars)
+        test_vars = dict(SANITIZER=sanitizer)
+        if python:
+            test_vars["UV_PYTHON"] = python
+            tags.append("free-threaded")
+        test_func = FunctionCall(func="run sanitizer tests", vars=test_vars)
+        name = f"test-sanitizer-{sanitizer}"
+        tasks.append(
+            EvgTask(
+                name=name,
+                tags=tags,
+                commands=[server_func, test_func],
+            )
+        )
+    return tasks
+
+
 def create_test_non_standard_tasks():
     """For variants that set a TEST_NAME."""
     tasks = []
@@ -1362,6 +1393,13 @@ def create_run_tests_func():
     return "run tests", [setup_cmd, test_cmd]
 
 
+def create_run_sanitizer_tests_func():
+    includes = ["SANITIZER", "UV_PYTHON"]
+    args = [".evergreen/just.sh", "run-sanitizer-tests"]
+    sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
+    return "run sanitizer tests", [sub_cmd]
+
+
 def create_test_numpy_func():
     includes = ["TOOLCHAIN_VERSION", "COVERAGE"]
     test_cmd = get_subprocess_exec(

.evergreen/scripts/run-sanitizer-tests.sh

@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+# Clean sanitizer rebuild of the C extensions, run the fixed BSON/client
+# test files under the matching sanitizer runtime, and fail on any
+# sanitizer diagnostic even if pytest itself exits 0.
+#
+# ASan runs against a prebuilt interpreter with libasan.so LD_PRELOADed.
+# TSan builds a fully instrumented free-threaded CPython from source
+# instead: TSan cannot see synchronization in code compiled without
+# -fsanitize=thread, so LD_PRELOADing it onto a prebuilt interpreter
+# reports false races inside CPython's own free-threading internals.
+set -eu
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+
+SANITIZER=${SANITIZER:?"SANITIZER must be set to 'asan' or 'tsan'"}
+UV_PYTHON=${UV_PYTHON:-3.13}
+TEST_FILES=(test/test_bson.py test/test_message.py test/test_raw_bson.py test/test_raw_bson_shared.py test/test_client.py)
+
+# The free-threaded CPython the TSan task builds. Kept in step with the
+# "3.14t" entry in .evergreen/scripts/generate_config_utils.py's CPYTHONS.
+CPYTHON_TAG=v3.14.0
+CPYTHON_SRC=.tsan-cpython-src
+CPYTHON_INSTALL=.tsan-cpython-install
+
+# A stale build/ or venv can leave a .so or install linked against the wrong
+# sanitizer's runtime without a build error, so always start clean.
+rm -rf build
+rm -f bson/*.so pymongo/*.so
+
+export CC=${CC:-clang}
+export CXX=${CXX:-clang++}
+export PYMONGO_C_EXT_MUST_BUILD=1
+
+case "$SANITIZER" in
+  asan)
+    rm -rf .sanitizer-venv
+    # PYMONGO_TEST_ALLOC_FAILURE compiles the test-only fail-once realloc
+    # hook (bson/buffer.c) so the realloc-failure regression tests run.
+    export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g -O0 -DPYMONGO_TEST_ALLOC_FAILURE"
+    export LDFLAGS="-fsanitize=address,undefined"
+
+    # clang's ASan runtime also provides the UBSan handlers, so preloading
+    # libasan alone resolves the symbols from both -fsanitize=address and
+    # -fsanitize=undefined. Preloading a separate libubsan_standalone aborts
+    # during init: the two runtimes both bring sanitizer_common and fight
+    # over sigaction. clang names the runtime libclang_rt.asan-x86_64.so, and
+    # -print-file-name can also return a bare name or a linker script, which
+    # ld.so refuses to preload, so resolve the candidates and require a real
+    # ELF shared object.
+    resolve_runtime() {
+      local name path
+      for name in "$@"; do
+        path=$("$CC" -print-file-name="$name")
+        if [ -f "$path" ] && [ "$(head -c 4 "$path" | od -An -tx1 | tr -d " \n")" = "7f454c46" ]; then
+          printf "%s\n" "$path"
+          return 0
+        fi
+      done
+      return 1
+    }
+    RUNTIME_LIB=$(resolve_runtime libclang_rt.asan-x86_64.so libasan.so) || {
+      echo "Could not locate a loadable ASan runtime through $CC -print-file-name. Is the matching sanitizer runtime package installed?" >&2
+      exit 1
+    }
+    export ASAN_OPTIONS="detect_leaks=0"
+    export UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1"
+    # Route CPython's own allocations through the system allocator so ASan's
+    # redzones can see them; pymalloc otherwise hides real bugs and adds
+    # noise. The free-threaded TSan interpreter doesn't accept this value,
+    # so it's scoped to ASan only.
+    export PYTHONMALLOC=malloc
+
+    uv venv --python "$UV_PYTHON" .sanitizer-venv
+    VENV_PYTHON=.sanitizer-venv/bin/python3
+    uv pip install --python "$VENV_PYTHON" -e . --reinstall
+    uv pip install --python "$VENV_PYTHON" -r requirements/test.txt
+
+    PYTEST_CMD=(env "LD_PRELOAD=$RUNTIME_LIB" "$VENV_PYTHON" -m pytest)
+
+    # The C-specific tests skip silently when the extensions are missing,
+    # which would turn a broken rebuild into a false-green task. Import the
+    # extension modules directly so a failure shows the real ImportError.
+    env "LD_PRELOAD=$RUNTIME_LIB" "$VENV_PYTHON" -c "import bson._cbson, pymongo._cmessage"
+    ;;
+  tsan)
+    rm -rf "$CPYTHON_SRC" "$CPYTHON_INSTALL"
+
+    # TSan's shadow mapping can fail to reserve its address ranges under the
+    # default ASLR entropy on recent kernels, which crashes the process at
+    # startup. CPython's own CI lowers the entropy the same way. Evergreen
+    # hosts may not permit it, so this is best effort.
+    sudo sysctl -w vm.mmap_rnd_bits=28 || true
+
+    # Fail here rather than after the 20-30 minute build: without these
+    # headers _ssl won't build and the pip installs below cannot reach PyPI.
+    if [ ! -f /usr/include/openssl/ssl.h ]; then
+      echo "OpenSSL development headers not found at /usr/include/openssl/ssl.h. This host has no way to install them: apt-get isn't available here, so building CPython with a working ssl module isn't possible. Aborting before the CPython build." >&2
+      exit 1
+    fi
+
+    CPYTHON_INSTALL_ABS="$(pwd)/$CPYTHON_INSTALL"
+    git clone --depth 1 --branch "$CPYTHON_TAG" https://github.com/python/cpython.git "$CPYTHON_SRC"
+
+    # TSan accepts one suppressions file, so concatenate the pinned tag's own
+    # suppressions with this repo's additions. The 3.14 branch's file is not
+    # empty: even a fully instrumented build needs its ~24 entries.
+    CPYTHON_SUPPRESSIONS="$CPYTHON_SRC/Tools/tsan/suppressions_free_threading.txt"
+    if [ ! -f "$CPYTHON_SUPPRESSIONS" ]; then
+      echo "Expected CPython's own TSan suppressions at $CPYTHON_SUPPRESSIONS, but the file is missing from the $CPYTHON_TAG source tree." >&2
+      exit 1
+    fi
+    COMBINED_SUPPRESSIONS="$(pwd)/.tsan-suppressions-combined.txt"
+    cat "$CPYTHON_SUPPRESSIONS" .evergreen/tsan-suppressions.txt > "$COMBINED_SUPPRESSIONS"
+
+    # Flags mirror CPython's own TSan CI job (.github/workflows/reusable-san.yml).
+    # CFLAGS/LDFLAGS are deliberately left unset here: --with-thread-sanitizer
+    # and --with-pydebug already supply the right flags, and overriding them
+    # would fight configure.
+    #
+    # Unlike CPython's CI this does not rebuild OpenSSL with TSan, which is
+    # only needed to keep the ssl tests quiet. The tests below don't use ssl,
+    # but pip does need it to reach PyPI, so _ssl still has to build against
+    # the system OpenSSL.
+    (
+      cd "$CPYTHON_SRC"
+      ./configure \
+        --with-thread-sanitizer \
+        --with-pydebug \
+        --disable-gil \
+        --prefix="$CPYTHON_INSTALL_ABS"
+      make -j"$(nproc 2>/dev/null || echo 4)"
+      make install
+    )
+
+    # --disable-gil adds a "t" suffix and --with-pydebug adds a "d", so this
+    # build installs as pythonX.Ytd. bin/python3 is the last-resort fallback.
+    TSAN_PYTHON=""
+    for candidate in "$CPYTHON_INSTALL"/bin/python3.*td "$CPYTHON_INSTALL"/bin/python3.*t "$CPYTHON_INSTALL"/bin/python3; do
+      if [ -x "$candidate" ]; then
+        TSAN_PYTHON="$candidate"
+        break
+      fi
+    done
+    if [ -z "$TSAN_PYTHON" ]; then
+      echo "Could not find the interpreter built from source under $CPYTHON_INSTALL/bin:" >&2
+      ls -l "$CPYTHON_INSTALL/bin" >&2 || true
+      exit 1
+    fi
+    echo "Using TSan-instrumented interpreter: $TSAN_PYTHON"
+    "$TSAN_PYTHON" -VV
+    "$TSAN_PYTHON" -c 'import sysconfig, sys; sys.exit(0 if sysconfig.get_config_var("Py_GIL_DISABLED") else "interpreter is not free-threaded")'
+    if ! "$TSAN_PYTHON" -c 'import ssl' >/dev/null 2>&1; then
+      echo "Warning: the interpreter built from source has no working ssl module, so pip cannot reach PyPI. Install the system OpenSSL development headers on this host." >&2
+    fi
+
+    # Build PyMongo's C extensions with the same instrumentation as the
+    # interpreter. These are plain shell env vars so pip's isolated build
+    # subprocess inherits them; build isolation is left on so pip resolves
+    # hatchling's build dependencies itself.
+    export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0"
+    export LDFLAGS="-fsanitize=thread"
+    "$TSAN_PYTHON" -m pip install -e .
+    "$TSAN_PYTHON" -m pip install -r requirements/test.txt
+
+    # Set after the build: halt_on_error=1 would abort the CPython build and
+    # the installs on any diagnostic raised by those tools themselves.
+    # handle_segv=0 matches CPython's own TSan CI job.
+    TSAN_OPTIONS="halt_on_error=1:handle_segv=0:suppressions=$COMBINED_SUPPRESSIONS"
+    export TSAN_OPTIONS
+
+    # No LD_PRELOAD: both the interpreter and the extensions link the TSan
+    # runtime at build time.
+    PYTEST_CMD=("$TSAN_PYTHON" -m pytest)
+
+    # The C-specific tests skip silently when the extensions are missing,
+    # which would turn a broken rebuild into a false-green task. Import the
+    # extension modules directly so a failure shows the real ImportError.
+    "$TSAN_PYTHON" -c "import bson._cbson, pymongo._cmessage"
+    ;;
+  *)
+    echo "Unknown SANITIZER: $SANITIZER (expected 'asan' or 'tsan')" >&2
+    exit 1
+    ;;
+esac
+
+LOG_FILE=$(mktemp)
+set +e
+"${PYTEST_CMD[@]}" -v --capture=no "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE"
+PYTEST_STATUS=${PIPESTATUS[0]}
+set -e
+
+if grep -qE "ERROR: (AddressSanitizer|LeakSanitizer)|runtime error:|WARNING: ThreadSanitizer|SUMMARY: (Address|Undefined|ThreadSanitizer)" "$LOG_FILE"; then
+  echo "Sanitizer diagnostic detected in test output, failing task" >&2
+  exit 1
+fi
+
+exit "$PYTEST_STATUS"

.evergreen/tsan-suppressions.txt

@@ -0,0 +1,24 @@
+# PyMongo-specific ThreadSanitizer suppressions.
+#
+# .evergreen/scripts/run-sanitizer-tests.sh concatenates this file with the
+# pinned CPython tag's own Tools/tsan/suppressions_free_threading.txt and
+# passes the result as TSan's single suppressions= path. That upstream file
+# is not empty on the 3.14 branch: it carries roughly two dozen entries
+# (assign_version_tag, update_one_slot, _PyFrame_GetCode, rangeiter_next,
+# list_ass_slice_lock_held, PyObject_Realloc, mi_block_set_nextx,
+# pthread_create, and others) that CPython's own fully instrumented TSan CI
+# job needs to run clean. Do not duplicate those here.
+#
+# Add an entry below only for a race that has been investigated and judged
+# benign, with a comment saying why: either a race in PyMongo's own C
+# extensions, or a CPython-internal race (both stacks within CPython) that
+# upstream's own suppressions do not yet cover.
+#
+# Reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions
+
+# CPython-internal: under free-threading the GC's delete_garbage path
+# (memory_clear) and a worker thread's deferred refcount merge
+# (memory_dealloc) can both enter _memory_release for the same
+# memoryview. Both stacks are entirely within CPython; pymongo only
+# creates and releases the memoryviews concurrently.
+race_top:_memory_release

.gitignore

@@ -46,3 +46,9 @@ xunit-results/
 coverage.xml
 server.log
 .coverage
+
+# sanitizer test task scratch (see .evergreen/scripts/run-sanitizer-tests.sh)
+.sanitizer-venv/
+.tsan-cpython-src/
+.tsan-cpython-install/
+.tsan-suppressions-combined.txt

bson/_cbsonmodule.c

@@ -161,6 +161,15 @@ extern int cbson_long_long_to_str(long long num, char* str, size_t size) {
     return 0;
 }
 
+static PyObject* _test_fail_next_realloc(PyObject* self, PyObject* Py_UNUSED(ignored)) {
+#ifdef PYMONGO_TEST_ALLOC_FAILURE
+    pymongo_buffer_test_fail_next_realloc();
+    Py_RETURN_TRUE;
+#else
+    Py_RETURN_FALSE;
+#endif
+}
+
 static PyObject* _test_long_long_to_str(PyObject* self, PyObject* args) {
     // Test extreme values
     Py_ssize_t maxNum = PY_SSIZE_T_MAX;
@@ -3516,6 +3525,7 @@ static PyMethodDef _CBSONMethods[] = {
      "Decode all elements of a raw BSON document into a result mapping."},
     {"_array_of_documents_to_buffer", _cbson_array_of_documents_to_buffer, METH_VARARGS, "Convert raw array of documents to a stream of BSON documents"},
     {"_test_long_long_to_str", _test_long_long_to_str, METH_VARARGS, "Test conversion of extreme and common Py_ssize_t values to str."},
+    {"_test_fail_next_realloc", _test_fail_next_realloc, METH_NOARGS, "Test-only hook: fail the next buffer growth exactly once. Returns False if the hook is not compiled in."},
     {NULL, NULL, 0, NULL}
 };