diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 2bcc195224..be4adf52c4 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -213,6 +213,19 @@ tasks: TEST_NAME: fuzz SUB_TEST_NAME: fuzz UV_PYTHON: "3.13" + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/capture-fuzz-corpus.sh + working_dir: src + include_expansions_in_env: + - FUZZ_CORPUS_MAX_UNITS + - FUZZ_CORPUS_MAX_UNIT_BYTES + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + type: test tags: [fuzz, fuzz-nightly] # Getdata tests diff --git a/.evergreen/scripts/capture-fuzz-corpus.sh b/.evergreen/scripts/capture-fuzz-corpus.sh new file mode 100755 index 0000000000..438c7f6754 --- /dev/null +++ b/.evergreen/scripts/capture-fuzz-corpus.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# PYTHON-6178: capture coverage-increasing inputs from a fuzz run and open +# an automated PR growing fuzz/corpus/. Runs as the last command of the +# nightly "fuzz" task, while the scratch corpus exists; manual +# `evergreen patch --variant fuzz-rhel8 --task fuzz` runs capture too. +set -eu + +# Same environment as just.sh: venv on PATH for a modern Python. +. "$(dirname "${BASH_SOURCE:-$0}")/setup-dev-env.sh" + +# No new units means no PR, and the github secrets flow is never touched. +uv run --no-sync python fuzz/capture_corpus.py +if [ -z "$(ls -A fuzz-scratch/capture 2>/dev/null || true)" ]; then + echo "No new fuzz corpus units; nothing to do." + exit 0 +fi + +# A unit found by a later target never ran through the earlier ones, so +# replay every unit through all targets before publishing. +replay=(fuzz-scratch/capture/*) +for target in fuzz_decode fuzz_decode_all fuzz_is_valid; do + echo "Replaying captured units against ${target}..." + if ! uv run --no-sync python "fuzz/${target}.py" "${replay[@]}" \ + -artifact_prefix="$PWD/fuzz-scratch/" -atheris_runs="${#replay[@]}" + then + echo "Captured unit fails ${target}; no corpus PR." >&2 + exit 1 + fi +done + +bash .evergreen/scripts/create-fuzz-corpus-pr.sh \ + fuzz-scratch/capture fuzz-scratch/capture-summary.md diff --git a/.evergreen/scripts/create-fuzz-corpus-pr.sh b/.evergreen/scripts/create-fuzz-corpus-pr.sh new file mode 100755 index 0000000000..d24e677bff --- /dev/null +++ b/.evergreen/scripts/create-fuzz-corpus-pr.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -eu + +# Open (or update) the automated pull request that grows fuzz/corpus/ +# with freshly captured units. Arguments: 1 = captured units directory, +# 2 = capture summary markdown to use as the PR body. + +capture_dir="$1" +summary_file="$2" +today="$(date '+%m-%d-%Y')" + +# Shared github-app authentication, git branch and PR plumbing. +# shellcheck disable=SC1091 +. "$(dirname "${BASH_SOURCE[0]}")/github-pr-util.sh" + +acquire_github_token + +echo "Creating the git checkout..." +branch="fuzz-corpus-${today}" +prepare_bot_branch "$branch" + +mkdir -p fuzz/corpus +cp "$capture_dir"/* fuzz/corpus/ +if ! stage_and_commit "PYTHON-6178 Grow the BSON fuzz corpus (${today})" fuzz/corpus; then + echo "No new corpus units to commit; skipping PR creation." + exit 0 +fi +echo "Creating the git checkout... done." + +push_branch "$branch" +create_or_update_pr "$branch" "[Fuzz Corpus] ${today}" "$summary_file" diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index fb2dea1640..51561ef29f 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,127 +1,30 @@ #!/usr/bin/env bash set -eu -# Default to the drivers-evergreen-tools submodule when unset; a -# caller-provided DRIVERS_TOOLS wins, as in the other consumers. -tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")" -pushd $tools/.evergreen/github_app || exit +# Open (or update) the automated pull request resyncing the spec tests. +# Arguments: 1 = resync summary text to use as the PR body. -owner="mongodb" -repo="mongo-python-driver" +summary_file="$1" +today="$(date '+%m-%d-%Y')" -# Bootstrap the app. -echo "bootstrapping" -source utils.sh -bootstrap drivers/comment-bot +# Shared github-app authentication, git branch and PR plumbing. +# shellcheck disable=SC1091 +. "$(dirname "${BASH_SOURCE[0]}")/github-pr-util.sh" -# Run the app. -source ./secrets-export.sh +acquire_github_token -# Get a github access token for the git checkout. -echo "Getting github token..." - -token=$(bash ./get-access-token.sh $repo $owner) -if [ -z "${token}" ]; then - echo "Failed to get github access token!" - popd || exit - exit 1 -fi -echo "Getting github token... done." -popd || exit - -# Make the git checkout and create a new branch. echo "Creating the git checkout..." -branch="spec-resync-"$(date '+%m-%d-%Y') - -git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git -# -B resets the branch if a reused checkout already has it, keeping reruns idempotent. -git checkout -B "$branch" "origin/main" +branch="spec-resync-${today}" +prepare_bot_branch "$branch" -# Attribute the commit to the bot user; fall back to the host identity. -bot_login="mongodb-drivers-pr-bot[bot]" -bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true) -if [ -n "$bot_id" ]; then - git config user.name "$bot_login" - git config user.email "${bot_id}+${bot_login}@users.noreply.github.com" -fi - -git add ./test - -if git diff --cached --quiet && git diff --quiet --ignore-submodules=dirty; then +# The spec sync also updates the vendored public suffix list (cp_psl in +# resync-specs.sh), a tracked file outside test/. +if ! stage_and_commit "resyncing specs ${today}" ./test pymongo/public_suffix_list.dat; then echo "No spec changes to commit; skipping PR creation." - cat "$1" + cat "$summary_file" exit 0 fi - -git commit -am "resyncing specs $(date '+%m-%d-%Y')" echo "Creating the git checkout... done." -# Date-stamped branch: force-push makes same-day reruns idempotent; retry -# absorbs GitHub's intermittent 403s on the receive-pack endpoint. -push_ok=false -for _attempt in 1 2 3; do - if git push --force origin $branch; then - push_ok=true - break - fi - if [ "$_attempt" -lt 3 ]; then - echo "Push attempt ${_attempt} failed; retrying in 30s..." - sleep 30 - fi -done -if [ "$push_ok" != true ]; then - echo "Failed to push $branch after 3 attempts!" >&2 - exit 1 -fi - -# Build the payload as a file so the body content is always properly -# JSON-escaped, rather than interpolated into the command line as text. -payload_file=$(mktemp) -trap 'rm -f "$payload_file"' EXIT -jq -n \ - --arg title "[Spec Resync] $(date '+%m-%d-%Y')" \ - --arg head "${branch}" \ - --rawfile body "$1" \ - '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" - -# Reuse an open PR for this branch (same-day rerun) instead of duplicating it. -existing_pr_json=$(curl -gs \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $token" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - --url "https://api.github.com/repos/$owner/$repo/pulls?head=$owner:$branch&state=open") -existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty') -if [ -n "$existing_pr_url" ]; then - existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty') - if ! curl -sfX PATCH \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $token" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -d "{\"title\": \"[Spec Resync] $(date '+%m-%d-%Y')\", \"body\": $(jq -Rs . < "$1")}" \ - --url "https://api.github.com/repos/$owner/$repo/pulls/${existing_pr_number}" > /dev/null - then - echo "Failed to update PR summary!" >&2 - exit 1 - fi - echo "$existing_pr_url" - echo "Creating the PR... done. (PR already existed; branch and summary were updated)" - rm -rf $tools - exit 0 -fi - -resp=$(curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $token" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -d "@${payload_file}" \ - --url https://api.github.com/repos/$owner/$repo/pulls) - -pr_url=$(echo "$resp" | jq -r '.html_url // empty') -if [ -z "$pr_url" ]; then - echo "Failed to create PR! API response:" - echo "$resp" | jq . - exit 1 -fi -echo "$pr_url" -echo "Creating the PR... done." +push_branch "$branch" +create_or_update_pr "$branch" "[Spec Resync] ${today}" "$summary_file" diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 58299f7f33..123362a89a 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -520,14 +520,21 @@ def create_doctests_variants(): def create_fuzz_variants(): + # Two variants for the two fuzz flows: the regression reproducers run + # on every PR, the time-boxed fuzzing runs nightly. The nightly variant + # carries no "pr" tag, so PR patch builds never select it, while its + # default patchability keeps it manually runnable + # (`evergreen patch --variant fuzz-rhel8 --task fuzz`). The host needs + # CPython, which the rhel8 toolchain provides. host = HOSTS["rhel8"] - # Regression test on every PR. - name = get_variant_name("Fuzz Regression", host) - pr_variant = create_variant([".fuzz-regression"], name, host=host, tags=["pr"]) - # Fuzzing requires CPython on Linux/macOS; use a pinned CPython on RHEL8. - name = get_variant_name("Fuzz", host) - nightly = create_variant([".fuzz-nightly"], name, host=host, batchtime=BATCHTIME_DAY) - return [pr_variant, nightly] + return [ + create_variant( + [".fuzz-regression"], get_variant_name("Fuzz Regression", host), host=host, tags=["pr"] + ), + create_variant( + [".fuzz-nightly"], get_variant_name("Fuzz", host), host=host, batchtime=BATCHTIME_DAY + ), + ] def create_sanitizer_variants() -> list[BuildVariant]: @@ -960,7 +967,24 @@ def create_fuzz_tasks(): tags = ["fuzz", "fuzz-nightly"] test_vars = dict(TEST_NAME="fuzz", SUB_TEST_NAME="fuzz", UV_PYTHON="3.13") test_func = FunctionCall(func="run tests", vars=test_vars) - tasks.append(EvgTask(name="fuzz", tags=tags, commands=[test_func])) + # PYTHON-6178: while the scratch corpus still exists, capture the new + # coverage-increasing inputs, replay them through every target so a + # unit that fails one is never published, and open an automated PR + # growing fuzz/corpus/. + capture_cmd = get_subprocess_exec( + args=[".evergreen/scripts/capture-fuzz-corpus.sh"], + include_expansions_in_env=[ + "FUZZ_CORPUS_MAX_UNITS", + "FUZZ_CORPUS_MAX_UNIT_BYTES", + # The capture bootstraps the comment-bot github app, whose + # secrets live in AWS Secrets Manager; pass the assumed-role + # credentials as resync_specs and backport-pr also do. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + ], + ) + tasks.append(EvgTask(name="fuzz", tags=tags, commands=[test_func, capture_cmd])) return tasks diff --git a/.evergreen/scripts/github-pr-util.sh b/.evergreen/scripts/github-pr-util.sh new file mode 100755 index 0000000000..e2b2d87de4 --- /dev/null +++ b/.evergreen/scripts/github-pr-util.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Shared plumbing for the automated PR workflows (spec resync and fuzz +# corpus growth): github-app authentication, branch preparation, force +# pushing with retries, and create-or-update against the GitHub API. +# +# Source this file from a wrapper script run from the repository root +# (where the drivers-evergreen-tools submodule lives); it is not meant to +# be executed directly. Defines OWNER, REPO and the functions below; +# requires curl, jq and git. + +OWNER="mongodb" +REPO="mongo-python-driver" + +# Authenticate with the github app in drivers-evergreen-tools and set +# GITHUB_TOKEN. Exits the caller if no token can be obtained. +acquire_github_token() { + # Default to the drivers-evergreen-tools submodule when unset; a + # caller-provided DRIVERS_TOOLS wins, as in the other consumers. + local tools + tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")" + pushd "$tools/.evergreen/github_app" || exit + + # Bootstrap the app. + echo "bootstrapping" + # shellcheck disable=SC1091 + source utils.sh + bootstrap drivers/comment-bot + + # Run the app. + # shellcheck disable=SC1091 + source ./secrets-export.sh + + # Get a github access token for the git checkout. + echo "Getting github token..." + + GITHUB_TOKEN=$(bash ./get-access-token.sh "$REPO" "$OWNER") + if [ -z "${GITHUB_TOKEN}" ]; then + echo "Failed to get github access token!" + popd || exit + exit 1 + fi + echo "Getting github token... done." + popd || exit +} + +# Prepare the git checkout: point origin at the repo with token auth, +# check out fresh from origin/main (-B resets a reused checkout, +# keeping reruns idempotent), and attribute commits to the drivers PR bot +# when its GitHub id is resolvable. +prepare_bot_branch() { + local branch="$1" + + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${OWNER}/${REPO}.git" + git checkout -B "$branch" "origin/main" + + local bot_login="mongodb-drivers-pr-bot[bot]" bot_id + bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true) + if [ -n "$bot_id" ]; then + git config user.name "$bot_login" + git config user.email "${bot_id}+${bot_login}@users.noreply.github.com" + fi +} + +# Stage the given paths and commit them. Arguments: the commit message, +# then the paths to stage. Returns 1 when the tree was already clean, so +# the caller can skip PR creation. Git failures exit the caller: errexit +# is suspended while this function runs in an "if" condition, so every +# fallible command is guarded explicitly. +stage_and_commit() { + local message="$1" + shift + + git add "$@" || exit + if git diff --cached --quiet && git diff --quiet --ignore-submodules=dirty; then + return 1 + fi + git commit -m "$message" || exit +} + +# Push to origin. Force-pushing makes same-day reruns of a +# date-stamped branch idempotent; the retries absorb GitHub's +# intermittent 403s on the receive-pack endpoint. Exits the caller after +# 3 failed attempts. +push_branch() { + local branch="$1" + + local push_ok=false + local attempt + for attempt in 1 2 3; do + if git push --force origin "$branch"; then + push_ok=true + break + fi + if [ "$attempt" -lt 3 ]; then + echo "Push attempt ${attempt} failed; retrying in 30s..." + sleep 30 + fi + done + if [ "$push_ok" != true ]; then + echo "Failed to push $branch after 3 attempts!" + exit 1 + fi +} + +# Create a PR from , or, when an open PR is already attached to +# that branch (a same-day rerun), update its title and body instead. +# Arguments: the branch name, the PR title, and the path to a file with +# the PR body. Echoes the PR url and exits the caller if GitHub rejects +# the request. +create_or_update_pr() { + local branch="$1" title="$2" summary_file="$3" + + # Build the payload as a file so the body content is always properly + # JSON-escaped, rather than interpolated into the command line as text. + # Not local: the EXIT trap below must still see it after this function + # returns. + payload_file=$(mktemp) + trap 'rm -f "$payload_file"' EXIT + jq -n \ + --arg title "$title" \ + --arg head "${branch}" \ + --rawfile body "$summary_file" \ + '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" + + # Reuse an open PR for this branch (same-day rerun) instead of duplicating it. + local existing_pr_json existing_pr_url existing_pr_number + existing_pr_json=$(curl -gs \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --url "https://api.github.com/repos/$OWNER/$REPO/pulls?head=$OWNER:$branch&state=open") + existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty') + if [ -n "$existing_pr_url" ]; then + existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty') + if ! curl -sfX PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "$(jq -n --arg title "$title" --rawfile body "$summary_file" '{title: $title, body: $body}')" \ + --url "https://api.github.com/repos/$OWNER/$REPO/pulls/${existing_pr_number}" > /dev/null + then + echo "Failed to update PR summary!" + exit 1 + fi + echo "$existing_pr_url" + echo "Creating the PR... done. (PR already existed; branch and summary were updated)" + return 0 + fi + + local resp pr_url + resp=$(curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "@${payload_file}" \ + --url "https://api.github.com/repos/$OWNER/$REPO/pulls") + + pr_url=$(echo "$resp" | jq -r '.html_url // empty') + if [ -z "$pr_url" ]; then + echo "Failed to create PR! API response:" + echo "$resp" | jq . + exit 1 + fi + echo "$pr_url" + echo "Creating the PR... done." +} diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index c12599cbc0..e5525a5281 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import hashlib import os import platform import shutil @@ -547,6 +548,24 @@ def handle_test_env() -> None: with zipfile.ZipFile(zip_path) as zf: zf.extractall(corpus_dir) zip_path.unlink() + if sub_test_name == "fuzz": + # PYTHON-6178 Also seed from the committed corpus of previously + # captured inputs, so the nightly builds on prior discoveries + # instead of the spec seeds alone. Regression runs (PRs) keep + # running only the reproducers and the spec seed corpus. + committed_dir = ROOT / "fuzz" / "corpus" + if committed_dir.is_dir(): + for path in sorted(committed_dir.iterdir()): + if path.name.startswith("."): + continue + shutil.copyfile(path, corpus_dir / path.name) + # Manifest of every input this run starts from; capture_corpus.py + # treats these digests as already-known, so only units the fuzzer + # discovered during the run get captured. + with open(scratch_dir / "baseline.txt", "w", encoding="utf-8") as fp: + for path in sorted(corpus_dir.iterdir()): + if path.is_file(): + fp.write(hashlib.sha256(path.read_bytes()).hexdigest() + "\n") write_env("FUZZ_CORPUS_DIR", str(corpus_dir)) if test_name == "sanitizer": diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 67f9e8a639..8d299054ff 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -292,6 +292,37 @@ def test_non_standard_cov_tasks_have_no_cov_companions(): assert companion in no_cov, f"missing no-cov companion for {task.name}" +def test_fuzz_task_wiring(): + """Two fuzz variants: the pr-tagged regression variant runs the + reproducers on every PR; the nightly variant (which also captures the + corpus) carries no "pr" tag, so PR patch builds never select it, while + its default patchability keeps it manually runnable + (`evergreen patch --variant fuzz-rhel8 --task fuzz`). + """ + tasks = [task for task in all_tasks() if "fuzz" in (task.tags or [])] + assert [task.name for task in tasks if "pr" in (task.tags or [])] == ["fuzz-regression"] + assert [task.name for task in tasks if "pr" not in (task.tags or [])] == ["fuzz"] + variants = generate_config.create_fuzz_variants() + assert len(variants) == 2 + regression, nightly = variants + assert variant_selects(regression, tasks[0]) + assert "pr" in (regression.tags or []) + assert variant_selects(nightly, tasks[1]) + assert nightly.batchtime == generate_config_utils.BATCHTIME_DAY + # Webhook PR patch builds only select pr-tagged variants. + assert not nightly.tags + # The capture subprocess bootstraps the comment-bot github app, whose + # secrets live in AWS Secrets Manager: it needs the assumed-role + # credentials (as resync_specs and backport-pr also pass them). + capture = tasks[1].commands[-1] + env_vars = capture.params["include_expansions_in_env"] + for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"): + assert var in env_vars, f"capture subprocess missing {var}" + # The capture no longer keys off is_patch: manual patch runs must + # capture too, and the nightly variant never runs in a PR build. + assert "is_patch" not in env_vars + + def test_load_balancer_marker_selection(monkeypatch): """The load balancer tasks select the load_balancer marker explicitly: pytest_conf.py only grants the implicit default markers to unmarked diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 007b73cfd7..dd62d8d2f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -457,6 +457,10 @@ wheels (CPython 3.12-3.14). - Time-boxed fuzzing (runs daily on Evergreen): `just setup-tests fuzz fuzz` then `just run-tests`. Set `FUZZ_MAX_TIME` (seconds, default 300) to change the per-target budget. +- Nightly fuzzing captures coverage-increasing inputs into `fuzz/corpus/` + via an automated PR (PYTHON-6178); see [fuzz/README.md](fuzz/README.md). + To run that flow on demand, patch the task: + `evergreen patch --variant fuzz-rhel8 --task fuzz`. ## Testing time-dependent operations diff --git a/fuzz/README.md b/fuzz/README.md index 1dfee074be..464601ac06 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -95,6 +95,34 @@ after the corpus directory. The `Failed to find function __sanitizer_*` startup warnings are benign libFuzzer symbol probes on uninstrumented runs; native ASan-instrumented fuzzing is tracked separately (PYTHON-6169). +### Captured corpus (nightly) + +The nightly run seeds from the spec seed corpus plus every unit in +`corpus/`, and libFuzzer writes each coverage-increasing ("interesting") +input it discovers back into its corpus directory. After the targets +finish, the `fuzz` task captures those new units, replays them through +every target once so a unit that fails any target is never published (it +stays in `fuzz-scratch/` for triage), and opens an automated +"[Fuzz Corpus]" PR against main (PYTHON-6178). Later nights +build on prior discoveries instead of restarting from the spec seeds. + +Growth is bounded by per-unit and per-run caps, overridable on Evergreen: + +- `FUZZ_CORPUS_MAX_UNIT_BYTES` (default 65536): skip units larger than this. +- `FUZZ_CORPUS_MAX_UNITS` (default 50): keep at most this many new units + per run, smallest first. + +Units over a cap are discarded, not deferred: they are never persisted, so +the fuzzer may re-derive them on later nights. A run that finds nothing new +opens no PR. The capture deduplicates against `main` only, so while a +`[Fuzz Corpus]` PR is open, later nights can re-capture its still-pending +units and the PRs overlap until one merges (digest-named files keep that +conflict-free). Crashing inputs are excluded: +they land in `fuzz-scratch/` as `crash-*` artifacts and follow the +`regression/` triage flow above. Corpus entries are fuzzer-generated +inputs, not code: skim the PR for size and for inputs that look like a +bug rather than a plain coverage gain before merging. + ## Coverage report Measure which lines the seed corpus reaches: @@ -113,9 +141,11 @@ gives atheris a graceful exit so coverage.py can flush its report. ``` build_corpus.py Seed corpus builder (mirrors the Go driver's build-oss-fuzz-corpus) +capture_corpus.py Nightly capture of new coverage-increasing inputs (PYTHON-6178) fuzz_decode.py Fuzz target: bson.decode() fuzz_decode_all.py Fuzz target: bson.decode_all() fuzz_is_valid.py Fuzz target: bson.is_valid() fuzz_shared.py Code shared by the fuzz targets +corpus/ Captured coverage-increasing inputs; grown by automated nightly PRs regression/ Known bug reproducers; run with the seed corpus on PRs ``` diff --git a/fuzz/capture_corpus.py b/fuzz/capture_corpus.py new file mode 100755 index 0000000000..41c076e71c --- /dev/null +++ b/fuzz/capture_corpus.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +""" +Capture coverage-increasing inputs from a finished fuzzing run (PYTHON-6178). + +Compares the scratch corpus directory (where libFuzzer writes each new +"interesting" input) against the baseline manifest written by +setup_tests.py and the committed fuzz/corpus/ contents, copies the new +units into an output directory named by their SHA-256 digests, and writes +a Markdown summary for the automated corpus PR body. + +Pure stdlib: this runs on the Evergreen nightly host after the fuzz +targets finish, and its unit tests must run everywhere (no atheris, no +bson). + +Usage: python capture_corpus.py [--corpus-dir DIR] [--baseline FILE] +""" + +from __future__ import annotations + +import argparse +import os +import shutil +from hashlib import sha256 +from pathlib import Path + +DEFAULT_CORPUS_DIR = "fuzz-scratch/corpus" +DEFAULT_EXISTING = "fuzz/corpus" +DEFAULT_OUT = "fuzz-scratch/capture" +DEFAULT_SUMMARY = "fuzz-scratch/capture-summary.md" +# Growth caps keep a runaway run from flooding the repository; both are +# overridable from Evergreen. Defaults chosen in PYTHON-6178: small inputs +# carry the most coverage per committed byte. +DEFAULT_MAX_UNITS = 50 +DEFAULT_MAX_UNIT_BYTES = 64 * 1024 + + +def digest_of(path: Path) -> str: + """The SHA-256 hex digest of a file's contents.""" + digest = sha256() + with open(path, "rb") as fp: + for chunk in iter(lambda: fp.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def digest_set(directory: Path) -> set[str]: + """Content digests of every file in directory, skipping dotfiles.""" + if not directory.is_dir(): + return set() + return { + digest_of(path) + for path in sorted(directory.iterdir()) + if path.is_file() and not path.name.startswith(".") + } + + +def read_baseline(path: Path) -> set[str]: + """Read digests from a baseline manifest (one hex digest per line).""" + if not path.is_file(): + return set() + with open(path, encoding="utf-8") as fp: + return {line.strip() for line in fp if line.strip()} + + +def run( + corpus_dir: Path, + baseline: Path, + existing: Path, + out: Path, + summary: Path, + max_units: int, + max_unit_bytes: int, +) -> int: + """Copy the new units into out/ and write the PR summary. + + Returns the number of units captured; zero means the run found + nothing new and the PR step can no-op. + """ + known = read_baseline(baseline) | digest_set(existing) + + # Sorted (size, digest, path): the size key prefers the smallest + # inputs when the per-run cap bites. + candidates: list[tuple[int, str, Path]] = [] + oversize = 0 + for path in sorted(corpus_dir.iterdir()): + if path.name.startswith(".") or not path.is_file(): + continue + digest = digest_of(path) + if digest in known: + continue + # Track accepted digests: same contents under two names count once. + known.add(digest) + size = path.stat().st_size + if size > max_unit_bytes: + oversize += 1 + continue + candidates.append((size, digest, path)) + candidates.sort() + + kept = candidates[:max_units] + over_cap = len(candidates) - len(kept) + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + for _, digest, path in kept: + shutil.copyfile(path, out / digest) + + write_summary(summary, kept, oversize, over_cap, max_units, max_unit_bytes) + print( + f"Captured {len(kept)} new units " + f"({oversize} oversize skipped, {over_cap} over the per-run cap)." + ) + return len(kept) + + +def write_summary( + path: Path, + kept: list[tuple[int, str, Path]], + oversize: int, + over_cap: int, + max_units: int, + max_unit_bytes: int, +) -> None: + """Write the Markdown capture summary used as the automated PR body.""" + lines = [ + "Captured coverage-increasing inputs from last night's BSON fuzz run.", + "", + f"- New units added: **{len(kept)}**", + f"- Units skipped over the per-unit cap ({max_unit_bytes} bytes): {oversize}", + f"- Units dropped over the per-run cap ({max_units}): {over_cap}", + "", + ( + "These are fuzzer-generated inputs, not reviewed code. Before merging, " + "skim them for size and for inputs that look like a bug rather than a " + "plain coverage gain; anything crashing belongs in `fuzz/regression/`." + ), + ] + if kept: + lines.extend(["", "Largest units added:"]) + for size, digest, _ in sorted(kept, reverse=True)[:5]: + lines.append(f"- `{digest[:16]}` ({size} bytes)") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture new coverage-increasing inputs from a fuzz run." + ) + parser.add_argument( + "--corpus-dir", + default=os.environ.get("FUZZ_CORPUS_DIR", DEFAULT_CORPUS_DIR), + ) + parser.add_argument( + "--baseline", + default=None, + help="Digest manifest; defaults to baseline.txt next to the corpus dir.", + ) + parser.add_argument("--existing", default=DEFAULT_EXISTING) + parser.add_argument("--out", default=DEFAULT_OUT) + parser.add_argument("--summary", default=DEFAULT_SUMMARY) + parser.add_argument( + "--max-units", + type=int, + default=int(os.environ.get("FUZZ_CORPUS_MAX_UNITS", DEFAULT_MAX_UNITS)), + ) + parser.add_argument( + "--max-unit-bytes", + type=int, + default=int(os.environ.get("FUZZ_CORPUS_MAX_UNIT_BYTES", DEFAULT_MAX_UNIT_BYTES)), + ) + args = parser.parse_args() + # Negative or zero caps are operator error (e.g. a mis-set Evergreen + # expansion): fail with an actionable message rather than a nonsense + # keep-set like "all but the last N". + if args.max_units < 1: + parser.error("--max-units must be at least 1") + if args.max_unit_bytes < 1: + parser.error("--max-unit-bytes must be at least 1") + corpus_dir = Path(args.corpus_dir) + baseline = Path(args.baseline) if args.baseline else corpus_dir.parent / "baseline.txt" + run( + corpus_dir=corpus_dir, + baseline=baseline, + existing=Path(args.existing), + out=Path(args.out), + summary=Path(args.summary), + max_units=args.max_units, + max_unit_bytes=args.max_unit_bytes, + ) + + +if __name__ == "__main__": + main() diff --git a/fuzz/corpus/.gitkeep b/fuzz/corpus/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/test_fuzz_capture.py b/test/test_fuzz_capture.py new file mode 100644 index 0000000000..b7fcb57567 --- /dev/null +++ b/test/test_fuzz_capture.py @@ -0,0 +1,152 @@ +"""Tests for fuzz/capture_corpus.py (PYTHON-6178).""" + +from __future__ import annotations + +import sys +from hashlib import sha256 +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent / "fuzz")) + +import capture_corpus + + +def write_unit(directory: Path, data: bytes) -> Path: + """Write a unit named by its content digest, like libFuzzer does.""" + path = directory / sha256(data).hexdigest() + path.write_bytes(data) + return path + + +@pytest.fixture +def dirs(tmp_path: Path) -> dict[str, Path]: + corpus = tmp_path / "corpus" + corpus.mkdir() + return { + "corpus": corpus, + "baseline": tmp_path / "baseline.txt", + "existing": tmp_path / "committed", + "out": tmp_path / "capture", + "summary": tmp_path / "summary.md", + } + + +def capture(dirs: dict[str, Path], max_units: int = 50, max_unit_bytes: int = 65536) -> int: + return capture_corpus.run( + corpus_dir=dirs["corpus"], + baseline=dirs["baseline"], + existing=dirs["existing"], + out=dirs["out"], + summary=dirs["summary"], + max_units=max_units, + max_unit_bytes=max_unit_bytes, + ) + + +def test_captures_new_units(dirs: dict[str, Path]) -> None: + write_unit(dirs["corpus"], b"new-unit") + + count = capture(dirs) + + assert count == 1 + captured = list(dirs["out"].iterdir()) + assert [p.name for p in captured] == [sha256(b"new-unit").hexdigest()] + assert captured[0].read_bytes() == b"new-unit" + + +def test_skips_baseline_units(dirs: dict[str, Path]) -> None: + write_unit(dirs["corpus"], b"seeded") + dirs["baseline"].write_text(sha256(b"seeded").hexdigest() + "\n") + + assert capture(dirs) == 0 + assert list(dirs["out"].iterdir()) == [] + + +def test_skips_committed_units(dirs: dict[str, Path]) -> None: + dirs["existing"].mkdir() + write_unit(dirs["existing"], b"already-committed") + write_unit(dirs["corpus"], b"already-committed") + + assert capture(dirs) == 0 + + +def test_skips_oversize_units(dirs: dict[str, Path]) -> None: + write_unit(dirs["corpus"], b"x" * 100) + write_unit(dirs["corpus"], b"y" * 50) + + assert capture(dirs, max_unit_bytes=99) == 1 + assert len(list(dirs["out"].iterdir())) == 1 + assert "per-unit cap" in dirs["summary"].read_text() + + +def test_per_run_cap_prefers_smallest(dirs: dict[str, Path]) -> None: + write_unit(dirs["corpus"], b"a" * 30) + write_unit(dirs["corpus"], b"b" * 10) + write_unit(dirs["corpus"], b"c" * 20) + + assert capture(dirs, max_units=2) == 2 + assert {p.read_bytes() for p in dirs["out"].iterdir()} == {b"b" * 10, b"c" * 20} + assert "per-run cap (2)" in dirs["summary"].read_text() + + +def test_dedups_identical_candidates_within_run(dirs: dict[str, Path]) -> None: + """Same contents under two names count once against the cap. + + libFuzzer digest-names corpus files, so the second name simulates a + crash artifact. + """ + dup = b"same" * 4 + distinct = b"distinct-unit-input" + write_unit(dirs["corpus"], dup) + (dirs["corpus"] / f"crash-{sha256(dup).hexdigest()[:8]}").write_bytes(dup) + write_unit(dirs["corpus"], distinct) + + assert capture(dirs, max_units=2) == 2 + assert {p.read_bytes() for p in dirs["out"].iterdir()} == {dup, distinct} + assert "Units dropped over the per-run cap (2): 0" in dirs["summary"].read_text() + + +def test_no_new_units_writes_summary_and_empty_out(dirs: dict[str, Path]) -> None: + assert capture(dirs) == 0 + assert list(dirs["out"].iterdir()) == [] + assert dirs["summary"].is_file() + + +def test_rejects_nonpositive_max_units(monkeypatch, capsys, dirs: dict[str, Path]) -> None: + """A cap of 0 or fewer is operator error; fail with an actionable message.""" + monkeypatch.setenv("FUZZ_CORPUS_MAX_UNITS", "0") + _argv(monkeypatch, dirs) + with pytest.raises(SystemExit): + capture_corpus.main() + assert "--max-units must be at least 1" in capsys.readouterr().err + + +def test_rejects_nonpositive_max_unit_bytes(monkeypatch, capsys, dirs: dict[str, Path]) -> None: + monkeypatch.setenv("FUZZ_CORPUS_MAX_UNIT_BYTES", "-1") + _argv(monkeypatch, dirs) + with pytest.raises(SystemExit): + capture_corpus.main() + assert "--max-unit-bytes must be at least 1" in capsys.readouterr().err + + +def _argv(monkeypatch, dirs: dict[str, Path]) -> None: + """Point main()'s argparse at the fixture directories.""" + monkeypatch.setattr( + sys, + "argv", + [ + "capture_corpus.py", + "--corpus-dir", + str(dirs["corpus"]), + "--baseline", + str(dirs["baseline"]), + "--existing", + str(dirs["existing"]), + "--out", + str(dirs["out"]), + "--summary", + str(dirs["summary"]), + ], + )