diff --git a/docs/validation.md b/docs/validation.md index 4a64295..3847468 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -43,3 +43,33 @@ When adding a live gate: 5. assert positive evidence and relevant negative filtering; 6. require a complete footer with zero unexpected failures; 7. print the retained trace path for debugging. + +"Zero unexpected failures" means every counter the gate can control. A gate +that traces the kernel machinery skbx itself uses may see counters move for +reasons that belong to the host rather than to the code, and must scope the +assertion to say which counter that is and why, instead of either demanding a +clean footer it cannot guarantee or dropping the check. + +A gate whose capture is not scoped to its own traffic has the related +problem: system-wide events fill the event budget, the evidence it needs is +crowded out, and it passes or fails on host load. Scope the capture with a +filter so the assertion is deterministic, rather than widening the assertion +to tolerate the noise. + +`make live-bpf-helper` and `make live-stack-lifetime` are the current +examples. Helper tracking attaches kprobes +to the map helpers the tracer calls, so any concurrent BPF hash activity on the +host re-enters the tracer and trips the kernel recursion guard. Those misses +are genuine missed observations and the footer is right to report them, but +they scale with what else is running: on an otherwise idle machine they are +zero, and on a workstation they are reliably in the thousands. `make live-stack-lifetime` kprobes `kmem_cache_free`, which the tracer's own +teardown path reaches, and had the same problem twice over: it also captured +`consume_skb` unfiltered, so on a busy host 1024 unrelated events filled the +budget before the lifetime triple it asserts on could appear. + +Both gates therefore accept an incomplete footer only when +`kernel_recursion_misses` explains it, and keep every other reliability +counter at zero. `live-stack-lifetime` additionally scopes its capture to +`icmp`, which is the traffic it generates: measured under four concurrent +iperf3 streams that produced 30 events and 20 stack associations on every +run, against 1024 capped events and a 15-to-30 spread unfiltered. diff --git a/scripts/live-bpf-helper-test.sh b/scripts/live-bpf-helper-test.sh index 0d7f73c..5ec2f05 100755 --- a/scripts/live-bpf-helper-test.sh +++ b/scripts/live-bpf-helper-test.sh @@ -118,11 +118,20 @@ jq -e ' jq -e ' select( .kind == "capture_end" and - .complete == true and .events > 0 and .reliability.kernel_reserve_failures == 0 and .reliability.kernel_read_failures == 0 and - .reliability.userspace_decode_failures == 0 + .reliability.userspace_decode_failures == 0 and + .reliability.userspace_enrichment_failures == 0 and + .reliability.output_failures == 0 and + # Helper tracking kprobes the map helpers that the tracer itself + # calls, so any concurrent BPF hash activity anywhere on the host + # re-enters the tracer and trips the kernel recursion guard. Those + # misses are real missed observations and the footer is right to + # report them, but they depend on what else is running rather than + # on this code. They are the only incompleteness this gate accepts; + # every other counter above stays strict. + (.complete == true or .reliability.kernel_recursion_misses > 0) ) ' "${TRACE}" >/dev/null diff --git a/scripts/live-stack-lifetime-test.sh b/scripts/live-stack-lifetime-test.sh index 715458c..cd6cecd 100755 --- a/scripts/live-stack-lifetime-test.sh +++ b/scripts/live-stack-lifetime-test.sh @@ -28,7 +28,8 @@ trap cleanup EXIT --duration 4 \ --max-events 1024 \ --ready-file "${READY}" \ - --output "${TRACE}" & + --output "${TRACE}" \ + icmp & CAPTURE_PID=$! for _ in $(seq 1 50); do @@ -74,10 +75,19 @@ jq -s -e ' jq -e ' select( .kind == "capture_end" and - .complete == true and .events > 0 and .reliability.kernel_reserve_failures == 0 and - .reliability.userspace_decode_failures == 0 + .reliability.kernel_read_failures == 0 and + .reliability.userspace_decode_failures == 0 and + .reliability.userspace_enrichment_failures == 0 and + .reliability.output_failures == 0 and + # kmem_cache_free is on the teardown path the tracer itself walks, so + # concurrent kernel allocator traffic re-enters the tracer and trips + # the kernel recursion guard. Same reasoning as the BPF-helper gate: + # the misses are real and correctly reported, but they measure host + # load rather than this code, and are the only incompleteness + # accepted here. + (.complete == true or .reliability.kernel_recursion_misses > 0) ) ' "${TRACE}" >/dev/null