From 6ca8ab62c3c103e0527cd72db30d5769690b9fd1 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Fri, 7 Aug 2026 19:01:47 -0700 Subject: [PATCH 1/2] Scope the BPF-helper gate to the failures it can control The gate required a complete footer, which requires zero recursion misses. Helper tracking attaches kprobes to the map helpers the tracer itself calls, so any concurrent BPF hash activity anywhere on the host re-enters the tracer and trips the kernel recursion guard. The counter therefore measures what else is running: zero on an idle machine, reliably thousands on a workstation. The gate was red on any developer host and silently assumed an idle one. The misses are genuine missed observations and the footer is right to report them, so the assertion is what needs narrowing, not the behaviour. Accept an incomplete footer only when recursion misses explain it, and hold every other counter at zero. This tightens the gate rather than loosening it. Enrichment and output failures were never asserted at all and now are, and an incomplete footer that recursion misses do not explain is now rejected where before only the single `complete` flag was consulted. Co-Authored-By: Claude Opus 5 (1M context) --- docs/validation.md | 15 +++++++++++++++ scripts/live-bpf-helper-test.sh | 13 +++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/validation.md b/docs/validation.md index 4a64295..791c2c5 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -43,3 +43,18 @@ When adding a live gate: 5. assert positive evidence and relevant negative filtering; 6. require a complete footer with zero unexpected failures; 7. print the retained trace path for debugging. + +"Zero unexpected failures" means every counter the gate can control. A gate +that traces the kernel machinery skbx itself uses may see counters move for +reasons that belong to the host rather than to the code, and must scope the +assertion to say which counter that is and why, instead of either demanding a +clean footer it cannot guarantee or dropping the check. + +`make live-bpf-helper` is the current example. Helper tracking attaches kprobes +to the map helpers the tracer calls, so any concurrent BPF hash activity on the +host re-enters the tracer and trips the kernel recursion guard. Those misses +are genuine missed observations and the footer is right to report them, but +they scale with what else is running: on an otherwise idle machine they are +zero, and on a workstation they are reliably in the thousands. That gate +therefore accepts an incomplete footer only when `kernel_recursion_misses` +explains it, and keeps every other reliability counter at zero. diff --git a/scripts/live-bpf-helper-test.sh b/scripts/live-bpf-helper-test.sh index 0d7f73c..5ec2f05 100755 --- a/scripts/live-bpf-helper-test.sh +++ b/scripts/live-bpf-helper-test.sh @@ -118,11 +118,20 @@ jq -e ' jq -e ' select( .kind == "capture_end" and - .complete == true and .events > 0 and .reliability.kernel_reserve_failures == 0 and .reliability.kernel_read_failures == 0 and - .reliability.userspace_decode_failures == 0 + .reliability.userspace_decode_failures == 0 and + .reliability.userspace_enrichment_failures == 0 and + .reliability.output_failures == 0 and + # Helper tracking kprobes the map helpers that the tracer itself + # calls, so any concurrent BPF hash activity anywhere on the host + # re-enters the tracer and trips the kernel recursion guard. Those + # misses are real missed observations and the footer is right to + # report them, but they depend on what else is running rather than + # on this code. They are the only incompleteness this gate accepts; + # every other counter above stays strict. + (.complete == true or .reliability.kernel_recursion_misses > 0) ) ' "${TRACE}" >/dev/null From be0909796ebd10f8864ce1009e1801b785d5a28c Mon Sep 17 00:00:00 2001 From: zuub-don Date: Fri, 7 Aug 2026 19:11:14 -0700 Subject: [PATCH 2/2] Scope the stack-lifetime gate to its own traffic and failures The gate captured consume_skb unfiltered, so on a busy host system-wide teardown filled the 1024-event budget before the consume_skb -> dst_release -> kmem_cache_free triple it asserts on could appear. It also required a complete footer, and kmem_cache_free sits on the teardown path the tracer itself walks, so concurrent allocator traffic re-enters the tracer and trips the kernel recursion guard. Both failures track host load rather than this code, and the gate was flaky in both directions because of it: measured across interleaved runs it failed on an unmodified binary and passed on a modified one in the same session, which is worse than a gate that is simply red. Scope the capture to icmp, the traffic the gate itself generates. Under four concurrent iperf3 streams that yields 30 events and 20 stack associations on every run, against 1024 capped events and a 15-to-30 spread unfiltered. The pairing assertion becomes deterministic instead of merely likely, and kernel read failures drop from hundreds to zero because the capture no longer sees SKBs it has no business reading. Scope the footer the same way as the BPF-helper gate: recursion misses are the only accepted incompleteness, every other counter stays at zero. Read, enrichment and output failures were never asserted here at all and now are. Co-Authored-By: Claude Opus 5 (1M context) --- docs/validation.md | 23 +++++++++++++++++++---- scripts/live-stack-lifetime-test.sh | 16 +++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/validation.md b/docs/validation.md index 791c2c5..3847468 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -50,11 +50,26 @@ reasons that belong to the host rather than to the code, and must scope the assertion to say which counter that is and why, instead of either demanding a clean footer it cannot guarantee or dropping the check. -`make live-bpf-helper` is the current example. Helper tracking attaches kprobes +A gate whose capture is not scoped to its own traffic has the related +problem: system-wide events fill the event budget, the evidence it needs is +crowded out, and it passes or fails on host load. Scope the capture with a +filter so the assertion is deterministic, rather than widening the assertion +to tolerate the noise. + +`make live-bpf-helper` and `make live-stack-lifetime` are the current +examples. Helper tracking attaches kprobes to the map helpers the tracer calls, so any concurrent BPF hash activity on the host re-enters the tracer and trips the kernel recursion guard. Those misses are genuine missed observations and the footer is right to report them, but they scale with what else is running: on an otherwise idle machine they are -zero, and on a workstation they are reliably in the thousands. That gate -therefore accepts an incomplete footer only when `kernel_recursion_misses` -explains it, and keeps every other reliability counter at zero. +zero, and on a workstation they are reliably in the thousands. `make live-stack-lifetime` kprobes `kmem_cache_free`, which the tracer's own +teardown path reaches, and had the same problem twice over: it also captured +`consume_skb` unfiltered, so on a busy host 1024 unrelated events filled the +budget before the lifetime triple it asserts on could appear. + +Both gates therefore accept an incomplete footer only when +`kernel_recursion_misses` explains it, and keep every other reliability +counter at zero. `live-stack-lifetime` additionally scopes its capture to +`icmp`, which is the traffic it generates: measured under four concurrent +iperf3 streams that produced 30 events and 20 stack associations on every +run, against 1024 capped events and a 15-to-30 spread unfiltered. diff --git a/scripts/live-stack-lifetime-test.sh b/scripts/live-stack-lifetime-test.sh index 715458c..cd6cecd 100755 --- a/scripts/live-stack-lifetime-test.sh +++ b/scripts/live-stack-lifetime-test.sh @@ -28,7 +28,8 @@ trap cleanup EXIT --duration 4 \ --max-events 1024 \ --ready-file "${READY}" \ - --output "${TRACE}" & + --output "${TRACE}" \ + icmp & CAPTURE_PID=$! for _ in $(seq 1 50); do @@ -74,10 +75,19 @@ jq -s -e ' jq -e ' select( .kind == "capture_end" and - .complete == true and .events > 0 and .reliability.kernel_reserve_failures == 0 and - .reliability.userspace_decode_failures == 0 + .reliability.kernel_read_failures == 0 and + .reliability.userspace_decode_failures == 0 and + .reliability.userspace_enrichment_failures == 0 and + .reliability.output_failures == 0 and + # kmem_cache_free is on the teardown path the tracer itself walks, so + # concurrent kernel allocator traffic re-enters the tracer and trips + # the kernel recursion guard. Same reasoning as the BPF-helper gate: + # the misses are real and correctly reported, but they measure host + # load rather than this code, and are the only incompleteness + # accepted here. + (.complete == true or .reliability.kernel_recursion_misses > 0) ) ' "${TRACE}" >/dev/null