File each lost observation against the packet it belonged to

crates/skbx-cli/src/main.rs

@@ -5,16 +5,17 @@ use skbx_contract::{
     BpfMapOperation, BpfMapOperationKind, BpfProgramAction, BpfProgramKind, BpfProgramPhase,
     BpfProgramRef, BtfDump, CONTRACT_VERSION, CaptureEnd, CaptureFilters, CaptureLimits,
     CaptureStart, Describe, Envelope, EventAssociation, FunctionRef, KernelCpuLoss,
-    KernelProbeLoss, MatchOrigin, MetadataEncoding, MetadataScalar, MetadataValue, PacketMeta,
-    PacketTuple, PresentedTimestamp, Reliability, StopReason, TimestampMode, TraceEvent,
+    KernelProbeLoss, KernelSkbLoss, MatchOrigin, MetadataEncoding, MetadataScalar, MetadataValue,
+    PacketMeta, PacketTuple, PresentedTimestamp, Reliability, StopReason, TimestampMode,
+    TraceEvent,
 };
 use skbx_core::{
     BoundedMap, DEFAULT_BTF_PATH, DropReasonTable, SymbolTable, build_dynamic_probe_plan,
     build_probe_plan_with_bpf_helpers, capture_id, discover_bpf_helpers, doctor,
     ensure_btf_dump_support, event_handle, explain_with_context, replay, resolve_skb_filter,
     resolve_skb_metadata, resolve_xdp_filter, resolve_xdp_metadata,
 };
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::ffi::CString;
 use std::fs::{self, File, OpenOptions};
 use std::io::{BufRead, BufReader, BufWriter, Write};
@@ -572,6 +573,28 @@ fn run(cli: Cli) -> Result<u8> {
                             summary.reliability.kernel_unattributed_reserve_failures
                         );
                     }
+                    let affected = summary
+                        .reliability
+                        .kernel_loss_by_skb
+                        .iter()
+                        .map(|loss| loss.skb.as_str())
+                        .collect::<BTreeSet<_>>()
+                        .len();
+                    if summary.reliability.loss_is_fully_attributed() {
+                        println!(
+                            "Absence is evidence: every loss was filed against a packet, so any packet not among the {affected} affected provably lost nothing"
+                        );
+                        if summary.skbs_lost_entirely != 0 {
+                            println!(
+                                "Packets lost entirely: {} (every observation dropped; absent from the {} packets observed above)",
+                                summary.skbs_lost_entirely, summary.distinct_skbs
+                            );
+                        }
+                    } else if !summary.complete {
+                        println!(
+                            "Absence is not evidence: loss remains that no packet can be cleared of"
+                        );
+                    }
                 }
             }
             Ok(if summary.complete { 0 } else { 3 })
@@ -1140,11 +1163,13 @@ fn capture(
                             // be attributed without being counted, leaving the
                             // breakdown larger than the total it refines.
                             let probes = kernel_loss_by_probe(&sensor, &symbols)?;
+                            let skbs = kernel_loss_by_skb(&sensor, &symbols)?;
                             let current = sensor.stats()?.into_reliability(
                                 sensor.recursion_misses()?,
                                 sensor.decode_failures(),
                                 sensor.enrichment_failures(),
                                 probes,
+                                skbs,
                             );
                             let segment = reliability_delta(&current, &reliability_checkpoint);
                             reliability_checkpoint = current;
@@ -1159,12 +1184,14 @@ fn capture(
         // Attribution before total, so the breakdown can only ever undercount
         // the number it refines. See the segment boundary above.
         let probes = kernel_loss_by_probe(&sensor, &symbols)?;
+        let skbs = kernel_loss_by_skb(&sensor, &symbols)?;
         let stats = sensor.stats()?;
         let reliability = stats.into_reliability(
             sensor.recursion_misses()?,
             sensor.decode_failures(),
             sensor.enrichment_failures(),
             probes,
+            skbs,
         );
         let complete = reliability.complete();
         let end = CaptureEnd {
@@ -1728,9 +1755,50 @@ fn reliability_delta(current: &Reliability, previous: &Reliability) -> Reliabili
         kernel_unattributed_reserve_failures: current
             .kernel_unattributed_reserve_failures
             .saturating_sub(previous.kernel_unattributed_reserve_failures),
+        kernel_loss_by_skb: kernel_loss_by_skb_delta(
+            &current.kernel_loss_by_skb,
+            &previous.kernel_loss_by_skb,
+        ),
+        kernel_skb_loss_unattributed: current
+            .kernel_skb_loss_unattributed
+            .saturating_sub(previous.kernel_skb_loss_unattributed),
     }
 }
 
+/// Matched on packet identity and probe together, since one packet can lose
+/// observations at several probes and each is its own ledger line.
+fn kernel_loss_by_skb_delta(
+    current: &[KernelSkbLoss],
+    previous: &[KernelSkbLoss],
+) -> Vec<KernelSkbLoss> {
+    let site = |loss: &KernelSkbLoss| {
+        (
+            loss.skb.clone(),
+            loss.function
+                .as_ref()
+                .map(|function| function.address.clone()),
+            loss.program_id,
+        )
+    };
+    let mut segment: Vec<KernelSkbLoss> = current
+        .iter()
+        .filter_map(|now| {
+            let before = previous.iter().find(|earlier| site(earlier) == site(now));
+            let reserve_failures = now
+                .reserve_failures
+                .saturating_sub(before.map_or(0, |earlier| earlier.reserve_failures));
+            (reserve_failures != 0).then(|| KernelSkbLoss {
+                skb: now.skb.clone(),
+                function: now.function.clone(),
+                program_id: now.program_id,
+                reserve_failures,
+            })
+        })
+        .collect();
+    segment.sort_by_key(site);
+    segment
+}
+
 /// Resolves each losing probe site to a name. A kernel site carries its
 /// instruction pointer symbolized against kallsyms; a TC/XDP site carries the
 /// BPF program id, which needs no lookup.
@@ -1752,6 +1820,30 @@ fn kernel_loss_by_probe(
         .collect())
 }
 
+/// Resolves each lost observation to the packet and probe it belonged to. The
+/// identity is formatted exactly as events format theirs, so a reader can join
+/// this against a replayed chain without reformatting either side.
+fn kernel_loss_by_skb(
+    sensor: &skbx_sensor::LiveSensor,
+    symbols: &SymbolTable,
+) -> Result<Vec<KernelSkbLoss>> {
+    Ok(sensor
+        .skb_loss()?
+        .into_iter()
+        .map(|loss| KernelSkbLoss {
+            skb: format!("{:#x}", loss.key.identity),
+            function: (loss.key.site.function_ip != 0).then(|| FunctionRef {
+                address: format!("{:#x}", loss.key.site.function_ip),
+                symbol: symbols
+                    .resolve(loss.key.site.function_ip)
+                    .map(str::to_owned),
+            }),
+            program_id: (loss.key.site.program_id != 0).then_some(loss.key.site.program_id),
+            reserve_failures: loss.reserve_failures,
+        })
+        .collect())
+}
+
 /// Probes are matched on their site, not their position: the list is ordered by
 /// loss, so a probe can move between checkpoints as other probes overtake it.
 fn kernel_loss_by_probe_delta(

crates/skbx-contract/src/lib.rs

@@ -678,6 +678,26 @@ pub struct KernelProbeLoss {
     pub reserve_failures: u64,
 }
 
+/// A lost observation, named down to the packet it belonged to.
+///
+/// This is the attribution that can restore absence as evidence. Given a
+/// capture where `kernel_skb_loss_unattributed` is zero, this table is
+/// exhaustive: a packet that does not appear in it lost nothing, so a function
+/// missing from that packet's chain was genuinely never reached rather than
+/// merely unobserved. The kernel table is a plain hash that refuses inserts
+/// when full rather than an LRU that evicts silently, which is what makes that
+/// negative claim provable instead of merely likely.
+///
+/// `skb` is the same identity emitted on every event, so this joins directly
+/// against a replayed chain.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct KernelSkbLoss {
+    pub skb: String,
+    pub function: Option<FunctionRef>,
+    pub program_id: Option<u32>,
+    pub reserve_failures: u64,
+}
+
 #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Reliability {
     pub kernel_reserve_failures: u64,
@@ -707,6 +727,58 @@ pub struct Reliability {
     /// `kernel_loss_by_probe` undercounts by exactly this much.
     #[serde(default)]
     pub kernel_unattributed_reserve_failures: u64,
+    /// Which observation of which packet was lost, ordered by packet.
+    #[serde(default)]
+    pub kernel_loss_by_skb: Vec<KernelSkbLoss>,
+    /// Losses the kernel could not file against a packet because the per-skb
+    /// table was full. While this is zero, `kernel_loss_by_skb` is exhaustive
+    /// and a packet absent from it provably lost nothing; once it is non-zero,
+    /// no such claim can be made about any packet.
+    #[serde(default)]
+    pub kernel_skb_loss_unattributed: u64,
+}
+
+impl Reliability {
+    /// Whether a packet absent from `kernel_loss_by_skb` can be treated as
+    /// having lost nothing.
+    ///
+    /// This is the difference between "this function was never hit" and "we
+    /// lost that one". It holds only while every loss was filed against a
+    /// packet; a single unfiled loss means any packet could be the one missing
+    /// an observation, so the claim collapses for all of them at once.
+    pub fn skb_loss_is_exhaustive(&self) -> bool {
+        self.kernel_skb_loss_unattributed == 0
+    }
+
+    /// Whether every lost observation in this capture could be filed against a
+    /// specific packet.
+    ///
+    /// Reserve failures are filed per packet, but the other loss kinds are not:
+    /// a recursion miss never reaches the emit path, and a decode, enrichment,
+    /// or output failure discards a record after the kernel has already handed
+    /// it over, by which point nothing knows which packet it described. Any of
+    /// those leaves a hole no packet can be cleared of, so all of them must be
+    /// zero before an absence anywhere in this capture may be read as evidence.
+    ///
+    /// Read failures are deliberately not included: they degrade fields within
+    /// an event that was still emitted, and each one is already visible on the
+    /// event that carries it.
+    pub fn loss_is_fully_attributed(&self) -> bool {
+        self.skb_loss_is_exhaustive()
+            && self.kernel_recursion_misses == 0
+            && self.userspace_decode_failures == 0
+            && self.userspace_enrichment_failures == 0
+            && self.output_failures == 0
+    }
+
+    /// Observations lost for one packet, by identity, across every probe.
+    pub fn losses_for_skb(&self, skb: &str) -> u64 {
+        self.kernel_loss_by_skb
+            .iter()
+            .filter(|loss| loss.skb == skb)
+            .map(|loss| loss.reserve_failures)
+            .sum()
+    }
 }
 
 impl Reliability {
@@ -762,6 +834,13 @@ pub struct TraceSummary {
     pub complete: bool,
     pub events: u64,
     pub distinct_skbs: usize,
+    /// Packets that lost every observation, so they appear in the footer's
+    /// per-packet loss ledger but nowhere in the capture itself. Previously
+    /// invisible: a packet whose whole chain was dropped left no trace at all.
+    /// Not counted in `distinct_skbs`, which only counts packets that were
+    /// actually observed.
+    #[serde(default)]
+    pub skbs_lost_entirely: usize,
     pub functions: BTreeMap<String, u64>,
     pub processes: BTreeMap<String, u64>,
     #[serde(default)]
@@ -1215,7 +1294,25 @@ pub fn json_schema() -> serde_json::Value {
                         "type": "array",
                         "items": {"$ref": "#/$defs/KernelProbeLoss"}
                     },
-                    "kernel_unattributed_reserve_failures": {"type": "integer", "minimum": 0}
+                    "kernel_unattributed_reserve_failures": {"type": "integer", "minimum": 0},
+                    "kernel_loss_by_skb": {
+                        "type": "array",
+                        "items": {"$ref": "#/$defs/KernelSkbLoss"}
+                    },
+                    "kernel_skb_loss_unattributed": {"type": "integer", "minimum": 0}
+                },
+                "additionalProperties": false
+            },
+            "KernelSkbLoss": {
+                "type": "object",
+                "required": ["skb", "function", "program_id", "reserve_failures"],
+                "properties": {
+                    "skb": {"type": "string"},
+                    "function": {
+                        "oneOf": [{"$ref": "#/$defs/FunctionRef"}, {"type": "null"}]
+                    },
+                    "program_id": {"type": ["integer", "null"], "minimum": 0},
+                    "reserve_failures": {"type": "integer", "minimum": 0}
                 },
                 "additionalProperties": false
             },
@@ -1359,6 +1456,89 @@ mod tests {
         assert!(!reliability.complete());
     }
 
+    fn skb_loss(skb: &str, symbol: &str, reserve_failures: u64) -> KernelSkbLoss {
+        KernelSkbLoss {
+            skb: skb.into(),
+            function: Some(FunctionRef {
+                address: "0xffffffff81234560".into(),
+                symbol: Some(symbol.into()),
+            }),
+            program_id: None,
+            reserve_failures,
+        }
+    }
+
+    #[test]
+    fn an_exhaustive_skb_table_clears_the_packets_absent_from_it() {
+        let reliability = Reliability {
+            kernel_reserve_failures: 3,
+            kernel_loss_by_skb: vec![skb_loss("0xaaa", "nf_hook_slow", 3)],
+            ..Reliability::default()
+        };
+
+        assert!(reliability.loss_is_fully_attributed());
+        assert_eq!(reliability.losses_for_skb("0xaaa"), 3);
+        // The packet nobody filed a loss against is the one whose absences can
+        // be read as evidence.
+        assert_eq!(reliability.losses_for_skb("0xbbb"), 0);
+    }
+
+    #[test]
+    fn one_unfiled_loss_collapses_the_claim_for_every_packet() {
+        let reliability = Reliability {
+            kernel_reserve_failures: 4,
+            kernel_loss_by_skb: vec![skb_loss("0xaaa", "nf_hook_slow", 3)],
+            kernel_skb_loss_unattributed: 1,
+            ..Reliability::default()
+        };
+
+        assert!(!reliability.skb_loss_is_exhaustive());
+        assert!(!reliability.loss_is_fully_attributed());
+        // 0xbbb still reports zero filed losses, which is exactly why the
+        // exhaustiveness flag has to be consulted first.
+        assert_eq!(reliability.losses_for_skb("0xbbb"), 0);
+    }
+
+    #[test]
+    fn loss_that_no_packet_owns_defeats_attribution() {
+        // Each of these discards a record after the kernel handed it over, or
+        // before it ever reached the emit path, so no packet can be cleared.
+        for defeat in [
+            Reliability {
+                kernel_recursion_misses: 1,
+                ..Reliability::default()
+            },
+            Reliability {
+                userspace_decode_failures: 1,
+                ..Reliability::default()
+            },
+            Reliability {
+                userspace_enrichment_failures: 1,
+                ..Reliability::default()
+            },
+            Reliability {
+                output_failures: 1,
+                ..Reliability::default()
+            },
+        ] {
+            assert!(defeat.skb_loss_is_exhaustive());
+            assert!(!defeat.loss_is_fully_attributed(), "{defeat:?}");
+        }
+    }
+
+    #[test]
+    fn read_failures_do_not_defeat_attribution() {
+        // A read failure degrades fields on an event that was still emitted,
+        // and is already visible on that event.
+        let reliability = Reliability {
+            kernel_read_failures: 9,
+            kernel_filtered_events: 500,
+            ..Reliability::default()
+        };
+
+        assert!(reliability.loss_is_fully_attributed());
+    }
+
     #[test]
     fn per_probe_loss_names_the_leg_and_admits_undercounting() {
         let reliability = Reliability {