Cross-OS portability: fix BPF load failures, guard/symbol drift, arm64 gaps, and installer breakage

.github/scripts/bpf_smoke.py

@@ -18,6 +18,17 @@
 BPF_FILE = os.path.join(REPO_ROOT, "src", "tracer", "prober", "prober.c")
 
 
+def tracepoint_format(category, name):
+    """Read a tracepoint format file from debugfs or tracefs (either mount)."""
+    for base in ("/sys/kernel/debug/tracing", "/sys/kernel/tracing"):
+        try:
+            with open(f"{base}/events/{category}/{name}/format") as f:
+                return f.read()
+        except OSError:
+            continue
+    return ""
+
+
 def build_cflags():
     """Mirror IOTracer._init_bpf so CI compiles with the real flags."""
     cflags = [
@@ -26,11 +37,18 @@ def build_cflags():
         "-mllvm",
         "-bpf-stack-size=4096",
     ]
-    tp_format = "/sys/kernel/debug/tracing/events/block/block_rq_complete/format"
-    if os.path.exists(tp_format):
-        with open(tp_format) as f:
-            if "cmd_flags" in f.read():
-                cflags.append("-DHAS_CMD_FLAGS")
+    if "cmd_flags" in tracepoint_format("block", "block_rq_complete"):
+        cflags.append("-DHAS_CMD_FLAGS")
+    return cflags
+
+
+def network_cflags():
+    """Mirror IOTracer._init_bpf's --network feature gates."""
+    cflags = ["-DENABLE_NETWORK"]
+    if " reason;" in tracepoint_format("skb", "kfree_skb"):
+        cflags.append("-DHAS_SKB_DROP_REASON")
+    if " state;" in tracepoint_format("tcp", "tcp_retransmit_skb"):
+        cflags.append("-DHAS_TCP_RETRANSMIT_STATE")
     return cflags
 
 
@@ -72,8 +90,11 @@ def main():
     conditional_probes = [
         (b"__x64_sys_mremap", [("kprobe", "__x64_sys_mremap", "trace_mremap_entry_x64"),
                                ("kretprobe", "__x64_sys_mremap", "trace_mremap_ret")]),
+        (b"__arm64_sys_mremap", [("kprobe", "__arm64_sys_mremap", "trace_mremap_entry_arm64"),
+                                 ("kretprobe", "__arm64_sys_mremap", "trace_mremap_ret")]),
         (b"__x64_sys_openat", [("kprobe", "__x64_sys_openat", "trace_openat_entry_x64")]),
         (b"__x64_sys_io_uring_enter", [("kprobe", "__x64_sys_io_uring_enter", "trace_io_uring_enter_x64")]),
+        (b"__arm64_sys_io_uring_enter", [("kprobe", "__arm64_sys_io_uring_enter", "trace_io_uring_enter_arm64")]),
         (b"iomap_dio_rw", [("kprobe", "iomap_dio_rw", "trace_dio_entry_iomap"),
                            ("kretprobe", "iomap_dio_rw", "trace_dio_return")]),
         (b"__blockdev_direct_IO", [("kprobe", "__blockdev_direct_IO", "trace_dio_entry_blockdev")]),
@@ -84,6 +105,41 @@ def main():
         else:
             print(f"SKIP: {symbol.decode()} not present on this kernel")
 
+    # Cache-probe guard/symbol alignment, expressed as ORDERED fallback chains
+    # exactly like KernelProbeTracker: the first present symbol wins and the
+    # rest are skipped. This matters because old page-API symbols
+    # (mark_page_accessed, add_to_page_cache_lru, ...) still exist on modern
+    # kernels as exported folio-compat wrappers while their page-variant
+    # handlers are compiled OUT (>= 5.16/5.17 guards) — attaching every
+    # existing symbol unconditionally would fail CI on every modern kernel.
+    conditional_probe_chains = [
+        [(b"folio_mark_accessed", "trace_folio_mark_accessed"),
+         (b"mark_page_accessed", "trace_hit")],
+        [(b"filemap_add_folio", "trace_filemap_add_folio"),
+         (b"add_to_page_cache_lru", "trace_miss")],
+        [(b"__folio_mark_dirty", "trace_folio_mark_dirty"),
+         (b"account_page_dirtied", "trace_account_page_dirtied")],
+        [(b"folio_clear_dirty_for_io", "trace_folio_clear_dirty_for_io"),
+         (b"clear_page_dirty_for_io", "trace_clear_page_dirty_for_io")],
+        [(b"folio_end_writeback", "trace_folio_end_writeback"),
+         (b"test_clear_page_writeback", "trace_test_clear_page_writeback")],
+        [(b"filemap_remove_folio", "trace_filemap_remove_folio"),
+         (b"__filemap_remove_folio", "trace_filemap_remove_folio"),
+         (b"__delete_from_page_cache", "trace_delete_from_page_cache")],
+        [(b"__do_page_cache_readahead", "trace_do_page_cache_readahead"),
+         (b"do_page_cache_ra", "trace_page_cache_ra"),
+         (b"page_cache_ra_order", "trace_page_cache_ra_order")],
+        [(b"shrink_folio_list", "trace_shrink_folio_list"),
+         (b"shrink_page_list", "trace_shrink_folio_list")],
+    ]
+    for chain in conditional_probe_chains:
+        for symbol, fn in chain:
+            if BPF.get_kprobe_functions(symbol):
+                probes.append(("kprobe", symbol.decode(), fn))
+                break
+        else:
+            print(f"SKIP: no symbol of chain {[s.decode() for s, _ in chain]} present")
+
     for kind, event, fn in probes:
         if kind == "kprobe":
             b.attach_kprobe(event=event, fn_name=fn)
@@ -99,7 +155,7 @@ def main():
     # connection/sockopt/drop probes get verifier coverage too. Their
     # TRACEPOINT_PROBE handlers auto-attach on load, so a successful BPF()
     # construction validates both compile and attach.
-    net_cflags = build_cflags() + ["-DENABLE_NETWORK"]
+    net_cflags = build_cflags() + network_cflags()
     print(f"cflags (network): {net_cflags}")
     b_net = BPF(src_file=BPF_FILE.encode(), cflags=net_cflags)
     print("OK: prober.c compiled with ENABLE_NETWORK (verifier passed, "

README.md

@@ -41,14 +41,18 @@ pacman -S bcc bcc-tools python-bcc
 
 For more distros, visit the official [BCC's installation guide](https://github.com/iovisor/bcc/blob/master/INSTALL.md)
 
-3. Finally, install the Python dependencies. The simplest way is to install
-them all at once from `requirements.txt`:
+3. Finally, install the Python dependencies. Prefer your distro's packages
+(the tracer runs under the system Python, which is what the distro `bcc`
+bindings are built for — see the next code block). If you use pip instead,
+note that Debian 12 / Ubuntu 23.04+ mark the system interpreter as
+externally managed (PEP 668), so a bare `pip install` fails; append
+`--break-system-packages` or use the distro packages:
 
 ```bash
 pip install -r requirements.txt
 ```
 
-Or, if you prefer your distro's package manager:
+Distro package manager equivalents:
 
 ```bash
 # Ubuntu / Debian
@@ -68,7 +72,7 @@ To run the test suite you'll also need `pytest` (`pip install pytest`).
 
 ## Usage
 ```
-usage: sudo iotrc [-h] [-v] [-a] [--cache] [--network] [--computer-id] [--reward] [--no-upload] {dev} ...
+usage: sudo iotrc [-h] [-v] [-a] [--cache] [--network] [--computer-id] [--reward] [--no-upload] [--output DIR] {dev} ...
 
 Trace IO syscalls
 
@@ -79,6 +83,7 @@ options:
   --computer-id    Print this machine ID and exit
   --reward         Show your reward code (unlocked after uploading traces)
   --no-upload      Disable automatic upload of traces (for testing)
+  --output DIR     Base directory for trace output (default: system temp dir)
 
 subcommands:
   {dev}            Run in developer mode with extra logs and checks

docs/COMPATIBILITY_FIXES.md

@@ -18,13 +18,26 @@ We implemented selective compilation using standard kernel version macros (`#if
 ## 2. Missing `cmd_flags` in `block_rq_complete`
 
 ### The Problem
-The `block_rq_complete` tracepoint arguments vary between kernel versions. On some older kernels, the `cmd_flags` variable is missing from the tracepoint format definition entirely, leading to a direct compilation failure when `args->cmd_flags` was accessed in `prober.c`.
+Referencing a tracepoint `args->` field that the running kernel's format file
+does not define is a direct compilation failure that aborts the whole BPF
+load. (Note: mainline `block_rq_complete` has never exposed `cmd_flags` — the
+field only exists on patched/vendor kernels, so on stock kernels the
+`cmd_flags`/`op_code` trace columns are empty by design and classification
+comes from `rwbs`.)
 
 ### The Solution
-Instead of relying on a hardcoded kernel version macro, which can be unreliable across backported distribution kernels, `src/tracer/IOTracer.py` now dynamically checks for the presence of `cmd_flags` by parsing the format file directly:
-`/sys/kernel/debug/tracing/events/block/block_rq_complete/format`
-
-If the keyword `cmd_flags` is found, the Python script injects a `-DHAS_CMD_FLAGS` definition into the BPF compiler (`cflags`). In `prober.c`, `cmd_flags` collection is now wrapped in an `#ifdef HAS_CMD_FLAGS` block, ensuring safe access.
+Instead of relying on a hardcoded kernel version macro, which can be unreliable
+across backported distribution kernels, `src/tracer/IOTracer.py` dynamically
+checks for the presence of a field by parsing the tracepoint format file
+directly (checking both `/sys/kernel/debug/tracing` and `/sys/kernel/tracing`
+mounts).
+
+If the field is found, the Python script injects a feature define into the BPF
+compiler `cflags`, and the corresponding `args->` access in `prober.c` is
+wrapped in an `#ifdef`. The same mechanism now also gates
+`skb:kfree_skb`'s `reason` field (`-DHAS_SKB_DROP_REASON`, kernel >= 5.17 or
+5.15.58+ LTS backports) and `tcp:tcp_retransmit_skb`'s `state` field
+(`-DHAS_TCP_RETRANSMIT_STATE`, kernel >= 4.20) for `--network` runs.
 
 ## 3. "Too many open files" (File Descriptor Exhaustion)
 
@@ -71,8 +84,9 @@ dump. The captured data includes:
 * **Kernel config**: a curated set of BPF-relevant `CONFIG_*` values read from
   `/proc/config.gz` or `/boot/config-<release>` (`CONFIG_BPF_SYSCALL`,
   `CONFIG_DEBUG_INFO_BTF`, `CONFIG_KPROBES`, …).
-* **Toolchain**: Python, `bcc`, `clang`/`llc`, `gcc`, and `ld` versions (`clang`
-  is what BCC shells out to when compiling the prober).
+* **Toolchain**: Python, `bcc`, `clang`/`llc`, `gcc`, and `ld` versions (BCC
+  compiles the prober in-process via libclang; the installed clang version
+  still indicates the LLVM generation in play).
 * **Kernel headers**: presence of `/lib/modules/<release>/build` and
   `/usr/src/linux-headers-<release>` (BCC's fallback when BTF is absent).
 * **tracefs**: whether debugfs/tracefs is mounted and whether the

install.sh

@@ -55,20 +55,40 @@ check_root() {
 
 check_python() {
     if ! command -v python3 &> /dev/null; then
-        log_error "python3 is not installed. Please install Python 3.6+ and re-run."
+        log_error "python3 is not installed. Please install Python 3.7+ and re-run."
         exit 1
     fi
 
+    # The tracer uses time.time_ns / subprocess(text=...) (3.7+). Annotations
+    # are PEP 563-lazy, so 3.7-3.9 work; RHEL 8's stock 3.6 does NOT — use the
+    # python38+ AppStream there (with the matching python3X-bcc bindings).
     PY_VERSION=$(python3 -c 'import sys; print("%d%02d" % sys.version_info[:2])')
-    if [ "$PY_VERSION" -lt 306 ]; then
+    if [ "$PY_VERSION" -lt 307 ]; then
         PY_LABEL=$(python3 --version 2>&1)
-        log_error "Python 3.6+ is required (found $PY_LABEL)"
+        log_error "Python 3.7+ is required (found $PY_LABEL)"
         exit 1
     fi
 
     log_success "Python $(python3 --version 2>&1 | awk '{print $2}') detected"
 }
 
+# Kernel headers are needed by BCC to compile the eBPF program at runtime,
+# but the exact linux-headers-$(uname -r) package is often unavailable (WSL2
+# kernels, cloud images whose running kernel left the mirrors). Never let a
+# missing headers package abort the whole install: BCC can also compile from
+# the kernel's embedded headers (CONFIG_IKHEADERS, /sys/kernel/kheaders.tar.xz).
+install_kernel_headers_apt() {
+    apt-get install -y "linux-headers-$(uname -r)" || {
+        log_warning "linux-headers-$(uname -r) is not available from apt (normal on WSL2 and stale cloud images)."
+        if [ -d "/lib/modules/$(uname -r)/build" ] || [ -e /sys/kernel/kheaders.tar.xz ]; then
+            log_info "Kernel headers are available another way (build dir or CONFIG_IKHEADERS); continuing."
+        else
+            log_warning "No kernel headers found: the tracer will fail to compile until headers are provided."
+            log_warning "On WSL2, build headers from https://github.com/microsoft/WSL2-Linux-Kernel or enable CONFIG_IKHEADERS."
+        fi
+    }
+}
+
 detect_distro() {
     if [ -f /etc/os-release ]; then
         . /etc/os-release
@@ -95,25 +115,46 @@ detect_distro() {
 install_bcc_ubuntu() {
     log_info "Installing BCC for Ubuntu/Debian-based system..."
     apt-get update -qq
-    apt-get install -y bpfcc-tools linux-headers-$(uname -r)
+    # bcc itself is a hard requirement (fatal); headers are best-effort.
+    apt-get install -y bpfcc-tools
+    install_kernel_headers_apt
 }
 
 install_bcc_debian() {
     log_info "Installing BCC for Debian..."
-
-    # Check if sid repo is already added
-    if ! grep -q "debian sid main" /etc/apt/sources.list 2>/dev/null; then
-        log_info "Adding Debian sid repository for BCC..."
-        echo "deb http://cloudfront.debian.net/debian sid main" >> /etc/apt/sources.list
-    fi
-
+    # bpfcc-tools/libbpfcc have shipped in Debian stable main since buster —
+    # no sid repository needed. (An earlier version of this script appended
+    # the sid repo to /etc/apt/sources.list, which risks partial upgrades to
+    # unstable on any later `apt upgrade`. If a previous run added it, remove
+    # the "deb http://cloudfront.debian.net/debian sid main" line.)
     apt-get update -qq
-    apt-get install -y bpfcc-tools libbpfcc libbpfcc-dev linux-headers-$(uname -r)
+    apt-get install -y bpfcc-tools libbpfcc libbpfcc-dev
+    install_kernel_headers_apt
 }
 
 install_bcc_fedora() {
-    log_info "Installing BCC for Fedora..."
+    log_info "Installing BCC for Fedora/RHEL-family..."
     dnf install -y bcc bcc-tools python3-bcc
+    # Match the running kernel where possible; plain kernel-devel as fallback
+    # (also covers install_weak_deps=False setups where the bcc RPM's
+    # "Recommends: kernel-devel" is not honored).
+    dnf install -y "kernel-devel-$(uname -r)" || dnf install -y kernel-devel || \
+        log_warning "kernel-devel unavailable; BCC will rely on embedded headers (CONFIG_IKHEADERS) if present"
+}
+
+install_bcc_amazon() {
+    # Amazon Linux 2023 ships dnf + bcc in the base repos; Amazon Linux 2
+    # (EOL 2026-06-30) needs amazon-linux-extras and is not supported —
+    # rejected outright, even if dnf happens to be installed on it (its
+    # repos still lack bcc, so the install would fail mid-flight anyway).
+    if [ "${VERSION_ID%%.*}" = "2" ]; then
+        log_error "Amazon Linux 2 is past end-of-life and not supported; use Amazon Linux 2023."
+        exit 1
+    fi
+    log_info "Installing BCC for Amazon Linux 2023..."
+    dnf install -y bcc bcc-tools python3-bcc
+    dnf install -y "kernel-devel-$(uname -r)" || dnf install -y kernel-devel || \
+        log_warning "kernel-devel unavailable; BCC will rely on embedded headers (CONFIG_IKHEADERS) if present"
 }
 
 install_bcc_arch() {
@@ -150,7 +191,7 @@ install_git_if_needed() {
             ubuntu|debian|linuxmint|pop)
                 apt-get install -y git
                 ;;
-            fedora|rhel|centos)
+            fedora|rhel|centos|rocky|almalinux|amzn)
                 dnf install -y git
                 ;;
             arch|manjaro)
@@ -175,10 +216,11 @@ clone_repo() {
 install_bin() {
     log_info "Installing $BIN_NAME wrapper to $BIN_DIR..."
 
-    # Write a wrapper script so that iotrc.py is always executed from inside
-    # the repo directory. This is required because iotrc.py uses package-relative
-    # imports (from src.tracer.IOTracer import ...) which only resolve when
-    # Python's working directory is the repo root.
+    # The wrapper execs iotrc.py by absolute path from whatever directory the
+    # user is in: imports resolve via sys.path[0] (the script's directory) and
+    # iotrc.py resolves the BPF source relative to __file__, so no cd is
+    # needed. (An earlier comment here claimed the CWD had to be the repo
+    # root — that was never enforced and is not required.)
     cat > "$BIN_DIR/$BIN_NAME" << EOF
 #!/bin/bash
 exec python3 "$INSTALL_DIR/iotrc.py" "\$@"
@@ -204,7 +246,11 @@ install_dependencies() {
             ;;
         rhel|centos|rocky|almalinux)
             log_warning "RHEL-based distro detected. Using dnf..."
-            dnf install -y bcc bcc-tools python3-bcc
+            install_bcc_fedora
+            install_python_deps_dnf
+            ;;
+        amzn)
+            install_bcc_amazon
             install_python_deps_dnf
             ;;
         arch|manjaro)

iotrc.py

@@ -90,6 +90,9 @@ def maximize_fd_limit():
     parser.add_argument('--computer-id', action='store_true', help='Print this machine ID and exit')
     parser.add_argument('--reward', action='store_true', help='Show your reward code (unlocked after uploading traces)')
     parser.add_argument('--no-upload', action='store_true', help='Disable automatic upload of traces (for testing)')
+    parser.add_argument('--output', type=str, default=tempfile.gettempdir(), metavar='DIR',
+                        help='Base directory for trace output (default: system temp dir). '
+                             'The systemd service passes /var/log/iotracer/traces here.')
 
     subparsers = parser.add_subparsers(dest='subcommand')
     dev_parser = subparsers.add_parser('dev', help='Run in developer mode with extra logs and checks')
@@ -99,9 +102,13 @@ def maximize_fd_limit():
     dev_parser.add_argument('--network', action='store_true', help='Force-enable network event tracing: connection lifecycle, sockopt, drops (otherwise auto-enabled when the host has enough CPU, DRAM and network)')
     dev_parser.add_argument('--no-upload', action='store_true', help='Disable automatic upload of traces (for testing)')
     dev_parser.add_argument('--trace-bucket', type=str, default=None, help='Override upload bucket name (default: linux_v1)')
+    # SUPPRESS (not a real default): a subparser default would CLOBBER a value
+    # already parsed by the main parser ('iotrc --output /x dev' must keep /x).
+    dev_parser.add_argument('--output', type=str, default=argparse.SUPPRESS, metavar='DIR',
+                            help='Base directory for trace output (default: system temp dir)')
 
     parse_args = parser.parse_args()
-    output_dir = tempfile.gettempdir()
+    output_dir = parse_args.output
 
     # Handle --computer-id flag: print machine ID and exit
     if parse_args.computer_id:
@@ -132,10 +139,19 @@ def maximize_fd_limit():
         trace_cache, trace_network, verbose=verbose
     )
 
+    # Resolve the BPF source relative to this file, not the CWD. BCC's
+    # _find_file has an argv[0]-relative fallback that usually rescues a
+    # CWD-relative path, but it checks the CWD FIRST — so running iotrc from
+    # inside a different/stale checkout would silently compile that foreign
+    # prober.c. An absolute path removes both the fallback reliance and the
+    # shadowing hazard.
+    bpf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                            'src', 'tracer', 'prober', 'prober.c')
+
     # Initialize and start the IO tracer
     tracer = IOTracer(
         output_dir=output_dir,
-        bpf_file='./src/tracer/prober/prober.c',
+        bpf_file=bpf_file,
         page_cnt=8,
         verbose=verbose,
         anonymous=anonimize,