diff --git a/.github/scripts/bpf_smoke.py b/.github/scripts/bpf_smoke.py index 4a7e5b3..2573745 100644 --- a/.github/scripts/bpf_smoke.py +++ b/.github/scripts/bpf_smoke.py @@ -18,6 +18,17 @@ BPF_FILE = os.path.join(REPO_ROOT, "src", "tracer", "prober", "prober.c") +def tracepoint_format(category, name): + """Read a tracepoint format file from debugfs or tracefs (either mount).""" + for base in ("/sys/kernel/debug/tracing", "/sys/kernel/tracing"): + try: + with open(f"{base}/events/{category}/{name}/format") as f: + return f.read() + except OSError: + continue + return "" + + def build_cflags(): """Mirror IOTracer._init_bpf so CI compiles with the real flags.""" cflags = [ @@ -26,11 +37,18 @@ def build_cflags(): "-mllvm", "-bpf-stack-size=4096", ] - tp_format = "/sys/kernel/debug/tracing/events/block/block_rq_complete/format" - if os.path.exists(tp_format): - with open(tp_format) as f: - if "cmd_flags" in f.read(): - cflags.append("-DHAS_CMD_FLAGS") + if "cmd_flags" in tracepoint_format("block", "block_rq_complete"): + cflags.append("-DHAS_CMD_FLAGS") + return cflags + + +def network_cflags(): + """Mirror IOTracer._init_bpf's --network feature gates.""" + cflags = ["-DENABLE_NETWORK"] + if " reason;" in tracepoint_format("skb", "kfree_skb"): + cflags.append("-DHAS_SKB_DROP_REASON") + if " state;" in tracepoint_format("tcp", "tcp_retransmit_skb"): + cflags.append("-DHAS_TCP_RETRANSMIT_STATE") return cflags @@ -72,8 +90,11 @@ def main(): conditional_probes = [ (b"__x64_sys_mremap", [("kprobe", "__x64_sys_mremap", "trace_mremap_entry_x64"), ("kretprobe", "__x64_sys_mremap", "trace_mremap_ret")]), + (b"__arm64_sys_mremap", [("kprobe", "__arm64_sys_mremap", "trace_mremap_entry_arm64"), + ("kretprobe", "__arm64_sys_mremap", "trace_mremap_ret")]), (b"__x64_sys_openat", [("kprobe", "__x64_sys_openat", "trace_openat_entry_x64")]), (b"__x64_sys_io_uring_enter", [("kprobe", "__x64_sys_io_uring_enter", "trace_io_uring_enter_x64")]), + (b"__arm64_sys_io_uring_enter", [("kprobe", "__arm64_sys_io_uring_enter", "trace_io_uring_enter_arm64")]), (b"iomap_dio_rw", [("kprobe", "iomap_dio_rw", "trace_dio_entry_iomap"), ("kretprobe", "iomap_dio_rw", "trace_dio_return")]), (b"__blockdev_direct_IO", [("kprobe", "__blockdev_direct_IO", "trace_dio_entry_blockdev")]), @@ -84,6 +105,41 @@ def main(): else: print(f"SKIP: {symbol.decode()} not present on this kernel") + # Cache-probe guard/symbol alignment, expressed as ORDERED fallback chains + # exactly like KernelProbeTracker: the first present symbol wins and the + # rest are skipped. This matters because old page-API symbols + # (mark_page_accessed, add_to_page_cache_lru, ...) still exist on modern + # kernels as exported folio-compat wrappers while their page-variant + # handlers are compiled OUT (>= 5.16/5.17 guards) — attaching every + # existing symbol unconditionally would fail CI on every modern kernel. + conditional_probe_chains = [ + [(b"folio_mark_accessed", "trace_folio_mark_accessed"), + (b"mark_page_accessed", "trace_hit")], + [(b"filemap_add_folio", "trace_filemap_add_folio"), + (b"add_to_page_cache_lru", "trace_miss")], + [(b"__folio_mark_dirty", "trace_folio_mark_dirty"), + (b"account_page_dirtied", "trace_account_page_dirtied")], + [(b"folio_clear_dirty_for_io", "trace_folio_clear_dirty_for_io"), + (b"clear_page_dirty_for_io", "trace_clear_page_dirty_for_io")], + [(b"folio_end_writeback", "trace_folio_end_writeback"), + (b"test_clear_page_writeback", "trace_test_clear_page_writeback")], + [(b"filemap_remove_folio", "trace_filemap_remove_folio"), + (b"__filemap_remove_folio", "trace_filemap_remove_folio"), + (b"__delete_from_page_cache", "trace_delete_from_page_cache")], + [(b"__do_page_cache_readahead", "trace_do_page_cache_readahead"), + (b"do_page_cache_ra", "trace_page_cache_ra"), + (b"page_cache_ra_order", "trace_page_cache_ra_order")], + [(b"shrink_folio_list", "trace_shrink_folio_list"), + (b"shrink_page_list", "trace_shrink_folio_list")], + ] + for chain in conditional_probe_chains: + for symbol, fn in chain: + if BPF.get_kprobe_functions(symbol): + probes.append(("kprobe", symbol.decode(), fn)) + break + else: + print(f"SKIP: no symbol of chain {[s.decode() for s, _ in chain]} present") + for kind, event, fn in probes: if kind == "kprobe": b.attach_kprobe(event=event, fn_name=fn) @@ -99,7 +155,7 @@ def main(): # connection/sockopt/drop probes get verifier coverage too. Their # TRACEPOINT_PROBE handlers auto-attach on load, so a successful BPF() # construction validates both compile and attach. - net_cflags = build_cflags() + ["-DENABLE_NETWORK"] + net_cflags = build_cflags() + network_cflags() print(f"cflags (network): {net_cflags}") b_net = BPF(src_file=BPF_FILE.encode(), cflags=net_cflags) print("OK: prober.c compiled with ENABLE_NETWORK (verifier passed, " diff --git a/README.md b/README.md index 6a8bee3..51ee693 100644 --- a/README.md +++ b/README.md @@ -41,14 +41,18 @@ pacman -S bcc bcc-tools python-bcc For more distros, visit the official [BCC's installation guide](https://github.com/iovisor/bcc/blob/master/INSTALL.md) -3. Finally, install the Python dependencies. The simplest way is to install -them all at once from `requirements.txt`: +3. Finally, install the Python dependencies. Prefer your distro's packages +(the tracer runs under the system Python, which is what the distro `bcc` +bindings are built for — see the next code block). If you use pip instead, +note that Debian 12 / Ubuntu 23.04+ mark the system interpreter as +externally managed (PEP 668), so a bare `pip install` fails; append +`--break-system-packages` or use the distro packages: ```bash pip install -r requirements.txt ``` -Or, if you prefer your distro's package manager: +Distro package manager equivalents: ```bash # Ubuntu / Debian @@ -68,7 +72,7 @@ To run the test suite you'll also need `pytest` (`pip install pytest`). ## Usage ``` -usage: sudo iotrc [-h] [-v] [-a] [--cache] [--network] [--computer-id] [--reward] [--no-upload] {dev} ... +usage: sudo iotrc [-h] [-v] [-a] [--cache] [--network] [--computer-id] [--reward] [--no-upload] [--output DIR] {dev} ... Trace IO syscalls @@ -79,6 +83,7 @@ options: --computer-id Print this machine ID and exit --reward Show your reward code (unlocked after uploading traces) --no-upload Disable automatic upload of traces (for testing) + --output DIR Base directory for trace output (default: system temp dir) subcommands: {dev} Run in developer mode with extra logs and checks diff --git a/docs/COMPATIBILITY_FIXES.md b/docs/COMPATIBILITY_FIXES.md index ec58e2c..23fcb78 100644 --- a/docs/COMPATIBILITY_FIXES.md +++ b/docs/COMPATIBILITY_FIXES.md @@ -18,13 +18,26 @@ We implemented selective compilation using standard kernel version macros (`#if ## 2. Missing `cmd_flags` in `block_rq_complete` ### The Problem -The `block_rq_complete` tracepoint arguments vary between kernel versions. On some older kernels, the `cmd_flags` variable is missing from the tracepoint format definition entirely, leading to a direct compilation failure when `args->cmd_flags` was accessed in `prober.c`. +Referencing a tracepoint `args->` field that the running kernel's format file +does not define is a direct compilation failure that aborts the whole BPF +load. (Note: mainline `block_rq_complete` has never exposed `cmd_flags` — the +field only exists on patched/vendor kernels, so on stock kernels the +`cmd_flags`/`op_code` trace columns are empty by design and classification +comes from `rwbs`.) ### The Solution -Instead of relying on a hardcoded kernel version macro, which can be unreliable across backported distribution kernels, `src/tracer/IOTracer.py` now dynamically checks for the presence of `cmd_flags` by parsing the format file directly: -`/sys/kernel/debug/tracing/events/block/block_rq_complete/format` - -If the keyword `cmd_flags` is found, the Python script injects a `-DHAS_CMD_FLAGS` definition into the BPF compiler (`cflags`). In `prober.c`, `cmd_flags` collection is now wrapped in an `#ifdef HAS_CMD_FLAGS` block, ensuring safe access. +Instead of relying on a hardcoded kernel version macro, which can be unreliable +across backported distribution kernels, `src/tracer/IOTracer.py` dynamically +checks for the presence of a field by parsing the tracepoint format file +directly (checking both `/sys/kernel/debug/tracing` and `/sys/kernel/tracing` +mounts). + +If the field is found, the Python script injects a feature define into the BPF +compiler `cflags`, and the corresponding `args->` access in `prober.c` is +wrapped in an `#ifdef`. The same mechanism now also gates +`skb:kfree_skb`'s `reason` field (`-DHAS_SKB_DROP_REASON`, kernel >= 5.17 or +5.15.58+ LTS backports) and `tcp:tcp_retransmit_skb`'s `state` field +(`-DHAS_TCP_RETRANSMIT_STATE`, kernel >= 4.20) for `--network` runs. ## 3. "Too many open files" (File Descriptor Exhaustion) @@ -71,8 +84,9 @@ dump. The captured data includes: * **Kernel config**: a curated set of BPF-relevant `CONFIG_*` values read from `/proc/config.gz` or `/boot/config-` (`CONFIG_BPF_SYSCALL`, `CONFIG_DEBUG_INFO_BTF`, `CONFIG_KPROBES`, …). -* **Toolchain**: Python, `bcc`, `clang`/`llc`, `gcc`, and `ld` versions (`clang` - is what BCC shells out to when compiling the prober). +* **Toolchain**: Python, `bcc`, `clang`/`llc`, `gcc`, and `ld` versions (BCC + compiles the prober in-process via libclang; the installed clang version + still indicates the LLVM generation in play). * **Kernel headers**: presence of `/lib/modules//build` and `/usr/src/linux-headers-` (BCC's fallback when BTF is absent). * **tracefs**: whether debugfs/tracefs is mounted and whether the diff --git a/install.sh b/install.sh index a4dd0cf..47defe2 100644 --- a/install.sh +++ b/install.sh @@ -55,20 +55,40 @@ check_root() { check_python() { if ! command -v python3 &> /dev/null; then - log_error "python3 is not installed. Please install Python 3.6+ and re-run." + log_error "python3 is not installed. Please install Python 3.7+ and re-run." exit 1 fi + # The tracer uses time.time_ns / subprocess(text=...) (3.7+). Annotations + # are PEP 563-lazy, so 3.7-3.9 work; RHEL 8's stock 3.6 does NOT — use the + # python38+ AppStream there (with the matching python3X-bcc bindings). PY_VERSION=$(python3 -c 'import sys; print("%d%02d" % sys.version_info[:2])') - if [ "$PY_VERSION" -lt 306 ]; then + if [ "$PY_VERSION" -lt 307 ]; then PY_LABEL=$(python3 --version 2>&1) - log_error "Python 3.6+ is required (found $PY_LABEL)" + log_error "Python 3.7+ is required (found $PY_LABEL)" exit 1 fi log_success "Python $(python3 --version 2>&1 | awk '{print $2}') detected" } +# Kernel headers are needed by BCC to compile the eBPF program at runtime, +# but the exact linux-headers-$(uname -r) package is often unavailable (WSL2 +# kernels, cloud images whose running kernel left the mirrors). Never let a +# missing headers package abort the whole install: BCC can also compile from +# the kernel's embedded headers (CONFIG_IKHEADERS, /sys/kernel/kheaders.tar.xz). +install_kernel_headers_apt() { + apt-get install -y "linux-headers-$(uname -r)" || { + log_warning "linux-headers-$(uname -r) is not available from apt (normal on WSL2 and stale cloud images)." + if [ -d "/lib/modules/$(uname -r)/build" ] || [ -e /sys/kernel/kheaders.tar.xz ]; then + log_info "Kernel headers are available another way (build dir or CONFIG_IKHEADERS); continuing." + else + log_warning "No kernel headers found: the tracer will fail to compile until headers are provided." + log_warning "On WSL2, build headers from https://github.com/microsoft/WSL2-Linux-Kernel or enable CONFIG_IKHEADERS." + fi + } +} + detect_distro() { if [ -f /etc/os-release ]; then . /etc/os-release @@ -95,25 +115,46 @@ detect_distro() { install_bcc_ubuntu() { log_info "Installing BCC for Ubuntu/Debian-based system..." apt-get update -qq - apt-get install -y bpfcc-tools linux-headers-$(uname -r) + # bcc itself is a hard requirement (fatal); headers are best-effort. + apt-get install -y bpfcc-tools + install_kernel_headers_apt } install_bcc_debian() { log_info "Installing BCC for Debian..." - - # Check if sid repo is already added - if ! grep -q "debian sid main" /etc/apt/sources.list 2>/dev/null; then - log_info "Adding Debian sid repository for BCC..." - echo "deb http://cloudfront.debian.net/debian sid main" >> /etc/apt/sources.list - fi - + # bpfcc-tools/libbpfcc have shipped in Debian stable main since buster — + # no sid repository needed. (An earlier version of this script appended + # the sid repo to /etc/apt/sources.list, which risks partial upgrades to + # unstable on any later `apt upgrade`. If a previous run added it, remove + # the "deb http://cloudfront.debian.net/debian sid main" line.) apt-get update -qq - apt-get install -y bpfcc-tools libbpfcc libbpfcc-dev linux-headers-$(uname -r) + apt-get install -y bpfcc-tools libbpfcc libbpfcc-dev + install_kernel_headers_apt } install_bcc_fedora() { - log_info "Installing BCC for Fedora..." + log_info "Installing BCC for Fedora/RHEL-family..." dnf install -y bcc bcc-tools python3-bcc + # Match the running kernel where possible; plain kernel-devel as fallback + # (also covers install_weak_deps=False setups where the bcc RPM's + # "Recommends: kernel-devel" is not honored). + dnf install -y "kernel-devel-$(uname -r)" || dnf install -y kernel-devel || \ + log_warning "kernel-devel unavailable; BCC will rely on embedded headers (CONFIG_IKHEADERS) if present" +} + +install_bcc_amazon() { + # Amazon Linux 2023 ships dnf + bcc in the base repos; Amazon Linux 2 + # (EOL 2026-06-30) needs amazon-linux-extras and is not supported — + # rejected outright, even if dnf happens to be installed on it (its + # repos still lack bcc, so the install would fail mid-flight anyway). + if [ "${VERSION_ID%%.*}" = "2" ]; then + log_error "Amazon Linux 2 is past end-of-life and not supported; use Amazon Linux 2023." + exit 1 + fi + log_info "Installing BCC for Amazon Linux 2023..." + dnf install -y bcc bcc-tools python3-bcc + dnf install -y "kernel-devel-$(uname -r)" || dnf install -y kernel-devel || \ + log_warning "kernel-devel unavailable; BCC will rely on embedded headers (CONFIG_IKHEADERS) if present" } install_bcc_arch() { @@ -150,7 +191,7 @@ install_git_if_needed() { ubuntu|debian|linuxmint|pop) apt-get install -y git ;; - fedora|rhel|centos) + fedora|rhel|centos|rocky|almalinux|amzn) dnf install -y git ;; arch|manjaro) @@ -175,10 +216,11 @@ clone_repo() { install_bin() { log_info "Installing $BIN_NAME wrapper to $BIN_DIR..." - # Write a wrapper script so that iotrc.py is always executed from inside - # the repo directory. This is required because iotrc.py uses package-relative - # imports (from src.tracer.IOTracer import ...) which only resolve when - # Python's working directory is the repo root. + # The wrapper execs iotrc.py by absolute path from whatever directory the + # user is in: imports resolve via sys.path[0] (the script's directory) and + # iotrc.py resolves the BPF source relative to __file__, so no cd is + # needed. (An earlier comment here claimed the CWD had to be the repo + # root — that was never enforced and is not required.) cat > "$BIN_DIR/$BIN_NAME" << EOF #!/bin/bash exec python3 "$INSTALL_DIR/iotrc.py" "\$@" @@ -204,7 +246,11 @@ install_dependencies() { ;; rhel|centos|rocky|almalinux) log_warning "RHEL-based distro detected. Using dnf..." - dnf install -y bcc bcc-tools python3-bcc + install_bcc_fedora + install_python_deps_dnf + ;; + amzn) + install_bcc_amazon install_python_deps_dnf ;; arch|manjaro) diff --git a/iotrc.py b/iotrc.py index e862fa0..a13697b 100755 --- a/iotrc.py +++ b/iotrc.py @@ -90,6 +90,9 @@ def maximize_fd_limit(): parser.add_argument('--computer-id', action='store_true', help='Print this machine ID and exit') parser.add_argument('--reward', action='store_true', help='Show your reward code (unlocked after uploading traces)') parser.add_argument('--no-upload', action='store_true', help='Disable automatic upload of traces (for testing)') + parser.add_argument('--output', type=str, default=tempfile.gettempdir(), metavar='DIR', + help='Base directory for trace output (default: system temp dir). ' + 'The systemd service passes /var/log/iotracer/traces here.') subparsers = parser.add_subparsers(dest='subcommand') dev_parser = subparsers.add_parser('dev', help='Run in developer mode with extra logs and checks') @@ -99,9 +102,13 @@ def maximize_fd_limit(): dev_parser.add_argument('--network', action='store_true', help='Force-enable network event tracing: connection lifecycle, sockopt, drops (otherwise auto-enabled when the host has enough CPU, DRAM and network)') dev_parser.add_argument('--no-upload', action='store_true', help='Disable automatic upload of traces (for testing)') dev_parser.add_argument('--trace-bucket', type=str, default=None, help='Override upload bucket name (default: linux_v1)') + # SUPPRESS (not a real default): a subparser default would CLOBBER a value + # already parsed by the main parser ('iotrc --output /x dev' must keep /x). + dev_parser.add_argument('--output', type=str, default=argparse.SUPPRESS, metavar='DIR', + help='Base directory for trace output (default: system temp dir)') parse_args = parser.parse_args() - output_dir = tempfile.gettempdir() + output_dir = parse_args.output # Handle --computer-id flag: print machine ID and exit if parse_args.computer_id: @@ -132,10 +139,19 @@ def maximize_fd_limit(): trace_cache, trace_network, verbose=verbose ) + # Resolve the BPF source relative to this file, not the CWD. BCC's + # _find_file has an argv[0]-relative fallback that usually rescues a + # CWD-relative path, but it checks the CWD FIRST — so running iotrc from + # inside a different/stale checkout would silently compile that foreign + # prober.c. An absolute path removes both the fallback reliance and the + # shadowing hazard. + bpf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'src', 'tracer', 'prober', 'prober.c') + # Initialize and start the IO tracer tracer = IOTracer( output_dir=output_dir, - bpf_file='./src/tracer/prober/prober.c', + bpf_file=bpf_file, page_cnt=8, verbose=verbose, anonymous=anonimize, diff --git a/src/tracer/IOTracer.py b/src/tracer/IOTracer.py index 3dca2ab..b2b5670 100644 --- a/src/tracer/IOTracer.py +++ b/src/tracer/IOTracer.py @@ -16,9 +16,15 @@ tracer.trace() """ +# PEP 563: keep all annotations lazy so PEP 604 (`X | None`) and PEP 585 +# (`list[str]`) syntax import cleanly on Python 3.7-3.9 (RHEL 9, Debian 11, +# Ubuntu 20.04, Amazon Linux stock interpreters). +from __future__ import annotations + import shutil import signal import os +import tempfile import ctypes import threading import json @@ -268,34 +274,126 @@ def __init__( self.bpf_file = bpf_file self._attempted_cflags: list[str] = [] try: + def _tracepoint_format(category: str, name: str) -> str: + # Tracepoint format files live under debugfs on older setups and + # under tracefs (/sys/kernel/tracing) on modern ones that no + # longer mount debugfs; check both. + for base in ("/sys/kernel/debug/tracing", "/sys/kernel/tracing"): + try: + with open(f"{base}/events/{category}/{name}/format", "r") as f: + return f.read() + except OSError: + continue + return "" + def _init_bpf(): + # KNOWN FOLLOW-UP: -bpf-stack-size=4096 only lifts LLVM's + # compile-time diagnostic; the kernel verifier still enforces + # 512 bytes per frame. struct data_t (~408 bytes) is + # stack-allocated in ~26 handlers, so frames sit close enough + # to 512 that a different LLVM's stack layout can cross it at + # load time on some distros. The durable fix is moving data_t + # into a per-CPU scratch map (like dual_data_buffer) handler by + # handler and THEN dropping this flag so oversize frames fail + # at compile again — do not drop the flag alone. cflags = ["-Wno-duplicate-decl-specifier", "-Wno-macro-redefined", "-mllvm", "-bpf-stack-size=4096"] - tp_format = "/sys/kernel/debug/tracing/events/block/block_rq_complete/format" - if os.path.exists(tp_format): - with open(tp_format, "r") as f: - if "cmd_flags" in f.read(): - cflags.append("-DHAS_CMD_FLAGS") + # Feature-detect optional tracepoint fields by sniffing the same + # format files BCC generates the args structs from, so the -D + # gates can never disagree with what actually compiles. This + # (rather than LINUX_VERSION_CODE) also does the right thing on + # kernels with backports (e.g. 'reason' backported to 5.15.58). + if "cmd_flags" in _tracepoint_format("block", "block_rq_complete"): + cflags.append("-DHAS_CMD_FLAGS") # Compile the network probe subset only when requested. The # connection/sockopt/drop probes auto-attach when compiled, # so gating at compile time keeps overhead at zero when off. if self.trace_network: cflags.append("-DENABLE_NETWORK") + # skb:kfree_skb 'reason' exists on mainline 5.17+ (and + # 5.15.58+ LTS); tcp:tcp_retransmit_skb 'state' on 4.20+. + # Referencing a missing args-> field is a compile error + # that aborts the whole load, so gate each on its format. + if " reason;" in _tracepoint_format("skb", "kfree_skb"): + cflags.append("-DHAS_SKB_DROP_REASON") + if " state;" in _tracepoint_format("tcp", "tcp_retransmit_skb"): + cflags.append("-DHAS_TCP_RETRANSMIT_STATE") # Record the cflags before compiling so the diagnostics dump can # report them even when the BPF() call itself raises. self._attempted_cflags = cflags + # A native compiler abort (LLVM report_fatal_error -> SIGABRT) + # kills the process inside libbcc WITHOUT raising, so the + # except-path below never runs for it. Leave a breadcrumb + # diagnostics file before compiling and delete it on success — + # an abort then still leaves a shareable report on disk. + self._write_compile_breadcrumb(cflags) self.b = BPF(src_file=bpf_file.encode(), cflags=cflags) self.probe_tracker = KernelProbeTracker(self.b, developer_mode, trace_cache=self.trace_cache) + self._remove_compile_breadcrumb() run_with_spinner("Loading BPF program", _init_bpf) except Exception as e: + # The full dump below supersedes the pre-compile breadcrumb. + self._remove_compile_breadcrumb() logger("error", f"failed to initialize BPF: {e}") - print("Your device is incompatible with this version of IO Tracer.") + print("IO Tracer could not compile or load its eBPF program on this host.") # Dump as much OS/kernel/toolchain information as possible so the # incompatibility can actually be diagnosed (rather than emailing us # a bare "it doesn't work"). Never let the dump mask the real error. + # The dump also prints remediation hints (missing kernel headers, + # Secure Boot lockdown, old bcc) when it can tell the causes apart. self._dump_failure_diagnostics(e, "BPF program failed to compile or load") sys.exit(1) + def _write_compile_breadcrumb(self, cflags: list[str]) -> None: + """Write a pre-compile diagnostics file (no console output). + + Deleted again on successful load (and on a handled failure, where the + full dump supersedes it); if the compiler aborts the process natively + the file survives as the failure report. Created with + NamedTemporaryFile (O_CREAT|O_EXCL, mode 0600, random suffix) — the + tracer runs as root, so an open() of a fixed, predictable name in + world-writable /tmp would follow an attacker-planted symlink. Never + raises. + """ + self._compile_breadcrumb_path = None + try: + diagnostics = self.system_snapper.collect_diagnostics( + error=None, + attempted_cflags=cflags, + bpf_file=getattr(self, "bpf_file", None), + # Happy-path cost control: skip the subprocess toolchain + # probes here; the full failure dump still collects them. + include_toolchain_probes=False, + context=( + "BPF compile started. If this file still exists, the " + "compiler terminated the process natively (e.g. 'LLVM " + "ERROR: ...' followed by an abort) before the tracer " + "could handle the failure — share this file with the " + "maintainers." + ), + ) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix="io-tracer-pending-compile-", + suffix=".json", + delete=False, + ) as f: + json.dump(diagnostics, f, indent=2, default=str) + self._compile_breadcrumb_path = f.name + except Exception: # noqa: BLE001 - breadcrumb is best effort + self._compile_breadcrumb_path = None + + def _remove_compile_breadcrumb(self) -> None: + """Delete the pre-compile breadcrumb, if one was written. Never raises.""" + path = getattr(self, "_compile_breadcrumb_path", None) + self._compile_breadcrumb_path = None + if path: + try: + os.unlink(path) + except OSError: + pass + def _dump_failure_diagnostics(self, error: BaseException, context: str) -> None: """Best-effort OS-information dump for a compile/run failure. @@ -1102,7 +1200,10 @@ def _print_event_block(self, cpu, data, size): cmd_flags_str = self.flag_mapper.decode_block_req_flags(cmd_flags) if cmd_flags else "" # Decode raw operation code (REQ_OP_READ, REQ_OP_WRITE, etc.) - # Note: op_code is 0 on kernel 5.17+ where cmd_flags is unavailable - don't decode it + # Note: mainline block_rq_complete has never exposed cmd_flags, so + # op_code/cmd_flags are 0 on stock kernels (the rwbs-derived + # `operation` column covers classification); they only populate on + # kernels whose tracepoint format actually carries cmd_flags. op_code = event.op_code if hasattr(event, 'op_code') else 0 op_code_str = self.flag_mapper.decode_block_op_code(op_code) if (op_code is not None and op_code != 0) else "" diff --git a/src/tracer/KernelProbeTracker.py b/src/tracer/KernelProbeTracker.py index afe6f48..c907b6d 100644 --- a/src/tracer/KernelProbeTracker.py +++ b/src/tracer/KernelProbeTracker.py @@ -225,10 +225,16 @@ def attach_probes(self): self.add_kprobe("__vm_munmap", "trace_munmap") # mremap probes — kernel may export the arch wrapper or the generic - # symbol. The wrapper needs the pt_regs-unwrapping variant. + # symbol. The wrappers need the per-arch pt_regs-unwrapping variant + # (get_kprobe_functions fullmatches, so the arch branches are + # mutually exclusive; the kretprobe reads only PT_REGS_RC, which is + # correct on every arch). if BPF.get_kprobe_functions(b'__x64_sys_mremap'): self.add_kprobe("__x64_sys_mremap", "trace_mremap_entry_x64") self.add_kretprobe("__x64_sys_mremap", "trace_mremap_ret") + elif BPF.get_kprobe_functions(b'__arm64_sys_mremap'): + self.add_kprobe("__arm64_sys_mremap", "trace_mremap_entry_arm64") + self.add_kretprobe("__arm64_sys_mremap", "trace_mremap_ret") elif BPF.get_kprobe_functions(b'sys_mremap'): self.add_kprobe("sys_mremap", "trace_mremap_entry") self.add_kretprobe("sys_mremap", "trace_mremap_ret") @@ -374,14 +380,18 @@ def attach_probes(self): elif self.developer_mode: logger("warning", "No cache drop function found, drop events will not be traced") - # Cache readahead probes - track prefetch operations + # Cache readahead probes - track prefetch operations. Each + # symbol has a different signature, so each attaches to its + # own correctly-typed handler. if BPF.get_kprobe_functions(b'__do_page_cache_readahead'): + # (mapping, file, offset, nr_to_read, ...) — kernel < 5.10 self.add_kprobe("__do_page_cache_readahead", "trace_do_page_cache_readahead") elif BPF.get_kprobe_functions(b'do_page_cache_ra'): - self.add_kprobe("do_page_cache_ra", "trace_do_page_cache_readahead") + # (readahead_control, nr_to_read, ...) — kernel >= 5.10 + self.add_kprobe("do_page_cache_ra", "trace_page_cache_ra") elif BPF.get_kprobe_functions(b'page_cache_ra_order'): - # Newer kernels (5.16+) - self.add_kprobe("page_cache_ra_order", "trace_do_page_cache_readahead") + # (readahead_control, file_ra_state, order) — kernel >= 5.18 + self.add_kprobe("page_cache_ra_order", "trace_page_cache_ra_order") elif self.developer_mode: logger("warning", "No readahead probe available, readahead events will not be traced") @@ -402,20 +412,22 @@ def attach_probes(self): # The tracepoints are automatically attached via TRACEPOINT_PROBE macros in BPF code # We also attach kprobes as fallback for kernels without stable tracepoints - # io_uring_enter syscall probe - if BPF.get_kprobe_functions(b'__io_uring_enter'): - self.add_kprobe("__io_uring_enter", "trace_io_uring_enter") - if self.developer_mode: - logger("info", "io_uring tracing enabled via __io_uring_enter") - elif BPF.get_kprobe_functions(b'__x64_sys_io_uring_enter'): + # io_uring_enter syscall probe. The syscall is only reachable via + # the per-arch SYSCALL_DEFINE wrappers, so each arch needs its + # pt_regs-unwrapping variant. (Earlier fallbacks __io_uring_enter / + # __sys_io_uring_enter never existed in any mainline kernel — + # only SYSCALL_DEFINE6 in fs/io_uring.c / io_uring/io_uring.c — + # so those branches were dead on every arch and arm64 silently + # got no ENTER events at all.) + if BPF.get_kprobe_functions(b'__x64_sys_io_uring_enter'): # Syscall wrapper: needs the pt_regs-unwrapping variant. self.add_kprobe("__x64_sys_io_uring_enter", "trace_io_uring_enter_x64") if self.developer_mode: logger("info", "io_uring tracing enabled via __x64_sys_io_uring_enter") - elif BPF.get_kprobe_functions(b'__sys_io_uring_enter'): - self.add_kprobe("__sys_io_uring_enter", "trace_io_uring_enter") + elif BPF.get_kprobe_functions(b'__arm64_sys_io_uring_enter'): + self.add_kprobe("__arm64_sys_io_uring_enter", "trace_io_uring_enter_arm64") if self.developer_mode: - logger("info", "io_uring tracing enabled via __sys_io_uring_enter") + logger("info", "io_uring tracing enabled via __arm64_sys_io_uring_enter") else: if self.developer_mode: logger("warning", "io_uring_enter probe not available - ENTER events disabled") diff --git a/src/tracer/ObjectStorageManager.py b/src/tracer/ObjectStorageManager.py index c2e0b31..0b627c2 100644 --- a/src/tracer/ObjectStorageManager.py +++ b/src/tracer/ObjectStorageManager.py @@ -16,6 +16,11 @@ manager.put_object("/path/to/trace.tar.zst") # Upload a file """ +# PEP 563: keep all annotations lazy so PEP 604 (`X | None`) and PEP 585 +# (`list[str]`) syntax import cleanly on Python 3.7-3.9 (RHEL 9, Debian 11, +# Ubuntu 20.04, Amazon Linux stock interpreters). +from __future__ import annotations + import mimetypes import os import threading diff --git a/src/tracer/PathResolver.py b/src/tracer/PathResolver.py index d8d5fd8..eddae09 100644 --- a/src/tracer/PathResolver.py +++ b/src/tracer/PathResolver.py @@ -16,6 +16,11 @@ path = resolver.resolve_path(inode=12345, pid=1234, filename="unknown") """ +# PEP 563: keep all annotations lazy so PEP 604 (`X | None`) and PEP 585 +# (`list[str]`) syntax import cleanly on Python 3.7-3.9 (RHEL 9, Debian 11, +# Ubuntu 20.04, Amazon Linux stock interpreters). +from __future__ import annotations + import os import time from pathlib import Path diff --git a/src/tracer/WriterManager.py b/src/tracer/WriterManager.py index 8a06f70..77a95b7 100644 --- a/src/tracer/WriterManager.py +++ b/src/tracer/WriterManager.py @@ -21,6 +21,11 @@ writer.force_flush() # Flush all buffers on shutdown """ +# PEP 563: keep all annotations lazy so PEP 604 (`X | None`) and PEP 585 +# (`list[str]`) syntax import cleanly on Python 3.7-3.9 (RHEL 9, Debian 11, +# Ubuntu 20.04, Amazon Linux stock interpreters). +from __future__ import annotations + import os import sys import json diff --git a/src/tracer/prober/prober.c b/src/tracer/prober/prober.c index 1c11cac..a22c255 100644 --- a/src/tracer/prober/prober.c +++ b/src/tracer/prober/prober.c @@ -41,23 +41,33 @@ * incomplete type". The program never instantiates these structs, so an empty * placeholder is enough to complete the type for the compiler. * - * We can only declare the ones the headers leave incomplete: defining a - * placeholder for a struct the headers already define fully is a redefinition - * error, and there is no preprocessor test for "is this type complete?". - * Hence the per-version guards below. + * Whether each struct is already complete depends on BOTH the kernel version + * AND the installed BCC release: BCC force-includes its own vendored uapi + * snapshot (virtual_bpf.h) BEFORE this file, and that snapshot fully defines + * e.g. struct bpf_timer since bcc 0.23. A LINUX_VERSION_CODE guard therefore + * cannot be right for every combination — `struct bpf_timer {};` under + * `< 5.17` was a hard "redefinition" compile error on stock Ubuntu 22.04 + * (kernel 5.15 + bcc 0.24), and there is no preprocessor test for "is this + * type complete?". + * + * Instead, RENAME the struct tags from this point on: the vendored uapi + * definitions processed before this file keep their real names, while every + * later reference (the kernel headers' btf_field_type_size() and friends) is + * rewritten to a private placeholder tag that we define completely. This is + * correct on every bcc/kernel pair regardless of which side defines the real + * struct, because the renamed tag is ours alone. * * MAINTENANCE: if a future kernel fails to compile with * "invalid application of 'sizeof' to an incomplete type 'struct bpf_'" - * add `struct bpf_ {};` here under the matching version guard. The - * authoritative list lives in btf_field_type_size() in . + * add a rename + placeholder pair for bpf_ here. The authoritative list + * lives in btf_field_type_size() in . */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0) -struct bpf_timer {}; /* bpf_timer field, forward-declared from 5.17 on */ -#endif -#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0) -struct bpf_wq {}; /* workqueue field, added in 6.14 */ -struct bpf_task_work {}; /* task_work field, added in 6.14 */ -#endif +#define bpf_timer __iotracer_ph_bpf_timer +#define bpf_wq __iotracer_ph_bpf_wq +#define bpf_task_work __iotracer_ph_bpf_task_work +struct __iotracer_ph_bpf_timer {}; +struct __iotracer_ph_bpf_wq {}; +struct __iotracer_ph_bpf_task_work {}; /* BPF atomic load/store instructions - fallback definitions */ #ifndef BPF_LOAD_ACQ @@ -91,6 +101,7 @@ struct bpf_task_work {}; /* task_work field, added in 6.14 */ #include /* IPv4 socket address structures */ #include /* IPv6 socket address structures */ #include /* Memory management (page, vm_area_struct) */ +#include /* Page cache (readahead_control, address_space) */ #include /* Process/task structures */ #include /* File mode/permission macros (S_ISREG, etc.) */ #include /* TCP protocol structures */ @@ -111,6 +122,45 @@ struct bpf_task_work {}; /* task_work field, added in 6.14 */ #endif #endif +/* ---------------------------------------------------------------------------- + * bpf_probe_read_kernel/user compatibility (kernel < 5.5) + * ---------------------------------------------------------------------------- + * The split probe_read helpers only exist from kernel 5.5; on older kernels + * the generic bpf_probe_read/bpf_probe_read_str ARE the correct helpers and + * every BCC release declares them. BCC >= 0.15 self-heals on such kernels by + * injecting its own object-like `#define bpf_probe_read_kernel bpf_probe_read` + * into the prologue (which makes the #ifndef below skip ours); bcc 0.12-0.14 + * (stock Ubuntu 20.04) declare the new names but emit helper ids a < 5.5 + * verifier rejects at load ("invalid func unknown#113"), so these + * function-like macros rewrite the calls to the legacy helpers instead. + * The kernel-version guard matters: on >= 5.5 kernels the legacy + * bpf_probe_read may not even exist (s390x/riscv64 omit it), and the + * kernel/user distinction must be preserved there. + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#ifndef bpf_probe_read_kernel +#define bpf_probe_read_kernel(dst, sz, src) bpf_probe_read(dst, sz, src) +#endif +#ifndef bpf_probe_read_kernel_str +#define bpf_probe_read_kernel_str(dst, sz, src) bpf_probe_read_str(dst, sz, src) +#endif +#ifndef bpf_probe_read_user +#define bpf_probe_read_user(dst, sz, src) bpf_probe_read(dst, sz, src) +#endif +#ifndef bpf_probe_read_user_str +#define bpf_probe_read_user_str(dst, sz, src) bpf_probe_read_str(dst, sz, src) +#endif +#endif /* LINUX_VERSION_CODE < 5.5 */ + +/* bpf_get_current_cgroup_id() exists from kernel 4.18 (commit bf6fa2c893c5); + * referencing it on older kernels rejects every program that calls it at + * load time. 0 is the "unknown cgroup" sentinel userspace already accepts. */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) +#define iotracer_get_cgroup_id() bpf_get_current_cgroup_id() +#else +#define iotracer_get_cgroup_id() 0 +#endif + /* ============================================================================ * CONSTANTS AND CONFIGURATION * ============================================================================ @@ -119,6 +169,14 @@ struct bpf_task_work {}; /* task_work field, added in 6.14 */ /** Maximum length for captured filenames (including null terminator) */ #define FILENAME_MAX_LEN 256 +/* The BPF backend inlines constant memset/memcpy only up to 1024 bytes; above + * that, clang fails with "A call to built-in function 'memset' is not + * supported" on every LLVM version. Several buffers of this size are + * memset/memcpy'd as a unit, so keep the constant under the cliff. */ +_Static_assert(FILENAME_MAX_LEN <= 1024, + "FILENAME_MAX_LEN must stay <= 1024: larger constant " + "memset/memcpy cannot be inlined by the BPF backend"); + /** openat() dirfd sentinel meaning "resolve relative to the cwd". Defined here * in case the BPF include set doesn't pull in . */ #ifndef AT_FDCWD @@ -272,21 +330,14 @@ struct data_dual_t { u64 latency_ns; /**< Operation latency */ }; -/** - * @brief Kernel renamedata structure (kernel 5.12+) - * - * Used by vfs_rename() in modern kernels. We define a minimal version - * to extract the dentry pointers we need. - */ -struct renamedata_bpf { - void *old_mnt_idmap; - struct inode *old_dir; - struct dentry *old_dentry; - void *new_mnt_idmap; - struct inode *new_dir; - struct dentry *new_dentry; - /* remaining fields not needed */ -}; +/* NOTE: vfs_rename's renamedata argument is read via the REAL struct + * renamedata from (see trace_vfs_rename). A hand-rolled mirror + * of its layout used to live here and silently broke whenever the kernel + * reshuffled the struct — most recently in 6.18, which merged the two + * mnt_idmap fields and moved new_dentry from offset 40 to 32, so the stale + * offset read landed on delegated_inode and emitted garbage rename events. + * BCC recompiles against the running kernel's headers, so using the real + * type keeps the offsets correct on every kernel automatically. */ /** * @brief Staging struct for sys_mremap arguments @@ -322,9 +373,10 @@ struct block_event { u32 dev; /**< Device number (major:minor encoded) for partition ID */ u64 queue_time_ns; /**< Time from insert to issue (scheduler latency) */ u32 op_code; /**< Raw block operation code (REQ_OP_*) */ - u64 req_id; /**< Monotonic per-request id, unique within a trace - * session. Disambiguates distinct I/Os that reuse - * the same (dev, sector). */ + u64 req_id; /**< Per-request id, unique within a trace session + * (CPU id in the high 16 bits, per-CPU sequence + * in the low 48). Disambiguates distinct I/Os + * that reuse the same (dev, sector). */ }; /* ============================================================================ @@ -636,7 +688,7 @@ struct vfs_info { */ struct block_issue_ctx { u64 ts; /**< Issue timestamp (device latency baseline) */ - u64 req_id; /**< Monotonic per-request id (see block_event.req_id) */ + u64 req_id; /**< Unique per-request id (see block_event.req_id) */ u32 pid; /**< Submitting process ID */ u32 tid; /**< Submitting thread ID */ u32 ppid; /**< Submitter's parent PID */ @@ -676,10 +728,15 @@ struct block_rq_key_t { BPF_TABLE("lru_hash", struct block_rq_key_t, struct block_issue_ctx, block_start_times, 65536); /**< Issue time + submitter, keyed by dev+sector */ BPF_TABLE("lru_hash", struct block_rq_key_t, u64, block_insert_times, 65536); /**< Tracks block request insert time (queue latency) */ -/* Monotonic generator for per-request IDs (see block_event.req_id). A single - * u64 bumped atomically at issue time; lets consumers disambiguate repeated - * I/O to the same (dev, sector) and correlate a request across its lifecycle. */ -BPF_ARRAY(block_req_id_gen, u64, 1); +/* Per-request ID generator (see block_event.req_id). A per-CPU u64 counter + * bumped at issue time; the emitted req_id folds the CPU id into the high bits + * so ids stay unique across CPUs without an atomic fetch-and-add. A global + * atomic counter is not portable here: the BPF XADD instruction can't return + * its old value on all kernels/LLVM ("Invalid usage of the XADD return value"), + * and a read-then-lock_xadd would race and hand out duplicate ids. Lets + * consumers disambiguate repeated I/O to the same (dev, sector) and correlate a + * request across its lifecycle. */ +BPF_PERCPU_ARRAY(block_req_id_gen, u64, 1); /* Block-tracing diagnostics counters (per-CPU, summed in userspace at exit): * [0] requests issued, [1] requests completed (emitted), @@ -1126,6 +1183,16 @@ BPF_PERCPU_ARRAY(dpath_scratch_map, struct dpath_scratch, 1); static __always_inline void build_dentry_path(struct dentry *dentry, char *buf, int buf_size) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0) + /* Pre-5.3 verifiers have no bounded-loop support: if the compiler leaves + * either loop below rolled (LLVM-version dependent — a real deployment has + * logged "loop not unrolled" here), the back-edge rejects the WHOLE + * program; and when the loops do unroll, the ~thousands of resulting + * instructions can breach the pre-5.2 4096-insn cap. Only the readdir + * probe reaches this path, so degrade it to basename-only on old kernels + * rather than risk the entire tracer. */ + get_file_path_from_dentry(dentry, buf, buf_size); +#else buf[0] = '\0'; if (!dentry) return; /* The assembly buffer is masked into a FILENAME_MAX_LEN window and copied out @@ -1181,6 +1248,7 @@ static __always_inline void build_dentry_path(struct dentry *dentry, * buffer; out[off] is already NUL-terminated within that window. */ __builtin_memcpy(buf, out, FILENAME_MAX_LEN); buf[FILENAME_MAX_LEN - 1] = '\0'; +#endif /* LINUX_VERSION_CODE >= 5.3 */ } /** @@ -1276,7 +1344,7 @@ int trace_vfs_read(struct pt_regs *ctx, struct file *file, char __user *buf, // Provenance metadata: parent PID, container (cgroup) id, backing // device, and filesystem magic for source classification. data.ppid = get_ppid(); - data.cgroup_id = bpf_get_current_cgroup_id(); + data.cgroup_id = iotracer_get_cgroup_id(); get_file_source(file, &data.dev, &data.fs_magic); // Defer submission to the kretprobe, which records the return value @@ -1357,7 +1425,7 @@ int trace_vfs_write(struct pt_regs *ctx, struct file *file, // Provenance metadata: parent PID, container (cgroup) id, backing // device, and filesystem magic for source classification. data.ppid = get_ppid(); - data.cgroup_id = bpf_get_current_cgroup_id(); + data.cgroup_id = iotracer_get_cgroup_id(); get_file_source(file, &data.dev, &data.fs_magic); // Defer submission to the kretprobe, which records the return value @@ -1521,7 +1589,7 @@ int trace_vfs_open(struct pt_regs *ctx, const struct path *path, // Provenance metadata: parent PID, container (cgroup) id, backing // device, and filesystem magic for source classification. data.ppid = get_ppid(); - data.cgroup_id = bpf_get_current_cgroup_id(); + data.cgroup_id = iotracer_get_cgroup_id(); get_file_source(file, &data.dev, &data.fs_magic); // Prefer the full path captured from the syscall entry (trace_do_sys_openat2_entry). @@ -1965,6 +2033,45 @@ int trace_mremap_entry_x64(struct pt_regs *ctx) { } #endif /* __x86_64__ */ +#if defined(__aarch64__) || defined(bpf_target_arm64) +/** + * @brief kprobe entry for the __arm64_sys_mremap syscall wrapper. + * + * arm64 has CONFIG_ARCH_HAS_SYSCALL_WRAPPER since 4.19: __arm64_sys_* + * functions receive a single struct pt_regs * holding the user registers; + * the syscall arguments live in regs[0..3] (x0-x3 per SC_ARM64_REGS_TO_ARGS), + * mirroring the x86-64 variant above. Without this variant (and its Python + * attach branch) arm64 silently lost all MREMAP events. + * + * @param ctx BPF context (PARM1 = user pt_regs) + * @return 0 + */ +int trace_mremap_entry_arm64(struct pt_regs *ctx) { + u64 pid_tgid = bpf_get_current_pid_tgid(); + u32 pid = pid_tgid >> 32; + + u32 config_key = 0; + u32 *tracer_pid = tracer_config.lookup(&config_key); + if (tracer_pid && pid == *tracer_pid) { + return 0; + } + + struct pt_regs *uregs = (struct pt_regs *)PT_REGS_PARM1(ctx); + if (!uregs) { + return 0; + } + + struct mremap_args args = {}; + bpf_probe_read_kernel(&args.old_addr, sizeof(args.old_addr), &uregs->regs[0]); + bpf_probe_read_kernel(&args.old_len, sizeof(args.old_len), &uregs->regs[1]); + bpf_probe_read_kernel(&args.new_len, sizeof(args.new_len), &uregs->regs[2]); + bpf_probe_read_kernel(&args.flags, sizeof(args.flags), &uregs->regs[3]); + + mremap_staging.update(&pid_tgid, &args); + return 0; +} +#endif /* __aarch64__ */ + /** * @brief kretprobe return for sys_mremap - emit event with old + new addresses * @@ -2399,17 +2506,14 @@ int trace_ksys_sync(struct pt_regs *ctx) { */ /** - * @brief Trace vfs_rename() - File/directory rename + * @brief Shared body for trace_vfs_rename (both signature variants below). * * Captures rename()/renameat() operations with source and destination paths. * Uses per-CPU buffer for the 572-byte data_dual_t structure. - * Kernel 6.x signature: vfs_rename(struct renamedata *rd) - * - * @param ctx BPF context - * @param rd Rename data structure containing old/new dentry info - * @return 0 */ -int trace_vfs_rename(struct pt_regs *ctx, struct renamedata_bpf *rd) { +static __always_inline int emit_vfs_rename(struct pt_regs *ctx, + struct dentry *old_dentry, + struct dentry *new_dentry) { u64 pid_tgid = bpf_get_current_pid_tgid(); u32 pid = pid_tgid >> 32; @@ -2419,16 +2523,6 @@ int trace_vfs_rename(struct pt_regs *ctx, struct renamedata_bpf *rd) { return 0; } - if (!rd) { - return 0; - } - - // Read dentry pointers from renamedata struct - struct dentry *old_dentry = NULL; - struct dentry *new_dentry = NULL; - bpf_probe_read_kernel(&old_dentry, sizeof(old_dentry), &rd->old_dentry); - bpf_probe_read_kernel(&new_dentry, sizeof(new_dentry), &rd->new_dentry); - if (!old_dentry || !new_dentry) { return 0; } @@ -2439,11 +2533,11 @@ int trace_vfs_rename(struct pt_regs *ctx, struct renamedata_bpf *rd) { if (!data) { return 0; } - + // Zero-initialize filename buffers to avoid stale data __builtin_memset(data->filename_old, 0, FILENAME_MAX_LEN); __builtin_memset(data->filename_new, 0, FILENAME_MAX_LEN); - + data->pid = pid; data->ts = bpf_ktime_get_ns(); bpf_get_current_comm(&data->comm, sizeof(data->comm)); @@ -2464,6 +2558,50 @@ int trace_vfs_rename(struct pt_regs *ctx, struct renamedata_bpf *rd) { return 0; } +/** + * @brief Trace vfs_rename() - File/directory rename + * + * Kernel >= 5.12 signature: vfs_rename(struct renamedata *rd). The dentries + * are read through the REAL struct renamedata from , so BCC + * computes the field offsets from the running kernel's headers — correct + * across the 6.3 user_namespace->mnt_idmap swap AND the 6.18 relayout that + * broke the previous fixed-offset mirror (see the note above mremap_args). + * The old_dentry/new_dentry field names are stable across 5.12..6.18+. + * + * @param ctx BPF context + * @param rd Rename data structure containing old/new dentry info + * @return 0 + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0) +int trace_vfs_rename(struct pt_regs *ctx, struct renamedata *rd) { + if (!rd) { + return 0; + } + + // Read dentry pointers from the kernel's own renamedata definition + struct dentry *old_dentry = NULL; + struct dentry *new_dentry = NULL; + bpf_probe_read_kernel(&old_dentry, sizeof(old_dentry), &rd->old_dentry); + bpf_probe_read_kernel(&new_dentry, sizeof(new_dentry), &rd->new_dentry); + + return emit_vfs_rename(ctx, old_dentry, new_dentry); +} +#else +/** + * Kernel < 5.12 signature: + * vfs_rename(struct inode *old_dir, struct dentry *old_dentry, + * struct inode *new_dir, struct dentry *new_dentry, + * struct inode **delegated_inode, unsigned int flags) + * The previous unguarded renamedata-typed handler read inode internals as + * dentry pointers on these kernels and emitted garbage rename events. + */ +int trace_vfs_rename(struct pt_regs *ctx, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry) { + return emit_vfs_rename(ctx, old_dentry, new_dentry); +} +#endif + /** * @brief Trace vfs_mkdir() - Directory creation * @@ -2881,11 +3019,19 @@ TRACEPOINT_PROBE(block, block_rq_issue) { ictx.ppid = get_ppid(); bpf_get_current_comm(&ictx.comm, sizeof(ictx.comm)); - // Assign a monotonic per-request id, carried to the completion event. + // Assign a unique per-request id, carried to the completion event. The + // generator is a per-CPU counter — BPF programs run with preemption disabled, + // so the read-increment-store needs no atomic — and the CPU id is folded into + // the high 16 bits to keep ids unique across CPUs. Using __sync_fetch_and_add + // on a global counter is not portable: its return value compiles to a BPF + // XADD that can't return its old value on all kernels/LLVM. u32 gen_key = 0; u64 *gen = block_req_id_gen.lookup(&gen_key); - if (gen) - ictx.req_id = __sync_fetch_and_add(gen, 1); + if (gen) { + u64 seq = *gen; + *gen = seq + 1; + ictx.req_id = ((u64)(bpf_get_smp_processor_id() & 0xFFFF) << 48) | (seq & 0xFFFFFFFFFFFFULL); + } block_start_times.update(&key, &ictx); @@ -3030,7 +3176,7 @@ TRACEPOINT_PROBE(block, block_rq_complete) { * folio_mark_accessed() is called when a cached page is accessed. * Indicates data was served from cache without disk I/O. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) int trace_folio_mark_accessed(struct pt_regs *ctx, struct folio *folio) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3115,7 +3261,7 @@ int trace_hit(struct pt_regs *ctx, struct page *page) { * filemap_add_folio() adds a new page to cache after disk read. * This indicates a cache miss that required actual disk I/O. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) int trace_filemap_add_folio(struct pt_regs *ctx, struct address_space *mapping, struct folio *folio, pgoff_t index, gfp_t gfp) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3226,11 +3372,11 @@ int trace_account_page_dirtied(struct pt_regs *ctx, struct page *page, #endif /** - * @brief Dirty page probe - folio version (kernel >= 5.17) + * @brief Dirty page probe - folio version (kernel >= 5.16) * * folio_mark_dirty() marks a folio as modified in newer kernels. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) int trace_folio_mark_dirty(struct pt_regs *ctx, struct folio *folio) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3308,11 +3454,11 @@ int trace_clear_page_dirty_for_io(struct pt_regs *ctx, struct page *page) { #endif /** - * @brief Writeback start probe - folio version (kernel >= 5.17) + * @brief Writeback start probe - folio version (kernel >= 5.16) * * folio_clear_dirty_for_io() starts writeback in newer kernels. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) int trace_folio_clear_dirty_for_io(struct pt_regs *ctx, struct folio *folio) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3390,11 +3536,11 @@ int trace_test_clear_page_writeback(struct pt_regs *ctx, struct page *page) { #endif /** - * @brief Writeback completion probe - folio version (kernel >= 5.17) + * @brief Writeback completion probe - folio version (kernel >= 5.16) * * folio_end_writeback() signals writeback completion. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) int trace_folio_end_writeback(struct pt_regs *ctx, struct folio *folio) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3681,13 +3827,17 @@ int trace_truncate_pages(struct pt_regs *ctx, struct address_space *mapping, } /** - * @brief Cache drop probe - folio version (kernel >= 5.18) + * @brief Cache drop probe - folio version (kernel >= 5.17) * - * Captures explicit cache drops (e.g., POSIX_FADV_DONTNEED). + * Captures explicit cache drops (e.g., POSIX_FADV_DONTNEED). Attached to + * __filemap_remove_folio(folio, shadow) — the folio is the FIRST argument + * and the mapping is read from folio->mapping. (The previous prototype, + * (mapping, folio), matched no kernel: it misread the folio pointer as the + * mapping and the shadow pointer as the folio, emitting garbage drop rows.) */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) -int trace_cache_drop_folio(struct pt_regs *ctx, struct address_space *mapping, - struct folio *folio) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +int trace_cache_drop_folio(struct pt_regs *ctx, struct folio *folio, + void *shadow) { u32 pid = bpf_get_current_pid_tgid() >> 32; u32 config_key = 0; @@ -3703,21 +3853,23 @@ int trace_cache_drop_folio(struct pt_regs *ctx, struct address_space *mapping, if (folio) { bpf_probe_read_kernel(&data.index, sizeof(data.index), &folio->index); - + // Get LRU type from folio/page flags unsigned long flags = 0; struct page *p = (struct page *)folio; bpf_probe_read_kernel(&flags, sizeof(flags), &p->flags); if (flags != 0) { } - } - if (mapping) { - struct inode *host = NULL; - bpf_probe_read_kernel(&host, sizeof(host), &mapping->host); - if (host) { - bpf_probe_read_kernel(&data.inode, sizeof(data.inode), &host->i_ino); - populate_cache_metadata(&data, host); + struct address_space *mapping = NULL; + bpf_probe_read_kernel(&mapping, sizeof(mapping), &folio->mapping); + if (mapping) { + struct inode *host = NULL; + bpf_probe_read_kernel(&host, sizeof(host), &mapping->host); + if (host) { + bpf_probe_read_kernel(&data.inode, sizeof(data.inode), &host->i_ino); + populate_cache_metadata(&data, host); + } } } @@ -3777,12 +3929,18 @@ int trace_cache_drop_page(struct pt_regs *ctx, struct page *page) { #endif /** - * @brief Cache readahead probe - prefetch tracking + * @brief Cache readahead probe - legacy version (kernel < 5.10) * * Captures kernel readahead (prefetch) operations that speculatively * load pages into cache. count field contains pages being prefetched. + * Matches __do_page_cache_readahead(mapping, file, offset, nr_to_read, + * lookahead_size); kernel 5.10 replaced it with do_page_cache_ra() taking + * a readahead_control (see trace_page_cache_ra below). The old >= 5.17 + * guard on this prototype matched no kernel at all: on >= 5.10 the first + * argument is the readahead_control, so reading it as the mapping emitted + * garbage readahead rows, and on < 5.17 the handler didn't compile. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) int trace_do_page_cache_readahead(struct pt_regs *ctx, struct address_space *mapping, struct file *file, pgoff_t index, unsigned long nr_pages) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3815,14 +3973,113 @@ int trace_do_page_cache_readahead(struct pt_regs *ctx, struct address_space *map } #endif +/** + * @brief Cache readahead probe - readahead_control version (kernel >= 5.10) + * + * do_page_cache_ra(ractl, nr_to_read, lookahead_size) carries the mapping + * and the start index inside the readahead_control. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +int trace_page_cache_ra(struct pt_regs *ctx, struct readahead_control *ractl, + unsigned long nr_to_read) { + u32 pid = bpf_get_current_pid_tgid() >> 32; + + u32 config_key = 0; + u32 *tracer_pid = tracer_config.lookup(&config_key); + if (tracer_pid && pid == *tracer_pid) + return 0; + + struct cache_data data = {}; + data.ts = bpf_ktime_get_ns(); + data.pid = pid; + data.type = CACHE_READAHEAD; + data.count = (u32)nr_to_read; // Number of pages in readahead window + bpf_get_current_comm(&data.comm, sizeof(data.comm)); + + if (ractl) { + struct address_space *mapping = NULL; + bpf_probe_read_kernel(&mapping, sizeof(mapping), &ractl->mapping); + // Set index before calling populate_cache_metadata + bpf_probe_read_kernel(&data.index, sizeof(data.index), &ractl->_index); + if (mapping) { + struct inode *host = NULL; + bpf_probe_read_kernel(&host, sizeof(host), &mapping->host); + if (host) { + bpf_probe_read_kernel(&data.inode, sizeof(data.inode), &host->i_ino); + populate_cache_metadata(&data, host); + } + } + } + + data.cpu_id = bpf_get_smp_processor_id(); + cache_events.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +#endif + +/** + * @brief Cache readahead probe - page_cache_ra_order fallback (kernel >= 5.18) + * + * page_cache_ra_order(ractl, ra, new_order) has no nr_to_read argument; the + * requested window size lives in ra->size. Only attached when + * do_page_cache_ra is unavailable (e.g. inlined) on a given kernel. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +int trace_page_cache_ra_order(struct pt_regs *ctx, struct readahead_control *ractl, + struct file_ra_state *ra) { + u32 pid = bpf_get_current_pid_tgid() >> 32; + + u32 config_key = 0; + u32 *tracer_pid = tracer_config.lookup(&config_key); + if (tracer_pid && pid == *tracer_pid) + return 0; + + struct cache_data data = {}; + data.ts = bpf_ktime_get_ns(); + data.pid = pid; + data.type = CACHE_READAHEAD; + bpf_get_current_comm(&data.comm, sizeof(data.comm)); + + if (ra) { + unsigned int ra_size = 0; + bpf_probe_read_kernel(&ra_size, sizeof(ra_size), &ra->size); + data.count = ra_size; // Requested readahead window, in pages + } + + if (ractl) { + struct address_space *mapping = NULL; + bpf_probe_read_kernel(&mapping, sizeof(mapping), &ractl->mapping); + // Set index before calling populate_cache_metadata + bpf_probe_read_kernel(&data.index, sizeof(data.index), &ractl->_index); + if (mapping) { + struct inode *host = NULL; + bpf_probe_read_kernel(&host, sizeof(host), &mapping->host); + if (host) { + bpf_probe_read_kernel(&data.inode, sizeof(data.inode), &host->i_ino); + populate_cache_metadata(&data, host); + } + } + } + + data.cpu_id = bpf_get_smp_processor_id(); + cache_events.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +#endif + /** * @brief Cache reclaim probe - memory pressure tracking * - * shrink_folio_list() is called during memory reclaim. + * shrink_folio_list() (shrink_page_list() before the folio conversion) is + * called during memory reclaim. * kswapd = background reclaim, other processes = direct reclaim. * Direct reclaim indicates memory pressure affecting performance. + * + * The handler reads no function arguments, so it works attached to either + * symbol on any kernel — no version guard needed (the old >= 5.17 guard + * compiled it out on kernels where shrink_page_list exists, silently + * disabling reclaim tracing there). */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) int trace_shrink_folio_list(struct pt_regs *ctx) { u32 pid = bpf_get_current_pid_tgid() >> 32; @@ -3852,7 +4109,6 @@ int trace_shrink_folio_list(struct pt_regs *ctx) { cache_events.perf_submit(ctx, &data, sizeof(data)); return 0; } -#endif /* ============================================================================ * DIRECT I/O TRACING @@ -4161,6 +4417,33 @@ int trace_io_uring_enter_x64(struct pt_regs *ctx) { } #endif /* __x86_64__ */ +#if defined(__aarch64__) || defined(bpf_target_arm64) +/** + * @brief Kprobe entry for the __arm64_sys_io_uring_enter syscall wrapper. + * + * Unwraps the user pt_regs (see trace_mremap_entry_arm64): fd/to_submit/ + * min_complete/flags live in regs[0..3] per the arm64 syscall ABI. + * + * @param ctx BPF context (PARM1 = user pt_regs) + * @return 0 + */ +int trace_io_uring_enter_arm64(struct pt_regs *ctx) { + struct pt_regs *uregs = (struct pt_regs *)PT_REGS_PARM1(ctx); + if (!uregs) { + return 0; + } + + unsigned long fd = 0, to_submit = 0, min_complete = 0, flags = 0; + bpf_probe_read_kernel(&fd, sizeof(fd), &uregs->regs[0]); + bpf_probe_read_kernel(&to_submit, sizeof(to_submit), &uregs->regs[1]); + bpf_probe_read_kernel(&min_complete, sizeof(min_complete), &uregs->regs[2]); + bpf_probe_read_kernel(&flags, sizeof(flags), &uregs->regs[3]); + + return emit_io_uring_enter(ctx, (u32)fd, (u32)to_submit, (u32)min_complete, + (u32)flags); +} +#endif /* __aarch64__ */ + /** * @brief ABI-stable subset of the io_uring SQE (uapi/linux/io_uring.h). * @@ -4998,7 +5281,13 @@ TRACEPOINT_PROBE(tcp, tcp_retransmit_skb) { bpf_probe_read_kernel(&e.saddr_v6, sizeof(e.saddr_v6), args->saddr_v6); bpf_probe_read_kernel(&e.daddr_v6, sizeof(e.daddr_v6), args->daddr_v6); + /* The 'state' field was added to this tracepoint in kernel 4.20 (never + * backported to RHEL 8's 4.18). Gated on the loader's format-file sniff for + * the same reason as HAS_SKB_DROP_REASON above; without it, state stays 0 + * (not a valid TCP state, reads as "unknown"). */ +#ifdef HAS_TCP_RETRANSMIT_STATE e.state = args->state; +#endif e.ipver = (e.saddr_v4 != 0 || e.daddr_v4 != 0) ? 4 : 6; net_drop_events.perf_submit(args, &e, sizeof(e)); @@ -5025,8 +5314,15 @@ TRACEPOINT_PROBE(skb, kfree_skb) { eth_proto = bpf_ntohs(eth_proto); if (eth_proto != 0x0800 && eth_proto != 0x86dd) return 0; /* not IPv4/IPv6 */ - /* Read drop reason (kernel 5.17+ has this field) */ + /* Read drop reason. The 'reason' field only exists where the running + * kernel's tracepoint format has it (mainline 5.17+, backported to 5.15.58+ + * LTS). Referencing a missing args-> field is a compile error that aborts + * the whole load, so the loader sniffs the format file and defines + * HAS_SKB_DROP_REASON — the gate can never disagree with the args struct. + * Without it, drop_reason stays 0 (SKB_DROP_REASON_NOT_SPECIFIED). */ +#ifdef HAS_SKB_DROP_REASON bpf_probe_read_kernel(&e.drop_reason, sizeof(e.drop_reason), &args->reason); +#endif /* Read packet length */ bpf_probe_read_kernel(&e.skb_len, sizeof(e.skb_len), &skb->len); diff --git a/src/tracer/snappers/FilesystemSnapper.py b/src/tracer/snappers/FilesystemSnapper.py index 780293e..dec9ca8 100644 --- a/src/tracer/snappers/FilesystemSnapper.py +++ b/src/tracer/snappers/FilesystemSnapper.py @@ -164,22 +164,27 @@ def get_birth_time(path: str, fallback: float) -> float: class _Statfs(ctypes.Structure): - # Layout of glibc `struct statfs` on 64-bit Linux. Only f_type (the first - # word, carrying the superblock magic) is read; the rest is declared so the - # buffer is correctly sized for the syscall to write into. + # Layout of glibc `struct statfs`. The __fsword_t fields are `long` and + # the count fields `unsigned long` in glibc, so declaring them as + # c_long/c_ulong matches the ABI on BOTH 32- and 64-bit userlands + # (hard-coding c_int64 misread f_type on 32-bit, notably big-endian ones). + # Only f_type (the superblock magic) is read; the rest is declared so the + # buffer is correctly sized for the syscall to write into. The trailing + # pad keeps the buffer comfortably larger than any glibc variant. _fields_ = [ - ("f_type", ctypes.c_int64), - ("f_bsize", ctypes.c_int64), - ("f_blocks", ctypes.c_uint64), - ("f_bfree", ctypes.c_uint64), - ("f_bavail", ctypes.c_uint64), - ("f_files", ctypes.c_uint64), - ("f_ffree", ctypes.c_uint64), + ("f_type", ctypes.c_long), + ("f_bsize", ctypes.c_long), + ("f_blocks", ctypes.c_ulong), + ("f_bfree", ctypes.c_ulong), + ("f_bavail", ctypes.c_ulong), + ("f_files", ctypes.c_ulong), + ("f_ffree", ctypes.c_ulong), ("f_fsid", ctypes.c_int32 * 2), - ("f_namelen", ctypes.c_int64), - ("f_frsize", ctypes.c_int64), - ("f_flags", ctypes.c_int64), - ("f_spare", ctypes.c_int64 * 4), + ("f_namelen", ctypes.c_long), + ("f_frsize", ctypes.c_long), + ("f_flags", ctypes.c_long), + ("f_spare", ctypes.c_long * 4), + ("_pad", ctypes.c_byte * 64), ] diff --git a/src/tracer/snappers/SystemSnapper.py b/src/tracer/snappers/SystemSnapper.py index 4c12e4f..9dca277 100644 --- a/src/tracer/snappers/SystemSnapper.py +++ b/src/tracer/snappers/SystemSnapper.py @@ -23,6 +23,11 @@ snapper.capture_spec_snapshot() # Capture and write specs """ +# PEP 563: keep all annotations lazy so PEP 604 (`X | None`) and PEP 585 +# (`list[str]`) syntax import cleanly on Python 3.7-3.9 (RHEL 9, Debian 11, +# Ubuntu 20.04, Amazon Linux stock interpreters). +from __future__ import annotations + from ..WriterManager import WriteManager from ...utility.utils import logger import subprocess @@ -619,6 +624,9 @@ def get_kernel_headers_info(self) -> dict: "modules_build_symlink_target": build_target, "usr_src_headers_path": headers_dir, "usr_src_headers_present": os.path.exists(headers_dir), + # CONFIG_IKHEADERS: BCC can extract headers embedded in the kernel + # when no headers package is installed (common on WSL2/cloud). + "kheaders_present": os.path.exists("/sys/kernel/kheaders.tar.xz"), } def get_tracefs_info(self) -> dict: @@ -630,17 +638,27 @@ def get_tracefs_info(self) -> dict: """ debug_tracing = "/sys/kernel/debug/tracing" tracefs = "/sys/kernel/tracing" - tp_format = f"{debug_tracing}/events/block/block_rq_complete/format" + # Check both mounts — modern systems increasingly expose only tracefs + # at /sys/kernel/tracing (mirrors IOTracer._init_bpf's sniffing). has_cmd_flags = None - if os.path.exists(tp_format): - fmt = self._read_text_file(tp_format) - has_cmd_flags = ("cmd_flags" in fmt) if fmt is not None else None + for base in (debug_tracing, tracefs): + tp_format = f"{base}/events/block/block_rq_complete/format" + if os.path.exists(tp_format): + fmt = self._read_text_file(tp_format) + has_cmd_flags = ("cmd_flags" in fmt) if fmt is not None else None + break return { "debugfs_tracing_mounted": os.path.isdir(debug_tracing), "tracefs_mounted": os.path.isdir(tracefs), "block_rq_complete_has_cmd_flags": has_cmd_flags, } + def get_lockdown_info(self) -> dict: + """Kernel lockdown state — Secure Boot lockdown blocks kprobes without + ever appearing on the boot cmdline, so read the runtime state too.""" + state = self._read_text_file("/sys/kernel/security/lockdown") + return {"lockdown": state.strip() if state else None} + def get_cpu_info(self) -> dict: """CPU brand and core counts (no frequency probe — kept dependency-light).""" return { @@ -665,6 +683,7 @@ def collect_diagnostics( attempted_cflags: list[str] | None = None, bpf_file: str | None = None, context: str | None = None, + include_toolchain_probes: bool = True, ) -> dict: """ Gather as much OS / kernel / toolchain context as possible. @@ -716,9 +735,17 @@ def collect_diagnostics( "kernel": self._safe(self.get_kernel_info), "btf": self._safe(self.get_btf_info), "kernel_config": self._safe(self.get_kernel_config), - "toolchain": self._safe(self.get_toolchain_info), + # The toolchain probe shells out to clang/llc/gcc/ld --version + # (2s timeout each); callers on the happy path (the pre-compile + # breadcrumb, written on EVERY startup) skip it so a hung/slow + # toolchain can't add seconds to tracer startup. The full + # failure-path dump always includes it. + "toolchain": (self._safe(self.get_toolchain_info) + if include_toolchain_probes + else {"skipped": "pre-compile breadcrumb omits subprocess probes"}), "kernel_headers": self._safe(self.get_kernel_headers_info), "tracefs": self._safe(self.get_tracefs_info), + "lockdown": self._safe(self.get_lockdown_info), } diagnostics["system"] = { @@ -782,8 +809,58 @@ def dump_failure_diagnostics( continue self._print_diagnostics_summary(diagnostics, path) + self._print_remediation_hints(diagnostics) return path + @staticmethod + def _print_remediation_hints(diagnostics: dict) -> None: + """Print actionable next steps when the collected diagnostics can + distinguish the common failure causes. Never raises.""" + try: + def _get(d, *keys): + for k in keys: + if not isinstance(d, dict): + return None + d = d.get(k) + return d + + hints = [] + release = _get(diagnostics, "bpf_environment", "kernel", "release") or "$(uname -r)" + headers = _get(diagnostics, "bpf_environment", "kernel_headers") or {} + if headers and not headers.get("modules_build_present") and not headers.get("kheaders_present"): + hints.append( + "Matching kernel headers were not found. Install them " + f"(apt: linux-headers-{release} / dnf: kernel-devel-{release}) " + "or boot a kernel with CONFIG_IKHEADERS; BCC cannot compile " + "the tracer without them." + ) + lockdown = _get(diagnostics, "bpf_environment", "lockdown", "lockdown") or "" + if "[integrity]" in lockdown or "[confidentiality]" in lockdown: + hints.append( + "The kernel is in Secure Boot lockdown mode " + f"({lockdown}), which blocks kprobes/eBPF tracing. Disable " + "lockdown or Secure Boot to run the tracer." + ) + bcc_version = _get(diagnostics, "bpf_environment", "toolchain", "bcc_version") + if bcc_version: + try: + major_minor = tuple(int(x) for x in str(bcc_version).split(".")[:2]) + if major_minor < (0, 15): + hints.append( + f"The installed bcc ({bcc_version}) is very old and " + "lacks compatibility shims this tracer relies on; " + "upgrade bcc/python3-bpfcc." + ) + except (TypeError, ValueError): + pass + + if hints: + print("\nPossible fixes for this host:") + for hint in hints: + print(f" * {hint}") + except Exception: # noqa: BLE001 - hints must never mask the real error + pass + @staticmethod def _print_diagnostics_summary(diagnostics: dict, path: str | None) -> None: """Print a short, human-readable highlight of the collected diagnostics.""" diff --git a/src/utility/utils.py b/src/utility/utils.py index 1fdc38d..c1f0246 100644 --- a/src/utility/utils.py +++ b/src/utility/utils.py @@ -18,8 +18,14 @@ hashed = simple_hash("sensitive_data") """ +# PEP 563: keep all annotations lazy so PEP 604 (`X | None`) and PEP 585 +# (`list[str]`) syntax import cleanly on Python 3.7-3.9 (RHEL 9, Debian 11, +# Ubuntu 20.04, Amazon Linux stock interpreters). +from __future__ import annotations + import gzip import itertools +import uuid import shutil import sys import threading @@ -359,20 +365,49 @@ def compress_log(input_file: str): def capture_machine_id() -> str: """ Capture and hash the machine's unique identifier. - - Reads /etc/machine-id and returns a 16-character hash. - This provides a consistent anonymous machine identifier. - + + Tries /etc/machine-id, then /var/lib/dbus/machine-id (hosts without + systemd), treating empty or "uninitialized" content as missing (container + images where systemd never booted ship an empty /etc/machine-id, and + hashing "" would collide every such host onto one ID). As a last resort a + random ID is generated once and persisted so the machine keeps a stable + identity across runs — previously a missing /etc/machine-id crashed the + tracer at startup with an unhandled FileNotFoundError. + Returns: str: 16-character hash of the machine ID - + Example: >>> capture_machine_id() 'a1b2c3d4e5f6g7h8' """ - with open("/etc/machine-id") as f: - machine_id = f.read().strip() - return simple_hash(machine_id, 16) + for candidate in ("/etc/machine-id", "/var/lib/dbus/machine-id"): + try: + with open(candidate) as f: + machine_id = f.read().strip() + except OSError: + continue + if machine_id and machine_id != "uninitialized": + return simple_hash(machine_id, 16) + + # No usable system machine-id: generate one and persist it so repeated + # runs (and upload grouping/reward attribution) keep a stable identity. + state_path = Path("/var/lib/iotracer/machine-id") + try: + machine_id = state_path.read_text().strip() + if machine_id: + return simple_hash(machine_id, 16) + except OSError: + pass + machine_id = uuid.uuid4().hex + try: + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text(machine_id + "\n") + except OSError: + # Unwritable filesystem: fall through with the in-memory ID; the ID + # will differ on the next run, which is still better than crashing. + pass + return simple_hash(machine_id, 16) # Reward code for Prolific submissions REWARD_CODE = "CKXDRTBX"