support RHEL 7 / CentOS 7 (kernel 3.10 with the eBPF backport)

README.md

@@ -5,8 +5,9 @@
 
 The agent gathers metrics related to a node and the containers running on it, and it exposes them in the Prometheus format.
 
-It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 5.1.
+It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 4.16.
 The kernel must also be built with `CONFIG_BPF_EVENTS=y` (kprobe and tracepoint BPF programs); some embedded and vendor kernels disable it.
+RHEL 7 and CentOS 7 (kernel 3.10 with the eBPF backport) are supported starting from 7.6 with reduced functionality, see [RHEL 7 and CentOS 7](#rhel-7-and-centos-7).
 
 <img src="https://coroot.com/static/img/blog/ebpf.svg" width="800" />
 
@@ -85,6 +86,28 @@ Related blog posts:
 
 Follow the Coroot [documentation](https://docs.coroot.com/)
 
+### RHEL 7 and CentOS 7
+
+The agent runs on RHEL 7.6 and later (kernel 3.10.0-957 or later, which includes the eBPF backport) with a reduced set of eBPF programs.
+The following features are not available on these kernels:
+* Protocol-level (L7) metrics and TLS tracing
+* Network latency measurements (`container_net_latency_seconds`)
+* Python GIL and Node.js event loop metrics for processes running in containers
+
+Node and container resource metrics, TCP connections, listening sockets, retransmits, OOM kills and log parsing work as usual.
+
+The RHEL 7 SELinux policy doesn't allow systemd services and containers to use eBPF. The denial is not logged, the agent just fails to load its programs.
+The install script takes care of it for the systemd service. When running the agent as a Docker container, install the following policy module:
+
+```bash
+echo '(allow spc_t self (bpf (map_create map_read map_write prog_load prog_run)))' > coroot-node-agent-docker.cil
+semodule -i coroot-node-agent-docker.cil
+```
+
+Also, replace the `/sys/kernel/tracing` volume from the documented `docker run` command with `/sys/kernel/debug:/sys/kernel/debug`. Kernel 3.10 has no `/sys/kernel/tracing`, so the mount fails.
+
+Docker 1.13 from the CentOS repositories can't pull the agent image (it is published in the OCI format). Use Docker CE instead.
+
 ## Metrics
 
 The collected metrics are described [here](https://docs.coroot.com/metrics/node-agent).

cgroup/cgroup_linux.go

@@ -12,8 +12,30 @@ import (
 )
 
 func Init() error {
+	if err := initBaseCgroupPath(); err != nil {
+		return err
+	}
+	if _, err := os.Stat(path.Join(cgRoot, "unified")); err == nil {
+		if data, err := os.ReadFile("/proc/self/mounts"); err == nil {
+			for _, line := range strings.Split(string(data), "\n") {
+				if strings.Contains(line, "cgroup/unified") {
+					cg2Root = path.Join(cgRoot, "unified")
+					break
+				}
+			}
+		}
+	}
+	klog.Infoln("cgroup v2 root is", cg2Root)
+	return nil
+}
+
+func initBaseCgroupPath() error {
 	selfNs, err := netns.GetFromPath("/proc/self/ns/cgroup")
 	if err != nil {
+		if os.IsNotExist(err) { // kernels without cgroup namespaces (e.g., RHEL 7): all cgroup paths are host paths
+			klog.Infoln("cgroup namespaces are not supported by the kernel")
+			return nil
+		}
 		return err
 	}
 	defer selfNs.Close()
@@ -38,16 +60,5 @@ func Init() error {
 			return err
 		}
 	}
-	if _, err := os.Stat(path.Join(cgRoot, "unified")); err == nil {
-		if data, err := os.ReadFile("/proc/self/mounts"); err == nil {
-			for _, line := range strings.Split(string(data), "\n") {
-				if strings.Contains(line, "cgroup/unified") {
-					cg2Root = path.Join(cgRoot, "unified")
-					break
-				}
-			}
-		}
-	}
-	klog.Infoln("cgroup v2 root is", cg2Root)
 	return nil
 }

containers/dockerd.go

@@ -11,7 +11,9 @@ import (
 	"github.com/coroot/logparser"
 	"github.com/moby/moby/api/types/network"
 	"github.com/moby/moby/client"
+	"github.com/moby/moby/client/pkg/versions"
 	"inet.af/netaddr"
+	"k8s.io/klog/v2"
 )
 
 const dockerdTimeout = 30 * time.Second
@@ -21,16 +23,23 @@ var (
 )
 
 func DockerdInit() error {
-	c, err := client.New(
-		client.WithHost("unix://" + proc.HostPath("/run/docker.sock")),
-	)
+	host := client.WithHost("unix://" + proc.HostPath("/run/docker.sock"))
+	c, err := client.New(host)
 	if err != nil {
 		return err
 	}
 	ctx, cancelFn := context.WithTimeout(context.Background(), dockerdTimeout)
 	defer cancelFn()
-	if _, err := c.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true}); err != nil {
-		return err
+	ping, err := c.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true})
+	if err != nil {
+		if ping.APIVersion == "" || !versions.LessThan(ping.APIVersion, client.MinAPIVersion) {
+			return err
+		}
+		_ = c.Close()
+		if c, err = client.New(host, client.WithAPIVersion(ping.APIVersion)); err != nil {
+			return err
+		}
+		klog.Warningf("dockerd API version %s is older than %s, using it as is", ping.APIVersion, client.MinAPIVersion)
 	}
 	dockerdClient = c
 	return nil

ebpftracer/Dockerfile

@@ -5,11 +5,13 @@ RUN apk add llvm clang libbpf-dev linux-headers
 COPY ebpf /tmp/ebpf
 WORKDIR /tmp/ebpf
 
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=310 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf310x86.o && llvm-strip --strip-debug ebpf310x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf416x86.o && llvm-strip --strip-debug ebpf416x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf420x86.o && llvm-strip --strip-debug ebpf420x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf506x86.o && llvm-strip --strip-debug ebpf506x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf512x86.o && llvm-strip --strip-debug ebpf512x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -D__CTX_EXTRA_PADDING -c ebpf.c -o ebpf512x86cep.o && llvm-strip --strip-debug ebpf512x86cep.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=310 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf310arm64.o && llvm-strip --strip-debug ebpf310arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf416arm64.o && llvm-strip --strip-debug ebpf416arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf420arm64.o && llvm-strip --strip-debug ebpf420arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf506arm64.o && llvm-strip --strip-debug ebpf506arm64.o
@@ -27,12 +29,14 @@ RUN echo -en '// generated - do not edit\npackage ebpftracer\n\nvar ebpfProgs =
 	&& echo -en '\t\t{"5.6", "", []byte("' >> ebpf.go && gzip -c ebpf506x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.20", "", []byte("' >> ebpf.go && gzip -c ebpf420x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.16", "", []byte("' >> ebpf.go && gzip -c ebpf416x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"3.10", "", []byte("' >> ebpf.go && gzip -c ebpf310x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t},\n'>> ebpf.go \
 	&& echo -en '\t"arm64": {\n' >> ebpf.go \
 	&& echo -en '\t\t{"5.12", "ctx-extra-padding", []byte("' >> ebpf.go && gzip -c ebpf512arm64cep.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"5.12", "", []byte("' >> ebpf.go && gzip -c ebpf512arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"5.6", "", []byte("' >> ebpf.go && gzip -c ebpf506arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.20", "", []byte("' >> ebpf.go && gzip -c ebpf420arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.16", "", []byte("' >> ebpf.go && gzip -c ebpf416arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"3.10", "", []byte("' >> ebpf.go && gzip -c ebpf310arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t},\n'>> ebpf.go \
 	&& echo -en '}\n'>> ebpf.go

ebpftracer/ebpf/ebpf.c

@@ -46,10 +46,14 @@ struct trace_event_raw_sys_exit__stub {
 #include "tcp/conntrack.c"
 #include "tcp/state.c"
 #include "tcp/retransmit.c"
+#if __KERNEL_FROM >= 416
 #include "l7/l7.c"
 #include "l7/gotls.c"
 #include "l7/openssl.c"
 #include "l7/rustls.c"
 #include "l7/java_tls.c"
+#else
+#include "tcp/bytes.c"
+#endif
 
 char _license[] SEC("license") = "GPL";

ebpftracer/ebpf/proc.c

@@ -103,6 +103,7 @@ int sched_process_exit(struct trace_event_raw_sched_process_template__stub *args
     return 0;
 }
 
+#if __KERNEL_FROM >= 416
 struct trace_event_raw_mark_victim__stub {
     __u64 unused;
 #if defined(__CTX_EXTRA_PADDING)
@@ -118,3 +119,52 @@ int oom_mark_victim(struct trace_event_raw_mark_victim__stub *args)
     bpf_map_update_elem(&oom_info, &pid, &pid, BPF_ANY);
     return 0;
 }
+#else
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(key_size, sizeof(__u64));
+    __uint(value_size, sizeof(__u8));
+    __uint(max_entries, 1024);
+} oom_killer_running SEC(".maps");
+
+SEC("kprobe/oom_kill_process")
+int oom_kill_process(struct pt_regs *ctx)
+{
+    __u64 id = bpf_get_current_pid_tgid();
+    __u8 one = 1;
+    bpf_map_update_elem(&oom_killer_running, &id, &one, BPF_ANY);
+    return 0;
+}
+
+SEC("kretprobe/oom_kill_process")
+int oom_kill_process_ret(struct pt_regs *ctx)
+{
+    __u64 id = bpf_get_current_pid_tgid();
+    bpf_map_delete_elem(&oom_killer_running, &id);
+    return 0;
+}
+
+struct trace_event_raw_signal_generate__stub {
+    __u64 unused;
+    int sig;
+    int errno;
+    int code;
+    char comm[TASK_COMM_LEN];
+    int pid;
+};
+
+SEC("tracepoint/signal/signal_generate")
+int signal_generate(struct trace_event_raw_signal_generate__stub *args)
+{
+    if (args->sig != 9) { // SIGKILL
+        return 0;
+    }
+    __u64 id = bpf_get_current_pid_tgid();
+    if (!bpf_map_lookup_elem(&oom_killer_running, &id)) {
+        return 0;
+    }
+    __u32 pid = args->pid;
+    bpf_map_update_elem(&oom_info, &pid, &pid, BPF_ANY);
+    return 0;
+}
+#endif

ebpftracer/ebpf/tcp/bytes.c

@@ -0,0 +1,84 @@
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(key_size, sizeof(__u64));
+    __uint(value_size, sizeof(__u64));
+    __uint(max_entries, 10240);
+} rw_fd_by_pid_tgid SEC(".maps");
+
+static inline __attribute__((__always_inline__))
+int count_bytes_enter(struct trace_event_raw_args_with_fd__stub *ctx) {
+    struct trace_event_raw_args_with_fd__stub args = {};
+    if (bpf_probe_read(&args, sizeof(args), ctx) < 0) {
+        return 0;
+    }
+    __u64 id = bpf_get_current_pid_tgid();
+    struct connection_id cid = {};
+    cid.pid = id >> 32;
+    cid.fd = args.fd;
+    if (!bpf_map_lookup_elem(&active_connections, &cid)) {
+        return 0;
+    }
+    bpf_map_update_elem(&rw_fd_by_pid_tgid, &id, &args.fd, BPF_ANY);
+    return 0;
+}
+
+static inline __attribute__((__always_inline__))
+int count_bytes_exit(struct trace_event_raw_sys_exit__stub *ctx, __u8 is_write) {
+    __u64 id = bpf_get_current_pid_tgid();
+    __u64 *fdp = bpf_map_lookup_elem(&rw_fd_by_pid_tgid, &id);
+    if (!fdp) {
+        return 0;
+    }
+    struct connection_id cid = {};
+    cid.pid = id >> 32;
+    cid.fd = *fdp;
+    bpf_map_delete_elem(&rw_fd_by_pid_tgid, &id);
+    long int ret = ctx->ret;
+    if (ret <= 0) {
+        return 0;
+    }
+    struct connection *conn = bpf_map_lookup_elem(&active_connections, &cid);
+    if (!conn) {
+        return 0;
+    }
+    if (is_write) {
+        __sync_fetch_and_add(&conn->bytes_sent, ret);
+    } else {
+        __sync_fetch_and_add(&conn->bytes_received, ret);
+    }
+    return 0;
+}
+
+SEC("tracepoint/syscalls/sys_enter_write")
+int count_enter_write(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_writev")
+int count_enter_writev(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_sendto")
+int count_enter_sendto(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_sendmsg")
+int count_enter_sendmsg(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_read")
+int count_enter_read(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_readv")
+int count_enter_readv(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_recvfrom")
+int count_enter_recvfrom(void *ctx) { return count_bytes_enter(ctx); }
+SEC("tracepoint/syscalls/sys_enter_recvmsg")
+int count_enter_recvmsg(void *ctx) { return count_bytes_enter(ctx); }
+
+SEC("tracepoint/syscalls/sys_exit_write")
+int count_exit_write(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 1); }
+SEC("tracepoint/syscalls/sys_exit_writev")
+int count_exit_writev(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 1); }
+SEC("tracepoint/syscalls/sys_exit_sendto")
+int count_exit_sendto(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 1); }
+SEC("tracepoint/syscalls/sys_exit_sendmsg")
+int count_exit_sendmsg(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 1); }
+SEC("tracepoint/syscalls/sys_exit_read")
+int count_exit_read(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 0); }
+SEC("tracepoint/syscalls/sys_exit_readv")
+int count_exit_readv(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 0); }
+SEC("tracepoint/syscalls/sys_exit_recvfrom")
+int count_exit_recvfrom(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 0); }
+SEC("tracepoint/syscalls/sys_exit_recvmsg")
+int count_exit_recvmsg(struct trace_event_raw_sys_exit__stub *ctx) { return count_bytes_exit(ctx, 0); }

ebpftracer/ebpf/tcp/conntrack.c

@@ -73,6 +73,7 @@ int handle_ct(struct pt_regs *ctx, struct nf_conn conn)
     return 0;
 }
 
+#if __KERNEL_FROM >= 416
 SEC("kprobe/nf_ct_deliver_cached_events")
 int nf_ct_deliver_cached_events(struct pt_regs *ctx) {
     struct nf_conn conn;
@@ -81,3 +82,4 @@ int nf_ct_deliver_cached_events(struct pt_regs *ctx) {
     }
     return handle_ct(ctx, conn);
 }
+#endif

ebpftracer/ebpf/tcp/retransmit.c

@@ -25,6 +25,7 @@ struct trace_event_raw_tcp_event_sk_skb__stub {
     __u8 daddr_v6[16];
 };
 
+#if __KERNEL_FROM >= 416
 SEC("tracepoint/tcp/tcp_retransmit_skb")
 int tcp_retransmit_skb(struct trace_event_raw_tcp_event_sk_skb__stub *args)
 {
@@ -40,3 +41,24 @@ int tcp_retransmit_skb(struct trace_event_raw_tcp_event_sk_skb__stub *args)
 
     return 0;
 }
+#else
+SEC("kprobe/tcp_retransmit_skb")
+int tcp_retransmit_skb(struct pt_regs *ctx)
+{
+    struct trace_event_raw_inet_sock_set_state__stub args = {};
+    if (read_sock((void *)PT_REGS_PARM1(ctx), &args) < 0) {
+        return 0;
+    }
+    struct tcp_event e = {
+        .type = EVENT_TYPE_TCP_RETRANSMIT,
+        .sport = args.sport,
+        .dport = args.dport,
+    };
+    __builtin_memcpy(&e.saddr, &args.saddr_v6, sizeof(e.saddr));
+    __builtin_memcpy(&e.daddr, &args.daddr_v6, sizeof(e.daddr));
+
+    bpf_perf_event_output(ctx, &tcp_retransmit_events, BPF_F_CURRENT_CPU, &e, sizeof(e));
+
+    return 0;
+}
+#endif