From abbfb3376ff29d34e75aeeb2567d591abe11f189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Treffer?= Date: Wed, 26 Aug 2026 12:14:20 +0200 Subject: [PATCH] Add resctrl collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose the Linux resctrl monitoring counters (Intel RDT, AMD PQoS, ARM MPAM): per L3 domain memory bandwidth (mbm_total_bytes, mbm_local_bytes) and last level cache occupancy (llc_occupancy). Parsing is done by the procfs resctrlfs package. The collector is disabled by default because the resctrl filesystem must be mounted first (mount -t resctrl resctrl /sys/fs/resctrl). The mountpoint is configurable via --collector.resctrl.path. Counters a CPU does not support (for example mbm_local_bytes on Arm MPAM) or that read "Unavailable" are skipped per sample. The hardware counters wrap; Prometheus treats the wrap as a counter reset, so no correction is applied. Depends on a procfs release that contains the resctrlfs package. Signed-off-by: René Treffer --- README.md | 1 + collector/fixtures/e2e-64k-page-output.txt | 10 ++ collector/fixtures/e2e-output.txt | 10 ++ .../resctrl/mon_data/mon_L3_00/llc_occupancy | 1 + .../mon_data/mon_L3_00/mbm_local_bytes | 1 + .../mon_data/mon_L3_00/mbm_total_bytes | 1 + .../resctrl/mon_data/mon_L3_01/llc_occupancy | 1 + .../mon_data/mon_L3_01/mbm_local_bytes | 1 + .../mon_data/mon_L3_01/mbm_total_bytes | 1 + collector/resctrl_linux.go | 110 ++++++++++++++++++ collector/resctrl_linux_test.go | 91 +++++++++++++++ end-to-end-test.sh | 2 + 12 files changed, 230 insertions(+) create mode 100644 collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy create mode 100644 collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes create mode 100644 collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes create mode 100644 collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy create mode 100644 collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes create mode 100644 collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes create mode 100644 collector/resctrl_linux.go create mode 100644 collector/resctrl_linux_test.go diff --git a/README.md b/README.md index 8abf2cdf0f..da24ef5b52 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,7 @@ pcidevice | Exposes pci devices' information including their link status and par perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux processes | Exposes aggregate process statistics from `/proc`. | Linux qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux +resctrl | Exposes Intel RDT / AMD PQoS / ARM MPAM memory bandwidth and cache occupancy monitoring counters from the resctrl filesystem (must be mounted). | Linux slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux softirqs | Exposes detailed softirq statistics from `/proc/softirqs`. | Linux sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 7b8e4be5e4..0506336da7 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3708,6 +3708,15 @@ node_rapl_core_joules_total{index="0",path="collector/fixtures/sys/class/powerca # HELP node_rapl_package_joules_total Current RAPL package value in joules # TYPE node_rapl_package_joules_total counter node_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 240422.366267 +# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain. +# TYPE node_resctrl_llc_occupancy_bytes gauge +node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07 +node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06 +# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain. +# TYPE node_resctrl_memory_bandwidth_bytes_total counter +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11 # HELP node_schedstat_running_seconds_total Number of seconds CPU spent running a process. # TYPE node_schedstat_running_seconds_total counter node_schedstat_running_seconds_total{cpu="0"} 2.045936778163039e+06 @@ -3768,6 +3777,7 @@ node_scrape_collector_success{collector="pressure"} 1 node_scrape_collector_success{collector="processes"} 1 node_scrape_collector_success{collector="qdisc"} 1 node_scrape_collector_success{collector="rapl"} 1 +node_scrape_collector_success{collector="resctrl"} 1 node_scrape_collector_success{collector="schedstat"} 1 node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index d03fb08f44..a779728d77 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3740,6 +3740,15 @@ node_rapl_core_joules_total{index="0",path="collector/fixtures/sys/class/powerca # HELP node_rapl_package_joules_total Current RAPL package value in joules # TYPE node_rapl_package_joules_total counter node_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 240422.366267 +# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain. +# TYPE node_resctrl_llc_occupancy_bytes gauge +node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07 +node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06 +# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain. +# TYPE node_resctrl_memory_bandwidth_bytes_total counter +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11 # HELP node_schedstat_running_seconds_total Number of seconds CPU spent running a process. # TYPE node_schedstat_running_seconds_total counter node_schedstat_running_seconds_total{cpu="0"} 2.045936778163039e+06 @@ -3800,6 +3809,7 @@ node_scrape_collector_success{collector="pressure"} 1 node_scrape_collector_success{collector="processes"} 1 node_scrape_collector_success{collector="qdisc"} 1 node_scrape_collector_success{collector="rapl"} 1 +node_scrape_collector_success{collector="resctrl"} 1 node_scrape_collector_success{collector="schedstat"} 1 node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy b/collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy new file mode 100644 index 0000000000..317ea811d9 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy @@ -0,0 +1 @@ +44040192 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes new file mode 100644 index 0000000000..5cb8136235 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes @@ -0,0 +1 @@ +210196273664 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes new file mode 100644 index 0000000000..d91f52ebae --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes @@ -0,0 +1 @@ +315294410752 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy b/collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy new file mode 100644 index 0000000000..16edb4cc19 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy @@ -0,0 +1 @@ +8388608 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes new file mode 100644 index 0000000000..ba54161f35 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes @@ -0,0 +1 @@ +Unavailable diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes new file mode 100644 index 0000000000..47cf3d0c1b --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes @@ -0,0 +1 @@ +105098136832 diff --git a/collector/resctrl_linux.go b/collector/resctrl_linux.go new file mode 100644 index 0000000000..9995950210 --- /dev/null +++ b/collector/resctrl_linux.go @@ -0,0 +1,110 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noresctrl + +package collector + +import ( + "errors" + "log/slog" + "os" + + "github.com/alecthomas/kingpin/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs/resctrlfs" +) + +const resctrlSubsystem = "resctrl" + +// The resctrl filesystem exposes the cache and memory bandwidth monitoring of +// the CPU (Intel RDT, AMD PQoS, Arm MPAM), documented in +// https://docs.kernel.org/filesystems/resctrl.html and +// https://docs.kernel.org/arch/arm64/mpam.html. +// +// It is not mounted by default, hence this collector is disabled by default. +// Mount it with: +// +// mount -t resctrl resctrl /sys/fs/resctrl +var resctrlPath = kingpin.Flag("collector.resctrl.path", "resctrl filesystem mountpoint.").Default(resctrlfs.DefaultMountPoint).String() + +type resctrlCollector struct{} + +func init() { + registerCollector("resctrl", defaultDisabled, NewResctrlCollector) +} + +var ( + resctrlMemoryBandwidth = prometheus.NewDesc( + prometheus.BuildFQName(namespace, resctrlSubsystem, "memory_bandwidth_bytes_total"), + "Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain.", + []string{"domain", "scope"}, nil, + ) + resctrlLLCOccupancy = prometheus.NewDesc( + prometheus.BuildFQName(namespace, resctrlSubsystem, "llc_occupancy_bytes"), + "Last level cache bytes occupied in this domain.", + []string{"domain"}, nil, + ) +) + +// NewResctrlCollector returns a new Collector exposing resctrl monitoring counters. +func NewResctrlCollector(_ *slog.Logger) (Collector, error) { + return &resctrlCollector{}, nil +} + +func (c *resctrlCollector) Update(ch chan<- prometheus.Metric) error { + // The filesystem is opened on every scrape because it can be mounted and + // unmounted while node_exporter runs. + fs, err := resctrlfs.NewFS(*resctrlPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrNoData + } + return err + } + + domains, err := fs.MonData() + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrNoData + } + return err + } + + found := false + for _, domain := range domains { + // The metrics describe the last level cache, so only L3 fits them. + if domain.Resource != "L3" { + continue + } + found = true + + // The bandwidth counters wrap. No correction is applied: Prometheus + // reads a wrap as a counter reset, which is the closest available + // approximation. + if v := domain.MBMTotalBytes; v != nil { + ch <- prometheus.MustNewConstMetric(resctrlMemoryBandwidth, prometheus.CounterValue, float64(*v), domain.ID, "total") + } + if v := domain.MBMLocalBytes; v != nil { + ch <- prometheus.MustNewConstMetric(resctrlMemoryBandwidth, prometheus.CounterValue, float64(*v), domain.ID, "local") + } + if v := domain.LLCOccupancy; v != nil { + ch <- prometheus.MustNewConstMetric(resctrlLLCOccupancy, prometheus.GaugeValue, float64(*v), domain.ID) + } + } + + if !found { + return ErrNoData + } + return nil +} diff --git a/collector/resctrl_linux_test.go b/collector/resctrl_linux_test.go new file mode 100644 index 0000000000..fd5598a92d --- /dev/null +++ b/collector/resctrl_linux_test.go @@ -0,0 +1,91 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noresctrl + +package collector + +import ( + "io" + "log/slog" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +type testResctrlCollector struct { + rc Collector +} + +func (c testResctrlCollector) Collect(ch chan<- prometheus.Metric) { + c.rc.Update(ch) +} + +func (c testResctrlCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(c, ch) +} + +func newTestResctrlCollector(logger *slog.Logger) (prometheus.Collector, error) { + rc, err := NewResctrlCollector(logger) + if err != nil { + return testResctrlCollector{}, err + } + return testResctrlCollector{rc}, nil +} + +func TestResctrlMetrics(t *testing.T) { + *resctrlPath = "fixtures/resctrl" + + // Domain 1 reports "Unavailable" for mbm_local_bytes, so that sample is + // absent. + testcase := `# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain. +# TYPE node_resctrl_llc_occupancy_bytes gauge +node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07 +node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06 +# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain. +# TYPE node_resctrl_memory_bandwidth_bytes_total counter +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11 +` + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := newTestResctrlCollector(logger) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(testcase)); err != nil { + t.Fatal(err) + } +} + +func TestResctrlNotMounted(t *testing.T) { + *resctrlPath = t.TempDir() + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := NewResctrlCollector(logger) + if err != nil { + t.Fatal(err) + } + + ch := make(chan prometheus.Metric, 16) + err = c.Update(ch) + close(ch) + + if err != ErrNoData { + t.Fatalf("expected ErrNoData, got %v", err) + } +} diff --git a/end-to-end-test.sh b/end-to-end-test.sh index b439cea556..4002c82e59 100755 --- a/end-to-end-test.sh +++ b/end-to-end-test.sh @@ -74,6 +74,7 @@ enabled_collectors=$(cat << COLLECTORS processes qdisc rapl + resctrl schedstat slabinfo sockstat @@ -175,6 +176,7 @@ collector_flags=$(cat << FLAGS --collector.netdev.device-include=lo --collector.qdisc.device-include=(wlan0|eth0) --collector.qdisc.fixtures=collector/fixtures/qdisc/ + --collector.resctrl.path=collector/fixtures/resctrl --collector.stat.softirq --collector.sysctl.include-info=kernel.seccomp.actions_avail --collector.sysctl.include=fs.file-nr