Add resctrl collector

README.md

@@ -213,6 +213,7 @@ pcidevice | Exposes pci devices' information including their link status and par
 perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux
 processes | Exposes aggregate process statistics from `/proc`. | Linux
 qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux
+resctrl | Exposes Intel RDT / AMD PQoS / ARM MPAM memory bandwidth and cache occupancy monitoring counters from the resctrl filesystem (must be mounted). | Linux
 slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux
 softirqs | Exposes detailed softirq statistics from `/proc/softirqs`. | Linux
 sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux

collector/fixtures/e2e-64k-page-output.txt

@@ -3708,6 +3708,15 @@ node_rapl_core_joules_total{index="0",path="collector/fixtures/sys/class/powerca
 # HELP node_rapl_package_joules_total Current RAPL package value in joules
 # TYPE node_rapl_package_joules_total counter
 node_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 240422.366267
+# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain.
+# TYPE node_resctrl_llc_occupancy_bytes gauge
+node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07
+node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06
+# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain.
+# TYPE node_resctrl_memory_bandwidth_bytes_total counter
+node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11
+node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11
+node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11
 # HELP node_schedstat_running_seconds_total Number of seconds CPU spent running a process.
 # TYPE node_schedstat_running_seconds_total counter
 node_schedstat_running_seconds_total{cpu="0"} 2.045936778163039e+06
@@ -3768,6 +3777,7 @@ node_scrape_collector_success{collector="pressure"} 1
 node_scrape_collector_success{collector="processes"} 1
 node_scrape_collector_success{collector="qdisc"} 1
 node_scrape_collector_success{collector="rapl"} 1
+node_scrape_collector_success{collector="resctrl"} 1
 node_scrape_collector_success{collector="schedstat"} 1
 node_scrape_collector_success{collector="slabinfo"} 1
 node_scrape_collector_success{collector="sockstat"} 1

collector/fixtures/e2e-output.txt

@@ -3740,6 +3740,15 @@ node_rapl_core_joules_total{index="0",path="collector/fixtures/sys/class/powerca
 # HELP node_rapl_package_joules_total Current RAPL package value in joules
 # TYPE node_rapl_package_joules_total counter
 node_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 240422.366267
+# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain.
+# TYPE node_resctrl_llc_occupancy_bytes gauge
+node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07
+node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06
+# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain.
+# TYPE node_resctrl_memory_bandwidth_bytes_total counter
+node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11
+node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11
+node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11
 # HELP node_schedstat_running_seconds_total Number of seconds CPU spent running a process.
 # TYPE node_schedstat_running_seconds_total counter
 node_schedstat_running_seconds_total{cpu="0"} 2.045936778163039e+06
@@ -3800,6 +3809,7 @@ node_scrape_collector_success{collector="pressure"} 1
 node_scrape_collector_success{collector="processes"} 1
 node_scrape_collector_success{collector="qdisc"} 1
 node_scrape_collector_success{collector="rapl"} 1
+node_scrape_collector_success{collector="resctrl"} 1
 node_scrape_collector_success{collector="schedstat"} 1
 node_scrape_collector_success{collector="slabinfo"} 1
 node_scrape_collector_success{collector="sockstat"} 1

collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy

@@ -0,0 +1 @@
+44040192

collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes

@@ -0,0 +1 @@
+210196273664

collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes

@@ -0,0 +1 @@
+315294410752

collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy

@@ -0,0 +1 @@
+8388608

collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes

@@ -0,0 +1 @@
+Unavailable

collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes

@@ -0,0 +1 @@
+105098136832

collector/resctrl_linux.go

@@ -0,0 +1,110 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noresctrl
+
+package collector
+
+import (
+	"errors"
+	"log/slog"
+	"os"
+
+	"github.com/alecthomas/kingpin/v2"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs/resctrlfs"
+)
+
+const resctrlSubsystem = "resctrl"
+
+// The resctrl filesystem exposes the cache and memory bandwidth monitoring of
+// the CPU (Intel RDT, AMD PQoS, Arm MPAM), documented in
+// https://docs.kernel.org/filesystems/resctrl.html and
+// https://docs.kernel.org/arch/arm64/mpam.html.
+//
+// It is not mounted by default, hence this collector is disabled by default.
+// Mount it with:
+//
+//	mount -t resctrl resctrl /sys/fs/resctrl
+var resctrlPath = kingpin.Flag("collector.resctrl.path", "resctrl filesystem mountpoint.").Default(resctrlfs.DefaultMountPoint).String()
+
+type resctrlCollector struct{}
+
+func init() {
+	registerCollector("resctrl", defaultDisabled, NewResctrlCollector)
+}
+
+var (
+	resctrlMemoryBandwidth = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, resctrlSubsystem, "memory_bandwidth_bytes_total"),
+		"Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain.",
+		[]string{"domain", "scope"}, nil,
+	)
+	resctrlLLCOccupancy = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, resctrlSubsystem, "llc_occupancy_bytes"),
+		"Last level cache bytes occupied in this domain.",
+		[]string{"domain"}, nil,
+	)
+)
+
+// NewResctrlCollector returns a new Collector exposing resctrl monitoring counters.
+func NewResctrlCollector(_ *slog.Logger) (Collector, error) {
+	return &resctrlCollector{}, nil
+}
+
+func (c *resctrlCollector) Update(ch chan<- prometheus.Metric) error {
+	// The filesystem is opened on every scrape because it can be mounted and
+	// unmounted while node_exporter runs.
+	fs, err := resctrlfs.NewFS(*resctrlPath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return ErrNoData
+		}
+		return err
+	}
+
+	domains, err := fs.MonData()
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return ErrNoData
+		}
+		return err
+	}
+
+	found := false
+	for _, domain := range domains {
+		// The metrics describe the last level cache, so only L3 fits them.
+		if domain.Resource != "L3" {
+			continue
+		}
+		found = true
+
+		// The bandwidth counters wrap. No correction is applied: Prometheus
+		// reads a wrap as a counter reset, which is the closest available
+		// approximation.
+		if v := domain.MBMTotalBytes; v != nil {
+			ch <- prometheus.MustNewConstMetric(resctrlMemoryBandwidth, prometheus.CounterValue, float64(*v), domain.ID, "total")
+		}
+		if v := domain.MBMLocalBytes; v != nil {
+			ch <- prometheus.MustNewConstMetric(resctrlMemoryBandwidth, prometheus.CounterValue, float64(*v), domain.ID, "local")
+		}
+		if v := domain.LLCOccupancy; v != nil {
+			ch <- prometheus.MustNewConstMetric(resctrlLLCOccupancy, prometheus.GaugeValue, float64(*v), domain.ID)
+		}
+	}
+
+	if !found {
+		return ErrNoData
+	}
+	return nil
+}

collector/resctrl_linux_test.go

@@ -0,0 +1,91 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noresctrl
+
+package collector
+
+import (
+	"io"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+type testResctrlCollector struct {
+	rc Collector
+}
+
+func (c testResctrlCollector) Collect(ch chan<- prometheus.Metric) {
+	c.rc.Update(ch)
+}
+
+func (c testResctrlCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(c, ch)
+}
+
+func newTestResctrlCollector(logger *slog.Logger) (prometheus.Collector, error) {
+	rc, err := NewResctrlCollector(logger)
+	if err != nil {
+		return testResctrlCollector{}, err
+	}
+	return testResctrlCollector{rc}, nil
+}
+
+func TestResctrlMetrics(t *testing.T) {
+	*resctrlPath = "fixtures/resctrl"
+
+	// Domain 1 reports "Unavailable" for mbm_local_bytes, so that sample is
+	// absent.
+	testcase := `# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain.
+# TYPE node_resctrl_llc_occupancy_bytes gauge
+node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07
+node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06
+# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain.
+# TYPE node_resctrl_memory_bandwidth_bytes_total counter
+node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11
+node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11
+node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11
+`
+
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	c, err := newTestResctrlCollector(logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := testutil.CollectAndCompare(c, strings.NewReader(testcase)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestResctrlNotMounted(t *testing.T) {
+	*resctrlPath = t.TempDir()
+
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	c, err := NewResctrlCollector(logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch := make(chan prometheus.Metric, 16)
+	err = c.Update(ch)
+	close(ch)
+
+	if err != ErrNoData {
+		t.Fatalf("expected ErrNoData, got %v", err)
+	}
+}

end-to-end-test.sh

@@ -74,6 +74,7 @@ enabled_collectors=$(cat << COLLECTORS
   processes
   qdisc
   rapl
+  resctrl
   schedstat
   slabinfo
   sockstat
@@ -175,6 +176,7 @@ collector_flags=$(cat << FLAGS
   --collector.netdev.device-include=lo
   --collector.qdisc.device-include=(wlan0|eth0)
   --collector.qdisc.fixtures=collector/fixtures/qdisc/
+  --collector.resctrl.path=collector/fixtures/resctrl
   --collector.stat.softirq
   --collector.sysctl.include-info=kernel.seccomp.actions_avail
   --collector.sysctl.include=fs.file-nr