diff --git a/README.md b/README.md index 8abf2cdf0f..da24ef5b52 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,7 @@ pcidevice | Exposes pci devices' information including their link status and par perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux processes | Exposes aggregate process statistics from `/proc`. | Linux qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux +resctrl | Exposes Intel RDT / AMD PQoS / ARM MPAM memory bandwidth and cache occupancy monitoring counters from the resctrl filesystem (must be mounted). | Linux slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux softirqs | Exposes detailed softirq statistics from `/proc/softirqs`. | Linux sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 7b8e4be5e4..0506336da7 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3708,6 +3708,15 @@ node_rapl_core_joules_total{index="0",path="collector/fixtures/sys/class/powerca # HELP node_rapl_package_joules_total Current RAPL package value in joules # TYPE node_rapl_package_joules_total counter node_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 240422.366267 +# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain. +# TYPE node_resctrl_llc_occupancy_bytes gauge +node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07 +node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06 +# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain. +# TYPE node_resctrl_memory_bandwidth_bytes_total counter +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11 # HELP node_schedstat_running_seconds_total Number of seconds CPU spent running a process. # TYPE node_schedstat_running_seconds_total counter node_schedstat_running_seconds_total{cpu="0"} 2.045936778163039e+06 @@ -3768,6 +3777,7 @@ node_scrape_collector_success{collector="pressure"} 1 node_scrape_collector_success{collector="processes"} 1 node_scrape_collector_success{collector="qdisc"} 1 node_scrape_collector_success{collector="rapl"} 1 +node_scrape_collector_success{collector="resctrl"} 1 node_scrape_collector_success{collector="schedstat"} 1 node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index d03fb08f44..a779728d77 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3740,6 +3740,15 @@ node_rapl_core_joules_total{index="0",path="collector/fixtures/sys/class/powerca # HELP node_rapl_package_joules_total Current RAPL package value in joules # TYPE node_rapl_package_joules_total counter node_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 240422.366267 +# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain. +# TYPE node_resctrl_llc_occupancy_bytes gauge +node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07 +node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06 +# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain. +# TYPE node_resctrl_memory_bandwidth_bytes_total counter +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11 # HELP node_schedstat_running_seconds_total Number of seconds CPU spent running a process. # TYPE node_schedstat_running_seconds_total counter node_schedstat_running_seconds_total{cpu="0"} 2.045936778163039e+06 @@ -3800,6 +3809,7 @@ node_scrape_collector_success{collector="pressure"} 1 node_scrape_collector_success{collector="processes"} 1 node_scrape_collector_success{collector="qdisc"} 1 node_scrape_collector_success{collector="rapl"} 1 +node_scrape_collector_success{collector="resctrl"} 1 node_scrape_collector_success{collector="schedstat"} 1 node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy b/collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy new file mode 100644 index 0000000000..317ea811d9 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_00/llc_occupancy @@ -0,0 +1 @@ +44040192 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes new file mode 100644 index 0000000000..5cb8136235 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_local_bytes @@ -0,0 +1 @@ +210196273664 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes new file mode 100644 index 0000000000..d91f52ebae --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_00/mbm_total_bytes @@ -0,0 +1 @@ +315294410752 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy b/collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy new file mode 100644 index 0000000000..16edb4cc19 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_01/llc_occupancy @@ -0,0 +1 @@ +8388608 diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes new file mode 100644 index 0000000000..ba54161f35 --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_local_bytes @@ -0,0 +1 @@ +Unavailable diff --git a/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes new file mode 100644 index 0000000000..47cf3d0c1b --- /dev/null +++ b/collector/fixtures/resctrl/mon_data/mon_L3_01/mbm_total_bytes @@ -0,0 +1 @@ +105098136832 diff --git a/collector/resctrl_linux.go b/collector/resctrl_linux.go new file mode 100644 index 0000000000..9995950210 --- /dev/null +++ b/collector/resctrl_linux.go @@ -0,0 +1,110 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noresctrl + +package collector + +import ( + "errors" + "log/slog" + "os" + + "github.com/alecthomas/kingpin/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs/resctrlfs" +) + +const resctrlSubsystem = "resctrl" + +// The resctrl filesystem exposes the cache and memory bandwidth monitoring of +// the CPU (Intel RDT, AMD PQoS, Arm MPAM), documented in +// https://docs.kernel.org/filesystems/resctrl.html and +// https://docs.kernel.org/arch/arm64/mpam.html. +// +// It is not mounted by default, hence this collector is disabled by default. +// Mount it with: +// +// mount -t resctrl resctrl /sys/fs/resctrl +var resctrlPath = kingpin.Flag("collector.resctrl.path", "resctrl filesystem mountpoint.").Default(resctrlfs.DefaultMountPoint).String() + +type resctrlCollector struct{} + +func init() { + registerCollector("resctrl", defaultDisabled, NewResctrlCollector) +} + +var ( + resctrlMemoryBandwidth = prometheus.NewDesc( + prometheus.BuildFQName(namespace, resctrlSubsystem, "memory_bandwidth_bytes_total"), + "Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain.", + []string{"domain", "scope"}, nil, + ) + resctrlLLCOccupancy = prometheus.NewDesc( + prometheus.BuildFQName(namespace, resctrlSubsystem, "llc_occupancy_bytes"), + "Last level cache bytes occupied in this domain.", + []string{"domain"}, nil, + ) +) + +// NewResctrlCollector returns a new Collector exposing resctrl monitoring counters. +func NewResctrlCollector(_ *slog.Logger) (Collector, error) { + return &resctrlCollector{}, nil +} + +func (c *resctrlCollector) Update(ch chan<- prometheus.Metric) error { + // The filesystem is opened on every scrape because it can be mounted and + // unmounted while node_exporter runs. + fs, err := resctrlfs.NewFS(*resctrlPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrNoData + } + return err + } + + domains, err := fs.MonData() + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrNoData + } + return err + } + + found := false + for _, domain := range domains { + // The metrics describe the last level cache, so only L3 fits them. + if domain.Resource != "L3" { + continue + } + found = true + + // The bandwidth counters wrap. No correction is applied: Prometheus + // reads a wrap as a counter reset, which is the closest available + // approximation. + if v := domain.MBMTotalBytes; v != nil { + ch <- prometheus.MustNewConstMetric(resctrlMemoryBandwidth, prometheus.CounterValue, float64(*v), domain.ID, "total") + } + if v := domain.MBMLocalBytes; v != nil { + ch <- prometheus.MustNewConstMetric(resctrlMemoryBandwidth, prometheus.CounterValue, float64(*v), domain.ID, "local") + } + if v := domain.LLCOccupancy; v != nil { + ch <- prometheus.MustNewConstMetric(resctrlLLCOccupancy, prometheus.GaugeValue, float64(*v), domain.ID) + } + } + + if !found { + return ErrNoData + } + return nil +} diff --git a/collector/resctrl_linux_test.go b/collector/resctrl_linux_test.go new file mode 100644 index 0000000000..fd5598a92d --- /dev/null +++ b/collector/resctrl_linux_test.go @@ -0,0 +1,91 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noresctrl + +package collector + +import ( + "io" + "log/slog" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +type testResctrlCollector struct { + rc Collector +} + +func (c testResctrlCollector) Collect(ch chan<- prometheus.Metric) { + c.rc.Update(ch) +} + +func (c testResctrlCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(c, ch) +} + +func newTestResctrlCollector(logger *slog.Logger) (prometheus.Collector, error) { + rc, err := NewResctrlCollector(logger) + if err != nil { + return testResctrlCollector{}, err + } + return testResctrlCollector{rc}, nil +} + +func TestResctrlMetrics(t *testing.T) { + *resctrlPath = "fixtures/resctrl" + + // Domain 1 reports "Unavailable" for mbm_local_bytes, so that sample is + // absent. + testcase := `# HELP node_resctrl_llc_occupancy_bytes Last level cache bytes occupied in this domain. +# TYPE node_resctrl_llc_occupancy_bytes gauge +node_resctrl_llc_occupancy_bytes{domain="0"} 4.4040192e+07 +node_resctrl_llc_occupancy_bytes{domain="1"} 8.388608e+06 +# HELP node_resctrl_memory_bandwidth_bytes_total Bytes moved between the last level cache and memory. scope=total includes remote sockets, scope=local is memory attached to this domain. +# TYPE node_resctrl_memory_bandwidth_bytes_total counter +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="local"} 2.10196273664e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="0",scope="total"} 3.15294410752e+11 +node_resctrl_memory_bandwidth_bytes_total{domain="1",scope="total"} 1.05098136832e+11 +` + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := newTestResctrlCollector(logger) + if err != nil { + t.Fatal(err) + } + + if err := testutil.CollectAndCompare(c, strings.NewReader(testcase)); err != nil { + t.Fatal(err) + } +} + +func TestResctrlNotMounted(t *testing.T) { + *resctrlPath = t.TempDir() + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := NewResctrlCollector(logger) + if err != nil { + t.Fatal(err) + } + + ch := make(chan prometheus.Metric, 16) + err = c.Update(ch) + close(ch) + + if err != ErrNoData { + t.Fatalf("expected ErrNoData, got %v", err) + } +} diff --git a/end-to-end-test.sh b/end-to-end-test.sh index b439cea556..4002c82e59 100755 --- a/end-to-end-test.sh +++ b/end-to-end-test.sh @@ -74,6 +74,7 @@ enabled_collectors=$(cat << COLLECTORS processes qdisc rapl + resctrl schedstat slabinfo sockstat @@ -175,6 +176,7 @@ collector_flags=$(cat << FLAGS --collector.netdev.device-include=lo --collector.qdisc.device-include=(wlan0|eth0) --collector.qdisc.fixtures=collector/fixtures/qdisc/ + --collector.resctrl.path=collector/fixtures/resctrl --collector.stat.softirq --collector.sysctl.include-info=kernel.seccomp.actions_avail --collector.sysctl.include=fs.file-nr diff --git a/go.mod b/go.mod index 7fa9aaa664..ffffb0f0c5 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.70.1 github.com/prometheus/exporter-toolkit v0.19.0 - github.com/prometheus/procfs v0.21.1 + github.com/prometheus/procfs v0.21.2-0.20260827102831-a87803ee31e3 github.com/safchain/ethtool v0.7.0 golang.org/x/sys v0.47.0 howett.net/plist v1.0.1 diff --git a/go.sum b/go.sum index b6f9aef455..40ce869ec9 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/exporter-toolkit v0.19.0 h1:JljWCzE5naAiZ7Ukeb8PwjNbU+WwISuW0ktgdXMnMhc= github.com/prometheus/exporter-toolkit v0.19.0/go.mod h1:kOoEK/7wbe2Ns33l7wYHOXDZAZ/XGLyJqoGwmJxK+QU= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/procfs v0.21.2-0.20260827102831-a87803ee31e3 h1:ralsA1JiQYnRknAF3VGTLLJ2kK38MX+m8scg76t4Ujo= +github.com/prometheus/procfs v0.21.2-0.20260827102831-a87803ee31e3/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/safchain/ethtool v0.7.0 h1:rlJzfDetsVvT61uz8x1YIcFn12akMfuPulHtZjtb7Is= github.com/safchain/ethtool v0.7.0/go.mod h1:MenQKEjXdfkjD3mp2QdCk8B/hwvkrlOTm/FD4gTpFxQ= github.com/siebenmann/go-kstat v0.0.0-20210513183136-173c9b0a9973 h1:GfSdC6wKfTGcgCS7BtzF5694Amne1pGCSTY252WhlEY=