diff --git a/cmd/main.go b/cmd/main.go index e603732..942d7fd 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,6 +28,7 @@ import ( networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" "github.com/ironcore-dev/sonic-operator/internal/controller" "github.com/ironcore-dev/sonic-operator/internal/onie" + "github.com/ironcore-dev/sonic-operator/internal/sd" "github.com/ironcore-dev/sonic-operator/internal/ztp" // +kubebuilder:scaffold:imports ) @@ -208,6 +209,10 @@ func main() { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } + if err := mgr.AddMetricsServerExtraHandler("/switch-sd", sd.NewHandler(mgr.GetClient())); err != nil { + setupLog.Error(err, "unable to register switch-sd handler") + os.Exit(1) + } if !disableProvisionsingServer { setupLog.Info("starting HTTP server") provServer, err := setupProvisioningServer(httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile) diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 30caeb6..fd62002 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -76,7 +76,14 @@ export default withMermaid({ { text: 'Getting started', link: '/usage/getting-started' }, { text: 'Provisioning', link: '/usage/provisioning' }, { text: 'Agent', link: '/usage/agent' }, - { text: 'Agent Metrics', link: '/usage/metrics' }, + { text: 'Metrics', + collapsed: false, + items: [ + { text: 'Operator Metrics', link: '/usage/operator-metrics' }, + { text: 'Per-Switch Metrics', link: '/usage/metrics' }, + { text: 'Switch Metrics Discovery', link: '/usage/service-discovery' }, + ] + }, ] }, { diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index ef43e81..779c85d 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -1,4 +1,4 @@ -# Agent metrics +# Per-Switch Metrics The switch agent exposes a Prometheus-compatible `/metrics` endpoint for monitoring switch health, interface state, and transceiver optics. Metrics are collected just-in-time from SONiC Redis on every Prometheus scrape — there is no background polling or caching. @@ -34,7 +34,6 @@ These require custom logic (cross-database joins, aggregate counting, error fall | `sonic_switch_interface_admin_state` | gauge | `interface` | Admin state (1=up, 0=down) | | `sonic_switch_interfaces_total` | gauge | `operational_status` | Number of interfaces by status | | `sonic_switch_ports_total` | gauge | — | Total physical ports | -| `sonic_scrape_duration_seconds` | gauge | — | Duration of the last metrics scrape | ### Config-driven collectors diff --git a/docs/usage/operator-metrics.md b/docs/usage/operator-metrics.md new file mode 100644 index 0000000..39e2dd7 --- /dev/null +++ b/docs/usage/operator-metrics.md @@ -0,0 +1,69 @@ +# Operator Metrics + +The sonic-operator exposes standard [controller-runtime](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/metrics) metrics on its metrics server. These provide insight into the reconciliation performance and health of the operator itself. + +## Endpoint + +The metrics server is configured with `--metrics-bind-address`. By default it is disabled (`0`). To enable: + +``` +--metrics-bind-address=:8443 # HTTPS (default when non-zero) +--metrics-bind-address=:8080 --metrics-secure=false # HTTP +``` + +Metrics are served at `/metrics` on the configured port. + +## Available metrics + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `controller_runtime_reconcile_total` | counter | `controller`, `result` | Total reconciliations per controller | +| `controller_runtime_reconcile_errors_total` | counter | `controller` | Reconciliation errors per controller | +| `controller_runtime_terminal_reconcile_errors_total` | counter | `controller` | Terminal (non-retryable) errors per controller | +| `controller_runtime_reconcile_panics_total` | counter | `controller` | Reconciliation panics per controller | +| `controller_runtime_reconcile_time_seconds` | histogram | `controller` | Reconciliation duration per controller | +| `controller_runtime_active_workers` | gauge | `controller` | Currently active workers per controller | +| `controller_runtime_max_concurrent_reconciles` | gauge | `controller` | Maximum concurrent reconciles per controller | + +The `controller` label identifies which controller produced the metric (e.g. `switch`, `switchinterface`). + +## Scrape configuration + +The operator metrics server also hosts the [Metrics Discovery](/usage/service-discovery) endpoint at `/switch-sd`. You can scrape both the operator and the discovered switches from the same Prometheus job configuration: + +```yaml +scrape_configs: + # Operator itself + - job_name: sonic-operator + static_configs: + - targets: ["sonic-operator.sonic-operator-system:8080"] + + # Switches (auto-discovered) + - job_name: sonic-switches + http_sd_configs: + - url: http://sonic-operator.sonic-operator-system:8080/switch-sd + relabel_configs: + - source_labels: [__meta_sonic_switch_name] + target_label: switch +``` + +## Useful queries + +Reconciliation rate per controller: + +```promql +rate(controller_runtime_reconcile_total[5m]) +``` + +Error ratio: + +```promql +rate(controller_runtime_reconcile_errors_total[5m]) +/ rate(controller_runtime_reconcile_total[5m]) +``` + +p99 reconciliation latency: + +```promql +histogram_quantile(0.99, rate(controller_runtime_reconcile_time_seconds_bucket[5m])) +``` diff --git a/docs/usage/service-discovery.md b/docs/usage/service-discovery.md new file mode 100644 index 0000000..8d1b4a3 --- /dev/null +++ b/docs/usage/service-discovery.md @@ -0,0 +1,108 @@ +# Switch Metrics Discovery + +The sonic-operator exposes an HTTP Service Discovery (SD) endpoint that enables Prometheus and compatible scrapers to automatically discover and scrape metrics from all ready switches. This eliminates the need to maintain a static list of scrape targets. + +## How it works + +The operator watches all `Switch` resources in the cluster. Switches that have reached the `Ready` state and have a management address configured are served as scrape targets via the `/switch-sd` endpoint on the operator's metrics server. + +```mermaid +graph TD + Prometheus[Metrics Collector
e.g. Prometheus] -->|GET /switch-sd| Operator[sonic-operator
metrics server] + Operator -->|watches| CRs[Switch CRs
Kubernetes API] + Operator -->|"[{targets, labels}, ...]"| Prometheus + Prometheus -->|"scrape
:9100/metrics"| Switch1[switch-1] + Prometheus -->|"scrape
:9100/metrics"| Switch2[switch-2] + Prometheus -->|"scrape
:9100/metrics"| SwitchN[switch-N] +``` + +## Response format + +The endpoint returns a JSON array of [Prometheus HTTP SD target groups](https://prometheus.io/docs/prometheus/latest/http_sd/): + +```json +[ + { + "targets": ["10.0.1.1:9100"], + "labels": { + "__meta_sonic_switch_name": "leaf-1", + "__meta_sonic_switch_mac": "aa:bb:cc:dd:ee:ff", + "__meta_sonic_switch_sku": "Accton-AS7726-32X", + "__meta_sonic_switch_firmware": "11" + } + }, + { + "targets": ["[2001:db8::1]:9100"], + "labels": { + "__meta_sonic_switch_name": "spine-1", + "__meta_sonic_switch_mac": "11:22:33:44:55:66", + "__meta_sonic_switch_sku": "Accton-AS7726-32X", + "__meta_sonic_switch_firmware": "11" + } + } +] +``` + +IPv6 addresses are automatically wrapped in brackets as required by the Prometheus target format. + +## Available meta labels + +These labels are provided to the collector as discovery metadata. They are **not** automatically attached to scraped metrics — only labels promoted via `relabel_configs` become metric labels. This allows operators to choose the level of detail they need without inflating cardinality by default. + +| Label | Description | Always present | +|-------|-------------|:-:| +| `__meta_sonic_switch_name` | Name of the `Switch` resource | yes | +| `__meta_sonic_switch_mac` | MAC address | no | +| `__meta_sonic_switch_sku` | Hardware SKU | no | +| `__meta_sonic_switch_firmware` | Firmware version | no | + +## Operator configuration + +The SD endpoint is registered on the operator's metrics server. Enable the metrics server with: + +``` +--metrics-bind-address=:8443 +``` + +The endpoint is then available at `https://:8443/switch-sd`. For unsecured HTTP access (e.g. inside a trusted cluster network): + +``` +--metrics-bind-address=:8080 --metrics-secure=false +``` + +## Scrape configuration + +Any Prometheus-compatible scraper (Prometheus, VictoriaMetrics, Grafana Agent, etc.) can use the endpoint with the standard `http_sd_configs` directive: + +```yaml +scrape_configs: + - job_name: sonic-switches + http_sd_configs: + - url: http://sonic-operator.sonic-operator-system:8080/switch-sd + refresh_interval: 1m + relabel_configs: + - source_labels: [__meta_sonic_switch_name] + target_label: switch +``` + +The `relabel_configs` block copies the switch name into a `switch` label on all scraped metrics, making it easy to filter and group by switch in dashboards. + +## Target lifecycle + +- A switch appears as a target when its `status.state` becomes `Ready` and `spec.management.host` is set. +- A switch is removed from the target list when it is no longer `Ready` (e.g. during provisioning, on failure, or after deletion). +- Prometheus and vmagent poll the SD endpoint periodically (`refresh_interval`, default 1 minute) and automatically add or remove targets. + +## Extracting additional labels + +You can promote any meta label to a target label using `relabel_configs`: + +```yaml +relabel_configs: + - source_labels: [__meta_sonic_switch_name] + target_label: switch + - source_labels: [__meta_sonic_switch_sku] + target_label: hardware + - source_labels: [__meta_sonic_switch_firmware] + target_label: firmware +``` diff --git a/internal/agent/agent_client/client/client.go b/internal/agent/agent_client/client/client.go index fde3e0b..e1bab94 100644 --- a/internal/agent/agent_client/client/client.go +++ b/internal/agent/agent_client/client/client.go @@ -61,14 +61,7 @@ func NewDefaultSwitchAgentClient(address string, connectTimeout time.Duration) ( } func (c *defaultSwitchAgentClient) dial() (func() error, error) { - println("connect to ", c.Address) - - conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) - - // conn, err := grpc.DialContext(dialCtx, c.Address, - // grpc.WithTransportCredentials(insecure.NewCredentials()), - // grpc.WithBlock(), // Wait for connection to be ready - // ) + conn, err := grpc.NewClient("passthrough:///"+c.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, fmt.Errorf("failed to connect to switch proxy: %w", err) } diff --git a/internal/agent/agent_server/server.go b/internal/agent/agent_server/server.go index a732cba..216fe61 100644 --- a/internal/agent/agent_server/server.go +++ b/internal/agent/agent_server/server.go @@ -296,7 +296,7 @@ func NewProxyServer(switchAgentImpl switchAgent.SwitchAgent) pb.SwitchAgentServi func StartServer() { flag.Parse() - lis, err := net.Listen("tcp4", fmt.Sprintf("0.0.0.0:%d", *port)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) if err != nil { log.Fatalf("failed to listen: %v", err) } @@ -310,9 +310,9 @@ func StartServer() { } // Start Prometheus metrics HTTP server - metricsSrv := metrics.NewMetricsServer(fmt.Sprintf("0.0.0.0:%d", *metricsPort), swAgent, sonic.GetSonicVersionInfo, *metricsConfig) + metricsSrv := metrics.NewMetricsServer(fmt.Sprintf(":%d", *metricsPort), swAgent, sonic.GetSonicVersionInfo, *metricsConfig) go func() { - log.Printf("metrics server listening at 0.0.0.0:%d", *metricsPort) + log.Printf("metrics server listening at :%d", *metricsPort) if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("metrics server failed: %v", err) } diff --git a/internal/agent/metrics/config_collector.go b/internal/agent/metrics/config_collector.go index 6b1e993..eb6e38c 100644 --- a/internal/agent/metrics/config_collector.go +++ b/internal/agent/metrics/config_collector.go @@ -44,9 +44,10 @@ func NewConfigCollector(connector RedisConnector, mapping MetricMapping) *Config if f.Transform != nil && f.Transform.RegexCapture != nil { re := regexp.MustCompile(f.Transform.RegexCapture.Pattern) compiledRegex[i] = re - // Extract label names from named capture groups for _, name := range re.SubexpNames()[1:] { - labels = appendUnique(labels, name) + if name != "" { + labels = appendUnique(labels, name) + } } } descs[f.Metric] = prometheus.NewDesc(f.Metric, f.Help, labels, nil) @@ -216,7 +217,11 @@ func (c *ConfigCollector) collectFieldEntry( if m == nil { return // field doesn't match, skip } - captureLabels = append(captureLabels, m[1:]...) + for i, name := range re.SubexpNames()[1:] { + if name != "" { + captureLabels = append(captureLabels, m[i+1]) + } + } } // Handle parse_threshold_field transform diff --git a/internal/sd/sd.go b/internal/sd/sd.go new file mode 100644 index 0000000..b3705a3 --- /dev/null +++ b/internal/sd/sd.go @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package sd + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "sync" + "time" + + networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + metricsPort = 9100 + listTimeout = 10 * time.Second + cacheTTL = 10 * time.Second +) + +// TargetGroup is a Prometheus HTTP SD target group. +type TargetGroup struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} + +// NewHandler returns an http.Handler that serves Prometheus HTTP SD target +// groups for all Ready switches. Responses are cached for 10s to bound +// Kubernetes API load when many Prometheus instances scrape concurrently. +func NewHandler(c client.Reader) http.Handler { + return &handler{client: c} +} + +type handler struct { + client client.Reader + + mu sync.Mutex + cached []byte + cachedAt time.Time +} + +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + if time.Since(h.cachedAt) < cacheTTL && h.cached != nil { + data := h.cached + h.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(data) + return + } + h.mu.Unlock() + + ctx, cancel := context.WithTimeout(r.Context(), listTimeout) + defer cancel() + + var switches networkingv1alpha1.SwitchList + if err := h.client.List(ctx, &switches); err != nil { + log.Printf("switch-sd: failed to list switches: %v", err) + http.Error(w, "failed to list switches", http.StatusInternalServerError) + return + } + + groups := buildTargetGroups(switches.Items) + + data, err := json.Marshal(groups) + if err != nil { + log.Printf("switch-sd: failed to encode response: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + h.mu.Lock() + h.cached = data + h.cachedAt = time.Now() + h.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(data) +} + +func buildTargetGroups(switches []networkingv1alpha1.Switch) []TargetGroup { + groups := make([]TargetGroup, 0, len(switches)) + for _, sw := range switches { + if sw.Status.State != networkingv1alpha1.SwitchStateReady { + continue + } + if sw.Spec.Management.Host == "" { + continue + } + + labels := map[string]string{ + "__meta_sonic_switch_name": sw.Name, + } + if sw.Status.MACAddress != "" { + labels["__meta_sonic_switch_mac"] = sw.Status.MACAddress + } + if sw.Status.SKU != "" { + labels["__meta_sonic_switch_sku"] = sw.Status.SKU + } + if sw.Status.FirmwareVersion != "" { + labels["__meta_sonic_switch_firmware"] = sw.Status.FirmwareVersion + } + + groups = append(groups, TargetGroup{ + Targets: []string{net.JoinHostPort(sw.Spec.Management.Host, fmt.Sprintf("%d", metricsPort))}, + Labels: labels, + }) + } + return groups +} diff --git a/internal/sd/sd_test.go b/internal/sd/sd_test.go new file mode 100644 index 0000000..888af23 --- /dev/null +++ b/internal/sd/sd_test.go @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package sd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newScheme() *runtime.Scheme { + s := runtime.NewScheme() + utilruntime.Must(networkingv1alpha1.AddToScheme(s)) + return s +} + +func TestHandler_NoSwitches(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newScheme()).Build() + h := NewHandler(c) + + req := httptest.NewRequest(http.MethodGet, "/switch-sd", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", rec.Code) + } + + var groups []TargetGroup + if err := json.NewDecoder(rec.Body).Decode(&groups); err != nil { + t.Fatalf("decode: %v", err) + } + if len(groups) != 0 { + t.Fatalf("expected 0 groups, got %d", len(groups)) + } +} + +func TestHandler_ReadySwitchReturned(t *testing.T) { + sw := &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "leaf1", Namespace: "default"}, + Spec: networkingv1alpha1.SwitchSpec{ + Management: networkingv1alpha1.Management{Host: "10.0.0.1"}, + }, + Status: networkingv1alpha1.SwitchStatus{ + State: networkingv1alpha1.SwitchStateReady, + MACAddress: "aa:bb:cc:dd:ee:ff", + SKU: "AS7726-32X", + FirmwareVersion: "4.2.0", + }, + } + + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(sw).WithStatusSubresource(sw).Build() + // Set status after creation since fake client doesn't persist status on create. + sw.Status = networkingv1alpha1.SwitchStatus{ + State: networkingv1alpha1.SwitchStateReady, + MACAddress: "aa:bb:cc:dd:ee:ff", + SKU: "AS7726-32X", + FirmwareVersion: "4.2.0", + } + if err := c.Status().Update(context.Background(), sw); err != nil { + t.Fatalf("status update: %v", err) + } + + h := NewHandler(c) + req := httptest.NewRequest(http.MethodGet, "/switch-sd", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", rec.Code) + } + + var groups []TargetGroup + if err := json.NewDecoder(rec.Body).Decode(&groups); err != nil { + t.Fatalf("decode: %v", err) + } + if len(groups) != 1 { + t.Fatalf("expected 1 group, got %d", len(groups)) + } + if groups[0].Targets[0] != "10.0.0.1:9100" { + t.Errorf("expected target 10.0.0.1:9100, got %s", groups[0].Targets[0]) + } + if groups[0].Labels["__meta_sonic_switch_name"] != "leaf1" { + t.Errorf("expected label leaf1, got %s", groups[0].Labels["__meta_sonic_switch_name"]) + } + if groups[0].Labels["__meta_sonic_switch_mac"] != "aa:bb:cc:dd:ee:ff" { + t.Errorf("expected mac label, got %s", groups[0].Labels["__meta_sonic_switch_mac"]) + } +} + +func TestHandler_PendingSwitchExcluded(t *testing.T) { + sw := &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "leaf2", Namespace: "default"}, + Spec: networkingv1alpha1.SwitchSpec{ + Management: networkingv1alpha1.Management{Host: "10.0.0.2"}, + }, + Status: networkingv1alpha1.SwitchStatus{ + State: networkingv1alpha1.SwitchStatePending, + }, + } + + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(sw).WithStatusSubresource(sw).Build() + sw.Status = networkingv1alpha1.SwitchStatus{State: networkingv1alpha1.SwitchStatePending} + if err := c.Status().Update(context.Background(), sw); err != nil { + t.Fatalf("status update: %v", err) + } + + h := NewHandler(c) + req := httptest.NewRequest(http.MethodGet, "/switch-sd", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + var groups []TargetGroup + if err := json.NewDecoder(rec.Body).Decode(&groups); err != nil { + t.Fatalf("decode: %v", err) + } + if len(groups) != 0 { + t.Fatalf("expected 0 groups for non-ready switch, got %d", len(groups)) + } +} + +func TestHandler_IPv6Target(t *testing.T) { + sw := &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "spine1", Namespace: "default"}, + Spec: networkingv1alpha1.SwitchSpec{ + Management: networkingv1alpha1.Management{Host: "2001:db8::1"}, + }, + Status: networkingv1alpha1.SwitchStatus{ + State: networkingv1alpha1.SwitchStateReady, + }, + } + + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(sw).WithStatusSubresource(sw).Build() + sw.Status = networkingv1alpha1.SwitchStatus{State: networkingv1alpha1.SwitchStateReady} + if err := c.Status().Update(context.Background(), sw); err != nil { + t.Fatalf("status update: %v", err) + } + + h := NewHandler(c) + req := httptest.NewRequest(http.MethodGet, "/switch-sd", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + var groups []TargetGroup + if err := json.NewDecoder(rec.Body).Decode(&groups); err != nil { + t.Fatalf("decode: %v", err) + } + if len(groups) != 1 { + t.Fatalf("expected 1 group, got %d", len(groups)) + } + if groups[0].Targets[0] != "[2001:db8::1]:9100" { + t.Errorf("expected target [2001:db8::1]:9100, got %s", groups[0].Targets[0]) + } +} diff --git a/internal/switch_util/switch_util.go b/internal/switch_util/switch_util.go index ccbffdf..6f13ab3 100644 --- a/internal/switch_util/switch_util.go +++ b/internal/switch_util/switch_util.go @@ -5,6 +5,7 @@ package switchutil import ( "context" + "net" networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" v1 "k8s.io/api/core/v1" @@ -22,7 +23,7 @@ func NewAgentClientForSwitch(ctx context.Context, s *networkingv1alpha1.Switch) return agentcli, err } - address := s.Spec.Management.Host + ":" + s.Spec.Management.Port + address := net.JoinHostPort(s.Spec.Management.Host, s.Spec.Management.Port) agentcli, err := agentCli.NewDefaultSwitchAgentClient(address, 0) if err != nil {