Add Prometheus HTTP Service Discovery for switches

cmd/main.go

@@ -28,6 +28,7 @@ import (
 	networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1"
 	"github.com/ironcore-dev/sonic-operator/internal/controller"
 	"github.com/ironcore-dev/sonic-operator/internal/onie"
+	"github.com/ironcore-dev/sonic-operator/internal/sd"
 	"github.com/ironcore-dev/sonic-operator/internal/ztp"
 	// +kubebuilder:scaffold:imports
 )
@@ -208,6 +209,10 @@ func main() {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
+	if err := mgr.AddMetricsServerExtraHandler("/switch-sd", sd.NewHandler(mgr.GetClient())); err != nil {
+		setupLog.Error(err, "unable to register switch-sd handler")
+		os.Exit(1)
+	}
 	if !disableProvisionsingServer {
 		setupLog.Info("starting HTTP server")
 		provServer, err := setupProvisioningServer(httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile)

docs/.vitepress/config.mts

@@ -76,7 +76,14 @@ export default withMermaid({
           { text: 'Getting started', link: '/usage/getting-started' },
           { text: 'Provisioning', link: '/usage/provisioning' },
           { text: 'Agent', link: '/usage/agent' },
-          { text: 'Agent Metrics', link: '/usage/metrics' },
+          { text: 'Metrics',
+            collapsed: false,
+            items: [
+              { text: 'Operator Metrics', link: '/usage/operator-metrics' },
+              { text: 'Per-Switch Metrics', link: '/usage/metrics' },
+              { text: 'Switch Metrics Discovery', link: '/usage/service-discovery' },
+            ]
+          },
         ]
       },
       {

docs/usage/metrics.md

@@ -1,4 +1,4 @@
-# Agent metrics
+# Per-Switch Metrics
 
 The switch agent exposes a Prometheus-compatible `/metrics` endpoint for monitoring switch health, interface state, and transceiver optics. Metrics are collected just-in-time from SONiC Redis on every Prometheus scrape — there is no background polling or caching.
 
@@ -34,7 +34,6 @@ These require custom logic (cross-database joins, aggregate counting, error fall
 | `sonic_switch_interface_admin_state` | gauge | `interface` | Admin state (1=up, 0=down) |
 | `sonic_switch_interfaces_total` | gauge | `operational_status` | Number of interfaces by status |
 | `sonic_switch_ports_total` | gauge | — | Total physical ports |
-| `sonic_scrape_duration_seconds` | gauge | — | Duration of the last metrics scrape |
 
 ### Config-driven collectors
 

docs/usage/operator-metrics.md

@@ -0,0 +1,69 @@
+# Operator Metrics
+
+The sonic-operator exposes standard [controller-runtime](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/metrics) metrics on its metrics server. These provide insight into the reconciliation performance and health of the operator itself.
+
+## Endpoint
+
+The metrics server is configured with `--metrics-bind-address`. By default it is disabled (`0`). To enable:
+
+```
+--metrics-bind-address=:8443                   # HTTPS (default when non-zero)
+--metrics-bind-address=:8080 --metrics-secure=false  # HTTP
+```
+
+Metrics are served at `/metrics` on the configured port.
+
+## Available metrics
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `controller_runtime_reconcile_total` | counter | `controller`, `result` | Total reconciliations per controller |
+| `controller_runtime_reconcile_errors_total` | counter | `controller` | Reconciliation errors per controller |
+| `controller_runtime_terminal_reconcile_errors_total` | counter | `controller` | Terminal (non-retryable) errors per controller |
+| `controller_runtime_reconcile_panics_total` | counter | `controller` | Reconciliation panics per controller |
+| `controller_runtime_reconcile_time_seconds` | histogram | `controller` | Reconciliation duration per controller |
+| `controller_runtime_active_workers` | gauge | `controller` | Currently active workers per controller |
+| `controller_runtime_max_concurrent_reconciles` | gauge | `controller` | Maximum concurrent reconciles per controller |
+
+The `controller` label identifies which controller produced the metric (e.g. `switch`, `switchinterface`).
+
+## Scrape configuration
+
+The operator metrics server also hosts the [Metrics Discovery](/usage/service-discovery) endpoint at `/switch-sd`. You can scrape both the operator and the discovered switches from the same Prometheus job configuration:
+
+```yaml
+scrape_configs:
+  # Operator itself
+  - job_name: sonic-operator
+    static_configs:
+      - targets: ["sonic-operator.sonic-operator-system:8080"]
+
+  # Switches (auto-discovered)
+  - job_name: sonic-switches
+    http_sd_configs:
+      - url: http://sonic-operator.sonic-operator-system:8080/switch-sd
+    relabel_configs:
+      - source_labels: [__meta_sonic_switch_name]
+        target_label: switch
+```
+
+## Useful queries
+
+Reconciliation rate per controller:
+
+```promql
+rate(controller_runtime_reconcile_total[5m])
+```
+
+Error ratio:
+
+```promql
+rate(controller_runtime_reconcile_errors_total[5m])
+/ rate(controller_runtime_reconcile_total[5m])
+```
+
+p99 reconciliation latency:
+
+```promql
+histogram_quantile(0.99, rate(controller_runtime_reconcile_time_seconds_bucket[5m]))
+```

docs/usage/service-discovery.md

@@ -0,0 +1,108 @@
+# Switch Metrics Discovery
+
+The sonic-operator exposes an HTTP Service Discovery (SD) endpoint that enables Prometheus and compatible scrapers to automatically discover and scrape metrics from all ready switches. This eliminates the need to maintain a static list of scrape targets.
+
+## How it works
+
+The operator watches all `Switch` resources in the cluster. Switches that have reached the `Ready` state and have a management address configured are served as scrape targets via the `/switch-sd` endpoint on the operator's metrics server.
+
+```mermaid
+graph TD
+    Prometheus[Metrics Collector<br/>e.g. Prometheus] -->|GET /switch-sd| Operator[sonic-operator<br/>metrics server]
+    Operator -->|watches| CRs[Switch CRs<br/>Kubernetes API]
+    Operator -->|"[{targets, labels}, ...]"| Prometheus
+    Prometheus -->|"scrape<br/>:9100/metrics"| Switch1[switch-1]
+    Prometheus -->|"scrape<br/>:9100/metrics"| Switch2[switch-2]
+    Prometheus -->|"scrape<br/>:9100/metrics"| SwitchN[switch-N]
+```
+
+## Response format
+
+The endpoint returns a JSON array of [Prometheus HTTP SD target groups](https://prometheus.io/docs/prometheus/latest/http_sd/):
+
+```json
+[
+  {
+    "targets": ["10.0.1.1:9100"],
+    "labels": {
+      "__meta_sonic_switch_name": "leaf-1",
+      "__meta_sonic_switch_mac": "aa:bb:cc:dd:ee:ff",
+      "__meta_sonic_switch_sku": "Accton-AS7726-32X",
+      "__meta_sonic_switch_firmware": "11"
+    }
+  },
+  {
+    "targets": ["[2001:db8::1]:9100"],
+    "labels": {
+      "__meta_sonic_switch_name": "spine-1",
+      "__meta_sonic_switch_mac": "11:22:33:44:55:66",
+      "__meta_sonic_switch_sku": "Accton-AS7726-32X",
+      "__meta_sonic_switch_firmware": "11"
+    }
+  }
+]
+```
+
+IPv6 addresses are automatically wrapped in brackets as required by the Prometheus target format.
+
+## Available meta labels
+
+These labels are provided to the collector as discovery metadata. They are **not** automatically attached to scraped metrics — only labels promoted via `relabel_configs` become metric labels. This allows operators to choose the level of detail they need without inflating cardinality by default.
+
+| Label | Description | Always present |
+|-------|-------------|:-:|
+| `__meta_sonic_switch_name` | Name of the `Switch` resource | yes |
+| `__meta_sonic_switch_mac` | MAC address | no |
+| `__meta_sonic_switch_sku` | Hardware SKU | no |
+| `__meta_sonic_switch_firmware` | Firmware version | no |
+
+## Operator configuration
+
+The SD endpoint is registered on the operator's metrics server. Enable the metrics server with:
+
+```
+--metrics-bind-address=:8443
+```
+
+The endpoint is then available at `https://<operator>:8443/switch-sd`. For unsecured HTTP access (e.g. inside a trusted cluster network):
+
+```
+--metrics-bind-address=:8080 --metrics-secure=false
+```
+
+## Scrape configuration
+
+Any Prometheus-compatible scraper (Prometheus, VictoriaMetrics, Grafana Agent, etc.) can use the endpoint with the standard `http_sd_configs` directive:
+
+```yaml
+scrape_configs:
+  - job_name: sonic-switches
+    http_sd_configs:
+      - url: http://sonic-operator.sonic-operator-system:8080/switch-sd
+        refresh_interval: 1m
+    relabel_configs:
+      - source_labels: [__meta_sonic_switch_name]
+        target_label: switch
+```
+
+The `relabel_configs` block copies the switch name into a `switch` label on all scraped metrics, making it easy to filter and group by switch in dashboards.
+
+## Target lifecycle
+
+- A switch appears as a target when its `status.state` becomes `Ready` and `spec.management.host` is set.
+- A switch is removed from the target list when it is no longer `Ready` (e.g. during provisioning, on failure, or after deletion).
+- Prometheus and vmagent poll the SD endpoint periodically (`refresh_interval`, default 1 minute) and automatically add or remove targets.
+
+## Extracting additional labels
+
+You can promote any meta label to a target label using `relabel_configs`:
+
+```yaml
+relabel_configs:
+  - source_labels: [__meta_sonic_switch_name]
+    target_label: switch
+  - source_labels: [__meta_sonic_switch_sku]
+    target_label: hardware
+  - source_labels: [__meta_sonic_switch_firmware]
+    target_label: firmware
+```

internal/agent/agent_client/client/client.go

@@ -61,14 +61,7 @@ func NewDefaultSwitchAgentClient(address string, connectTimeout time.Duration) (
 }
 
 func (c *defaultSwitchAgentClient) dial() (func() error, error) {
-	println("connect to ", c.Address)
-
-	conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
-
-	// conn, err := grpc.DialContext(dialCtx, c.Address,
-	// 	grpc.WithTransportCredentials(insecure.NewCredentials()),
-	// 	grpc.WithBlock(), // Wait for connection to be ready
-	// )
+	conn, err := grpc.NewClient("passthrough:///"+c.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to switch proxy: %w", err)
 	}

internal/agent/agent_server/server.go

@@ -296,7 +296,7 @@ func NewProxyServer(switchAgentImpl switchAgent.SwitchAgent) pb.SwitchAgentServi
 func StartServer() {
 	flag.Parse()
 
-	lis, err := net.Listen("tcp4", fmt.Sprintf("0.0.0.0:%d", *port))
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
 	}
@@ -310,9 +310,9 @@ func StartServer() {
 	}
 
 	// Start Prometheus metrics HTTP server
-	metricsSrv := metrics.NewMetricsServer(fmt.Sprintf("0.0.0.0:%d", *metricsPort), swAgent, sonic.GetSonicVersionInfo, *metricsConfig)
+	metricsSrv := metrics.NewMetricsServer(fmt.Sprintf(":%d", *metricsPort), swAgent, sonic.GetSonicVersionInfo, *metricsConfig)
 	go func() {
-		log.Printf("metrics server listening at 0.0.0.0:%d", *metricsPort)
+		log.Printf("metrics server listening at :%d", *metricsPort)
 		if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
 			log.Fatalf("metrics server failed: %v", err)
 		}

internal/agent/metrics/config_collector.go

@@ -44,9 +44,10 @@ func NewConfigCollector(connector RedisConnector, mapping MetricMapping) *Config
 		if f.Transform != nil && f.Transform.RegexCapture != nil {
 			re := regexp.MustCompile(f.Transform.RegexCapture.Pattern)
 			compiledRegex[i] = re
-			// Extract label names from named capture groups
 			for _, name := range re.SubexpNames()[1:] {
-				labels = appendUnique(labels, name)
+				if name != "" {
+					labels = appendUnique(labels, name)
+				}
 			}
 		}
 		descs[f.Metric] = prometheus.NewDesc(f.Metric, f.Help, labels, nil)
@@ -216,7 +217,11 @@ func (c *ConfigCollector) collectFieldEntry(
 		if m == nil {
 			return // field doesn't match, skip
 		}
-		captureLabels = append(captureLabels, m[1:]...)
+		for i, name := range re.SubexpNames()[1:] {
+			if name != "" {
+				captureLabels = append(captureLabels, m[i+1])
+			}
+		}
 	}
 
 	// Handle parse_threshold_field transform