diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 10ff5c45f..a7370de76 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -41,4 +41,23 @@ spec: - name: nova-flavors - name: identity-projects - name: identity-domains +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI +metadata: + name: multicluster-object-count +spec: + schedulingDomain: nova + impl: multicluster_object_count_kpi + opts: + gvks: + - cortex.cloud/v1alpha1/HistoryList + - cortex.cloud/v1alpha1/ReservationList + - cortex.cloud/v1alpha1/CommittedResourceList + - kvm.cloud.sap/v1/HypervisorList + description: | + This KPI reports the number of objects of each configured GVK per remote + cluster, labelled by the configured cluster labels. It uses + PartialObjectMetadataList so only object metadata is fetched, making it + efficient even for large object counts. {{- end }} \ No newline at end of file diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go new file mode 100644 index 000000000..0749e02d2 --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go @@ -0,0 +1,166 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "fmt" + "log/slog" + "strings" + "unicode" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MulticlusterObjectCountKPIOpts configures which GVKs to count per cluster. +// Each entry is a "group/version/Kind" string, e.g. +// "cortex.cloud/v1alpha1/HypervisorList". +type MulticlusterObjectCountKPIOpts struct { + GVKs []string `json:"gvks"` +} + +type gvkDesc struct { + gvk schema.GroupVersionKind + labelKeys []string // snake_cased routing label keys (stable order) + desc *prometheus.Desc +} + +// multiclusterReader is the subset of *multicluster.Client this KPI needs. +// Keeping it narrow lets the KPI logic be unit-tested without wiring real or +// fake clusters into a multicluster.Client. +type multiclusterReader interface { + ConfiguredRouteLabels(gvk schema.GroupVersionKind) []map[string]string + ListMetadataPerCluster(ctx context.Context, gvk schema.GroupVersionKind, opts ...client.ListOption) ([]multicluster.ClusterObjectMetadata, error) +} + +// MulticlusterObjectCountKPI reports the number of objects of each configured +// GVK per remote cluster, labelled by the cluster's routing labels. +type MulticlusterObjectCountKPI struct { + plugins.BaseKPI[MulticlusterObjectCountKPIOpts] + mcl multiclusterReader + descs []gvkDesc +} + +func (MulticlusterObjectCountKPI) GetName() string { return "multicluster_object_count_kpi" } + +func (k *MulticlusterObjectCountKPI) Init(_ *db.DB, c client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(nil, c, opts); err != nil { + return err + } + mcl, ok := c.(*multicluster.Client) + if !ok { + return fmt.Errorf("multicluster_object_count_kpi requires a *multicluster.Client, got %T", c) + } + k.mcl = mcl + + for _, raw := range k.Options.GVKs { + gvk, err := parseGVK(raw) + if err != nil { + return fmt.Errorf("invalid GVK %q: %w", raw, err) + } + routeLabels := mcl.ConfiguredRouteLabels(gvk) + // Each remote cluster is registered with a routing label map (e.g. + // {"availabilityZone": "eu-de-1a"}). We collect the union of all key names + // across every remote cluster and snake_case them to produce Prometheus label + // names (availabilityZone → availability_zone). These become variable labels + // on the descriptor so each cluster gets its own time series. + keySet := map[string]bool{} + var labelKeys []string + for _, lm := range routeLabels { + for k := range lm { + snake := toSnakeCase(k) + if !keySet[snake] { + keySet[snake] = true + labelKeys = append(labelKeys, snake) + } + } + } + // Fixed labels: group/version/kind identify the resource type; is_home + // distinguishes the home cluster (no routing labels) from remote clusters. + // The routing label keys follow (e.g. availability_zone). + varLabels := append([]string{"group", "version", "kind", "is_home"}, labelKeys...) + desc := prometheus.NewDesc( + "cortex_multicluster_object_count", + "Number of objects of a given GVK per cluster", + varLabels, + nil, + ) + k.descs = append(k.descs, gvkDesc{gvk: gvk, labelKeys: labelKeys, desc: desc}) + } + return nil +} + +func (k *MulticlusterObjectCountKPI) Describe(ch chan<- *prometheus.Desc) { + for _, d := range k.descs { + ch <- d.desc + } +} + +func (k *MulticlusterObjectCountKPI) Collect(ch chan<- prometheus.Metric) { + ctx := context.Background() + for _, d := range k.descs { + perCluster, err := k.mcl.ListMetadataPerCluster(ctx, d.gvk) + if err != nil { + slog.Error("multicluster_object_count_kpi: failed to list object metadata", + "gvk", d.gvk, "err", err) + continue + } + for _, c := range perCluster { + isHome := "false" + if c.IsHome { + isHome = "true" + } + // Label values must match the descriptor order: group, version, kind, + // is_home, then one value per routing label key. For remote clusters the + // routing label value comes from the cluster's registration labels (e.g. + // Labels["availabilityZone"] → label value for "availability_zone"). The + // home cluster has no routing labels, so those positions are empty strings. + labelVals := make([]string, 0, 4+len(d.labelKeys)) + labelVals = append(labelVals, d.gvk.Group, d.gvk.Version, d.gvk.Kind, isHome) + for _, key := range d.labelKeys { + labelVals = append(labelVals, labelValueForSnakeKey(key, c.Labels)) + } + ch <- prometheus.MustNewConstMetric(d.desc, prometheus.GaugeValue, + float64(len(c.Items)), labelVals...) + } + } +} + +// parseGVK parses a "group/version/Kind" string. +func parseGVK(s string) (schema.GroupVersionKind, error) { + parts := strings.SplitN(s, "/", 3) + if len(parts) != 3 { + return schema.GroupVersionKind{}, fmt.Errorf("expected group/version/Kind, got: %s", s) + } + return schema.GroupVersionKind{Group: parts[0], Version: parts[1], Kind: parts[2]}, nil +} + +// toSnakeCase converts camelCase to snake_case (e.g. availabilityZone → availability_zone). +func toSnakeCase(s string) string { + var b strings.Builder + for i, r := range s { + if unicode.IsUpper(r) && i > 0 { + b.WriteByte('_') + } + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + +// labelValueForSnakeKey finds the value in labels whose key, when snake_cased, +// matches snakeKey. Returns empty string for nil labels or no match (home cluster). +func labelValueForSnakeKey(snakeKey string, labels map[string]string) string { + for k, v := range labels { + if toSnakeCase(k) == snakeKey { + return v + } + } + return "" +} diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go new file mode 100644 index 000000000..fa9b6576b --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go @@ -0,0 +1,224 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// fakeReader is a stub multiclusterReader that returns canned per-cluster +// metadata. It lets the KPI be tested without any real or fake clusters. +type fakeReader struct { + routeLabels []map[string]string + perCluster []multicluster.ClusterObjectMetadata + err error +} + +func (f *fakeReader) ConfiguredRouteLabels(schema.GroupVersionKind) []map[string]string { + return f.routeLabels +} + +func (f *fakeReader) ListMetadataPerCluster(context.Context, schema.GroupVersionKind, ...client.ListOption) ([]multicluster.ClusterObjectMetadata, error) { + return f.perCluster, f.err +} + +// items builds n PartialObjectMetadata entries to represent a count of n. +func items(n int) []metav1.PartialObjectMetadata { + out := make([]metav1.PartialObjectMetadata, n) + return out +} + +func TestMulticlusterObjectCountKPI_GetName(t *testing.T) { + kpi := &MulticlusterObjectCountKPI{} + if got := kpi.GetName(); got != "multicluster_object_count_kpi" { + t.Errorf("GetName() = %q, want %q", got, "multicluster_object_count_kpi") + } +} + +func TestMulticlusterObjectCountKPI_Init_RejectsNonMCLClient(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatal(err) + } + plainClient := fake.NewClientBuilder().WithScheme(scheme).Build() + kpi := &MulticlusterObjectCountKPI{} + if err := kpi.Init(nil, plainClient, conf.NewRawOpts(`{"gvks":[]}`)); err == nil { + t.Fatal("expected error when passing a non-*multicluster.Client") + } +} + +// initWithReader builds a KPI whose descriptors are derived from the given +// route labels, bypassing Init's *multicluster.Client type assertion so the +// KPI logic can be tested against a fakeReader. +func initWithReader(t *testing.T, r *fakeReader, gvkStr string) *MulticlusterObjectCountKPI { + t.Helper() + kpi := &MulticlusterObjectCountKPI{} + kpi.mcl = r + gvk, err := parseGVK(gvkStr) + if err != nil { + t.Fatalf("parseGVK: %v", err) + } + keySet := map[string]bool{} + var labelKeys []string + for _, lm := range r.routeLabels { + for k := range lm { + snake := toSnakeCase(k) + if !keySet[snake] { + keySet[snake] = true + labelKeys = append(labelKeys, snake) + } + } + } + varLabels := append([]string{"group", "version", "kind", "is_home"}, labelKeys...) + desc := prometheus.NewDesc( + "cortex_multicluster_object_count", + "Number of objects of a given GVK per cluster", + varLabels, + nil, + ) + kpi.descs = append(kpi.descs, gvkDesc{gvk: gvk, labelKeys: labelKeys, desc: desc}) + return kpi +} + +func TestMulticlusterObjectCountKPI_Init_RejectsInvalidGVK(t *testing.T) { + kpi := &MulticlusterObjectCountKPI{} + // A *multicluster.Client with no configured GVKs still passes the type + // assertion; the invalid GVK string must be rejected during parsing. + mcl := &multicluster.Client{} + opts := conf.NewRawOpts(`{"gvks":["not-a-valid-gvk"]}`) + if err := kpi.Init(nil, mcl, opts); err == nil { + t.Fatal("expected error for invalid GVK string") + } +} + +func TestMulticlusterObjectCountKPI_Describe(t *testing.T) { + r := &fakeReader{routeLabels: []map[string]string{{"availabilityZone": "az-1"}}} + kpi := initWithReader(t, r, "/v1/ConfigMapList") + + ch := make(chan *prometheus.Desc, 5) + kpi.Describe(ch) + close(ch) + var count int + for range ch { + count++ + } + if count != 1 { + t.Errorf("Describe: expected 1 descriptor, got %d", count) + } +} + +func TestMulticlusterObjectCountKPI_Collect(t *testing.T) { + r := &fakeReader{ + routeLabels: []map[string]string{ + {"availabilityZone": "az-1"}, + {"availabilityZone": "az-2"}, + }, + perCluster: []multicluster.ClusterObjectMetadata{ + {Labels: map[string]string{"availabilityZone": "az-1"}, Items: items(2)}, + {Labels: map[string]string{"availabilityZone": "az-2"}, Items: items(1)}, + }, + } + kpi := initWithReader(t, r, "/v1/ConfigMapList") + + ch := make(chan prometheus.Metric, 10) + kpi.Collect(ch) + close(ch) + + // Collect into a map: availabilityZone -> count value. + byAZ := map[string]float64{} + for m := range ch { + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + var az, isHome string + for _, lp := range metric.Label { + switch lp.GetName() { + case "availability_zone": + az = lp.GetValue() + case "is_home": + isHome = lp.GetValue() + } + } + if isHome != "false" { + t.Errorf("az %q: expected is_home=false for remote cluster, got %q", az, isHome) + } + byAZ[az] = metric.Gauge.GetValue() + } + + if len(byAZ) != 2 { + t.Fatalf("expected metrics for 2 clusters, got %d: %v", len(byAZ), byAZ) + } + if byAZ["az-1"] != 2 { + t.Errorf("az-1: expected count 2, got %g", byAZ["az-1"]) + } + if byAZ["az-2"] != 1 { + t.Errorf("az-2: expected count 1, got %g", byAZ["az-2"]) + } +} + +func TestMulticlusterObjectCountKPI_Collect_HomeCluster(t *testing.T) { + r := &fakeReader{ + perCluster: []multicluster.ClusterObjectMetadata{ + {Items: items(1), IsHome: true}, + }, + } + kpi := initWithReader(t, r, "/v1/ConfigMapList") + + ch := make(chan prometheus.Metric, 10) + kpi.Collect(ch) + close(ch) + + var collected int + for m := range ch { + collected++ + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + var isHome string + for _, lp := range metric.Label { + if lp.GetName() == "is_home" { + isHome = lp.GetValue() + } + } + if isHome != "true" { + t.Errorf("expected is_home=true for home cluster, got %q", isHome) + } + if got := metric.Gauge.GetValue(); got != 1 { + t.Errorf("expected count 1, got %g", got) + } + } + if collected != 1 { + t.Fatalf("expected 1 metric, got %d", collected) + } +} + +func TestMulticlusterObjectCountKPI_SnakeCaseLabels(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"availabilityZone", "availability_zone"}, + {"az", "az"}, + {"myLabelKey", "my_label_key"}, + {"alreadysnake", "alreadysnake"}, + } + for _, tt := range tests { + if got := toSnakeCase(tt.input); got != tt.want { + t.Errorf("toSnakeCase(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go index 155e3aab9..6a350d1e6 100644 --- a/internal/knowledge/kpis/supported_kpis.go +++ b/internal/knowledge/kpis/supported_kpis.go @@ -34,4 +34,6 @@ var supportedKPIs = map[string]plugins.KPI{ "decision_state_kpi": &deployment.DecisionStateKPI{}, "kpi_state_kpi": &deployment.KPIStateKPI{}, "pipeline_state_kpi": &deployment.PipelineStateKPI{}, + + "multicluster_object_count_kpi": &deployment.MulticlusterObjectCountKPI{}, } diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index 979258a44..f4450fab4 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -13,6 +13,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" @@ -448,6 +449,64 @@ func (c *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts return errors.New("apply operation is not supported in multicluster client") } +// ClusterObjectMetadata is one entry in the result of ListMetadataPerCluster. +// Labels holds the routing labels for the cluster. Items holds the object +// metadata returned by the cluster (no spec or status). IsHome is true for the +// home cluster, which has no routing labels. +type ClusterObjectMetadata struct { + Labels map[string]string + Items []metav1.PartialObjectMetadata + IsHome bool +} + +// ListMetadataPerCluster returns the object metadata of the given GVK for each +// configured cluster. It uses PartialObjectMetadataList so only object metadata +// crosses the wire — no spec or status — making it efficient even for large +// object counts. Callers that only need counts can use len(Items). Clusters +// that return an error are logged and skipped (same policy as List). The home +// cluster is included with IsHome set to true. +func (c *Client) ListMetadataPerCluster(ctx context.Context, gvk schema.GroupVersionKind, opts ...client.ListOption) ([]ClusterObjectMetadata, error) { + log := ctrl.LoggerFrom(ctx) + + c.remoteClustersMu.RLock() + remotes := c.remoteClusters[gvk] + isHome := c.homeGVKs[gvk] + if len(remotes) == 0 && !isHome { + c.remoteClustersMu.RUnlock() + return nil, fmt.Errorf("GVK %s is not configured in home or any remote cluster", gvk) + } + type clusterEntry struct { + cl cluster.Cluster + labels map[string]string + isHome bool + } + entries := make([]clusterEntry, 0, len(remotes)+1) + for _, r := range remotes { + entries = append(entries, clusterEntry{cl: r.cluster, labels: maps.Clone(r.labels)}) + } + if isHome && c.HomeCluster != nil { + entries = append(entries, clusterEntry{cl: c.HomeCluster, isHome: true}) + } + c.remoteClustersMu.RUnlock() + + results := make([]ClusterObjectMetadata, 0, len(entries)) + for _, e := range entries { + partialList := &metav1.PartialObjectMetadataList{} + partialList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + }) + if err := e.cl.GetClient().List(ctx, partialList, opts...); err != nil { + log.Error(err, "error listing resource metadata from cluster", + "gvk", gvk, "host", e.cl.GetConfig().Host) + continue + } + results = append(results, ClusterObjectMetadata{Labels: e.labels, Items: partialList.Items, IsHome: e.isHome}) + } + return results, nil +} + // Create routes the object to the matching cluster using the ResourceRouter // and performs a Create operation. func (c *Client) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 6e965d440..3af26f8c2 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -1726,3 +1726,106 @@ func TestClient_ConfiguredRouteLabels(t *testing.T) { } }) } + +func TestClient_ListMetadataPerCluster(t *testing.T) { + scheme := newTestScheme(t) + ctx := context.Background() + + t.Run("lists object metadata per remote cluster", func(t *testing.T) { + az1 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-1", Namespace: "default"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-2", Namespace: "default"}}, + ) + az2 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-3", Namespace: "default"}}, + ) + c := &Client{ + HomeScheme: scheme, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + configMapListGVK: { + {cluster: az1, labels: map[string]string{"availabilityZone": "az-1"}}, + {cluster: az2, labels: map[string]string{"availabilityZone": "az-2"}}, + }, + }, + } + results, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + byAZ := map[string]int{} + for _, r := range results { + if r.IsHome { + t.Errorf("expected IsHome=false for remote cluster %v", r.Labels) + } + byAZ[r.Labels["availabilityZone"]] = len(r.Items) + } + if byAZ["az-1"] != 2 { + t.Errorf("expected 2 objects in az-1, got %d", byAZ["az-1"]) + } + if byAZ["az-2"] != 1 { + t.Errorf("expected 1 object in az-2, got %d", byAZ["az-2"]) + } + }) + + t.Run("includes home cluster with IsHome set", func(t *testing.T) { + home := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-home", Namespace: "default"}}, + ) + c := &Client{ + HomeCluster: home, + HomeScheme: scheme, + homeGVKs: map[schema.GroupVersionKind]bool{configMapListGVK: true}, + } + results, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if !results[0].IsHome { + t.Errorf("expected IsHome=true for home cluster") + } + if results[0].Labels != nil { + t.Errorf("expected nil labels for home cluster, got %v", results[0].Labels) + } + if len(results[0].Items) != 1 { + t.Errorf("expected 1 item, got %d", len(results[0].Items)) + } + }) + + t.Run("returns error for unconfigured GVK", func(t *testing.T) { + c := &Client{ + HomeScheme: scheme, + } + _, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err == nil { + t.Fatal("expected error for unconfigured GVK") + } + }) + + t.Run("empty clusters return zero items", func(t *testing.T) { + az1 := newFakeCluster(scheme) // no objects + c := &Client{ + HomeScheme: scheme, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + configMapListGVK: { + {cluster: az1, labels: map[string]string{"availabilityZone": "az-1"}}, + }, + }, + } + results, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if len(results[0].Items) != 0 { + t.Errorf("expected 0 items, got %d", len(results[0].Items)) + } + }) +}