diff --git a/.github/actions/setup-claude-code-action/action.yml b/.github/actions/setup-claude-code-action/action.yml index de25bab99..382435339 100644 --- a/.github/actions/setup-claude-code-action/action.yml +++ b/.github/actions/setup-claude-code-action/action.yml @@ -39,7 +39,7 @@ runs: echo "$HOME/.bun/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.13" diff --git a/.github/workflows/claude-assistant.yaml b/.github/workflows/claude-assistant.yaml index 23596cec9..0511412d6 100644 --- a/.github/workflows/claude-assistant.yaml +++ b/.github/workflows/claude-assistant.yaml @@ -62,7 +62,7 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version-file: 'go.mod' - name: Generate GitHub App token diff --git a/.github/workflows/claude-weekly.yaml b/.github/workflows/claude-weekly.yaml index 7d0e8a0bb..538773f49 100644 --- a/.github/workflows/claude-weekly.yaml +++ b/.github/workflows/claude-weekly.yaml @@ -17,7 +17,7 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version-file: 'go.mod' - name: Generate GitHub App token diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 47b415216..cf302e9ca 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -29,7 +29,7 @@ jobs: - name: Check out code uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: check-latest: true go-version-file: 'go.mod' diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index a186be03c..b25b08a6b 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -23,7 +23,7 @@ jobs: - name: Set up Helm uses: azure/setup-helm@v5.0.1 - - uses: actions/setup-python@v6.3.0 + - uses: actions/setup-python@v7.0.0 with: python-version: "3.13" check-latest: true diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index efc720e3a..fbf7917d3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,7 +18,7 @@ jobs: - name: Check out code uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version-file: 'go.mod' - name: Run make crds deepcopy lint-fix diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 48c91cbfd..2a1ded7ba 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,7 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version-file: 'go.mod' - name: Run tests and calculate coverage diff --git a/.github/workflows/update-appversion.yml b/.github/workflows/update-appversion.yml index a83623dfb..dbc0952ea 100644 --- a/.github/workflows/update-appversion.yml +++ b/.github/workflows/update-appversion.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.ref }} persist-credentials: false diff --git a/CHANGELOG.md b/CHANGELOG.md index bd7cc17f7..a4c302df4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,58 @@ # Changelog +## 2026-07-22 — [#1068](https://github.com/cobaltcore-dev/cortex/pull/1068) + +### cortex v0.3.2 (sha-378ee2f5) + +Non-breaking changes: +- Filter KVM hypervisors for non-KVM images — adds a new Nova scheduler filter that removes KVM hypervisors from scheduling candidates when the image declares a non-KVM hypervisor type ([#1067](https://github.com/cobaltcore-dev/cortex/pull/1067)) +- Improve FlavorGroupCapacity CRD observability with better metrics and status reporting ([#942](https://github.com/cobaltcore-dev/cortex/pull/942)) +- Protect pipeline maps with RWMutex to prevent concurrent map read/write data race ([#1066](https://github.com/cobaltcore-dev/cortex/pull/1066)) +- Fix broken relative links in quickstart and develop docs ([#1063](https://github.com/cobaltcore-dev/cortex/pull/1063)) +- Update `actions/setup-go` to v7 ([#1049](https://github.com/cobaltcore-dev/cortex/pull/1049)) +- Update `actions/setup-python` to v7 ([#1062](https://github.com/cobaltcore-dev/cortex/pull/1062)) +- Update `github.com/sapcc/go-bits` ([#1059](https://github.com/cobaltcore-dev/cortex/pull/1059), [#1071](https://github.com/cobaltcore-dev/cortex/pull/1071), [#1074](https://github.com/cobaltcore-dev/cortex/pull/1074)) +- Update external dependencies: `actions/checkout` v7.0.1, `go-logr/logr` v1.4.4, `prometheus/client_golang` v1.24.0 ([#1072](https://github.com/cobaltcore-dev/cortex/pull/1072)) + +### cortex-shim v0.1.8 (sha-378ee2f5) + +Includes updated image sha-378ee2f5. + +### cortex-postgres v0.6.10 (sha-e06153f8) + +Includes updated image sha-e06153f8. + +### cortex-nova v0.0.82 + +Includes updated charts cortex v0.3.2, cortex-postgres v0.6.10. + +- Add alert for missing hypervisor properties and KVM pipeline registration ([#1067](https://github.com/cobaltcore-dev/cortex/pull/1067)) +- Update `kube-prometheus-stack` to v87.19.0 ([#1061](https://github.com/cobaltcore-dev/cortex/pull/1061), [#1075](https://github.com/cobaltcore-dev/cortex/pull/1075)) + +### cortex-cinder v0.0.82 + +Includes updated charts cortex v0.3.2, cortex-postgres v0.6.10. + +### cortex-manila v0.0.82 + +Includes updated charts cortex v0.3.2, cortex-postgres v0.6.10. + +### cortex-crds v0.0.82 + +Includes updated chart cortex v0.3.2. + +### cortex-ironcore v0.0.82 + +Includes updated chart cortex v0.3.2. + +### cortex-pods v0.0.82 + +Includes updated chart cortex v0.3.2. + +### cortex-placement-shim v0.1.8 + +Includes updated chart cortex-shim v0.1.8. + ## 2026-07-16 — [#1051](https://github.com/cobaltcore-dev/cortex/pull/1051) ### cortex v0.3.1 (sha-fc47858e) diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index 4779eddeb..fba3a4cee 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -315,6 +315,48 @@ type NovaImageMeta struct { Properties NovaObject[map[string]any] `json:"properties"` } +type NovaImageMetaHVType string + +const ( + NovaImageMetaHVTypeVMware NovaImageMetaHVType = "vmware" + NovaImageMetaHVTypeBaremetal NovaImageMetaHVType = "baremetal" + NovaImageMetaHVTypeKVM NovaImageMetaHVType = "kvm" +) + +// GetHypervisorType determines the hypervisor type based on the image properties. +func (m NovaImageMeta) GetHypervisorType() (NovaImageMetaHVType, error) { + if m.Properties.Data == nil { + return "", errors.New("image properties are not set") + } + keys := []string{ + // See https://github.com/sapcc/nova/blob/27eb1f0/nova/scheduler/filters/image_props_filter.py#L55 + "img_hv_type", // Used by kvm images + // See https://github.com/sapcc/nova/blob/27eb1f0/doc/source/admin/configuration/hypervisor-vmware.rst#tag-vmware-images + "hypervisor_type", // Used by other images (e.g., VMware) + } + for _, key := range keys { + val, ok := m.Properties.Data[key] + if !ok { + continue + } + str, ok := val.(string) + if !ok { + return "", fmt.Errorf("hypervisor type in image properties is not a string: %v", val) + } + switch strings.ToLower(str) { + case "vmware": + return NovaImageMetaHVTypeVMware, nil + case "baremetal": + return NovaImageMetaHVTypeBaremetal, nil + case "kvm": + return NovaImageMetaHVTypeKVM, nil + default: + return "", fmt.Errorf("unsupported hypervisor type in image properties: %s", str) + } + } + return "", errors.New("hypervisor type not found in image properties") +} + // Nova flavor metadata for the specified VM. type NovaFlavor struct { ID int `json:"id"` diff --git a/api/external/nova/messages_test.go b/api/external/nova/messages_test.go index 13040da11..a9074bd3a 100644 --- a/api/external/nova/messages_test.go +++ b/api/external/nova/messages_test.go @@ -491,3 +491,81 @@ func TestNovaSpecUnmarshal(t *testing.T) { t.Errorf("Expected NumInstances to be 1, got %d", spec.Spec.Data.NumInstances) } } + +func TestNovaImageMeta_GetHypervisorType(t *testing.T) { + tests := []struct { + name string + properties map[string]any + propsNil bool + expected NovaImageMetaHVType + expectError bool + }{ + { + name: "kvm via img_hv_type", + properties: map[string]any{"img_hv_type": "kvm"}, + expected: NovaImageMetaHVTypeKVM, + }, + { + name: "vmware via hypervisor_type", + properties: map[string]any{"hypervisor_type": "vmware"}, + expected: NovaImageMetaHVTypeVMware, + }, + { + name: "baremetal via hypervisor_type", + properties: map[string]any{"hypervisor_type": "baremetal"}, + expected: NovaImageMetaHVTypeBaremetal, + }, + { + name: "case insensitive", + properties: map[string]any{"img_hv_type": "KVM"}, + expected: NovaImageMetaHVTypeKVM, + }, + { + name: "img_hv_type takes precedence over hypervisor_type", + properties: map[string]any{"img_hv_type": "kvm", "hypervisor_type": "vmware"}, + expected: NovaImageMetaHVTypeKVM, + }, + { + name: "properties nil", + propsNil: true, + expectError: true, + }, + { + name: "no hypervisor key present", + properties: map[string]any{"other_key": "value"}, + expectError: true, + }, + { + name: "unsupported hypervisor type", + properties: map[string]any{"img_hv_type": "xen"}, + expectError: true, + }, + { + name: "value not a string", + properties: map[string]any{"img_hv_type": 42}, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + meta := NovaImageMeta{} + if !tt.propsNil { + meta.Properties = NovaObject[map[string]any]{Data: tt.properties} + } + got, err := meta.GetHypervisorType() + if tt.expectError { + if err == nil { + t.Fatalf("expected an error, got nil (result: %q)", got) + } + return + } + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got != tt.expected { + t.Errorf("expected %q, got %q", tt.expected, got) + } + }) + } +} diff --git a/docs/develop.md b/docs/develop.md index 4bad0c4da..23ba7687f 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -56,7 +56,7 @@ The `testsum` target provides cleaner output by showing only full verbose output Helm charts bundle the application into a package, containing all the [Kubernetes](https://kubernetes.io/docs/tutorials/hello-minikube/) resources needed to run the application. The configuration for the application is specified in the [Helm `values.yaml`](cortex.secrets.example.yaml). -Read [the helm chart structure documentation](helm/README.md) for more information about the structure of the Helm charts used in this repository. +Read [the helm chart structure documentation](../helm/README.md) for more information about the structure of the Helm charts used in this repository. For local development, use the `cortex.secrets.example.yaml` file to override the default Helm values. You can write your OpenStack credentials here to include credentials, such as SSO certificates to access Prometheus metrics or OpenStack credentials to authenticate with Keystone. diff --git a/docs/quickstart.md b/docs/quickstart.md index ceda4eb90..2386a1305 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -2,7 +2,7 @@ ### 1. Tilt Values Setup -Copy the example secrets values file. This file is used for local development and overrides the Helm chart values provided in [values.yaml](helm/cortex/values.yaml) for your local testing setup. +Copy the example secrets values file. This file is used for local development and overrides the Helm chart values provided in [values.yaml](../helm/library/cortex/values.yaml) for your local testing setup. ```bash cp cortex.secrets.example.yaml "${HOME}/cortex.secrets.yaml" ``` diff --git a/go.mod b/go.mod index 5d1606e51..2ca48a470 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/go-gorp/gorp v2.2.0+incompatible github.com/gophercloud/gophercloud/v2 v2.13.0 github.com/ironcore-dev/ironcore v0.4.3 - github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_golang v1.24.0 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260709170107-f8ef1547bb62 + github.com/sapcc/go-bits v0.0.0-20260721134224-acf388eecf26 go.xyrillian.de/gg v1.11.1 k8s.io/api v0.36.2 k8s.io/apimachinery v0.36.2 @@ -55,7 +55,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.3 + github.com/go-logr/logr v1.4.4 github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect @@ -85,8 +85,8 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/poy/onpar v0.3.5 // indirect - github.com/prometheus/common v0.69.0 // indirect - github.com/prometheus/procfs v0.19.2 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/sapcc/go-api-declarations v1.24.0 github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.10.2 // indirect @@ -109,12 +109,12 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 // indirect golang.org/x/term v0.45.0 - golang.org/x/text v0.38.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect diff --git a/go.sum b/go.sum index 7d4fbb9a4..32aea00c9 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj2 github.com/go-gorp/gorp v2.2.0+incompatible h1:xAUh4QgEeqPPhK3vxZN+bzrim1z5Av6q837gtjUlshc= github.com/go-gorp/gorp v2.2.0+incompatible/go.mod h1:7IfkAQnO7jfT/9IQ3R9wL1dFhukN6aQxzKTHnkxzA/E= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -143,8 +143,8 @@ github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo79 github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -191,21 +191,21 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v0.3.5 h1:AYOT1AvNtucJhmEK6BSD1KvLvtYXvp/50vL5yaPAljo= github.com/poy/onpar v0.3.5/go.mod h1:Ae6eC73WNjI7NHdcughszLtcw+msOi0pb6eq6LzL97o= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= +github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.69.0 h1:OA85nJQS/T/MaYh/Q2CcgDKSGWqNIgrBDvDH85CuiNk= -github.com/prometheus/common v0.69.0/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= -github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= -github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.24.0 h1:sGBvOMVSM1olJlyvNoQSk7NX5uatXHKkztGDBPnTWMs= github.com/sapcc/go-api-declarations v1.24.0/go.mod h1:ZWRTijvgF8o8aHg5stgg7u4DF6jFrd0X97le/uGlZsA= -github.com/sapcc/go-bits v0.0.0-20260709170107-f8ef1547bb62 h1:HGIsxyetnws+88X5ZAY86AxYGR88vFWMVrXzxAMWg6U= -github.com/sapcc/go-bits v0.0.0-20260709170107-f8ef1547bb62/go.mod h1:uchHF8FNxncTCoQYDjGSO86AuxSzgIbN5mQzM95RYOA= +github.com/sapcc/go-bits v0.0.0-20260721134224-acf388eecf26 h1:H3m0oO3OKPKScPkXxPtSL7WifLvPVyyQZQKfeeQAKwE= +github.com/sapcc/go-bits v0.0.0-20260721134224-acf388eecf26/go.mod h1:McCHf83n0tsTgnTSMxkdyBys84LG/d7uqubQrjsoZ1U= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -267,10 +267,10 @@ go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBs go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= @@ -281,12 +281,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 858510351..ce7cd6675 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.81 +version: 0.0.82 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.9 + version: 0.6.10 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index c8f73b1c7..f911120d7 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.81 +version: 0.0.82 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 5e42a57aa..8ee47cd11 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.81 +version: 0.0.82 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 3be82952a..74f5be1bc 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.81 +version: 0.0.82 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.9 + version: 0.6.10 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index 7df970361..9ae708704 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.81 +version: 0.0.82 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.9 + version: 0.6.10 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 8db8bfb2e..6655c5528 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -303,6 +303,28 @@ spec: configuration. It is recommended to investigate the pipeline status and logs for more details. + - alert: CortexNovaImagePropertiesHypervisorTypeUndetermined + expr: | + sum by (pipeline, step) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="image_properties_hv_type_undetermined"}[5m])) > 0.1 + for: 15m + labels: + context: scheduling + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/scheduling + annotations: + summary: "Nova scheduling frequently cannot determine the hypervisor type from image properties" + description: > + The `filter_image_properties` step in pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` + is frequently unable to determine the hypervisor type from the image properties + of incoming scheduling requests. In this case the filter is skipped and all hosts + are returned, so kvm-only images may be placed on non-kvm hypervisors (or vice + versa). This may indicate that images are missing the expected hypervisor + type property, or that the property format has changed. Investigate the + image metadata of the affected requests. + {{- if .Values.kvm.enabled }} - alert: CortexNovaDoesntFindValidKVMHosts expr: sum by (az, hvtype) (increase(cortex_vm_faults{hvtype=~"CH|QEMU",faultmsg=~".*No valid host was found.*",faultmsg!~".*No such host.*"}[5m])) > 0 @@ -588,6 +610,28 @@ spec: This may mean hypervisors in that AZ are fully utilized for the corresponding flavor group and no further committed resources can be placed there. + - alert: CortexNovaCommittedResourceCapacityNotReady + expr: | + cortex_committed_resource_capacity_ready{service="cortex-nova-metrics"} == 0 + for: 10m + labels: + context: committed-resource-capacity + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity + annotations: + summary: "FlavorGroupCapacity for {{ "{{" }} $labels.flavor_group {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} has been not-ready for >10 minutes" + description: > + The FlavorGroupCapacity CRD for flavor group {{ "{{" }} $labels.flavor_group {{ "}}" }} + in availability zone {{ "{{" }} $labels.az {{ "}}" }} has had Ready=False for more than + 10 minutes. The capacity controller failed to complete all scheduler probes for this + (flavor group x AZ) pair. The capacity API (report-capacity) is serving stale total + capacity values for this group without usage data — Limes receives capacity but no + usage, causing silent staleness. Investigate the capacity controller logs for probe + errors and check scheduler availability. + # Committed Resource Usage API - alert: CortexNovaCommittedResourceUsageErrors expr: | diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 196973e1e..fa160c508 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -26,6 +26,11 @@ spec: This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_image_properties + description: | + This step will filter out hosts for which the hypervisor type does not match + the one specified in the image properties, for example, filtering out all + known KVM hypervisors if the image requires a different hypervisor type. - name: filter_status_conditions description: | This step will filter out hosts for which the hypervisor status conditions @@ -178,6 +183,11 @@ spec: This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_image_properties + description: | + This step will filter out hosts for which the hypervisor type does not match + the one specified in the image properties, for example, filtering out all + known KVM hypervisors if the image requires a different hypervisor type. - name: filter_status_conditions description: | This step will filter out hosts for which the hypervisor status conditions @@ -629,6 +639,11 @@ spec: This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_image_properties + description: | + This step will filter out hosts for which the hypervisor type does not match + the one specified in the image properties, for example, filtering out all + known KVM hypervisors if the image requires a different hypervisor type. - name: filter_status_conditions description: | This step will filter out hosts for which the hypervisor status conditions @@ -781,6 +796,11 @@ spec: This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_image_properties + description: | + This step will filter out hosts for which the hypervisor type does not match + the one specified in the image properties, for example, filtering out all + known KVM hypervisors if the image requires a different hypervisor type. - name: filter_status_conditions description: | This step will filter out hosts for which the hypervisor status conditions diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 36905ed39..6be1a1b69 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.7 +version: 0.1.8 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.7 + version: 0.1.8 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 864e88447..a7c1047aa 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.81 +version: 0.0.82 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.1 + version: 0.3.2 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index fc8798062..862d9378f 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 87.16.1 + version: 87.19.0 diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index b5abe1d7c..5925db38f 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.9 +version: 0.6.10 appVersion: "sha-e06153f8" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 5e5507ed7..2729d7145 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.7 -appVersion: "sha-7e36a65c" +version: 0.1.8 +appVersion: "sha-378ee2f5" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index bc45750c5..9cd948d99 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.3.1 -appVersion: "sha-fc47858e" +version: 0.3.2 +appVersion: "sha-4eba8400" icon: "https://example.com/icon.png" dependencies: [] diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go index 209e9e342..e746ca0b2 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go @@ -96,7 +96,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.Pipelines[decision.Spec.PipelineRef.Name] + pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) if !ok { log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) return errors.New("pipeline not found or not ready") diff --git a/internal/scheduling/lib/filter_weigher_pipeline_monitor.go b/internal/scheduling/lib/filter_weigher_pipeline_monitor.go index 191257a3a..341152322 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_monitor.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_monitor.go @@ -22,6 +22,8 @@ type FilterWeigherPipelineMonitor struct { stepReorderingsObserver *prometheus.HistogramVec // A histogram to observe the impact of the step on the hosts. stepImpactObserver *prometheus.HistogramVec + // A counter for named events reported by a step during its run. + stepEventCounter *prometheus.CounterVec // A histogram to measure how long the pipeline takes to run in total. pipelineRunTimer *prometheus.HistogramVec // A histogram to observe the number of hosts going into the scheduler pipeline. @@ -64,6 +66,10 @@ func NewPipelineMonitor() FilterWeigherPipelineMonitor { Help: "Impact of the step on the hosts", Buckets: prometheus.ExponentialBucketsRange(0.01, 1000, 20), }, []string{"pipeline", "step", "stat", "unit"}), + stepEventCounter: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_filter_weigher_pipeline_step_events_total", + Help: "Number of named events reported by a scheduler pipeline step", + }, []string{"pipeline", "step", "event"}), pipelineRunTimer: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "cortex_filter_weigher_pipeline_run_duration_seconds", Help: "Duration of scheduler pipeline run", @@ -121,6 +127,7 @@ func (m *FilterWeigherPipelineMonitor) Describe(ch chan<- *prometheus.Desc) { m.stepRemovedHostsObserver.Describe(ch) m.stepReorderingsObserver.Describe(ch) m.stepImpactObserver.Describe(ch) + m.stepEventCounter.Describe(ch) m.pipelineRunTimer.Describe(ch) m.hostNumberInObserver.Describe(ch) m.hostNumberOutObserver.Describe(ch) @@ -133,6 +140,7 @@ func (m *FilterWeigherPipelineMonitor) Collect(ch chan<- prometheus.Metric) { m.stepRemovedHostsObserver.Collect(ch) m.stepReorderingsObserver.Collect(ch) m.stepImpactObserver.Collect(ch) + m.stepEventCounter.Collect(ch) m.pipelineRunTimer.Collect(ch) m.hostNumberInObserver.Collect(ch) m.hostNumberOutObserver.Collect(ch) diff --git a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go index 3e64fa6ee..2ad84e9fd 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go @@ -36,6 +36,8 @@ type FilterWeigherPipelineStepMonitor[RequestType FilterWeigherPipelineRequest] stepReorderingsObserver *prometheus.HistogramVec // A metric measuring the impact of the step on the hosts. stepImpactObserver *prometheus.HistogramVec + // A counter for named events reported by the step during its run. + stepEventCounter *prometheus.CounterVec } // Schedule using the wrapped step and measure the time it takes. @@ -58,6 +60,7 @@ func monitorStep[RequestType FilterWeigherPipelineRequest](stepName string, m Fi removedHostsObserver: removedHostsObserver, stepReorderingsObserver: m.stepReorderingsObserver, stepImpactObserver: m.stepImpactObserver, + stepEventCounter: m.stepEventCounter, } } @@ -83,6 +86,15 @@ func (s *FilterWeigherPipelineStepMonitor[RequestType]) RunWrapped( "inWeights", inWeights, "outWeights", stepResult.Activations, ) + // Count named events reported by the step during its run. + if s.stepEventCounter != nil { + for _, event := range stepResult.Events { + s.stepEventCounter. + WithLabelValues(s.pipelineName, s.stepName, event). + Inc() + } + } + // Observe how much the step modifies the weights of the hosts. if s.stepHostWeight != nil { for host, weight := range stepResult.Activations { diff --git a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go index 7d7817abd..5ae402111 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go @@ -7,6 +7,9 @@ import ( "log/slog" "os" "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" ) type mockObserver struct { @@ -55,6 +58,37 @@ func TestStepMonitorRun(t *testing.T) { } } +func TestStepMonitorRunEvents(t *testing.T) { + stepEventCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_filter_weigher_pipeline_step_events_total", + Help: "Number of named events reported by a scheduler pipeline step", + }, []string{"pipeline", "step", "event"}) + monitor := &FilterWeigherPipelineStepMonitor[mockFilterWeigherPipelineRequest]{ + stepName: "mock_step", + pipelineName: "mock_pipeline", + stepEventCounter: stepEventCounter, + } + step := &mockWeigher[mockFilterWeigherPipelineRequest]{ + RunFunc: func(traceLog *slog.Logger, request mockFilterWeigherPipelineRequest) (*FilterWeigherPipelineStepResult, error) { + return &FilterWeigherPipelineStepResult{ + Activations: map[string]float64{"host1": 0.0}, + Events: []string{"hypervisor_type_undetermined"}, + }, nil + }, + } + request := mockFilterWeigherPipelineRequest{ + Hosts: []string{"host1"}, + Weights: map[string]float64{"host1": 0.0}, + } + if _, err := monitor.RunWrapped(slog.Default(), request, step); err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + got := testutil.ToFloat64(stepEventCounter.WithLabelValues("mock_pipeline", "mock_step", "hypervisor_type_undetermined")) + if got != 1 { + t.Errorf("stepEventCounter = %v, want 1", got) + } +} + func TestImpact(t *testing.T) { testcases := []struct { name string diff --git a/internal/scheduling/lib/filter_weigher_pipeline_step_result.go b/internal/scheduling/lib/filter_weigher_pipeline_step_result.go index cfa0dbbb0..8ccee7cf3 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_step_result.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_step_result.go @@ -23,6 +23,13 @@ type FilterWeigherPipelineStepResult struct { // These statistics are used to display the step's effect on the hosts. // For example: max cpu contention: before [ 100%, 50%, 40% ], after [ 40%, 50%, 100% ] Statistics map[string]FilterWeigherPipelineStepStatistics + + // Named events reported by the step during its run, e.g. + // "hypervisor_type_undetermined". Each event is counted by the pipeline + // monitor as cortex_filter_weigher_pipeline_step_events_total, labeled by + // pipeline, step and event. Use this to expose noteworthy step conditions + // (skipped filtering, missing data, ...) as Prometheus metrics. + Events []string } type FilterWeigherPipelineStepStatistics struct { diff --git a/internal/scheduling/lib/pipeline_controller.go b/internal/scheduling/lib/pipeline_controller.go index 94e61ed45..25f994ea6 100644 --- a/internal/scheduling/lib/pipeline_controller.go +++ b/internal/scheduling/lib/pipeline_controller.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "sync" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "k8s.io/apimachinery/pkg/api/meta" @@ -20,6 +21,8 @@ import ( // Base controller for decision pipelines. type BasePipelineController[PipelineType any] struct { + // Mutex protecting Pipelines and PipelineConfigs maps from concurrent access. + pipelinesMu sync.RWMutex // Initialized pipelines by their name. Pipelines map[string]PipelineType // The configured pipelines by their name. @@ -34,12 +37,30 @@ type BasePipelineController[PipelineType any] struct { HistoryManager HistoryClient } +// GetPipeline returns the initialized pipeline for the given name, if it exists. +func (c *BasePipelineController[PipelineType]) GetPipeline(name string) (PipelineType, bool) { + c.pipelinesMu.RLock() + defer c.pipelinesMu.RUnlock() + p, ok := c.Pipelines[name] + return p, ok +} + +// GetPipelineConfig returns the pipeline configuration for the given name, if it exists. +func (c *BasePipelineController[PipelineType]) GetPipelineConfig(name string) (v1alpha1.Pipeline, bool) { + c.pipelinesMu.RLock() + defer c.pipelinesMu.RUnlock() + p, ok := c.PipelineConfigs[name] + return p, ok +} + // Handle the startup of the manager by initializing the pipeline map. func (c *BasePipelineController[PipelineType]) InitAllPipelines(ctx context.Context) error { log := ctrl.LoggerFrom(ctx) log.Info("initializing pipeline map") + c.pipelinesMu.Lock() c.Pipelines = make(map[string]PipelineType) c.PipelineConfigs = make(map[string]v1alpha1.Pipeline) + c.pipelinesMu.Unlock() // List all existing pipelines and initialize them. var pipelines v1alpha1.PipelineList if err := c.List(ctx, &pipelines); err != nil { @@ -54,7 +75,6 @@ func (c *BasePipelineController[PipelineType]) InitAllPipelines(ctx context.Cont } log.Info("initializing existing pipeline", "pipelineName", pipelineConf.Name) c.handlePipelineChange(ctx, &pipelineConf, nil) - c.PipelineConfigs[pipelineConf.Name] = pipelineConf } return nil } @@ -67,8 +87,10 @@ func (c *BasePipelineController[PipelineType]) handlePipelineChange( ) { if obj.Spec.SchedulingDomain != c.SchedulingDomain { + c.pipelinesMu.Lock() delete(c.Pipelines, obj.Name) // Just to be sure. delete(c.PipelineConfigs, obj.Name) + c.pipelinesMu.Unlock() return } log := ctrl.LoggerFrom(ctx) @@ -96,8 +118,10 @@ func (c *BasePipelineController[PipelineType]) handlePipelineChange( if err := c.Status().Patch(ctx, obj, patch); err != nil { log.Error(err, "failed to patch pipeline status", "pipelineName", obj.Name) } + c.pipelinesMu.Lock() delete(c.Pipelines, obj.Name) delete(c.PipelineConfigs, obj.Name) + c.pipelinesMu.Unlock() return } @@ -153,8 +177,10 @@ func (c *BasePipelineController[PipelineType]) handlePipelineChange( }) } + c.pipelinesMu.Lock() c.Pipelines[obj.Name] = initResult.Pipeline c.PipelineConfigs[obj.Name] = *obj + c.pipelinesMu.Unlock() log.Info("pipeline created and ready", "pipelineName", obj.Name) meta.SetStatusCondition(&obj.Status.Conditions, metav1.Condition{ Type: v1alpha1.PipelineConditionReady, @@ -207,8 +233,10 @@ func (c *BasePipelineController[PipelineType]) HandlePipelineDeleted( ) { pipelineConf := evt.Object.(*v1alpha1.Pipeline) + c.pipelinesMu.Lock() delete(c.Pipelines, pipelineConf.Name) delete(c.PipelineConfigs, pipelineConf.Name) + c.pipelinesMu.Unlock() } // Handle a knowledge creation, readiness update, or delete event from watching knowledge resources. diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller.go b/internal/scheduling/machines/filter_weigher_pipeline_controller.go index 2bcf5ee0d..cbbf684c5 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller.go @@ -117,7 +117,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.Pipelines[decision.Spec.PipelineRef.Name] + pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) if !ok { log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) return errors.New("pipeline not found or not ready") diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller.go b/internal/scheduling/manila/filter_weigher_pipeline_controller.go index a0bc8b960..686ce4c2a 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller.go @@ -96,7 +96,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.Pipelines[decision.Spec.PipelineRef.Name] + pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) if !ok { log.Error(nil, "skipping decision, pipeline not found or not ready") return errors.New("pipeline not found or not ready") diff --git a/internal/scheduling/nova/detector_pipeline_controller.go b/internal/scheduling/nova/detector_pipeline_controller.go index dad1a6bf8..71f062cc2 100644 --- a/internal/scheduling/nova/detector_pipeline_controller.go +++ b/internal/scheduling/nova/detector_pipeline_controller.go @@ -70,7 +70,7 @@ func (c *DetectorPipelineController) CreateDeschedulingsPeriodically(ctx context return default: // Get the pipeline for the current configuration. - p, ok := c.Pipelines["kvm-descheduler"] + p, ok := c.GetPipeline("kvm-descheduler") if !ok { slog.Error("descheduler: pipeline not found or not ready yet") time.Sleep(jobloop.DefaultJitter(time.Minute)) diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index 0c6917e1a..2682c96b4 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -160,7 +160,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.Pipelines[decision.Spec.PipelineRef.Name] + pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) if !ok { log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) return nil, errors.New("pipeline not found or not ready") @@ -184,7 +184,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // If necessary gather all placement candidates before filtering. // This will override the hosts and weights in the nova request. - pipelineConf, ok := c.PipelineConfigs[decision.Spec.PipelineRef.Name] + pipelineConf, ok := c.GetPipelineConfig(decision.Spec.PipelineRef.Name) if !ok { log.Error(nil, "pipeline config not found", "pipelineName", decision.Spec.PipelineRef.Name) return &request, errors.New("pipeline config not found") diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties.go b/internal/scheduling/nova/plugins/filters/filter_image_properties.go new file mode 100644 index 000000000..7ce083f87 --- /dev/null +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties.go @@ -0,0 +1,53 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "context" + "log/slog" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +type FilterImagePropertiesStep struct { + lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] +} + +// Filter hosts based on image properties given in the request spec. +func (s *FilterImagePropertiesStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { + result := s.IncludeAllHostsFromRequest(request) + // If the image properties indicate any other hypervisor type than kvm, + // we filter out all known kvm hypervisors. + hvType, err := request.Spec.Data.Image.Data.GetHypervisorType() + if err != nil { + // Hypervisor type could not be determined. In this case, just + // ignore the filter and return all hosts. + traceLog.Warn("could not determine hypervisor type from image properties", + "error", err) + // Expose this event through the step monitor to alert on high-frequency + // occurrences of this situation. + result.Events = append(result.Events, "image_properties_hv_type_undetermined") + return result, nil + } + if hvType != api.NovaImageMetaHVTypeKVM { + traceLog.Info("filtering out all known kvm hypervisors since image properties indicate a different hypervisor type", + "image_hypervisor_type", hvType) + hvs := new(hv1.HypervisorList) + if err := s.Client.List(context.Background(), hvs); err != nil { + traceLog.Error("failed to list hypervisors", "error", err) + return nil, err + } + for _, hv := range hvs.Items { + delete(result.Activations, hv.Name) + traceLog.Debug("filtering host which is kvm hypervisor", "host", hv.Name) + } + } + return result, nil +} + +func init() { + Index["filter_image_properties"] = func() NovaFilter { return &FilterImagePropertiesStep{} } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go new file mode 100644 index 000000000..c4370e79e --- /dev/null +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go @@ -0,0 +1,115 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "log/slog" + "testing" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// requestWith builds an ExternalSchedulerRequest with the given hosts and image +// properties. +func requestWith(hosts []string, properties map[string]any) api.ExternalSchedulerRequest { + schedHosts := make([]api.ExternalSchedulerHost, 0, len(hosts)) + for _, h := range hosts { + schedHosts = append(schedHosts, api.ExternalSchedulerHost{ComputeHost: h}) + } + request := api.ExternalSchedulerRequest{Hosts: schedHosts} + request.Spec.Data.Image.Data.Properties = api.NovaObject[map[string]any]{Data: properties} + return request +} + +func TestFilterImagePropertiesStep_Run(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + // host1 and host2 are known kvm hypervisors, host3 is not registered. + hvs := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host2"}}, + } + + tests := []struct { + name string + request api.ExternalSchedulerRequest + expectedHosts []string + filteredHosts []string + }{ + { + name: "kvm image keeps all hosts", + request: requestWith([]string{"host1", "host2", "host3"}, map[string]any{"img_hv_type": "kvm"}), + expectedHosts: []string{"host1", "host2", "host3"}, + filteredHosts: []string{}, + }, + { + name: "vmware image filters out known kvm hypervisors", + request: requestWith([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}), + expectedHosts: []string{"host3"}, + filteredHosts: []string{"host1", "host2"}, + }, + { + name: "baremetal image filters out known kvm hypervisors", + request: requestWith([]string{"host1", "host3"}, map[string]any{"img_hv_type": "baremetal"}), + expectedHosts: []string{"host3"}, + filteredHosts: []string{"host1"}, + }, + { + name: "undeterminable hypervisor type keeps all hosts", + request: requestWith([]string{"host1", "host2", "host3"}, nil), + expectedHosts: []string{"host1", "host2", "host3"}, + filteredHosts: []string{}, + }, + { + name: "unsupported hypervisor type keeps all hosts", + request: requestWith([]string{"host1", "host2"}, map[string]any{"img_hv_type": "xen"}), + expectedHosts: []string{"host1", "host2"}, + filteredHosts: []string{}, + }, + { + name: "empty host list", + request: requestWith([]string{}, map[string]any{"hypervisor_type": "vmware"}), + expectedHosts: []string{}, + filteredHosts: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + step := &FilterImagePropertiesStep{} + step.Client = fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(hvs...). + Build() + result, err := step.Run(slog.Default(), tt.request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + for _, host := range tt.expectedHosts { + if _, ok := result.Activations[host]; !ok { + t.Errorf("expected host %s to be present in activations", host) + } + } + + for _, host := range tt.filteredHosts { + if _, ok := result.Activations[host]; ok { + t.Errorf("expected host %s to be filtered out", host) + } + } + + if len(result.Activations) != len(tt.expectedHosts) { + t.Errorf("expected %d hosts, got %d", len(tt.expectedHosts), len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller.go b/internal/scheduling/pods/filter_weigher_pipeline_controller.go index 898c5068d..c251c8b45 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller.go @@ -117,7 +117,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.Pipelines[decision.Spec.PipelineRef.Name] + pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) if !ok { log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) return errors.New("pipeline not found or not ready") diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 64a87befe..a6194d423 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -552,9 +552,14 @@ func (c *Reconciler) reconcileAZ( } } - // Write one CRD per group. Skip groups with failed probes — their CRDs retain last good state. + // Write one CRD per group. For groups with failed probes, mark Ready=False so the + // capacity API can detect staleness and return 5xx rather than serving stale data silently. for _, r := range results { if !r.allFresh { + if err := c.markCRDNotReady(ctx, r.groupName, az); err != nil { + logger.Error(err, "failed to mark FlavorGroupCapacity CRD not-ready", + "flavorGroup", r.groupName, "az", az) + } continue } if err := c.writeCRD(ctx, r.groupName, r.groupData, az, @@ -667,6 +672,32 @@ func (c *Reconciler) writeCRD( return nil } +// markCRDNotReady sets Ready=False on an existing FlavorGroupCapacity CRD without touching +// any other status fields, preserving the last-known capacity values for operator inspection. +// If the CRD does not exist yet it is a no-op. +func (c *Reconciler) markCRDNotReady(ctx context.Context, groupName, az string) error { + crdName := crdNameFor(groupName, az) + var existing v1alpha1.FlavorGroupCapacity + if err := c.client.Get(ctx, types.NamespacedName{Name: crdName}, &existing); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to get FlavorGroupCapacity %s: %w", crdName, err) + } + patch := client.MergeFrom(existing.DeepCopy()) + meta.SetStatusCondition(&existing.Status.Conditions, metav1.Condition{ + Type: v1alpha1.FlavorGroupCapacityConditionReady, + ObservedGeneration: existing.Generation, + Status: metav1.ConditionFalse, + Reason: "ReconcileFailed", + Message: "one or more scheduler probes failed; capacity data may be stale", + }) + if err := c.client.Status().Patch(ctx, &existing, patch); err != nil { + return fmt.Errorf("failed to patch FlavorGroupCapacity %s status: %w", crdName, err) + } + return nil +} + // probeScheduler calls the scheduler and returns slot count, host count, and candidate host names. // ignoreAllocations=true (total probe) uses raw effective capacity; false (placeable probe) subtracts allocations. func (c *Reconciler) probeScheduler( diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index d01110af0..4408433fb 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -81,7 +81,7 @@ func newFlavorGroupKnowledge(t *testing.T, groupName string, smallestMemoryMB ui } // newHypervisor creates a Hypervisor CRD with a topology AZ label, memory and CPU effective capacity. -func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *hv1.Hypervisor { +func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor { hv := &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -96,9 +96,6 @@ func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *h hv1.ResourceCPU: *cpuQty, } } - for _, id := range instanceIDs { - hv.Status.Instances = append(hv.Status.Instances, hv1.Instance{ID: id}) - } return hv } @@ -200,7 +197,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { ) scheme := newTestScheme(t) - hv := newHypervisor("host-1", az, memBytes, "vm1") + hv := newHypervisor("host-1", az, memBytes) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) fakeClient := fake.NewClientBuilder(). @@ -312,6 +309,85 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { } } +func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { + const ( + groupName = "hana-v2" + az = "qa-de-1a" + memMB = 2048 + memBytes = int64(memMB) * 1024 * 1024 + ) + + scheme := newTestScheme(t) + knowledge := newFlavorGroupKnowledge(t, groupName, memMB) + crdName := crdNameFor(groupName, az) + + // Pre-create a CRD that was previously Ready=True. + existing := &v1alpha1.FlavorGroupCapacity{ + ObjectMeta: metav1.ObjectMeta{Name: crdName}, + Spec: v1alpha1.FlavorGroupCapacitySpec{ + FlavorGroup: groupName, + AvailabilityZone: az, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(knowledge, existing). + WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). + Build() + + // Set Ready=True on the pre-existing CRD. + patch := client.MergeFrom(existing.DeepCopy()) + existing.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.FlavorGroupCapacityConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReconcileSucceeded", + }} + if err := fakeClient.Status().Patch(context.Background(), existing, patch); err != nil { + t.Fatalf("failed to set Ready=True: %v", err) + } + + // Scheduler returns 500 to simulate error. + failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer failServer.Close() + + ctrl := newController(t, fakeClient, Config{ + SchedulerURL: failServer.URL, + TotalPipeline: "kvm-report-capacity", + PlaceablePipeline: "kvm-general-purpose", + }) + + smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} + groupData := compute.FlavorGroupFeature{ + SmallestFlavor: smallFlavor, + Flavors: []compute.FlavorInGroup{smallFlavor}, + } + hv := newHypervisor("host-1", az, memBytes) + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + + ctrl.reconcileAZ(context.Background(), az, + map[string]compute.FlavorGroupFeature{groupName: groupData}, + hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) + + var crd v1alpha1.FlavorGroupCapacity + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { + t.Fatalf("failed to get CRD: %v", err) + } + + // CRD must now have Ready=False. + var readyStatus metav1.ConditionStatus + for _, c := range crd.Status.Conditions { + if c.Type == v1alpha1.FlavorGroupCapacityConditionReady { + readyStatus = c.Status + } + } + if readyStatus != metav1.ConditionFalse { + t.Errorf("Ready condition = %q, want False after scheduler error", readyStatus) + } +} + func TestReconcileAZ_IdempotentUpdate(t *testing.T) { const ( groupName = "hana-v2" @@ -842,7 +918,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { ) scheme := newTestScheme(t) - hv := newHypervisor("host-1", az, memBytes, "vm1") + hv := newHypervisor("host-1", az, memBytes) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) // Pre-create CRD with non-zero RunningInstances to simulate prior state. diff --git a/internal/scheduling/reservations/capacity/metrics.go b/internal/scheduling/reservations/capacity/metrics.go index 27293e0eb..2a34efe92 100644 --- a/internal/scheduling/reservations/capacity/metrics.go +++ b/internal/scheduling/reservations/capacity/metrics.go @@ -9,6 +9,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" + apimeta "k8s.io/apimachinery/pkg/api/meta" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -31,6 +32,7 @@ type Monitor struct { freeCapacityGiB *prometheus.GaugeVec exclusivelyFreeCapacityGiB *prometheus.GaugeVec exclusivelyFreeSlots *prometheus.GaugeVec + readyGauge *prometheus.GaugeVec } // NewMonitor creates a new Monitor that reads FlavorGroupCapacity CRDs. @@ -77,6 +79,10 @@ func NewMonitor(c client.Client) Monitor { Name: "cortex_committed_resource_exclusively_free_slots", Help: "Number of smallest-flavor VM slots available after the cross-group capacity split.", }, capacityFlavorLabels), + readyGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_committed_resource_capacity_ready", + Help: "1 if the FlavorGroupCapacity CRD is Ready (all scheduler probes succeeded), 0 otherwise.", + }, capacityLabels), } } @@ -92,6 +98,7 @@ func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { m.freeCapacityGiB.Describe(ch) m.exclusivelyFreeCapacityGiB.Describe(ch) m.exclusivelyFreeSlots.Describe(ch) + m.readyGauge.Describe(ch) } // Collect implements prometheus.Collector — lists all FlavorGroupCapacity CRDs and exports gauges. @@ -115,6 +122,7 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { m.freeCapacityGiB.Reset() m.exclusivelyFreeCapacityGiB.Reset() m.exclusivelyFreeSlots.Reset() + m.readyGauge.Reset() for _, crd := range list.Items { groupAZLabels := prometheus.Labels{ @@ -138,6 +146,12 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { } m.exclusivelyFreeSlots.With(groupAZFlavorLabels).Set(float64(crd.Status.ExclusivelyFreeSlots)) + readyVal := 0.0 + if apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { + readyVal = 1.0 + } + m.readyGauge.With(groupAZLabels).Set(readyVal) + for _, f := range crd.Status.Flavors { flavorLabels := prometheus.Labels{ "flavor_group": crd.Spec.FlavorGroup, @@ -161,4 +175,5 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { m.freeCapacityGiB.Collect(ch) m.exclusivelyFreeCapacityGiB.Collect(ch) m.exclusivelyFreeSlots.Collect(ch) + m.readyGauge.Collect(ch) } diff --git a/internal/scheduling/reservations/commitments/api/report_capacity.go b/internal/scheduling/reservations/commitments/api/report_capacity.go index 5277d51d0..4eb91b34c 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity.go @@ -5,6 +5,7 @@ package api import ( "encoding/json" + "errors" "net/http" "strconv" "time" @@ -76,8 +77,13 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request) calculator := commitments.NewCapacityCalculator(api.client, api.config) report, err := calculator.CalculateCapacity(ctx, req) if err != nil { - logger.Error(err, "failed to calculate capacity") - statusCode = http.StatusInternalServerError + if errors.Is(err, commitments.ErrCapacityNotReady) { + logger.Info("capacity data not ready, returning 503", "reason", err.Error()) + statusCode = http.StatusServiceUnavailable + } else { + logger.Error(err, "failed to calculate capacity") + statusCode = http.StatusInternalServerError + } http.Error(w, "Failed to calculate capacity: "+err.Error(), statusCode) api.recordCapacityMetrics(statusCode, startTime) return diff --git a/internal/scheduling/reservations/commitments/api/report_capacity_test.go b/internal/scheduling/reservations/commitments/api/report_capacity_test.go index 530fe319f..4bcb71056 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity_test.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "slices" @@ -184,7 +185,8 @@ func TestCapacityCalculator(t *testing.T) { wantCapacity uint64 wantUsage *uint64 // nil = expect absent cfg *commitments.APIConfig - wantResourceCount int // 0 = don't check + wantResourceCount int // 0 = don't check + wantNotReady bool // expect ErrCapacityNotReady } u := func(v uint64) *uint64 { return &v } @@ -196,10 +198,10 @@ func TestCapacityCalculator(t *testing.T) { checkAZ: "az-one", wantCapacity: 1000, wantUsage: u(200), }, { - // stale CRD: last-known capacity still reported, usage omitted - name: "stale CRD: capacity reported, usage absent", + // stale CRD: CalculateCapacity returns ErrCapacityNotReady → caller returns 503 + name: "stale CRD: returns ErrCapacityNotReady", runningInstances: 200, exclusiveFreeBytes: 800 * flavorMemBytes, ready: false, - checkAZ: "az-one", wantCapacity: 1000, wantUsage: nil, + checkAZ: "az-one", wantNotReady: true, }, { // CRD only covers az-one; az-two has no CRD → capacity=0 @@ -233,6 +235,12 @@ func TestCapacityCalculator(t *testing.T) { } report, err := calc.CalculateCapacity(context.Background(), liquid.ServiceCapacityRequest{AllAZs: allAZs}) + if tc.wantNotReady { + if !errors.Is(err, commitments.ErrCapacityNotReady) { + t.Fatalf("expected ErrCapacityNotReady, got %v", err) + } + return + } if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index ee3fc1a99..efb7d8b57 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -5,6 +5,7 @@ package commitments import ( "context" + "errors" "fmt" "github.com/sapcc/go-api-declarations/liquid" @@ -16,6 +17,11 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) +// ErrCapacityNotReady is returned by CalculateCapacity when one or more FlavorGroupCapacity +// CRDs have Ready=False, indicating the controller's last probe cycle failed. Callers should +// return 503 Service Unavailable rather than serving potentially stale data. +var ErrCapacityNotReady = errors.New("one or more FlavorGroupCapacity CRDs are not ready") + // CapacityCalculator computes capacity reports for Limes LIQUID API. type CapacityCalculator struct { client client.Client @@ -59,7 +65,6 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S Resources: make(map[liquid.ResourceName]*liquid.ResourceCapacityReport), } - logger := LoggerFromContext(ctx) for groupName, groupData := range flavorGroups { resCfg := c.conf.ResourceConfigForGroup(groupName) // Skip groups not configured for capacity reporting. @@ -84,8 +89,7 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S } if !apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { - logger.Info("FlavorGroupCapacity CRD is stale, reporting capacity without usage", - "flavorGroup", groupName, "az", az) + return liquid.ServiceCapacityReport{}, fmt.Errorf("%w: flavorGroup=%s az=%s", ErrCapacityNotReady, groupName, string(az)) } // ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots).