diff --git a/README.md b/README.md index f3df21fd..4c246728 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ For step-by-step setup, RBAC, image versions, and teardown see [docs/installatio - **[Installation](docs/installation.md)** — deploy the operator, create your first cluster, networking pitfalls, upgrades. - **[Concepts](docs/concepts.md)** — design rationale: locking pattern, single-seed bootstrap, GenerateName naming, scale-to-zero mechanics, conditions reference. - **[Operations](docs/operations.md)** — runbook for day-2: scaling, pausing/resuming, decoding conditions, escalating stuck reconciles, broken-member recovery. +- **[Defragmentation](docs/etcd-defrag.md)** — the `EtcdDefrag` resource: reclaiming etcd backend disk and its safety model (API type; reconciling controller is a follow-up). - **[Migration](docs/migration.md)** — moving onto this operator from the legacy aenix operator; tracks behavioural changes that need an explicit migration step — currently the BYO root-credentials requirement when enabling auth. ## Testing diff --git a/api/v1alpha2/cel_validation_test.go b/api/v1alpha2/cel_validation_test.go index 0ed601c8..827f98c6 100644 --- a/api/v1alpha2/cel_validation_test.go +++ b/api/v1alpha2/cel_validation_test.go @@ -794,3 +794,47 @@ func TestCEL_AuthAddOnExistingClusterRejected(t *testing.T) { t.Fatalf("error did not mention add/remove rejection: %v", err) } } + +// TestCEL_DefragRuleQuantityIntegerInput exercises the kubectl-style integer +// input path for EtcdDefrag's rule quantities (`freeSpaceAbove: 0` / a bare byte +// count, received as a JSON number, not a string). CEL's quantity() requires a +// string; the rules coerce with string(), so integer input must validate on its +// merits — a zero rejected with the human-readable message, a positive byte +// count accepted — rather than tripping a "no such overload" runtime error. +func TestCEL_DefragRuleQuantityIntegerInput(t *testing.T) { + skipIfNoEnvtest(t) + ctx := context.Background() + + mk := func(name string, freeSpace int64) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "etcd-operator.cozystack.io", + Version: "v1alpha2", + Kind: "EtcdDefrag", + }) + u.SetName(name) + u.SetNamespace("default") + u.Object["spec"] = map[string]any{ + "clusterRef": map[string]any{"name": "etcd"}, + "rule": map[string]any{"freeSpaceAbove": freeSpace}, // integer, not "200Mi" + } + return u + } + + // Zero must be rejected with the intended message, not a CEL overload error. + err := k8s.Create(ctx, mk("defrag-zero-int", 0)) + if err == nil { + _ = k8s.Delete(ctx, mk("defrag-zero-int", 0)) + t.Fatalf("apiserver accepted rule.freeSpaceAbove=0 (integer); expected rejection") + } + if !strings.Contains(err.Error(), "freeSpaceAbove must be greater than 0") { + t.Fatalf("error did not surface the intended message (CEL string() coercion missing?): %v", err) + } + + // A positive bare byte count is a legitimate Quantity and must be accepted. + valid := mk("defrag-bytes-int", 209715200) // 200Mi as a plain integer + if err := k8s.Create(ctx, valid); err != nil { + t.Fatalf("apiserver rejected a valid integer rule.freeSpaceAbove=209715200: %v", err) + } + _ = k8s.Delete(ctx, valid) +} diff --git a/api/v1alpha2/etcddefrag_types.go b/api/v1alpha2/etcddefrag_types.go new file mode 100644 index 00000000..7b11db9f --- /dev/null +++ b/api/v1alpha2/etcddefrag_types.go @@ -0,0 +1,227 @@ +/* +Copyright 2023 Timofey Larkin. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EtcdDefragSpec is the desired state of an EtcdDefrag: a one-shot request to +// defragment an EtcdCluster's members. +// +// +kubebuilder:validation:XValidation:rule="size(self.clusterRef.name) != 0",message="spec.clusterRef.name is required" +type EtcdDefragSpec struct { + // ClusterRef names the EtcdCluster (same namespace) to defragment. + ClusterRef corev1.LocalObjectReference `json:"clusterRef"` + + // Rule decides which members this run touches. Absent is equivalent to an + // empty rule: the default gate (freeSpaceAbove 200Mi). To defragment every + // member unconditionally, set rule.all: true. + // +optional + Rule *DefragRule `json:"rule,omitempty"` + + // TTLSecondsAfterFinished records how long after a terminal phase this + // object should be garbage-collected — meaningful for objects a scheduler + // stamps out. NOTE: acted on by the (not-yet-implemented) reconciling + // controller; the API server does not garbage-collect custom resources on + // its own. Absent means the record is kept. + // +kubebuilder:validation:Minimum=0 + // +optional + TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"` +} + +// DefragRule decides whether a member is worth defragmenting. A defrag can only +// reclaim DbSize-DbSizeInUse, so that reclaimable amount is the always-applied +// floor — what stops a full-but-unfragmented backend (DbSize ≈ DbSizeInUse near +// the quota) from being defragmented forever for nothing. +// +// +kubebuilder:validation:XValidation:rule="!(has(self.all) && self.all) || (!has(self.freeSpaceAbove) && !has(self.quotaUsageAbove) && !has(self.minReclaim))",message="rule.all cannot be combined with freeSpaceAbove/quotaUsageAbove/minReclaim" +// +kubebuilder:validation:XValidation:rule="!has(self.freeSpaceAbove) || quantity(string(self.freeSpaceAbove)).isGreaterThan(quantity('0'))",message="freeSpaceAbove must be greater than 0" +// +kubebuilder:validation:XValidation:rule="!has(self.minReclaim) || quantity(string(self.minReclaim)).isGreaterThan(quantity('0'))",message="minReclaim must be greater than 0" +// +kubebuilder:validation:XValidation:rule="!has(self.minReclaim) || has(self.quotaUsageAbove)",message="minReclaim is only meaningful with quotaUsageAbove" +// +kubebuilder:validation:XValidation:rule="!(has(self.minReclaim) && has(self.freeSpaceAbove)) || quantity(string(self.minReclaim)).compareTo(quantity(string(self.freeSpaceAbove))) <= 0",message="minReclaim must not exceed freeSpaceAbove" +type DefragRule struct { + // All defragments every member unconditionally, regardless of size — the + // explicit "do it now". Mutually exclusive with the threshold fields below. + // +optional + All bool `json:"all,omitempty"` + + // FreeSpaceAbove defragments a member whose reclaimable space + // (DbSize-DbSizeInUse) exceeds this. The primary, always-applied gate. + // Absent means the built-in default (200Mi). + // +optional + FreeSpaceAbove *resource.Quantity `json:"freeSpaceAbove,omitempty"` + + // QuotaUsageAbove: when DbSize exceeds this fraction of the backend quota + // (approaching NOSPACE), lower the reclaimable floor to MinReclaim so small + // wins are taken under pressure. A member is never defragmented when its + // reclaimable space is below MinReclaim. Integer percent 1..100 with a "%" + // suffix, e.g. "80%". + // +kubebuilder:validation:Pattern=`^([1-9][0-9]?|100)%$` + // +optional + QuotaUsageAbove string `json:"quotaUsageAbove,omitempty"` + + // MinReclaim floors the quota arm: even under quota pressure, skip a member + // that would reclaim less than this. Only meaningful with QuotaUsageAbove, + // and must not exceed FreeSpaceAbove. Absent means the built-in default + // (32Mi). + // +optional + MinReclaim *resource.Quantity `json:"minReclaim,omitempty"` +} + +// EtcdDefragPhase is the lifecycle phase of an EtcdDefrag. +type EtcdDefragPhase string + +const ( + // EtcdDefragPhasePending is the initial phase: the request is queued. + // Defragmentations are serialized per cluster, so a request waits here while + // another runs against the same EtcdCluster, or while the cluster is not yet + // healthy enough to defragment safely (surfaced as a condition). + EtcdDefragPhasePending EtcdDefragPhase = "Pending" + // EtcdDefragPhaseRunning means the member sweep is in progress. + EtcdDefragPhaseRunning EtcdDefragPhase = "Running" + // EtcdDefragPhaseComplete means the sweep finished; see status.members for + // per-member outcomes. + EtcdDefragPhaseComplete EtcdDefragPhase = "Complete" + // EtcdDefragPhaseFailed means the sweep could not complete. + EtcdDefragPhaseFailed EtcdDefragPhase = "Failed" +) + +// DefragOutcome is the result of processing a single member. +type DefragOutcome string + +const ( + // DefragOutcomePending: not yet processed. + DefragOutcomePending DefragOutcome = "Pending" + // DefragOutcomeSkipped: below the rule's threshold, nothing worth reclaiming. + DefragOutcomeSkipped DefragOutcome = "Skipped" + // DefragOutcomeDefragmented: successfully defragmented. + DefragOutcomeDefragmented DefragOutcome = "Defragmented" + // DefragOutcomeFailed: the Defragment RPC failed or timed out. + DefragOutcomeFailed DefragOutcome = "Failed" +) + +// MemberRole is a member's raft role at the time it was processed. +type MemberRole string + +const ( + MemberRoleLeader MemberRole = "leader" + MemberRoleFollower MemberRole = "follower" +) + +// EtcdDefragStatus is the observed state of an EtcdDefrag. +type EtcdDefragStatus struct { + // Phase is the high-level lifecycle phase. + // +optional + Phase EtcdDefragPhase `json:"phase,omitempty"` + + // StartedAt is when the sweep began. + // +optional + StartedAt *metav1.Time `json:"startedAt,omitempty"` + + // CompletedAt is when the sweep reached a terminal phase. + // +optional + CompletedAt *metav1.Time `json:"completedAt,omitempty"` + + // Defragmented counts members actually defragmented this run. + // +optional + Defragmented int32 `json:"defragmented,omitempty"` + + // Members holds the per-member outcome of the sweep, keyed by member name. + // +optional + // +listType=map + // +listMapKey=name + Members []MemberDefragStatus `json:"members,omitempty"` + + // Conditions represent the latest available observations — including why a + // Pending run is being deferred (e.g. the cluster is not fully healthy). + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// MemberDefragStatus is the outcome of defragmenting a single member. +type MemberDefragStatus struct { + // Name is the EtcdMember this row describes. + Name string `json:"name"` + + // Role is the member's raft role at the time it was processed. + // +optional + Role MemberRole `json:"role,omitempty"` + + // Outcome is the result of processing this member. + // +optional + Outcome DefragOutcome `json:"outcome,omitempty"` + + // Reason qualifies the outcome (e.g. BelowThreshold, ClusterNotHealthy, + // RPCError), in condition-reason style. + // +optional + Reason string `json:"reason,omitempty"` + + // DBSizeBefore is the member's physical backend size before defragmenting. + // +optional + DBSizeBefore int64 `json:"dbSizeBefore,omitempty"` + + // DBSizeAfter is the physical backend size after defragmenting. + // +optional + DBSizeAfter int64 `json:"dbSizeAfter,omitempty"` + + // ReclaimedBytes is DBSizeBefore-DBSizeAfter for a completed defrag. + // +optional + ReclaimedBytes int64 `json:"reclaimedBytes,omitempty"` + + // FinishedAt is when this member was processed. + // +optional + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Cluster",type=string,JSONPath=`.spec.clusterRef.name` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Defragmented",type=integer,JSONPath=`.status.defragmented` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// EtcdDefrag is the Schema for the etcddefrags API. It requests a one-shot, +// run-to-completion defragmentation of an EtcdCluster's members. Like +// EtcdSnapshot it is a record: the operator drives it through status.phase and +// it never re-runs. +// +// NOTE: this ships the API type ahead of its reconciling controller. Until that +// controller lands, an EtcdDefrag is inert — creating one records intent but +// nothing acts on it (no sweep runs, status stays empty, TTL does not fire). +type EtcdDefrag struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EtcdDefragSpec `json:"spec,omitempty"` + Status EtcdDefragStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EtcdDefragList contains a list of EtcdDefrag. +type EtcdDefragList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdDefrag `json:"items"` +} + +func init() { + SchemeBuilder.Register(&EtcdDefrag{}, &EtcdDefragList{}) +} diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index a3a5794e..0c2cca5b 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -146,6 +146,31 @@ func (in *ClientTLS) DeepCopy() *ClientTLS { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DefragRule) DeepCopyInto(out *DefragRule) { + *out = *in + if in.FreeSpaceAbove != nil { + in, out := &in.FreeSpaceAbove, &out.FreeSpaceAbove + x := (*in).DeepCopy() + *out = &x + } + if in.MinReclaim != nil { + in, out := &in.MinReclaim, &out.MinReclaim + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DefragRule. +func (in *DefragRule) DeepCopy() *DefragRule { + if in == nil { + return nil + } + out := new(DefragRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdCluster) DeepCopyInto(out *EtcdCluster) { *out = *in @@ -330,6 +355,128 @@ func (in *EtcdClusterTLS) DeepCopy() *EtcdClusterTLS { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdDefrag) DeepCopyInto(out *EtcdDefrag) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdDefrag. +func (in *EtcdDefrag) DeepCopy() *EtcdDefrag { + if in == nil { + return nil + } + out := new(EtcdDefrag) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdDefrag) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdDefragList) DeepCopyInto(out *EtcdDefragList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdDefrag, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdDefragList. +func (in *EtcdDefragList) DeepCopy() *EtcdDefragList { + if in == nil { + return nil + } + out := new(EtcdDefragList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdDefragList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdDefragSpec) DeepCopyInto(out *EtcdDefragSpec) { + *out = *in + out.ClusterRef = in.ClusterRef + if in.Rule != nil { + in, out := &in.Rule, &out.Rule + *out = new(DefragRule) + (*in).DeepCopyInto(*out) + } + if in.TTLSecondsAfterFinished != nil { + in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdDefragSpec. +func (in *EtcdDefragSpec) DeepCopy() *EtcdDefragSpec { + if in == nil { + return nil + } + out := new(EtcdDefragSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdDefragStatus) DeepCopyInto(out *EtcdDefragStatus) { + *out = *in + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.CompletedAt != nil { + in, out := &in.CompletedAt, &out.CompletedAt + *out = (*in).DeepCopy() + } + if in.Members != nil { + in, out := &in.Members, &out.Members + *out = make([]MemberDefragStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdDefragStatus. +func (in *EtcdDefragStatus) DeepCopy() *EtcdDefragStatus { + if in == nil { + return nil + } + out := new(EtcdDefragStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdMember) DeepCopyInto(out *EtcdMember) { *out = *in @@ -633,6 +780,25 @@ func (in *IssuerReference) DeepCopy() *IssuerReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemberDefragStatus) DeepCopyInto(out *MemberDefragStatus) { + *out = *in + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemberDefragStatus. +func (in *MemberDefragStatus) DeepCopy() *MemberDefragStatus { + if in == nil { + return nil + } + out := new(MemberDefragStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObservedClusterSpec) DeepCopyInto(out *ObservedClusterSpec) { *out = *in diff --git a/charts/etcd-operator/crd-bases/etcd-operator.cozystack.io_etcddefrags.yaml b/charts/etcd-operator/crd-bases/etcd-operator.cozystack.io_etcddefrags.yaml new file mode 100644 index 00000000..18934ae2 --- /dev/null +++ b/charts/etcd-operator/crd-bases/etcd-operator.cozystack.io_etcddefrags.yaml @@ -0,0 +1,282 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: etcddefrags.etcd-operator.cozystack.io +spec: + group: etcd-operator.cozystack.io + names: + kind: EtcdDefrag + listKind: EtcdDefragList + plural: etcddefrags + singular: etcddefrag + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.clusterRef.name + name: Cluster + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.defragmented + name: Defragmented + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: |- + EtcdDefrag is the Schema for the etcddefrags API. It requests a one-shot, + run-to-completion defragmentation of an EtcdCluster's members. Like + EtcdSnapshot it is a record: the operator drives it through status.phase and + it never re-runs. + + NOTE: this ships the API type ahead of its reconciling controller. Until that + controller lands, an EtcdDefrag is inert — creating one records intent but + nothing acts on it (no sweep runs, status stays empty, TTL does not fire). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EtcdDefragSpec is the desired state of an EtcdDefrag: a one-shot request to + defragment an EtcdCluster's members. + properties: + clusterRef: + description: ClusterRef names the EtcdCluster (same namespace) to + defragment. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + rule: + description: |- + Rule decides which members this run touches. Absent is equivalent to an + empty rule: the default gate (freeSpaceAbove 200Mi). To defragment every + member unconditionally, set rule.all: true. + properties: + all: + description: |- + All defragments every member unconditionally, regardless of size — the + explicit "do it now". Mutually exclusive with the threshold fields below. + type: boolean + freeSpaceAbove: + anyOf: + - type: integer + - type: string + description: |- + FreeSpaceAbove defragments a member whose reclaimable space + (DbSize-DbSizeInUse) exceeds this. The primary, always-applied gate. + Absent means the built-in default (200Mi). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minReclaim: + anyOf: + - type: integer + - type: string + description: |- + MinReclaim floors the quota arm: even under quota pressure, skip a member + that would reclaim less than this. Only meaningful with QuotaUsageAbove, + and must not exceed FreeSpaceAbove. Absent means the built-in default + (32Mi). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + quotaUsageAbove: + description: |- + QuotaUsageAbove: when DbSize exceeds this fraction of the backend quota + (approaching NOSPACE), lower the reclaimable floor to MinReclaim so small + wins are taken under pressure. A member is never defragmented when its + reclaimable space is below MinReclaim. Integer percent 1..100 with a "%" + suffix, e.g. "80%". + pattern: ^([1-9][0-9]?|100)%$ + type: string + type: object + x-kubernetes-validations: + - message: rule.all cannot be combined with freeSpaceAbove/quotaUsageAbove/minReclaim + rule: '!(has(self.all) && self.all) || (!has(self.freeSpaceAbove) + && !has(self.quotaUsageAbove) && !has(self.minReclaim))' + - message: freeSpaceAbove must be greater than 0 + rule: '!has(self.freeSpaceAbove) || quantity(string(self.freeSpaceAbove)).isGreaterThan(quantity(''0''))' + - message: minReclaim must be greater than 0 + rule: '!has(self.minReclaim) || quantity(string(self.minReclaim)).isGreaterThan(quantity(''0''))' + - message: minReclaim is only meaningful with quotaUsageAbove + rule: '!has(self.minReclaim) || has(self.quotaUsageAbove)' + - message: minReclaim must not exceed freeSpaceAbove + rule: '!(has(self.minReclaim) && has(self.freeSpaceAbove)) || quantity(string(self.minReclaim)).compareTo(quantity(string(self.freeSpaceAbove))) + <= 0' + ttlSecondsAfterFinished: + description: |- + TTLSecondsAfterFinished records how long after a terminal phase this + object should be garbage-collected — meaningful for objects a scheduler + stamps out. NOTE: acted on by the (not-yet-implemented) reconciling + controller; the API server does not garbage-collect custom resources on + its own. Absent means the record is kept. + format: int32 + minimum: 0 + type: integer + required: + - clusterRef + type: object + x-kubernetes-validations: + - message: spec.clusterRef.name is required + rule: size(self.clusterRef.name) != 0 + status: + description: EtcdDefragStatus is the observed state of an EtcdDefrag. + properties: + completedAt: + description: CompletedAt is when the sweep reached a terminal phase. + format: date-time + type: string + conditions: + description: |- + Conditions represent the latest available observations — including why a + Pending run is being deferred (e.g. the cluster is not fully healthy). + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + defragmented: + description: Defragmented counts members actually defragmented this + run. + format: int32 + type: integer + members: + description: Members holds the per-member outcome of the sweep, keyed + by member name. + items: + description: MemberDefragStatus is the outcome of defragmenting + a single member. + properties: + dbSizeAfter: + description: DBSizeAfter is the physical backend size after + defragmenting. + format: int64 + type: integer + dbSizeBefore: + description: DBSizeBefore is the member's physical backend size + before defragmenting. + format: int64 + type: integer + finishedAt: + description: FinishedAt is when this member was processed. + format: date-time + type: string + name: + description: Name is the EtcdMember this row describes. + type: string + outcome: + description: Outcome is the result of processing this member. + type: string + reason: + description: |- + Reason qualifies the outcome (e.g. BelowThreshold, ClusterNotHealthy, + RPCError), in condition-reason style. + type: string + reclaimedBytes: + description: ReclaimedBytes is DBSizeBefore-DBSizeAfter for + a completed defrag. + format: int64 + type: integer + role: + description: Role is the member's raft role at the time it was + processed. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + phase: + description: Phase is the high-level lifecycle phase. + type: string + startedAt: + description: StartedAt is when the sweep began. + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/docs/etcd-defrag.md b/docs/etcd-defrag.md new file mode 100644 index 00000000..60cd5a50 --- /dev/null +++ b/docs/etcd-defrag.md @@ -0,0 +1,151 @@ +# Defragmentation (`EtcdDefrag`) + +> **Status:** this ships the `EtcdDefrag` **API type** ahead of its reconciling +> controller. Until that controller lands the resource is **inert** — creating +> one records intent but nothing acts on it (no sweep runs, `status` stays +> empty, `ttlSecondsAfterFinished` does not fire). The "Safety model", +> "Timeouts and retries" and `status` sections below describe the **controller +> contract the follow-up implements**, not behaviour that exists today. + +etcd never reclaims backend disk on its own: compaction frees pages logically, +but the file — and the space counted against `--quota-backend-bytes` — stays +allocated until a **defragment** returns it. `EtcdDefrag` is how you ask the +operator to do that, safely. + +It is a one-shot, run-to-completion record, modeled on [`EtcdSnapshot`](concepts.md#snapshots--restore): +the operator drives it through `status.phase` and it never re-runs. + +**Scheduling.** `EtcdDefrag` is the *run*; what *triggers* a run is separate. +Today, recurring defragmentation is driven by creating `EtcdDefrag` objects from +outside (a `CronJob`, a GitOps cron). A companion `EtcdDefragPolicy` kind — a +cadence (`schedule`) and/or a condition (`when`) that stamps out `EtcdDefrag` +runs — is planned so the operator absorbs that scheduling itself; it is not part +of this API PR. + +## Why in the operator (not a bare CronJob) + +A defrag briefly blocks the member it runs on, so it must be sequenced. The +operator already holds each cluster's TLS/auth material and endpoints, has the +whole-cluster view, and runs under leader election, so it can defragment members +**one at a time, followers before the leader, and only while the cluster is +healthy** — deferring rather than forcing on a degraded cluster. A detached +`CronJob` calling `etcdctl defrag` cannot make those guarantees. + +## Usage + +Defragment every member of a cluster once, now (`rule.all` — the explicit, +unconditional form): + +```yaml +apiVersion: etcd-operator.cozystack.io/v1alpha2 +kind: EtcdDefrag +metadata: + name: etcd-now + namespace: team-a +spec: + clusterRef: + name: etcd + rule: + all: true +``` + +Guarded run (skips members that aren't worth defragmenting; with +`ttlSecondsAfterFinished` the controller GCs the record an hour after it +finishes — useful for objects a scheduler stamps out): + +```yaml +apiVersion: etcd-operator.cozystack.io/v1alpha2 +kind: EtcdDefrag +spec: + clusterRef: + name: etcd + ttlSecondsAfterFinished: 3600 + rule: + freeSpaceAbove: 200Mi # reclaimable (DbSize - DbSizeInUse) worth reclaiming + quotaUsageAbove: 80% # under quota pressure, take smaller wins too … + minReclaim: 32Mi # … but never a no-op defrag +``` + +Inspect progress and history (once the controller exists): + +```sh +kubectl get etcddefrag.etcd-operator.cozystack.io -n team-a +# NAME CLUSTER PHASE DEFRAGMENTED AGE +# etcd-now etcd Complete 2 5m + +kubectl get etcddefrag.etcd-operator.cozystack.io etcd-now -n team-a \ + -o jsonpath='{.status.members}' | jq +``` + +## The rule + +A defrag can only reclaim `DbSize - DbSizeInUse`, so that reclaimable amount is +the always-applied floor — this is what stops a full-but-unfragmented backend +(`DbSize ≈ DbSizeInUse` near the quota) from being defragmented over and over +for nothing. + +| Field | Default | Meaning | +|---|---|---| +| `all` | `false` | Defragment every member unconditionally. Mutually exclusive with the fields below. | +| `freeSpaceAbove` | `200Mi` | Defragment a member whose reclaimable space exceeds this. | +| `quotaUsageAbove` | unset | When `DbSize` exceeds this fraction of the backend quota, lower the reclaim floor to `minReclaim` so small wins are taken under pressure. | +| `minReclaim` | `32Mi` | The floor for the quota arm (only meaningful with `quotaUsageAbove`, and must not exceed `freeSpaceAbove`) — never a no-op defrag. | + +An **absent `rule`** is equivalent to an empty one: the default gate +(`freeSpaceAbove: 200Mi`). Unconditional defragmentation is something you ask +for explicitly with `rule.all: true`, never something you get by leaving a key +out. + +> **Compaction is a prerequisite you own.** Defrag reclaims what compaction +> freed. A cluster with no auto-compaction (`spec.options.autoCompactionMode` / +> `autoCompactionRetention`) has `DbSizeInUse ≈ DbSize` and little to reclaim. +> Set auto-compaction if you rely on defrag to hold the backend down. + +## Safety model (planned controller behaviour) + +- **One member at a time, followers before the leader**, only while the whole + cluster is healthy. A defrag due on a not-fully-healthy cluster is **deferred** + — the object stays `Pending` with a condition explaining why — never forced, + so quorum is never at risk. +- **Serialized per cluster:** at most one `EtcdDefrag` runs against a given + `EtcdCluster` at a time; others wait in `Pending`. +- Health is judged from more than "the member answered": a member replies to a + local status read while partitioned, alarmed (`NOSPACE`/`CORRUPT`), or behind + in raft, so those are checked before acting. + +## Status (planned controller behaviour) + +`status.phase` moves `Pending → Running → Complete | Failed`; a `Pending` run +waiting on cluster health carries a condition saying so. `status.members[]` +records, per member (keyed by name), the role at processing time, the outcome +(`Skipped` / `Defragmented` / `Failed`), the before/after `DbSize`, and the +bytes reclaimed — the run's full history, not a single rolled-up condition. + +## Timeouts and retries (planned controller behaviour) + +Following [`EtcdSnapshot`](concepts.md#snapshots--restore) — where the Job's +deadlines are controller constants and terminal phases are sticky — this needs +no `spec` knobs: + +- **Per-member timeout** bounds each `Defragment` RPC (a stop-the-world call on a + large backend), so one wedged member can't consume the whole run; on expiry + that member is `Failed`. +- **An overall active-deadline** bounds `Running` + waiting-while-`Pending` + together; on expiry the run is `Failed`. This also protects the per-cluster + serialization slot — a run stuck waiting on an unhealthy cluster can't block + the next one forever. +- **Retry within a run:** a deferred `Pending` re-checks cluster health with + backoff up to the deadline; a failed per-member RPC is retried a bounded number + of times then marked `Failed` (a failing leader fails the run); and a defrag + that doesn't shrink `DbSize` is backed off rather than repeated. +- **Retry across runs:** terminal phases (`Complete`/`Failed`) are sticky — an + `EtcdDefrag` never re-runs itself. A retry is a *new* `EtcdDefrag`: the external + scheduler's next tick for periodic use, or a re-create for a one-shot. Each + attempt is a discrete, auditable object (GC'd via `ttlSecondsAfterFinished`) + rather than hidden retry state. + +## Relationship to capacity metrics + +The capacity metrics and alert rules that tell you *when* a defrag is worth +running are tracked separately (see #357); `EtcdDefrag` records sizes in its own +`status` during a run rather than as continuously-scraped gauges.