From 46da209560c3928f353f24efc576fb51a9387249 Mon Sep 17 00:00:00 2001 From: "Per G. da Silva" Date: Fri, 13 Nov 2026 10:17:09 +0100 Subject: [PATCH] test(e2e): cover COD deadline recovery and completion latching Add dedicated ConfigMap-key-gated deadline scenarios for recovery before the original deadline and health regression after successful completion. Verify revision identity, ownership, current-generation status, and durable completion throughout a post-deadline observation window. Keep readiness control outside the managed manifest so gate changes neither create revisions nor conflict with reconciliation. Add focused helper regressions without changing the existing Secret-precondition scenarios or runtime behavior. Addresses OPRUN-4795. Signed-off-by: Per G. da Silva --- .../features/cod_progress_deadline.feature | 184 ++++++++++++++++++ .../object-controller/steps/deadline_steps.go | 69 +++++++ .../steps/deadline_steps_test.go | 123 ++++++++++++ .../e2e/object-controller/steps/steps.go | 2 + 4 files changed, 378 insertions(+) create mode 100644 test/e2e/object-controller/features/cod_progress_deadline.feature create mode 100644 test/internal/e2e/object-controller/steps/deadline_steps.go create mode 100644 test/internal/e2e/object-controller/steps/deadline_steps_test.go diff --git a/test/e2e/object-controller/features/cod_progress_deadline.feature b/test/e2e/object-controller/features/cod_progress_deadline.feature new file mode 100644 index 0000000000..688a43e2d8 --- /dev/null +++ b/test/e2e/object-controller/features/cod_progress_deadline.feature @@ -0,0 +1,184 @@ +@ObjectController @ClusterObjectDeployment @ProgressDeadline @CODDeadlineLifecycle +Feature: ClusterObjectDeployment progress deadline lifecycle + + A revision must recover within its deadline without creating another revision. + After completion, health may regress while completion remains recorded. + The deployment must keep reporting a successful rollout beyond the deadline. + + Background: + Given object-controller is available + And namespace "${TEST_NAMESPACE}" is available + + Scenario: Completed revision stays progressed when health regresses beyond the deadline + When resource is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterObjectDeployment + metadata: + name: ${NAME} + spec: + progressDeadlineMinutes: 1 + template: + spec: + collisionProtection: Prevent + progressionProbes: + - selector: + type: Label + label: + matchLabels: + probe: deadline-gate + assertions: + - type: FieldValue + fieldValue: + fieldPath: data.ready + value: "true" + phases: + - name: resources + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: deadline-gate + namespace: ${TEST_NAMESPACE} + labels: + probe: deadline-gate + data: + payload: unchanged + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as False with Reason ObjectsNotReady + And resource "clusterobjectset/${NAME}-1" matches + """ + status: + completedAt: null + """ + # Only the test owns data.ready, which is absent from the immutable manifest. + When resource is applied + """ + apiVersion: v1 + kind: ConfigMap + metadata: + name: deadline-gate + namespace: ${TEST_NAMESPACE} + data: + ready: "true" + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as True with Reason AllObjectsReady + And ClusterObjectDeployment "${NAME}" reports Progressing as False with Reason NewClusterObjectSetProgressed + And ClusterObjectDeployment "${NAME}" reports Available as True with Reason Available + And ClusterObjectDeployment "${NAME}" owns 1 ClusterObjectSets + And ClusterObjectSet "${NAME}-1" has a completion timestamp + When resource is applied + """ + apiVersion: v1 + kind: ConfigMap + metadata: + name: deadline-gate + namespace: ${TEST_NAMESPACE} + data: + ready: "false" + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as False with Reason ObjectsNotReady + And ClusterObjectDeployment "${NAME}" reports Available as False with Reason Unavailable + And ClusterObjectSet "${NAME}-1" has a completion timestamp + # Both controllers have reflected the health regression. The successful + # rollout reason must persist beyond the one-minute deadline. + And ClusterObjectDeployment "${NAME}" consistently reports Progressing as False with Reason NewClusterObjectSetProgressed for "70s" + When resource is applied + """ + apiVersion: v1 + kind: ConfigMap + metadata: + name: deadline-gate + namespace: ${TEST_NAMESPACE} + data: + ready: "true" + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as True with Reason AllObjectsReady + And ClusterObjectDeployment "${NAME}" reports Available as True with Reason Available + And ClusterObjectDeployment "${NAME}" reports Progressing as False with Reason NewClusterObjectSetProgressed + And ClusterObjectSet "${NAME}-1" has a completion timestamp + And ClusterObjectDeployment "${NAME}" owns 1 ClusterObjectSets + And resource "configmap/deadline-gate" matches + """ + data: + payload: unchanged + ready: "true" + """ + + Scenario: Revision recovers before its original progress deadline + When resource is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterObjectDeployment + metadata: + name: ${NAME} + spec: + progressDeadlineMinutes: 1 + template: + spec: + collisionProtection: Prevent + progressionProbes: + - selector: + type: Label + label: + matchLabels: + probe: deadline-gate + assertions: + - type: FieldValue + fieldValue: + fieldPath: data.ready + value: "true" + phases: + - name: resources + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: deadline-gate + namespace: ${TEST_NAMESPACE} + labels: + probe: deadline-gate + data: + payload: unchanged + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as False with Reason ObjectsNotReady + And ClusterObjectDeployment "${NAME}" reports Progressing as True with Reason NewClusterObjectSetProgressing + And ClusterObjectDeployment "${NAME}" reports Available as False with Reason Unavailable + And ClusterObjectDeployment "${NAME}" owns 1 ClusterObjectSets + And resource "clusterobjectset/${NAME}-1" matches + """ + status: + completedAt: null + """ + When resource is applied + """ + apiVersion: v1 + kind: ConfigMap + metadata: + name: deadline-gate + namespace: ${TEST_NAMESPACE} + data: + ready: "true" + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as True with Reason AllObjectsReady + And ClusterObjectDeployment "${NAME}" reports Progressing as False with Reason NewClusterObjectSetProgressed + And ClusterObjectDeployment "${NAME}" reports Available as True with Reason Available + And ClusterObjectDeployment "${NAME}" owns 1 ClusterObjectSets + And resource "clusterobjectdeployment/${NAME}" matches + """ + spec: + progressDeadlineMinutes: 1 + """ + # Compare the named revision's completion against its server-assigned creation + # timestamp plus the unchanged one-minute policy. + And ClusterObjectSet "${NAME}-1" completed before the deadline of ClusterObjectDeployment "${NAME}" + And resource "configmap/deadline-gate" matches + """ + data: + payload: unchanged + ready: "true" + """ diff --git a/test/internal/e2e/object-controller/steps/deadline_steps.go b/test/internal/e2e/object-controller/steps/deadline_steps.go new file mode 100644 index 0000000000..33ca539764 --- /dev/null +++ b/test/internal/e2e/object-controller/steps/deadline_steps.go @@ -0,0 +1,69 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + "time" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + shared "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func readDeadlineObject(ctx context.Context, kind, name string, into any) error { + name = shared.SubstituteScenarioVars(name, shared.Scenario(ctx)) + out, err := shared.K8sClient(ctx, "get", kind, name, "-o", "json") + if err != nil { + return fmt.Errorf("reading %s %q: %w", kind, name, err) + } + if err = json.Unmarshal([]byte(out), into); err != nil { + return fmt.Errorf("decoding %s %q: %w", kind, name, err) + } + return nil +} + +// ClusterObjectSetHasCompletion checks presence only. API validation separately +// guarantees that completedAt cannot be changed or removed once set. +func ClusterObjectSetHasCompletion(ctx context.Context, name string) error { + var cos ocv1.ClusterObjectSet + if err := readDeadlineObject(ctx, "clusterobjectset", name, &cos); err != nil { + return err + } + if cos.Status.CompletedAt.IsZero() { + return fmt.Errorf("revision %q has no completedAt", cos.Name) + } + return nil +} + +// ClusterObjectSetCompletedBeforeDeadline compares durable timestamps on the +// named revision. The feature separately checks sole ownership. +func ClusterObjectSetCompletedBeforeDeadline(ctx context.Context, name, deployment string) error { + var cos ocv1.ClusterObjectSet + if err := readDeadlineObject(ctx, "clusterobjectset", name, &cos); err != nil { + return err + } + var cod ocv1.ClusterObjectDeployment + if err := readDeadlineObject(ctx, "clusterobjectdeployment", deployment, &cod); err != nil { + return err + } + if cos.Spec.Group != cod.Name { + return fmt.Errorf("revision %q is not in deployment group %q", cos.Name, cod.Name) + } + if cos.CreationTimestamp.IsZero() { + return fmt.Errorf("revision %q has no creation timestamp", cos.Name) + } + if cos.Status.CompletedAt.IsZero() { + return fmt.Errorf("revision %q has no completedAt", cos.Name) + } + if cos.Status.CompletedAt.Before(&cos.CreationTimestamp) { + return fmt.Errorf("completedAt precedes revision creation") + } + if cod.Spec.ProgressDeadlineMinutes <= 0 { + return fmt.Errorf("deployment %q has no enabled deadline", cod.Name) + } + deadline := cos.CreationTimestamp.Add(time.Duration(cod.Spec.ProgressDeadlineMinutes) * time.Minute) + if !cos.Status.CompletedAt.Time.Before(deadline) { + return fmt.Errorf("revision completedAt %s must be before deadline %s", cos.Status.CompletedAt, deadline) + } + return nil +} diff --git a/test/internal/e2e/object-controller/steps/deadline_steps_test.go b/test/internal/e2e/object-controller/steps/deadline_steps_test.go new file mode 100644 index 0000000000..2f72caff59 --- /dev/null +++ b/test/internal/e2e/object-controller/steps/deadline_steps_test.go @@ -0,0 +1,123 @@ +package steps + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/cucumber/godog" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + shared "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +// Only the API transport is replaced. Completion checks use the real helpers; +// ordinary count/condition steps retain their existing tests. +func deadlineFixture(t *testing.T) (context.Context, *ocv1.ClusterObjectDeployment, *ocv1.ClusterObjectSet, string) { + t.Helper() + ctx, err := shared.CreateScenarioContext(t.Context(), &godog.Scenario{Id: "1", Uri: "cod_progress_deadline.feature"}) + require.NoError(t, err) + sc := shared.Scenario(ctx) + cod := &ocv1.ClusterObjectDeployment{ObjectMeta: metav1.ObjectMeta{Name: sc.ClusterExtensionName}} + cod.Spec.ProgressDeadlineMinutes = 1 + cos := &ocv1.ClusterObjectSet{ObjectMeta: metav1.ObjectMeta{Name: cod.Name + "-1", CreationTimestamp: metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC))}} + cos.Spec.Group = cod.Name + cos.Status.CompletedAt = metav1.NewTime(cos.CreationTimestamp.Add(10 * time.Second)) + dir := t.TempDir() + t.Setenv("DEADLINE_FIXTURES", dir) + script := `#!/bin/sh +if [ -f "$DEADLINE_FIXTURES/api-error" ]; then echo "API unavailable" >&2; exit 1; fi +case "$1 $2" in + "get clusterobjectdeployment") cat "$DEADLINE_FIXTURES/cod.json";; + "get clusterobjectset") cat "$DEADLINE_FIXTURES/cos.json";; + *) echo "unexpected command: $*" >&2; exit 1;; +esac +` + cli := filepath.Join(dir, "kubectl") + //nolint:gosec // Private executable fixture. + require.NoError(t, os.WriteFile(cli, []byte(script), 0o700)) + oldCLI := pflag.Lookup("k8s.cli").Value.String() + require.NoError(t, pflag.Set("k8s.cli", cli)) + t.Cleanup(func() { require.NoError(t, pflag.Set("k8s.cli", oldCLI)) }) + writeDeadlineFixture(t, dir, cod, cos) + return ctx, cod, cos, dir +} + +func writeDeadlineJSON(t *testing.T, dir, name string, obj any) { + t.Helper() + data, err := json.Marshal(obj) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o600)) +} +func writeDeadlineFixture(t *testing.T, dir string, cod *ocv1.ClusterObjectDeployment, cos *ocv1.ClusterObjectSet) { + t.Helper() + writeDeadlineJSON(t, dir, "cod.json", cod) + writeDeadlineJSON(t, dir, "cos.json", cos) +} + +func TestDeadlineTimestampComparison(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*ocv1.ClusterObjectDeployment, *ocv1.ClusterObjectSet) + want string + }{ + {name: "before expiry"}, + {name: "exact expiry", mutate: func(_ *ocv1.ClusterObjectDeployment, c *ocv1.ClusterObjectSet) { + c.Status.CompletedAt = metav1.NewTime(c.CreationTimestamp.Add(time.Minute)) + }, want: "before"}, + {name: "after expiry", mutate: func(_ *ocv1.ClusterObjectDeployment, c *ocv1.ClusterObjectSet) { + c.Status.CompletedAt = metav1.NewTime(c.CreationTimestamp.Add(61 * time.Second)) + }, want: "before"}, + {name: "before creation", mutate: func(_ *ocv1.ClusterObjectDeployment, c *ocv1.ClusterObjectSet) { + c.Status.CompletedAt = metav1.NewTime(c.CreationTimestamp.Add(-time.Second)) + }, want: "creation"}, + {name: "missing creation", mutate: func(_ *ocv1.ClusterObjectDeployment, c *ocv1.ClusterObjectSet) { c.CreationTimestamp = metav1.Time{} }, want: "creation"}, + {name: "missing completion", mutate: func(_ *ocv1.ClusterObjectDeployment, c *ocv1.ClusterObjectSet) { c.Status.CompletedAt = metav1.Time{} }, want: "completedAt"}, + {name: "disabled deadline", mutate: func(c *ocv1.ClusterObjectDeployment, _ *ocv1.ClusterObjectSet) { c.Spec.ProgressDeadlineMinutes = 0 }, want: "deadline"}, + {name: "different group", mutate: func(_ *ocv1.ClusterObjectDeployment, c *ocv1.ClusterObjectSet) { c.Spec.Group = "another" }, want: "group"}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cod, cos, dir := deadlineFixture(t) + if tc.mutate != nil { + tc.mutate(cod, cos) + } + writeDeadlineFixture(t, dir, cod, cos) + err := ClusterObjectSetCompletedBeforeDeadline(ctx, "${NAME}-1", "${NAME}") + if tc.want == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.want) + } + }) + } +} + +func TestClusterObjectSetHasCompletion(t *testing.T) { + for _, mode := range []string{"present", "missing", "API error", "invalid JSON"} { + t.Run(mode, func(t *testing.T) { + ctx, cod, cos, dir := deadlineFixture(t) + if mode == "missing" { + cos.Status.CompletedAt = metav1.Time{} + } + writeDeadlineFixture(t, dir, cod, cos) + if mode == "API error" { + require.NoError(t, os.WriteFile(filepath.Join(dir, "api-error"), nil, 0o600)) + } + if mode == "invalid JSON" { + require.NoError(t, os.WriteFile(filepath.Join(dir, "cos.json"), []byte("invalid"), 0o600)) + } + err := ClusterObjectSetHasCompletion(ctx, "${NAME}-1") + if mode == "present" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/test/internal/e2e/object-controller/steps/steps.go b/test/internal/e2e/object-controller/steps/steps.go index fba0f1eb8f..4c101e9cc9 100644 --- a/test/internal/e2e/object-controller/steps/steps.go +++ b/test/internal/e2e/object-controller/steps/steps.go @@ -22,6 +22,8 @@ import ( // RegisterSteps registers ClusterObjectSet and ClusterObjectDeployment assertions. func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^object-controller is available$`, ObjectControllerIsAvailable) + sc.Step(`^ClusterObjectSet "([^"]+)" has a completion timestamp$`, ClusterObjectSetHasCompletion) + sc.Step(`^ClusterObjectSet "([^"]+)" completed before the deadline of ClusterObjectDeployment "([^"]+)"$`, ClusterObjectSetCompletedBeforeDeadline) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" lifecycle is set to "([^"]+)"$`, ClusterObjectSetLifecycleUpdate) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+)$`, ClusterObjectSetReportsConditionWithoutMsg) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+) and Message:$`, ClusterObjectSetReportsConditionWithMsg)