diff --git a/test/e2e/README.md b/test/e2e/README.md index 4bb2677a0a..27b8c3fd20 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -65,7 +65,9 @@ Suites register additional component steps when existing features contain integr assertions: operator-controller uses object assertions, object-controller retains ClusterExtension integration coverage, and catalogd proxy tests use ClusterExtension steps. Platform registers ClusterExtension steps for demos. Feature contents, -scenario names, tags, and assertions are unchanged. +scenario names and assertions are preserved. Direct object-controller scenarios +use `@ObjectController` with `@ClusterObjectSet` or `@ClusterObjectDeployment`; +ClusterExtension integration scenarios retain `@BoxcutterRuntime`. `test/internal/e2e/steps` owns `World` and the `Scenario(ctx)` accessor, generic resource and condition helpers, catalog/registry prerequisites, RBAC templates, and cleanup. @@ -318,6 +320,51 @@ Select `./test/e2e//...` with the same flags to run just one suite: go test -count=1 -v ./test/e2e/object-controller/... -args --godog.tags="~@Serial && ~@demo" ``` +### Run Standalone Object-Controller Scenarios + +Direct scenarios use `Given object-controller is available`. Discovery finds +object-controller in its own namespace; readiness requires its Deployment and +the ClusterObjectSet CRD. Scenarios tagged `@ClusterObjectDeployment` also +require the ClusterObjectDeployment CRD. Permissions come from scenario tags, +independently of feature filenames. + +Install only object-controller on a disposable cluster using the existing chart: + +```bash +helm template olmv1 ./helm/olmv1 \ + --set options.featureSet=experimental \ + --set options.objectController.enabled=true \ + --set options.operatorController.enabled=false \ + --set options.catalogd.enabled=false | kubectl apply -f - +``` + +Run direct COS and COD scenarios separately, or select both with +`@ObjectController` (equivalently `@ClusterObjectSet,@ClusterObjectDeployment`; +Godog uses a comma for OR): + +```bash +go test -count=1 -v ./test/e2e/object-controller/... -args \ + --e2e.object-controller-only \ + --godog.tags="@ClusterObjectSet && ~@Serial && ~@demo" +go test -count=1 -v ./test/e2e/object-controller/... -args \ + --e2e.object-controller-only \ + --godog.tags="@ClusterObjectDeployment && ~@Serial && ~@demo" +``` + +Repeat each selector with `@Serial` and `--godog.concurrency=1` for any serial +scenarios. The standalone flag rejects discovered operator-controller/catalogd +Deployments and any installed ClusterExtension/ClusterCatalog CRDs. Missing +object-controller or a required API fails readiness instead of skipping. + +The two ClusterExtension integration features and the revision-pruning scenario +in `group.feature` stay in the object-controller suite, following the ownership +design. They retain `@BoxcutterRuntime`, require the integrated installation, and +are excluded by the direct COS/COD selectors. Aggregate standard runs skip direct +object scenarios when object-controller is absent; experimental Boxcutter runs +require it. Existing `@ProgressDeadline`, `@Serial`, and `@demo` filters still apply. +The legacy `go test ./test/e2e/features_test.go` entry point accepts these same +flags and feature paths for downstream Godog jobs. + ### Run Standard and Experimental in Parallel Both suites can run simultaneously on separate KIND clusters (see the Makefile diff --git a/test/e2e/object-controller/features/cos_object_lifecycle.feature b/test/e2e/object-controller/features/cos_object_lifecycle.feature new file mode 100644 index 0000000000..c6f9bd2429 --- /dev/null +++ b/test/e2e/object-controller/features/cos_object_lifecycle.feature @@ -0,0 +1,174 @@ +@ObjectController @ClusterObjectSet @COSObjectLifecycle +Feature: Standalone ClusterObjectSet object lifecycle + + Background: + Given object-controller is available + And namespace "${TEST_NAMESPACE}" is available + + Scenario: Objects omitted from a successor are collected while retained objects are preserved + # COS phases are immutable. Apply a new COS revision directly, without a COD. + When ClusterObjectSet is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterObjectSet + metadata: + name: ${NAME}-1 + spec: + group: ${NAME} + revision: 1 + lifecycleState: Active + collisionProtection: Prevent + phases: + - name: config + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: retained + namespace: ${TEST_NAMESPACE} + data: + value: unchanged + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: removed + namespace: ${TEST_NAMESPACE} + data: + value: obsolete + """ + Then ClusterObjectSet "${NAME}-1" reports Ready as True with Reason AllObjectsReady + And resource "configmap/removed" is installed + And resource "configmap/retained" UID is remembered + When ClusterObjectSet is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterObjectSet + metadata: + name: ${NAME}-2 + spec: + group: ${NAME} + revision: 2 + lifecycleState: Active + collisionProtection: Prevent + phases: + - name: config + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: retained + namespace: ${TEST_NAMESPACE} + data: + value: unchanged + """ + Then ClusterObjectSet "${NAME}-2" reports Ready as True with Reason AllObjectsReady + And ClusterObjectSet "${NAME}-1" is archived + And resource "configmap/removed" is eventually not found + And resource "configmap/retained" keeps its UID + And resource "configmap/retained" matches + """ + data: + value: unchanged + """ + And ClusterObjectSet "${NAME}-2" reports aggregate object counts: total=1, present=1, synced=1, ready=1 + + Scenario: Later phases stay blocked across reconciles until each earlier probe passes + When ClusterObjectSet is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterObjectSet + metadata: + name: ${COS_NAME} + spec: + group: ${NAME} + revision: 1 + lifecycleState: Active + collisionProtection: Prevent + progressionProbes: + - selector: + type: Label + label: + matchLabels: + probe: gated + assertions: + - type: FieldValue + fieldValue: + fieldPath: metadata.annotations.ready + value: "true" + phases: + - name: first + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: first + namespace: ${TEST_NAMESPACE} + labels: + probe: gated + data: + phase: first + - name: second + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: second + namespace: ${TEST_NAMESPACE} + labels: + probe: gated + data: + phase: second + - name: third + objects: + - type: InlineManifest + inlineManifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: third + namespace: ${TEST_NAMESPACE} + data: + phase: third + """ + Then resource "configmap/first" is installed + And ClusterObjectSet "${COS_NAME}" reports Ready as False with Reason ObjectsNotReady + And ClusterObjectSet "${COS_NAME}" keeps later phase resources absent for "5s": + | resource | + | configmap/second | + | configmap/third | + # Restoration proves the controller reconciled again with the probe blocked. + When resource "configmap/first" is removed + Then resource "configmap/first" is eventually restored + And ClusterObjectSet "${COS_NAME}" keeps later phase resources absent for "5s": + | resource | + | configmap/second | + | configmap/third | + # The readiness annotation is external to the immutable COS manifest. + When annotations are added to "configmap/first" + | key | value | + | ready | true | + Then resource "configmap/second" is installed + And ClusterObjectSet "${COS_NAME}" keeps later phase resources absent for "5s": + | resource | + | configmap/third | + When resource "configmap/second" is removed + Then resource "configmap/second" is eventually restored + And ClusterObjectSet "${COS_NAME}" keeps later phase resources absent for "5s": + | resource | + | configmap/third | + When annotations are added to "configmap/second" + | key | value | + | ready | true | + Then ClusterObjectSet "${COS_NAME}" reports Ready as True with Reason AllObjectsReady + And resource "configmap/third" is installed + And ClusterObjectSet "${COS_NAME}" reports aggregate object counts: total=3, present=3, synced=3, ready=3 diff --git a/test/e2e/object-controller/features/revision.feature b/test/e2e/object-controller/features/revision.feature index dcdbfaf709..acb76a323d 100644 --- a/test/e2e/object-controller/features/revision.feature +++ b/test/e2e/object-controller/features/revision.feature @@ -339,6 +339,7 @@ Feature: Install ClusterObjectSet And resource "configmap/test-configmap-3" is installed And ClusterObjectSet "${COS_NAME}" reports Ready as True with Reason AllObjectsReady + @COSObjectLifecycle Scenario: User can install a ClusterObjectSet with objects stored in Secrets Given namespace "${TEST_NAMESPACE}" is available When resource is applied diff --git a/test/internal/e2e/object-controller/steps/lifecycle_steps.go b/test/internal/e2e/object-controller/steps/lifecycle_steps.go new file mode 100644 index 0000000000..d930c7637a --- /dev/null +++ b/test/internal/e2e/object-controller/steps/lifecycle_steps.go @@ -0,0 +1,69 @@ +package steps + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/cucumber/godog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + shared "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +// ClusterObjectSetKeepsLaterPhaseResourcesAbsent observes a blocked COS and all +// listed resources throughout a window. An API error cannot establish absence. +func ClusterObjectSetKeepsLaterPhaseResourcesAbsent(ctx context.Context, name, durationStr string, table *godog.Table) error { + duration, err := time.ParseDuration(durationStr) + if err != nil || duration <= 0 { + return fmt.Errorf("expected a positive observation duration, got %q", durationStr) + } + if table == nil || len(table.Rows) < 2 || len(table.Rows[0].Cells) != 1 || table.Rows[0].Cells[0].Value != "resource" { + return fmt.Errorf("expected a resource table with a 'resource' header and at least one row") + } + + sc := shared.Scenario(ctx) + name = shared.SubstituteScenarioVars(name, sc) + type resourceRef struct{ kind, name string } + resources := make([]resourceRef, 0, len(table.Rows)-1) + for _, row := range table.Rows[1:] { + if len(row.Cells) != 1 { + return fmt.Errorf("expected one resource per row") + } + resource := shared.SubstituteScenarioVars(row.Cells[0].Value, sc) + kind, resourceName, found := strings.Cut(resource, "/") + if !found || kind == "" || resourceName == "" || strings.Contains(resourceName, "/") { + return fmt.Errorf("resource %q is not in the format /", resource) + } + resources = append(resources, resourceRef{kind, resourceName}) + } + + var start time.Time + return wait.PollUntilContextTimeout(ctx, shared.Tick, duration+shared.Timeout, true, func(ctx context.Context) (bool, error) { + resource, err := shared.ReadResourceConditions(ctx, "clusterobjectset", name) + if err != nil { + return false, err + } + ready := shared.CurrentCondition(resource.Metadata.Generation, resource.Status.Conditions, ocv1.ClusterObjectSetTypeReady) + if !shared.ConditionMatches(ready, metav1.ConditionFalse, ptr.To(ocv1.ClusterObjectSetReasonObjectsNotReady), nil) { + return false, fmt.Errorf("ClusterObjectSet %q no longer reports current-generation Ready=False/ObjectsNotReady: %v", name, ready) + } + for _, ref := range resources { + out, err := shared.K8sClient(ctx, "get", ref.kind, ref.name, "-n", sc.Namespace, "--ignore-not-found", "-o", "name") + if err != nil { + return false, fmt.Errorf("checking absence of %s/%s: %w", ref.kind, ref.name, err) + } + if strings.TrimSpace(out) != "" { + return false, fmt.Errorf("later phase resource %s/%s appeared while ClusterObjectSet %q was blocked", ref.kind, ref.name, name) + } + } + if start.IsZero() { + start = time.Now() + } + return time.Since(start) >= duration, nil + }) +} diff --git a/test/internal/e2e/object-controller/steps/lifecycle_steps_test.go b/test/internal/e2e/object-controller/steps/lifecycle_steps_test.go new file mode 100644 index 0000000000..c4deccea3d --- /dev/null +++ b/test/internal/e2e/object-controller/steps/lifecycle_steps_test.go @@ -0,0 +1,147 @@ +package steps + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cucumber/godog" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + shared "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func TestLaterPhaseResourcesStayAbsent(t *testing.T) { + for _, tc := range []struct { + name string + mode string + generation int64 + status metav1.ConditionStatus + reason string + duration string + wantError string + }{ + {name: "all resources absent", generation: 2, status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectsNotReady}, + {name: "resource already present", mode: "present", generation: 2, status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectsNotReady, wantError: "later phase resource configmap/second appeared"}, + {name: "resource appears later", mode: "late", generation: 2, status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectsNotReady, duration: "1s", wantError: "later phase resource configmap/third appeared"}, + {name: "stale condition", generation: 1, status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectsNotReady, wantError: "current-generation Ready=False/ObjectsNotReady"}, + {name: "probe no longer blocks", generation: 2, status: metav1.ConditionTrue, reason: ocv1.ClusterObjectSetReasonAllObjectsReady, wantError: "current-generation Ready=False/ObjectsNotReady"}, + {name: "different failure", generation: 2, status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectCollision, wantError: "current-generation Ready=False/ObjectsNotReady"}, + {name: "condition read fails", mode: "condition-error", wantError: "reading clusterobjectset"}, + {name: "condition response is invalid", mode: "invalid-json", wantError: "unmarshaling clusterobjectset"}, + {name: "absence read fails", mode: "resource-error", generation: 2, status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectsNotReady, wantError: "checking absence of configmap/second"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Setenv("E2E_LIFECYCLE_FIXTURES", dir) + t.Setenv("E2E_LIFECYCLE_MODE", tc.mode) + cli := filepath.Join(dir, "kubectl") + script := `#!/bin/sh +printf '%s\n' "$*" >> "$E2E_LIFECYCLE_FIXTURES/commands" +case "$1 $2" in + "get clusterobjectset") + case "$E2E_LIFECYCLE_MODE" in + condition-error) echo "API unavailable" >&2; exit 1;; + invalid-json) echo "invalid JSON"; exit 0;; + esac + count=0 + if [ -f "$E2E_LIFECYCLE_FIXTURES/reads" ]; then + count=$(cat "$E2E_LIFECYCLE_FIXTURES/reads") + fi + echo "$((count + 1))" > "$E2E_LIFECYCLE_FIXTURES/reads" + cat "$E2E_LIFECYCLE_FIXTURES/cos.json" + ;; + "get configmap") + case "$E2E_LIFECYCLE_MODE" in + present) printf 'configmap/%s\n' "$3";; + late) + if [ "$3" = third ] && [ "$(cat "$E2E_LIFECYCLE_FIXTURES/reads")" -gt 1 ]; then + echo "configmap/third" + fi + ;; + resource-error) echo "API unavailable" >&2; exit 1;; + esac + ;; + *) echo "unexpected command: $*" >&2; exit 1;; +esac +` + //nolint:gosec // The private test CLI must be executable. + require.NoError(t, os.WriteFile(cli, []byte(script), 0o700)) + oldCLI := pflag.Lookup("k8s.cli").Value.String() + require.NoError(t, pflag.Set("k8s.cli", cli)) + t.Cleanup(func() { require.NoError(t, pflag.Set("k8s.cli", oldCLI)) }) + + cos := ocv1.ClusterObjectSet{ObjectMeta: metav1.ObjectMeta{Generation: 2}} + cos.Status.Conditions = []metav1.Condition{{ + Type: ocv1.ClusterObjectSetTypeReady, ObservedGeneration: tc.generation, + Status: tc.status, Reason: tc.reason, + }} + data, err := json.Marshal(cos) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "cos.json"), data, 0o600)) + ctx, err := shared.CreateScenarioContext(t.Context(), &godog.Scenario{Id: "1", Uri: "cos_object_lifecycle.feature"}) + require.NoError(t, err) + table := new(godog.Table) + require.NoError(t, json.Unmarshal([]byte(`{"rows":[ + {"cells":[{"value":"resource"}]}, + {"cells":[{"value":"configmap/second"}]}, + {"cells":[{"value":"configmap/third"}]} + ]}`), table)) + duration := tc.duration + if duration == "" { + duration = "1ns" + } + err = ClusterObjectSetKeepsLaterPhaseResourcesAbsent(ctx, "${COS_NAME}", duration, table) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + return + } + require.NoError(t, err) + commands, err := os.ReadFile(filepath.Join(dir, "commands")) + require.NoError(t, err) + sc := shared.Scenario(ctx) + require.Contains(t, string(commands), "get clusterobjectset "+sc.ClusterObjectSetName+" -o json") + for _, name := range []string{"second", "third"} { + require.Contains(t, string(commands), "get configmap "+name+" -n "+sc.Namespace+" --ignore-not-found -o name") + } + }) + } +} + +func TestLaterPhaseObservationRejectsInvalidInput(t *testing.T) { + for _, tc := range []struct { + name string + duration string + rows string + wantError string + }{ + {name: "invalid duration", duration: "invalid", wantError: "positive observation duration"}, + {name: "zero duration", duration: "0s", wantError: "positive observation duration"}, + {name: "negative duration", duration: "-1s", wantError: "positive observation duration"}, + {name: "missing table", duration: "1s", wantError: "expected a resource table"}, + {name: "empty table", duration: "1s", rows: `[]`, wantError: "expected a resource table"}, + {name: "header only", duration: "1s", rows: `[{"cells":[{"value":"resource"}]}]`, wantError: "expected a resource table"}, + {name: "wrong header", duration: "1s", rows: `[{"cells":[{"value":"name"}]},{"cells":[{"value":"configmap/late"}]}]`, wantError: "expected a resource table"}, + {name: "extra cells", duration: "1s", rows: `[{"cells":[{"value":"resource"}]},{"cells":[{"value":"configmap/late"},{"value":"extra"}]}]`, wantError: "one resource per row"}, + {name: "missing separator", duration: "1s", rows: `[{"cells":[{"value":"resource"}]},{"cells":[{"value":"late"}]}]`, wantError: "/"}, + {name: "empty kind", duration: "1s", rows: `[{"cells":[{"value":"resource"}]},{"cells":[{"value":"/late"}]}]`, wantError: "/"}, + {name: "empty name", duration: "1s", rows: `[{"cells":[{"value":"resource"}]},{"cells":[{"value":"configmap/"}]}]`, wantError: "/"}, + {name: "extra separator", duration: "1s", rows: `[{"cells":[{"value":"resource"}]},{"cells":[{"value":"configmap/late/extra"}]}]`, wantError: "/"}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, err := shared.CreateScenarioContext(t.Context(), &godog.Scenario{Id: "1", Uri: "cos_object_lifecycle.feature"}) + require.NoError(t, err) + var table *godog.Table + if tc.rows != "" { + require.NoError(t, json.Unmarshal([]byte(`{"rows":`+strings.TrimSpace(tc.rows)+`}`), &table)) + } + err = ClusterObjectSetKeepsLaterPhaseResourcesAbsent(ctx, "cos", tc.duration, table) + require.ErrorContains(t, err, tc.wantError) + }) + } +} diff --git a/test/internal/e2e/object-controller/steps/steps.go b/test/internal/e2e/object-controller/steps/steps.go index 00adfbe81c..fba0f1eb8f 100644 --- a/test/internal/e2e/object-controller/steps/steps.go +++ b/test/internal/e2e/object-controller/steps/steps.go @@ -33,6 +33,7 @@ func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^(?i)ClusterObjectSet with Namespace owner "([^"]*)" is applied$`, shared.ResourceWithNamespaceOwnerApplied) sc.Step(`^(?i)ClusterObjectDeployment with Namespace owner "([^"]*)" is applied$`, shared.ResourceWithNamespaceOwnerApplied) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" reconciliation is triggered$`, TriggerClusterObjectSetReconciliation) + sc.Step(`^(?i)ClusterObjectSet "([^"]+)" keeps later phase resources absent for "([^"]+)":$`, ClusterObjectSetKeepsLaterPhaseResourcesAbsent) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" has observed phase "([^"]+)" with a non-empty digest$`, ClusterObjectSetHasObservedPhase) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" is archived$`, ClusterObjectSetIsArchived) sc.Step(`^(?i)ClusterObjectSet "([^"]+)" is eventually deleted$`, ClusterObjectSetIsEventuallyDeleted)