diff --git a/.github/workflows/update-demos.yaml b/.github/workflows/update-demos.yaml index 6dc13ebb54..37b83321d6 100644 --- a/.github/workflows/update-demos.yaml +++ b/.github/workflows/update-demos.yaml @@ -7,7 +7,7 @@ on: paths: - 'api/*' - 'config/*' - - 'test/e2e/operator-controller/features/demos.feature' + - 'test/e2e/platform/features/demos.feature' - 'test/internal/e2e/steps/demo_steps.go' - 'test/internal/e2e/steps/asciicast_*.go' - '.github/workflows/update-demos.yaml' @@ -15,7 +15,7 @@ on: paths: - 'api/*' - 'config/*' - - 'test/e2e/operator-controller/features/demos.feature' + - 'test/e2e/platform/features/demos.feature' - 'test/internal/e2e/steps/demo_steps.go' - 'test/internal/e2e/steps/asciicast_*.go' - '.github/workflows/update-demos.yaml' diff --git a/Makefile b/Makefile index ce033648e4..ff9ea51225 100644 --- a/Makefile +++ b/Makefile @@ -487,7 +487,10 @@ e2e/%: #EXHELP Run e2e scenario against a cluster created by e2e-setup or experi @feature=$$(echo "$*" | cut -d/ -f1); \ scenario=$$(echo "$*" | cut -d/ -f2-); \ if [ "$$scenario" = "$$feature" ]; then scenario=""; fi; \ - KUBECONFIG=$(KUBECONFIG) go test -count=1 -v ./test/e2e/operator-controller/... \ + set -- test/e2e/*/features/"$$feature.feature"; \ + if [ "$$#" -ne 1 ] || [ ! -f "$$1" ]; then echo "Expected one feature named $$feature" >&2; exit 1; fi; \ + suite=$$(dirname "$$(dirname "$$1")"); \ + KUBECONFIG=$(KUBECONFIG) go test -count=1 -v ./$$suite/... \ -timeout $(E2E_TIMEOUT) \ -args --godog.concurrency=1 --e2e.scenario="$$scenario" "features/$$feature.feature" diff --git a/test/e2e/README.md b/test/e2e/README.md index 1e369bded0..f5ee56682d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -26,13 +26,10 @@ test/ │ ├── features_test.go # Legacy file entry point │ ├── operator-controller/ │ │ ├── suite_test.go # Component suite entry point -│ │ └── features/ # All existing features during the first migration stage -│ ├── catalogd/ -│ │ └── features/ # Placeholder for the catalogd suite -│ ├── object-controller/ -│ │ └── features/ # Placeholder for the object-controller suite -│ └── platform/ -│ └── features/ # Placeholder for the cross-component suite +│ │ └── features/ # ClusterExtension scenarios +│ ├── catalogd/ # Catalog HA, proxy, and TLS scenarios +│ ├── object-controller/ # ClusterObjectSet/Deployment scenarios +│ └── platform/ # Cross-component metrics and demos └── internal/ ├── e2e/ │ ├── runner.go # Godog options, scenario selection, summary wiring @@ -55,12 +52,11 @@ by test suites without becoming public Go APIs. Envtest helpers used by API and controller unit tests live under `internal/testutil`, which is accessible throughout the repository. -This is the first stage of the [per-component suite design](../../docs/designs/e2e-per-component.md). -All existing features remain together under `operator-controller/features/` during -this stage. The catalogd, object-controller, and platform feature directories contain -only `.gitkeep` placeholders. Later branches will add their suite entry points and -move features to their owners, then separate component-specific steps. Feature -contents and step behavior are unchanged. +Features follow the ownership in the [per-component suite design](../../docs/designs/e2e-per-component.md). +Each component has a thin entry point that still registers the full shared step set; +step ownership is separated in the next stage. The newer `cos_*`, `cod_*`, and +`group` features belong to object-controller. Feature contents and step behavior +are unchanged, including ClusterExtension integration assertions in object features. ### 2. Feature Files (`operator-controller/features/*.feature`) @@ -186,7 +182,7 @@ Replaces `${TEST_NAMESPACE}`, `${NAME}`, `${SCENARIO_ID}`, `${PACKAGE:}`, ### 1. Create a Feature File -Create a new `.feature` file in `test/e2e/operator-controller/features/`: +Create a new `.feature` file in its owning component's `test/e2e//features/` directory: ```gherkin Feature: Your Feature Name @@ -297,8 +293,12 @@ go test -p 1 -count=1 -v ./test/e2e/... -args --godog.tags="@Serial && ~@demo" - `-p 1` also serializes packages, keeping serial scenarios exclusive as more suites are added. Demo scenarios run separately through `make update-demos`. -To select the current component suite, use `./test/e2e/operator-controller/...` -with the same flags. Additional component suites will be added in the next stage. +Select `./test/e2e//...` with the same flags to run just one suite: +`operator-controller`, `catalogd`, `object-controller`, or `platform`. For example: + +```bash +go test -count=1 -v ./test/e2e/object-controller/... -args --godog.tags="~@Serial && ~@demo" +``` ### Run Standard and Experimental in Parallel @@ -337,8 +337,10 @@ The cluster persists until explicitly torn down. **2. Run individual scenarios:** ```bash -# Run all scenarios in a feature file +# Run all scenarios in a feature file (its component is discovered automatically) make e2e/install +make e2e/revision +make e2e/ha # Run scenarios matching a name prefix (case-insensitive) make e2e/install/Install # all "Install ..." scenarios diff --git a/test/e2e/operator-controller/features/ha.feature b/test/e2e/catalogd/features/ha.feature similarity index 100% rename from test/e2e/operator-controller/features/ha.feature rename to test/e2e/catalogd/features/ha.feature diff --git a/test/e2e/operator-controller/features/proxy.feature b/test/e2e/catalogd/features/proxy.feature similarity index 100% rename from test/e2e/operator-controller/features/proxy.feature rename to test/e2e/catalogd/features/proxy.feature diff --git a/test/e2e/operator-controller/features/tls.feature b/test/e2e/catalogd/features/tls.feature similarity index 100% rename from test/e2e/operator-controller/features/tls.feature rename to test/e2e/catalogd/features/tls.feature diff --git a/test/e2e/catalogd/suite_test.go b/test/e2e/catalogd/suite_test.go new file mode 100644 index 0000000000..721ee038df --- /dev/null +++ b/test/e2e/catalogd/suite_test.go @@ -0,0 +1,21 @@ +package e2e + +import ( + "os" + "testing" + + "github.com/cucumber/godog" + + "github.com/operator-framework/operator-controller/test/internal/e2e" + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func TestMain(m *testing.M) { + os.Exit(e2e.Run("features", func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, func(sc *godog.ScenarioContext) { + steps.RegisterSteps(sc) + steps.RegisterDemoSteps(sc) + steps.RegisterHooks(sc) + })) +} diff --git a/test/e2e/features_test.go b/test/e2e/features_test.go index 5cdee2cb92..4be0828b24 100644 --- a/test/e2e/features_test.go +++ b/test/e2e/features_test.go @@ -7,6 +7,8 @@ package e2e import ( "log" "os" + "path/filepath" + "strings" "testing" "github.com/cucumber/godog" @@ -16,11 +18,26 @@ import ( ) func TestMain(m *testing.M) { - // Keep legacy paths such as features/install.feature relative to the suite. - if err := os.Chdir("operator-controller"); err != nil { - log.Fatal(err) + // Preserve legacy features/ arguments after features move to their owners. + for i, arg := range os.Args[1:] { + if arg == "features" || arg == "features/" { + os.Args[i+1] = "." + continue + } + if !strings.HasPrefix(arg, "features/") { + continue + } + path, line, hasLine := strings.Cut(arg, ":") + matches, err := filepath.Glob("*/" + path) + if err != nil || len(matches) != 1 { + log.Fatalf("expected one component feature for %q, found %d: %v", arg, len(matches), err) + } + os.Args[i+1] = matches[0] + if hasLine { + os.Args[i+1] += ":" + line + } } - os.Exit(e2e.Run("features", func(tc *godog.TestSuiteContext) { + os.Exit(e2e.Run(".", func(tc *godog.TestSuiteContext) { tc.BeforeSuite(steps.BeforeSuite) }, func(sc *godog.ScenarioContext) { steps.RegisterSteps(sc) diff --git a/test/e2e/operator-controller/features/cod_adoption_conflict.feature b/test/e2e/object-controller/features/cod_adoption_conflict.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_adoption_conflict.feature rename to test/e2e/object-controller/features/cod_adoption_conflict.feature diff --git a/test/e2e/operator-controller/features/cod_handoff.feature b/test/e2e/object-controller/features/cod_handoff.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_handoff.feature rename to test/e2e/object-controller/features/cod_handoff.feature diff --git a/test/e2e/operator-controller/features/cod_lifecycle.feature b/test/e2e/object-controller/features/cod_lifecycle.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_lifecycle.feature rename to test/e2e/object-controller/features/cod_lifecycle.feature diff --git a/test/e2e/operator-controller/features/cod_progressing_reasons.feature b/test/e2e/object-controller/features/cod_progressing_reasons.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_progressing_reasons.feature rename to test/e2e/object-controller/features/cod_progressing_reasons.feature diff --git a/test/e2e/operator-controller/features/cod_revision_history.feature b/test/e2e/object-controller/features/cod_revision_history.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_revision_history.feature rename to test/e2e/object-controller/features/cod_revision_history.feature diff --git a/test/e2e/operator-controller/features/cod_rollout.feature b/test/e2e/object-controller/features/cod_rollout.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_rollout.feature rename to test/e2e/object-controller/features/cod_rollout.feature diff --git a/test/e2e/operator-controller/features/cod_template_metadata.feature b/test/e2e/object-controller/features/cod_template_metadata.feature similarity index 100% rename from test/e2e/operator-controller/features/cod_template_metadata.feature rename to test/e2e/object-controller/features/cod_template_metadata.feature diff --git a/test/e2e/operator-controller/features/cos_object_collision_and_adoption.feature b/test/e2e/object-controller/features/cos_object_collision_and_adoption.feature similarity index 100% rename from test/e2e/operator-controller/features/cos_object_collision_and_adoption.feature rename to test/e2e/object-controller/features/cos_object_collision_and_adoption.feature diff --git a/test/e2e/operator-controller/features/cos_preflight.feature b/test/e2e/object-controller/features/cos_preflight.feature similarity index 100% rename from test/e2e/operator-controller/features/cos_preflight.feature rename to test/e2e/object-controller/features/cos_preflight.feature diff --git a/test/e2e/operator-controller/features/cos_revision_identity.feature b/test/e2e/object-controller/features/cos_revision_identity.feature similarity index 100% rename from test/e2e/operator-controller/features/cos_revision_identity.feature rename to test/e2e/object-controller/features/cos_revision_identity.feature diff --git a/test/e2e/operator-controller/features/cos_static_validation.feature b/test/e2e/object-controller/features/cos_static_validation.feature similarity index 100% rename from test/e2e/operator-controller/features/cos_static_validation.feature rename to test/e2e/object-controller/features/cos_static_validation.feature diff --git a/test/e2e/operator-controller/features/group.feature b/test/e2e/object-controller/features/group.feature similarity index 100% rename from test/e2e/operator-controller/features/group.feature rename to test/e2e/object-controller/features/group.feature diff --git a/test/e2e/operator-controller/features/revision.feature b/test/e2e/object-controller/features/revision.feature similarity index 100% rename from test/e2e/operator-controller/features/revision.feature rename to test/e2e/object-controller/features/revision.feature diff --git a/test/e2e/operator-controller/features/user-managed-fields.feature b/test/e2e/object-controller/features/user-managed-fields.feature similarity index 100% rename from test/e2e/operator-controller/features/user-managed-fields.feature rename to test/e2e/object-controller/features/user-managed-fields.feature diff --git a/test/e2e/object-controller/suite_test.go b/test/e2e/object-controller/suite_test.go new file mode 100644 index 0000000000..721ee038df --- /dev/null +++ b/test/e2e/object-controller/suite_test.go @@ -0,0 +1,21 @@ +package e2e + +import ( + "os" + "testing" + + "github.com/cucumber/godog" + + "github.com/operator-framework/operator-controller/test/internal/e2e" + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func TestMain(m *testing.M) { + os.Exit(e2e.Run("features", func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, func(sc *godog.ScenarioContext) { + steps.RegisterSteps(sc) + steps.RegisterDemoSteps(sc) + steps.RegisterHooks(sc) + })) +} diff --git a/test/e2e/operator-controller/features/demos.feature b/test/e2e/platform/features/demos.feature similarity index 100% rename from test/e2e/operator-controller/features/demos.feature rename to test/e2e/platform/features/demos.feature diff --git a/test/e2e/operator-controller/features/metrics.feature b/test/e2e/platform/features/metrics.feature similarity index 100% rename from test/e2e/operator-controller/features/metrics.feature rename to test/e2e/platform/features/metrics.feature diff --git a/test/e2e/platform/suite_test.go b/test/e2e/platform/suite_test.go new file mode 100644 index 0000000000..721ee038df --- /dev/null +++ b/test/e2e/platform/suite_test.go @@ -0,0 +1,21 @@ +package e2e + +import ( + "os" + "testing" + + "github.com/cucumber/godog" + + "github.com/operator-framework/operator-controller/test/internal/e2e" + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func TestMain(m *testing.M) { + os.Exit(e2e.Run("features", func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, func(sc *godog.ScenarioContext) { + steps.RegisterSteps(sc) + steps.RegisterDemoSteps(sc) + steps.RegisterHooks(sc) + })) +} diff --git a/test/internal/e2e/steps/hooks.go b/test/internal/e2e/steps/hooks.go index fb5980c726..7f319c83e0 100644 --- a/test/internal/e2e/steps/hooks.go +++ b/test/internal/e2e/steps/hooks.go @@ -2,6 +2,7 @@ package steps import ( "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -214,13 +215,18 @@ func CheckFeatureTags(ctx context.Context, sc *godog.Scenario) (context.Context, } func CreateScenarioContext(ctx context.Context, sc *godog.Scenario) (context.Context, error) { + // Godog IDs restart in each component process. Scope resource identities by + // feature so concurrent suites cannot delete or modify each other's resources. + featureName := strings.TrimSuffix(filepath.Base(sc.Uri), filepath.Ext(sc.Uri)) + featureHash := sha256.Sum256([]byte(featureName)) + id := fmt.Sprintf("%x-%s", featureHash[:4], sc.Id) scCtx := &scenarioContext{ - id: sc.Id, - featureName: strings.TrimSuffix(filepath.Base(sc.Uri), filepath.Ext(sc.Uri)), + id: id, + featureName: featureName, scenarioName: sc.Name, - namespace: fmt.Sprintf("ns-%s", sc.Id), - clusterExtensionName: fmt.Sprintf("ce-%s", sc.Id), - clusterObjectSetName: fmt.Sprintf("ce-%s-1", sc.Id), + namespace: fmt.Sprintf("ns-%s", id), + clusterExtensionName: fmt.Sprintf("ce-%s", id), + clusterObjectSetName: fmt.Sprintf("ce-%s-1", id), catalogs: make(map[string]string), catalogPackageNames: make(map[string]string), metricsResponse: make(map[string]string), diff --git a/test/internal/e2e/steps/isolation_test.go b/test/internal/e2e/steps/isolation_test.go new file mode 100644 index 0000000000..ca002e83af --- /dev/null +++ b/test/internal/e2e/steps/isolation_test.go @@ -0,0 +1,50 @@ +package steps_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/cucumber/godog" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" + + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func TestScenarioResourceIsolationAcrossSuites(t *testing.T) { + // Exercise substitution through the real apply step using a CLI that records + // its input. No cluster is needed to reproduce Godog IDs restarting at 1. + dir := t.TempDir() + cli := filepath.Join(dir, "kubectl") + require.NoError(t, os.WriteFile(cli, []byte("#!/bin/sh\ncat > \"$E2E_APPLIED_RESOURCE\"\n"), 0o700)) //nolint:gosec // The test CLI must be executable. + original := pflag.Lookup("k8s.cli").Value.String() + require.NoError(t, pflag.Set("k8s.cli", cli)) + t.Cleanup(func() { require.NoError(t, pflag.Set("k8s.cli", original)) }) + output := filepath.Join(dir, "applied.yaml") + t.Setenv("E2E_APPLIED_RESOURCE", output) + names := make([]string, 0, 3) + for _, uri := range []string{"features/install.feature", "features/revision.feature", "object-controller/features/revision.feature"} { + ctx, err := steps.CreateScenarioContext(context.Background(), &godog.Scenario{Id: "1", Name: "Stable scenario name", Uri: uri}) + require.NoError(t, err) + require.NoError(t, steps.ResourceIsApplied(ctx, &godog.DocString{Content: `apiVersion: v1 +kind: Namespace +metadata: + name: ${TEST_NAMESPACE} + labels: + scenario-id: ${SCENARIO_ID} +`})) + data, err := os.ReadFile(output) + require.NoError(t, err) + var obj unstructured.Unstructured + require.NoError(t, yaml.Unmarshal(data, &obj.Object)) + names = append(names, obj.GetName()) + require.Equal(t, "Stable scenario name", obj.GetAnnotations()["e2e.olm.operatorframework.io/scenario"]) + require.Equal(t, "ns-"+obj.GetLabels()["scenario-id"], obj.GetName()) + } + require.NotEqual(t, names[0], names[1], "different features with the same Godog ID need distinct resources") + require.Equal(t, names[1], names[2], "moving a feature must preserve its resource identity") +}