diff --git a/acceptance/README.md b/acceptance/README.md index 073778143..b4eda412d 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -157,3 +157,24 @@ podman machine settings. * Restart the vm * `podman machine stop` * `podman machine start` + +## ITS pipeline coverage + +`features/pipeline_validate_image.feature` runs the checked-out ITS pipeline in +kind using task bundles built from the same checkout. Only task bundle locations +are redirected to the test registry; pipeline parameters, task ordering, result +references, and task names are preserved. Scenarios cover trusted signed images, +image lookup failures, and untrusted signatures. Both failure cases exercise both +values of `STRICT`, asserting the pipeline's `TEST_OUTPUT` as well as its +completion status. + +Run these scenarios with the normal acceptance prerequisites: + +```bash +CGO_ENABLED=0 make feature_pipeline_validate_image +``` + +They also run through `make acceptance` in the existing PR Checks workflow, +including PRs that change only `pipelines/`. Keep task-test ConfigMap fixtures +away from `konflux-info/cluster-config`: the pipeline uses that default location, +so populating it with dummy keyless URLs would affect concurrent pipeline tests. diff --git a/acceptance/kubernetes/kind/kind.go b/acceptance/kubernetes/kind/kind.go index e63f93616..30ba39cb2 100644 --- a/acceptance/kubernetes/kind/kind.go +++ b/acceptance/kubernetes/kind/kind.go @@ -79,6 +79,7 @@ type testState struct { namespace string policy string taskRun string + pipelineRun string snapshot string registry string snapshotDigest string diff --git a/acceptance/kubernetes/kind/pipeline.go b/acceptance/kubernetes/kind/pipeline.go new file mode 100644 index 000000000..f9293e14a --- /dev/null +++ b/acceptance/kubernetes/kind/pipeline.go @@ -0,0 +1,132 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package kind + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/tektoncd/cli/pkg/formatted" + pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + tekton "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/typed/pipeline/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/yaml" + + "github.com/conforma/cli/acceptance/kubernetes/types" + "github.com/conforma/cli/acceptance/testenv" +) + +// localPipeline loads the checked-out definition, changing only bundle locations +// to use the tasks built from this checkout by buildTaskBundleImage. +func localPipeline(version, name, bundle string) (*pipeline.Pipeline, error) { + content, err := os.ReadFile(filepath.Join("pipelines", name, version, name+".yaml")) + if err != nil { + return nil, err + } + var definition pipeline.Pipeline + if err := yaml.UnmarshalStrict(content, &definition); err != nil { + return nil, err + } + for i := range definition.Spec.Tasks { + task := &definition.Spec.Tasks[i] + if task.TaskRef == nil || task.TaskRef.Resolver != "bundles" { + return nil, fmt.Errorf("pipeline task %q must use a bundle resolver", task.Name) + } + found := false + for j := range task.TaskRef.Params { + param := &task.TaskRef.Params[j] + if param.Name == "bundle" { + param.Value = pipeline.ParamValue{Type: pipeline.ParamTypeString, StringVal: bundle} + found = true + } + } + if !found { + return nil, fmt.Errorf("pipeline task %q has no bundle parameter", task.Name) + } + } + return &definition, nil +} + +// RunPipeline runs the repository pipeline against the local task bundles. +func (k *kindCluster) RunPipeline(ctx context.Context, version, name string, params map[string]string) error { + t := testenv.FetchState[testState](ctx) + bundle := fmt.Sprintf("registry.image-registry.svc.cluster.local:%d/ec-task-bundle:%s", k.registryPort, version) + definition, err := localPipeline(version, name, bundle) + if err != nil { + return err + } + tkn, err := tekton.NewForConfig(k.config) + if err != nil { + return err + } + tknParams := make([]pipeline.Param, 0, len(params)) + for n, v := range params { + tknParams = append(tknParams, stringParam(ctx, n, v, t)) + } + pr, err := tkn.PipelineRuns(t.namespace).Create(ctx, &pipeline.PipelineRun{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "acceptance-pipelinerun-"}, + Spec: pipeline.PipelineRunSpec{ + PipelineSpec: &definition.Spec, + Params: tknParams, + TaskRunTemplate: pipeline.PipelineTaskRunTemplate{ServiceAccountName: "default"}, + Timeouts: &pipeline.TimeoutFields{Pipeline: &metav1.Duration{Duration: 10 * time.Minute}}, + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + t.pipelineRun = pr.Name + return nil +} + +// AwaitUntilPipelineIsDone polls with a deadline. A timeout or API failure is +// always an error, never an expected negative validation result. +func (k *kindCluster) AwaitUntilPipelineIsDone(ctx context.Context) (*types.PipelineInfo, error) { + t := testenv.FetchState[testState](ctx) + tkn, err := tekton.NewForConfig(k.config) + if err != nil { + return nil, err + } + var pr *pipeline.PipelineRun + err = wait.PollUntilContextTimeout(ctx, time.Second, 11*time.Minute, true, func(ctx context.Context) (bool, error) { + var err error + pr, err = tkn.PipelineRuns(t.namespace).Get(ctx, t.pipelineRun, metav1.GetOptions{}) + if err != nil { + return false, err + } + return pr.IsDone(), nil + }) + if err != nil { + return nil, fmt.Errorf("waiting for PipelineRun %s/%s: %w", t.namespace, t.pipelineRun, err) + } + results := map[string]any{} + for _, result := range pr.Status.Results { + results[result.Name] = paramValue(result.Value) + } + // Preserve condition messages (including resolution/validation failures) in + // assertion errors instead of reporting just a boolean status. + return &types.PipelineInfo{ + Name: pr.Name, + Status: fmt.Sprintf("%s: %v", formatted.Condition(pr.Status.Conditions), pr.Status.Conditions), + Successful: pr.IsSuccessful(), + Results: results, + }, nil +} diff --git a/acceptance/kubernetes/kubernetes.go b/acceptance/kubernetes/kubernetes.go index 56c92cba1..583c6ad3e 100644 --- a/acceptance/kubernetes/kubernetes.go +++ b/acceptance/kubernetes/kubernetes.go @@ -494,6 +494,7 @@ func stepEnvVarShouldBe(ctx context.Context, stepName, envName, want string) err // AddStepsTo adds cluster-related steps to the context func AddStepsTo(sc *godog.ScenarioContext) { + addPipelineStepsTo(sc) sc.Step(`^a stub cluster running$`, startAndSetupState(stub.Start)) sc.Step(`^a cluster running$`, startAndSetupState(kind.Start)) sc.Step(`^a working namespace$`, createNamespace) diff --git a/acceptance/kubernetes/pipeline.go b/acceptance/kubernetes/pipeline.go new file mode 100644 index 000000000..c87c685df --- /dev/null +++ b/acceptance/kubernetes/pipeline.go @@ -0,0 +1,85 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package kubernetes + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/cucumber/godog" + + "github.com/conforma/cli/acceptance/testenv" +) + +func runPipeline(ctx context.Context, version, name string, params *godog.Table) error { + c := testenv.FetchState[ClusterState](ctx) + if err := mustBeUp(ctx, *c); err != nil { + return err + } + values := map[string]string{} + for _, row := range params.Rows { + values[row.Cells[0].Value] = row.Cells[1].Value + } + return c.cluster.RunPipeline(ctx, version, name, values) +} + +func pipelineShouldComplete(ctx context.Context, outcome string) error { + c := testenv.FetchState[ClusterState](ctx) + if err := mustBeUp(ctx, *c); err != nil { + return err + } + info, err := c.cluster.AwaitUntilPipelineIsDone(ctx) + if err != nil { + return err + } + if info.Successful != (outcome == "succeed") { + return fmt.Errorf("PipelineRun %s should %s, got %s", info.Name, outcome, info.Status) + } + return nil +} + +func pipelineReportShouldHaveResult(ctx context.Context, expected string) error { + c := testenv.FetchState[ClusterState](ctx) + if err := mustBeUp(ctx, *c); err != nil { + return err + } + info, err := c.cluster.AwaitUntilPipelineIsDone(ctx) + if err != nil { + return err + } + raw, ok := info.Results["TEST_OUTPUT"].(string) + if !ok { + return fmt.Errorf("PipelineRun %s has no string TEST_OUTPUT result: %s", info.Name, info.Status) + } + var report struct { + Result string `json:"result"` + } + if err := json.Unmarshal([]byte(raw), &report); err != nil { + return fmt.Errorf("invalid pipeline TEST_OUTPUT %q: %w", raw, err) + } + if report.Result != expected { + return fmt.Errorf("pipeline TEST_OUTPUT result: want %q, got %s", expected, raw) + } + return nil +} + +func addPipelineStepsTo(sc *godog.ScenarioContext) { + sc.Step(`^version ([\d.]+) of the pipeline named "([^"]*)" is run with parameters:$`, runPipeline) + sc.Step(`^the pipeline should (succeed|fail)$`, pipelineShouldComplete) + sc.Step(`^the pipeline TEST_OUTPUT should report "([^"]*)"$`, pipelineReportShouldHaveResult) +} diff --git a/acceptance/kubernetes/stub/stub.go b/acceptance/kubernetes/stub/stub.go index ca3100a2f..7b467edbd 100644 --- a/acceptance/kubernetes/stub/stub.go +++ b/acceptance/kubernetes/stub/stub.go @@ -214,3 +214,11 @@ func (s stubCluster) Registry(ctx context.Context) (string, error) { func (s stubCluster) BuildSnapshotArtifact(ctx context.Context, content string) (context.Context, error) { return ctx, nil } + +func (s stubCluster) RunPipeline(_ context.Context, _, _ string, _ map[string]string) error { + return errors.New("can't run pipelines when using the stub Kubernetes") +} + +func (s stubCluster) AwaitUntilPipelineIsDone(context.Context) (*types.PipelineInfo, error) { + return nil, errors.New("can't run pipelines when using the stub Kubernetes") +} diff --git a/acceptance/kubernetes/types/types.go b/acceptance/kubernetes/types/types.go index 4c4dd62d8..be6346815 100644 --- a/acceptance/kubernetes/types/types.go +++ b/acceptance/kubernetes/types/types.go @@ -27,6 +27,8 @@ type Cluster interface { CreateNamespace(context.Context) (context.Context, error) CreateNamedPolicy(context.Context, string, string) error CreatePolicy(context.Context, string) error + RunPipeline(context.Context, string, string, map[string]string) error + AwaitUntilPipelineIsDone(context.Context) (*PipelineInfo, error) RunTask(context.Context, string, string, string, map[string]string) error AwaitUntilTaskIsDone(context.Context) (bool, error) TaskInfo(context.Context) (*TaskInfo, error) @@ -52,3 +54,11 @@ type Step struct { Logs string EnvVars map[string]string } + +// PipelineInfo describes a completed PipelineRun. +type PipelineInfo struct { + Name string + Status string + Successful bool + Results map[string]any +} diff --git a/features/__snapshots__/task_validate_image.snap b/features/__snapshots__/task_validate_image.snap index b1fb8ea1f..f10131931 100644 --- a/features/__snapshots__/task_validate_image.snap +++ b/features/__snapshots__/task_validate_image.snap @@ -453,7 +453,7 @@ true --- [TestFeatures/Collect keyless signing parameters from ConfigMap:collect-signing-params - 1] -Reading ConfigMap konflux-info/cluster-config +Reading ConfigMap konflux-info/cluster-config-keyless ConfigMap found, extracting keyless signing parameters results.keylessSigningEnabled: true results.defaultOIDCIssuer: https://kubernetes.default.svc diff --git a/features/pipeline_validate_image.feature b/features/pipeline_validate_image.feature new file mode 100644 index 000000000..d37634498 --- /dev/null +++ b/features/pipeline_validate_image.feature @@ -0,0 +1,58 @@ +Feature: Verify images through the ITS pipeline + Exercise the repository pipeline with locally built task bundles in kind. + + Background: + Given a cluster running + And stub rekord running + And stub tuf running + And a working namespace + And a key pair named "known" + And a cluster policy with content: + ``` + { + "publicKey": ${known_PUBLIC_KEY} + } + ``` + + Scenario: ITS pipeline validates a signed image + Given an image named "acceptance/its-signed" + And a valid image signature of "acceptance/its-signed" image signed by the "known" key + And a valid attestation of "acceptance/its-signed" signed by the "known" key + When version 0.1 of the pipeline named "enterprise-contract" is run with parameters: + | SNAPSHOT | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-signed"}]} | + | PUBLIC_KEY | ${known_PUBLIC_KEY} | + | POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME} | + Then the pipeline should succeed + And the pipeline TEST_OUTPUT should report "SUCCESS" + + Scenario Outline: ITS pipeline handles image lookup failures according to STRICT + When version 0.1 of the pipeline named "enterprise-contract" is run with parameters: + | SNAPSHOT | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-missing"}]} | + | PUBLIC_KEY | ${known_PUBLIC_KEY} | + | POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME} | + | STRICT | | + Then the pipeline should + And the pipeline TEST_OUTPUT should report "FAILURE" + + Examples: + | strict | outcome | + | true | fail | + | false | succeed | + + Scenario Outline: ITS pipeline handles untrusted signatures according to STRICT + Given a key pair named "untrusted" + And an image named "acceptance/its-untrusted" + And a valid image signature of "acceptance/its-untrusted" image signed by the "untrusted" key + And a valid attestation of "acceptance/its-untrusted" signed by the "untrusted" key + When version 0.1 of the pipeline named "enterprise-contract" is run with parameters: + | SNAPSHOT | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-untrusted"}]} | + | PUBLIC_KEY | ${known_PUBLIC_KEY} | + | POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME} | + | STRICT | | + Then the pipeline should + And the pipeline TEST_OUTPUT should report "FAILURE" + + Examples: + | strict | outcome | + | true | fail | + | false | succeed | diff --git a/features/task_validate_image.feature b/features/task_validate_image.feature index 6642c6c89..d97dd367d 100644 --- a/features/task_validate_image.feature +++ b/features/task_validate_image.feature @@ -547,7 +547,7 @@ Feature: Verify Enterprise Contract Tekton Tasks # https://github.com/redhat-appstudio/tsf-cli/blob/84561ca6c9/installer/charts/tsf-konflux/templates/konflux.yaml#L51-L65 # Note: These scenarios might run in parallel so let's use a different config map # for each scenario so we don't have to worry about them clashing with each other - And a ConfigMap "cluster-config" in namespace "konflux-info" with content: + And a ConfigMap "cluster-config-keyless" in namespace "konflux-info" with content: # tufExternalUrl should be ignored here because tufInternalUrl takes precedence ``` { @@ -562,7 +562,7 @@ Feature: Verify Enterprise Contract Tekton Tasks } ``` When version 0.1 of the task named "collect-keyless-params" is run with parameters: - | configMapName | cluster-config | + | configMapName | cluster-config-keyless | Then the task should succeed And the task logs for step "collect-signing-params" should match the snapshot And the task result "keylessSigningEnabled" should equal "true"