test: add acceptance coverage for the ITS pipeline

acceptance/README.md

@@ -157,3 +157,24 @@ podman machine settings.
 * Restart the vm
   * `podman machine stop`
   * `podman machine start`
+
+## ITS pipeline coverage
+
+`features/pipeline_validate_image.feature` runs the checked-out ITS pipeline in
+kind using task bundles built from the same checkout. Only task bundle locations
+are redirected to the test registry; pipeline parameters, task ordering, result
+references, and task names are preserved. Scenarios cover trusted signed images,
+image lookup failures, and untrusted signatures. Both failure cases exercise both
+values of `STRICT`, asserting the pipeline's `TEST_OUTPUT` as well as its
+completion status.
+
+Run these scenarios with the normal acceptance prerequisites:
+
+```bash
+CGO_ENABLED=0 make feature_pipeline_validate_image
+```
+
+They also run through `make acceptance` in the existing PR Checks workflow,
+including PRs that change only `pipelines/`. Keep task-test ConfigMap fixtures
+away from `konflux-info/cluster-config`: the pipeline uses that default location,
+so populating it with dummy keyless URLs would affect concurrent pipeline tests.

acceptance/kubernetes/kind/kind.go

@@ -79,6 +79,7 @@ type testState struct {
 	namespace      string
 	policy         string
 	taskRun        string
+	pipelineRun    string
 	snapshot       string
 	registry       string
 	snapshotDigest string

acceptance/kubernetes/kind/pipeline.go

@@ -0,0 +1,132 @@
+// Copyright The Conforma Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package kind
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/tektoncd/cli/pkg/formatted"
+	pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
+	tekton "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/typed/pipeline/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/yaml"
+
+	"github.com/conforma/cli/acceptance/kubernetes/types"
+	"github.com/conforma/cli/acceptance/testenv"
+)
+
+// localPipeline loads the checked-out definition, changing only bundle locations
+// to use the tasks built from this checkout by buildTaskBundleImage.
+func localPipeline(version, name, bundle string) (*pipeline.Pipeline, error) {
+	content, err := os.ReadFile(filepath.Join("pipelines", name, version, name+".yaml"))
+	if err != nil {
+		return nil, err
+	}
+	var definition pipeline.Pipeline
+	if err := yaml.UnmarshalStrict(content, &definition); err != nil {
+		return nil, err
+	}
+	for i := range definition.Spec.Tasks {
+		task := &definition.Spec.Tasks[i]
+		if task.TaskRef == nil || task.TaskRef.Resolver != "bundles" {
+			return nil, fmt.Errorf("pipeline task %q must use a bundle resolver", task.Name)
+		}
+		found := false
+		for j := range task.TaskRef.Params {
+			param := &task.TaskRef.Params[j]
+			if param.Name == "bundle" {
+				param.Value = pipeline.ParamValue{Type: pipeline.ParamTypeString, StringVal: bundle}
+				found = true
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("pipeline task %q has no bundle parameter", task.Name)
+		}
+	}
+	return &definition, nil
+}
+
+// RunPipeline runs the repository pipeline against the local task bundles.
+func (k *kindCluster) RunPipeline(ctx context.Context, version, name string, params map[string]string) error {
+	t := testenv.FetchState[testState](ctx)
+	bundle := fmt.Sprintf("registry.image-registry.svc.cluster.local:%d/ec-task-bundle:%s", k.registryPort, version)
+	definition, err := localPipeline(version, name, bundle)
+	if err != nil {
+		return err
+	}
+	tkn, err := tekton.NewForConfig(k.config)
+	if err != nil {
+		return err
+	}
+	tknParams := make([]pipeline.Param, 0, len(params))
+	for n, v := range params {
+		tknParams = append(tknParams, stringParam(ctx, n, v, t))
+	}
+	pr, err := tkn.PipelineRuns(t.namespace).Create(ctx, &pipeline.PipelineRun{
+		ObjectMeta: metav1.ObjectMeta{GenerateName: "acceptance-pipelinerun-"},
+		Spec: pipeline.PipelineRunSpec{
+			PipelineSpec:    &definition.Spec,
+			Params:          tknParams,
+			TaskRunTemplate: pipeline.PipelineTaskRunTemplate{ServiceAccountName: "default"},
+			Timeouts:        &pipeline.TimeoutFields{Pipeline: &metav1.Duration{Duration: 10 * time.Minute}},
+		},
+	}, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	t.pipelineRun = pr.Name
+	return nil
+}
+
+// AwaitUntilPipelineIsDone polls with a deadline. A timeout or API failure is
+// always an error, never an expected negative validation result.
+func (k *kindCluster) AwaitUntilPipelineIsDone(ctx context.Context) (*types.PipelineInfo, error) {
+	t := testenv.FetchState[testState](ctx)
+	tkn, err := tekton.NewForConfig(k.config)
+	if err != nil {
+		return nil, err
+	}
+	var pr *pipeline.PipelineRun
+	err = wait.PollUntilContextTimeout(ctx, time.Second, 11*time.Minute, true, func(ctx context.Context) (bool, error) {
+		var err error
+		pr, err = tkn.PipelineRuns(t.namespace).Get(ctx, t.pipelineRun, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return pr.IsDone(), nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("waiting for PipelineRun %s/%s: %w", t.namespace, t.pipelineRun, err)
+	}
+	results := map[string]any{}
+	for _, result := range pr.Status.Results {
+		results[result.Name] = paramValue(result.Value)
+	}
+	// Preserve condition messages (including resolution/validation failures) in
+	// assertion errors instead of reporting just a boolean status.
+	return &types.PipelineInfo{
+		Name:       pr.Name,
+		Status:     fmt.Sprintf("%s: %v", formatted.Condition(pr.Status.Conditions), pr.Status.Conditions),
+		Successful: pr.IsSuccessful(),
+		Results:    results,
+	}, nil
+}

acceptance/kubernetes/kubernetes.go

@@ -494,6 +494,7 @@ func stepEnvVarShouldBe(ctx context.Context, stepName, envName, want string) err
 
 // AddStepsTo adds cluster-related steps to the context
 func AddStepsTo(sc *godog.ScenarioContext) {
+	addPipelineStepsTo(sc)
 	sc.Step(`^a stub cluster running$`, startAndSetupState(stub.Start))
 	sc.Step(`^a cluster running$`, startAndSetupState(kind.Start))
 	sc.Step(`^a working namespace$`, createNamespace)

acceptance/kubernetes/pipeline.go

@@ -0,0 +1,85 @@
+// Copyright The Conforma Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package kubernetes
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/cucumber/godog"
+
+	"github.com/conforma/cli/acceptance/testenv"
+)
+
+func runPipeline(ctx context.Context, version, name string, params *godog.Table) error {
+	c := testenv.FetchState[ClusterState](ctx)
+	if err := mustBeUp(ctx, *c); err != nil {
+		return err
+	}
+	values := map[string]string{}
+	for _, row := range params.Rows {
+		values[row.Cells[0].Value] = row.Cells[1].Value
+	}
+	return c.cluster.RunPipeline(ctx, version, name, values)
+}
+
+func pipelineShouldComplete(ctx context.Context, outcome string) error {
+	c := testenv.FetchState[ClusterState](ctx)
+	if err := mustBeUp(ctx, *c); err != nil {
+		return err
+	}
+	info, err := c.cluster.AwaitUntilPipelineIsDone(ctx)
+	if err != nil {
+		return err
+	}
+	if info.Successful != (outcome == "succeed") {
+		return fmt.Errorf("PipelineRun %s should %s, got %s", info.Name, outcome, info.Status)
+	}
+	return nil
+}
+
+func pipelineReportShouldHaveResult(ctx context.Context, expected string) error {
+	c := testenv.FetchState[ClusterState](ctx)
+	if err := mustBeUp(ctx, *c); err != nil {
+		return err
+	}
+	info, err := c.cluster.AwaitUntilPipelineIsDone(ctx)
+	if err != nil {
+		return err
+	}
+	raw, ok := info.Results["TEST_OUTPUT"].(string)
+	if !ok {
+		return fmt.Errorf("PipelineRun %s has no string TEST_OUTPUT result: %s", info.Name, info.Status)
+	}
+	var report struct {
+		Result string `json:"result"`
+	}
+	if err := json.Unmarshal([]byte(raw), &report); err != nil {
+		return fmt.Errorf("invalid pipeline TEST_OUTPUT %q: %w", raw, err)
+	}
+	if report.Result != expected {
+		return fmt.Errorf("pipeline TEST_OUTPUT result: want %q, got %s", expected, raw)
+	}
+	return nil
+}
+
+func addPipelineStepsTo(sc *godog.ScenarioContext) {
+	sc.Step(`^version ([\d.]+) of the pipeline named "([^"]*)" is run with parameters:$`, runPipeline)
+	sc.Step(`^the pipeline should (succeed|fail)$`, pipelineShouldComplete)
+	sc.Step(`^the pipeline TEST_OUTPUT should report "([^"]*)"$`, pipelineReportShouldHaveResult)
+}

acceptance/kubernetes/stub/stub.go

@@ -214,3 +214,11 @@ func (s stubCluster) Registry(ctx context.Context) (string, error) {
 func (s stubCluster) BuildSnapshotArtifact(ctx context.Context, content string) (context.Context, error) {
 	return ctx, nil
 }
+
+func (s stubCluster) RunPipeline(_ context.Context, _, _ string, _ map[string]string) error {
+	return errors.New("can't run pipelines when using the stub Kubernetes")
+}
+
+func (s stubCluster) AwaitUntilPipelineIsDone(context.Context) (*types.PipelineInfo, error) {
+	return nil, errors.New("can't run pipelines when using the stub Kubernetes")
+}

acceptance/kubernetes/types/types.go

@@ -27,6 +27,8 @@ type Cluster interface {
 	CreateNamespace(context.Context) (context.Context, error)
 	CreateNamedPolicy(context.Context, string, string) error
 	CreatePolicy(context.Context, string) error
+	RunPipeline(context.Context, string, string, map[string]string) error
+	AwaitUntilPipelineIsDone(context.Context) (*PipelineInfo, error)
 	RunTask(context.Context, string, string, string, map[string]string) error
 	AwaitUntilTaskIsDone(context.Context) (bool, error)
 	TaskInfo(context.Context) (*TaskInfo, error)
@@ -52,3 +54,11 @@ type Step struct {
 	Logs    string
 	EnvVars map[string]string
 }
+
+// PipelineInfo describes a completed PipelineRun.
+type PipelineInfo struct {
+	Name       string
+	Status     string
+	Successful bool
+	Results    map[string]any
+}

features/__snapshots__/task_validate_image.snap

@@ -453,7 +453,7 @@ true
 ---
 
 [TestFeatures/Collect keyless signing parameters from ConfigMap:collect-signing-params - 1]
-Reading ConfigMap konflux-info/cluster-config
+Reading ConfigMap konflux-info/cluster-config-keyless
 ConfigMap found, extracting keyless signing parameters
 results.keylessSigningEnabled: true
 results.defaultOIDCIssuer: https://kubernetes.default.svc

features/pipeline_validate_image.feature

@@ -0,0 +1,58 @@
+Feature: Verify images through the ITS pipeline
+  Exercise the repository pipeline with locally built task bundles in kind.
+
+  Background:
+    Given a cluster running
+      And stub rekord running
+      And stub tuf running
+      And a working namespace
+      And a key pair named "known"
+      And a cluster policy with content:
+      ```
+      {
+        "publicKey": ${known_PUBLIC_KEY}
+      }
+      ```
+
+  Scenario: ITS pipeline validates a signed image
+    Given an image named "acceptance/its-signed"
+      And a valid image signature of "acceptance/its-signed" image signed by the "known" key
+      And a valid attestation of "acceptance/its-signed" signed by the "known" key
+    When version 0.1 of the pipeline named "enterprise-contract" is run with parameters:
+      | SNAPSHOT             | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-signed"}]} |
+      | PUBLIC_KEY           | ${known_PUBLIC_KEY}                                                       |
+      | POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME}                                               |
+    Then the pipeline should succeed
+      And the pipeline TEST_OUTPUT should report "SUCCESS"
+
+  Scenario Outline: ITS pipeline handles image lookup failures according to STRICT
+    When version 0.1 of the pipeline named "enterprise-contract" is run with parameters:
+      | SNAPSHOT             | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-missing"}]} |
+      | PUBLIC_KEY           | ${known_PUBLIC_KEY}                                                        |
+      | POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME}                                                |
+      | STRICT               | <strict>                                                                   |
+    Then the pipeline should <outcome>
+      And the pipeline TEST_OUTPUT should report "FAILURE"
+
+    Examples:
+      | strict | outcome |
+      | true   | fail    |
+      | false  | succeed |
+
+  Scenario Outline: ITS pipeline handles untrusted signatures according to STRICT
+    Given a key pair named "untrusted"
+      And an image named "acceptance/its-untrusted"
+      And a valid image signature of "acceptance/its-untrusted" image signed by the "untrusted" key
+      And a valid attestation of "acceptance/its-untrusted" signed by the "untrusted" key
+    When version 0.1 of the pipeline named "enterprise-contract" is run with parameters:
+      | SNAPSHOT             | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-untrusted"}]} |
+      | PUBLIC_KEY           | ${known_PUBLIC_KEY}                                                          |
+      | POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME}                                                  |
+      | STRICT               | <strict>                                                                     |
+    Then the pipeline should <outcome>
+      And the pipeline TEST_OUTPUT should report "FAILURE"
+
+    Examples:
+      | strict | outcome |
+      | true   | fail    |
+      | false  | succeed |

features/task_validate_image.feature

@@ -547,7 +547,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
     #   https://github.com/redhat-appstudio/tsf-cli/blob/84561ca6c9/installer/charts/tsf-konflux/templates/konflux.yaml#L51-L65
     # Note: These scenarios might run in parallel so let's use a different config map
     # for each scenario so we don't have to worry about them clashing with each other
-    And a ConfigMap "cluster-config" in namespace "konflux-info" with content:
+    And a ConfigMap "cluster-config-keyless" in namespace "konflux-info" with content:
       # tufExternalUrl should be ignored here because tufInternalUrl takes precedence
       ```
       {
@@ -562,7 +562,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
       }
       ```
     When version 0.1 of the task named "collect-keyless-params" is run with parameters:
-      | configMapName      | cluster-config |
+      | configMapName | cluster-config-keyless |
     Then the task should succeed
      And the task logs for step "collect-signing-params" should match the snapshot
      And the task result "keylessSigningEnabled" should equal "true"