test(e2e): add FUNC_BIN/WithImage helpers and a middleware-rebuild spec

test/e2e/func_middleware_fixture_test.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	functionsdevv1alpha1 "github.com/functions-dev/func-operator/api/v1alpha1"
+	"github.com/functions-dev/func-operator/test/utils"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// STALE_FUNCTION_IMAGE must point at a pre-built, publicly pullable function image
+// whose "middleware-version" label is older than the operator's current version.
+// The operator reads that label off the deployed image, finds it stale, and
+// rebuilds the function from source. The spec is skipped when it is unset.
+//
+// Why a pre-built fixture (and not a build step): the middleware version is baked
+// by the func-utils image, which defaults to a floating "latest" tag, so there's
+// no way to produce a *stale* build on the fly. The fixture is built once as
+// `FROM <current function image>` + `LABEL middleware-version=<old>` and stored in
+// a durable public repo. The image ref is intentionally not hardcoded here so the
+// same test works across upstream/downstream with each CI supplying its own image.
+func staleFunctionImage() string {
+	return os.Getenv("STALE_FUNCTION_IMAGE")
+}
+
+var _ = Describe("Middleware Rebuild", func() {
+
+	Context("with a function deployed from a stale-middleware image", func() {
+		var repoURL string
+		var repoDir string
+		var functionName, functionNamespace string
+
+		BeforeEach(func() {
+			// Blocked on knative/func#4081: func's operator sync creates the
+			// registry auth secret as type Opaque / key "config.json", but
+			// func-operator's redeploy requires kubernetes.io/dockerconfigjson /
+			// ".dockerconfigjson", so the rebuild fails with
+			// "invalid registry auth secret type". The fixture/label detection
+			// itself works (operator correctly flags the stale middleware and
+			// starts the rebuild); re-enable this spec once #4081 is resolved.
+			Skip("blocked on knative/func#4081 (registry auth secret type mismatch on operator rebuild)")
+
+			// Opt-in: requires a pre-built stale-middleware fixture image.
+			if staleFunctionImage() == "" {
+				Skip("set STALE_FUNCTION_IMAGE to a function image whose middleware-version label is stale")
+			}
+
+			username, password, _, cleanup, err := repoProvider.CreateRandomUser()
+			Expect(err).NotTo(HaveOccurred())
+			utils.DeferCleanupOnSuccess(cleanup)
+
+			_, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false)
+			Expect(err).NotTo(HaveOccurred())
+			utils.DeferCleanupOnSuccess(cleanup)
+
+			repoDir, err = utils.InitializeRepoWithFunction(repoURL, username, password, "go")
+			Expect(err).NotTo(HaveOccurred())
+			utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir)
+
+			functionNamespace, err = utils.GetTestNamespace()
+			Expect(err).NotTo(HaveOccurred())
+			utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace)
+
+			// Seed the deployment with the pre-built stale image (func skips build
+			// because the ref carries a digest), so the operator sees outdated
+			// middleware and must rebuild from the repo.
+			out, err := utils.RunFuncDeploy(repoDir,
+				utils.WithNamespace(functionNamespace),
+				utils.WithImage(staleFunctionImage()))
+			Expect(err).NotTo(HaveOccurred())
+			_, _ = fmt.Fprint(GinkgoWriter, out)
+
+			utils.DeferCleanupOnSuccess(func() {
+				_, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace)
+			})
+
+			err = utils.CommitAndPush(repoDir, "Update func.yaml after deploy", "func.yaml")
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			logFailedTestDetails(functionName, functionNamespace)
+		})
+
+		It("should rebuild the function and mark middleware up to date", func() {
+			function := &functionsdevv1alpha1.Function{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "my-function-",
+					Namespace:    functionNamespace,
+				},
+				Spec: functionsdevv1alpha1.FunctionSpec{
+					Repository: functionsdevv1alpha1.FunctionSpecRepository{
+						URL: repoURL,
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, function)
+			Expect(err).NotTo(HaveOccurred())
+			utils.DeferCleanupOnSuccess(func() {
+				_, _ = utils.RunCmd("kubectl", "delete", "function", function.Name, "--namespace", function.Namespace)
+			})
+			functionName = function.Name
+
+			// The operator detects the stale middleware label, rebuilds from the
+			// repo, and the function becomes ready and up to date.
+			Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed())
+			Eventually(functionMiddlewareUpToDate(functionName, functionNamespace), 2*time.Minute).Should(Succeed())
+		})
+	})
+})

test/utils/func.go

@@ -29,10 +29,21 @@ import (
 	ginkgo "github.com/onsi/ginkgo/v2"
 )
 
+// funcBin returns the func binary to invoke. It honors the FUNC_BIN environment
+// variable (an absolute path or a name on PATH), defaulting to "func" on PATH.
+// This lets manual runs point at a local build and CI point at a binary extracted
+// from the operator image.
+func funcBin() string {
+	if b := os.Getenv("FUNC_BIN"); b != "" {
+		return b
+	}
+	return "func"
+}
+
 // RunFunc executes the func CLI with the current/latest version
 func RunFunc(command string, args ...string) (string, error) {
 	allArgs := append([]string{command}, args...)
-	cmd := exec.Command("func", allArgs...)
+	cmd := exec.Command(funcBin(), allArgs...)
 	return Run(cmd)
 }
 
@@ -49,6 +60,39 @@ func RunFuncWithVersion(version string, command string, args ...string) (string,
 	return Run(cmd)
 }
 
+// funcDeployArgs assembles the `func deploy` flags from the resolved options.
+func funcDeployArgs(functionDir string, opts *FuncDeployOptions) []string {
+	args := []string{"--path", functionDir}
+
+	// --image and --registry are mutually exclusive in func. With a pre-built
+	// image (digest) func skips build/push, so the registry flags don't apply.
+	if opts.Image != "" {
+		// Deploy a pre-built image as-is: no build, no push.
+		args = append(args, "--image", opts.Image, "--build=false", "--push=false")
+	} else {
+		args = append(args,
+			"--registry", opts.Registry,
+			fmt.Sprintf("--registry-insecure=%t", opts.RegistryInsecure),
+		)
+	}
+
+	if opts.Namespace != "" {
+		args = append(args, "--namespace", opts.Namespace)
+	}
+
+	// A pre-built image with a digest disables build/push, so the builder is
+	// irrelevant in that case.
+	if opts.Builder != "" && opts.Image == "" {
+		args = append(args, "--builder", opts.Builder)
+	}
+
+	if opts.Deployer != "" && opts.Deployer != "knative" {
+		args = append(args, "--deployer", opts.Deployer)
+	}
+
+	return args
+}
+
 // RunFuncDeploy runs func deploy with retry logic for transient network errors
 func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, error) {
 	opts := &FuncDeployOptions{
@@ -63,23 +107,7 @@ func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, erro
 		optFn(opts)
 	}
 
-	args := []string{
-		"--path", functionDir,
-		"--registry", opts.Registry,
-		fmt.Sprintf("--registry-insecure=%t", opts.RegistryInsecure),
-	}
-
-	if opts.Namespace != "" {
-		args = append(args, "--namespace", opts.Namespace)
-	}
-
-	if opts.Builder != "" {
-		args = append(args, "--builder", opts.Builder)
-	}
-
-	if opts.Deployer != "" && opts.Deployer != "knative" {
-		args = append(args, "--deployer", opts.Deployer)
-	}
+	args := funcDeployArgs(functionDir, opts)
 
 	// When using the pack builder, create a per-deploy PACK_HOME to prevent
 	// parallel builds from corrupting the shared ~/.pack/volume-keys.toml.
@@ -118,7 +146,7 @@ func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, erro
 				return "", err
 			}
 		} else {
-			funcBinary = "func"
+			funcBinary = funcBin()
 		}
 
 		cmd := exec.Command(funcBinary, append([]string{"deploy"}, args...)...)
@@ -143,6 +171,10 @@ type FuncDeployOptions struct {
 	Deployer         string
 	CliVersion       string
 	EnvVars          map[string]string
+	// Image, when set to a ref including a digest, makes func deploy the given
+	// pre-built image and skip the build/push phases. Used to seed a function
+	// with a specific (e.g. stale-middleware) image.
+	Image string
 }
 
 type FuncDeployOption func(*FuncDeployOptions)
@@ -165,6 +197,14 @@ func WithDeployer(deployer string) FuncDeployOption {
 	}
 }
 
+// WithImage deploys a pre-built image (ref should include a digest so func skips
+// build/push), rather than building from source.
+func WithImage(image string) FuncDeployOption {
+	return func(o *FuncDeployOptions) {
+		o.Image = image
+	}
+}
+
 func WithDeployCliVersion(version string) FuncDeployOption {
 	return func(opts *FuncDeployOptions) {
 		opts.CliVersion = version
@@ -211,7 +251,7 @@ func ensureFuncVersion(version string) (string, error) {
 	}
 	defer syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN) //nolint:errcheck
 
-	// Re-check after acquiring the lock - another process may have finished the download
+	// Re-check after acquiring the lock — another process may have finished the download
 	if _, err := os.Stat(funcBinary); err == nil {
 		return funcBinary, nil
 	}