diff --git a/cmd/oap/internal/installcmd/init.go b/cmd/oap/internal/installcmd/init.go index 16352ac7..87da6312 100644 --- a/cmd/oap/internal/installcmd/init.go +++ b/cmd/oap/internal/installcmd/init.go @@ -32,6 +32,7 @@ import ( "github.com/authzed/openagentprimitives/pkg/platform/cloud" idpregistry "github.com/authzed/openagentprimitives/pkg/platform/identity/idp/registry" "github.com/authzed/openagentprimitives/pkg/platform/manifests" + "github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay" ) // wizardMode controls how (or whether) the settings wizard runs after a @@ -99,6 +100,7 @@ func NewInitCmd(g *apcmd.Globals) *cobra.Command { var externalSpiceDBToken string var externalSpiceDBInsecure bool var acceptExisting bool + var overlayDirs []string // dm is populated below via settingscmd.RegisterDefaultModelFlags(cmd), after cmd // exists. The RunE closure captures the variable (not its zero value at // closure-literal time) — by the time cobra invokes RunE, dm has already @@ -112,6 +114,12 @@ func NewInitCmd(g *apcmd.Globals) *cobra.Command { if err := validateInitFlags(localMode, clusterKindFlag, developMode, externalSpiceDBEndpoint, externalSpiceDBToken, acceptExisting, skipInstall); err != nil { return err } + // Before the build step: a broken or conflicting overlay must not + // cost the user a full image build to discover. + overlays, err := loadOverlays(cmd.OutOrStdout(), overlay.Base, overlayDirs) + if err != nil { + return err + } // --local is sugar for --cluster-kind=local; validateInitFlags has // already refused a conflicting explicit --cluster-kind alongside it. clusterKind := clusterKindFlag @@ -138,11 +146,12 @@ func NewInitCmd(g *apcmd.Globals) *cobra.Command { // check are done, on every kind, instead of blocking on a // foreground tunnel whose death left that ConfigMap advertising a // URL that forwarded nothing. - return runInit(cmd.Context(), cmd.OutOrStdout(), g, skipBuild, skipInstall, timeout, tags, pinningMode, clusterKind, allowNonLocal, r, wizardFlag, defaultsFlag, imagePullSecret, mirrorDeps, createRegistry, noIdp, noMonitoring, *dm, artifactStoreURL, externalSpiceDBEndpoint, externalSpiceDBToken, externalSpiceDBInsecure, acceptExisting) + return runInit(cmd.Context(), cmd.OutOrStdout(), g, skipBuild, skipInstall, timeout, tags, pinningMode, clusterKind, allowNonLocal, r, wizardFlag, defaultsFlag, imagePullSecret, mirrorDeps, createRegistry, noIdp, noMonitoring, *dm, artifactStoreURL, externalSpiceDBEndpoint, externalSpiceDBToken, externalSpiceDBInsecure, acceptExisting, overlays) }, } cmd.Flags().BoolVar(&skipBuild, "skip-build", false, "Skip oap build all") cmd.Flags().BoolVar(&skipInstall, "skip-install", false, "Skip oap install") + cmd.Flags().StringArrayVar(&overlayDirs, "overlay", nil, overlayFlagUsage) cmd.Flags().DurationVar(&timeout, "timeout", timeout, "Overall deadline for the install + check loop") cmd.Flags().StringVar(&tags.Operator, "operator-image", "", "Override operator image") cmd.Flags().StringVar(&tags.Runner, "runner-image", "", "Override runner image") @@ -298,7 +307,7 @@ func validateInitFlags(localMode bool, clusterKind string, developMode bool, ext // prompts for it instead (see there). A non-empty result skips the bundled // SpiceDB install (RunInstall) and, after install completes, best-effort // applies the base schema to the caller's endpoint (applySchemaToEndpoint). -func runInit(ctx context.Context, out io.Writer, g *apcmd.Globals, skipBuild, skipInstall bool, timeout time.Duration, tags manifests.Tags, pinningMode string, clusterKind string, allowNonLocal bool, r WebdRoutingOpts, wizardFlag, defaultsFlag bool, imagePullSecret string, mirrorDeps, createRegistry bool, noIdp bool, noMonitoring bool, dm settingscmd.DefaultModelOpts, artifactStoreURL string, externalSpiceDBEndpoint, externalSpiceDBToken string, externalSpiceDBInsecure bool, acceptExisting bool) error { +func runInit(ctx context.Context, out io.Writer, g *apcmd.Globals, skipBuild, skipInstall bool, timeout time.Duration, tags manifests.Tags, pinningMode string, clusterKind string, allowNonLocal bool, r WebdRoutingOpts, wizardFlag, defaultsFlag bool, imagePullSecret string, mirrorDeps, createRegistry bool, noIdp bool, noMonitoring bool, dm settingscmd.DefaultModelOpts, artifactStoreURL string, externalSpiceDBEndpoint, externalSpiceDBToken string, externalSpiceDBInsecure bool, acceptExisting bool, overlays *overlay.Set) error { if err := validPinningMode(pinningMode); err != nil { return err } @@ -492,6 +501,7 @@ func runInit(ctx context.Context, out io.Writer, g *apcmd.Globals, skipBuild, sk WithoutBuilder: true, BuilderStarters: nil, Rail: nil, // today's linear `oap init` draws no wizard rail + Overlays: overlays, }); err != nil { return err } diff --git a/cmd/oap/internal/installcmd/init_test.go b/cmd/oap/internal/installcmd/init_test.go index 79781789..014ad9ea 100644 --- a/cmd/oap/internal/installcmd/init_test.go +++ b/cmd/oap/internal/installcmd/init_test.go @@ -106,7 +106,7 @@ func TestRunInit_DispatchesToWizard(t *testing.T) { false /*skipBuild*/, false /*skipInstall*/, 2*time.Minute, manifests.Tags{}, "" /*pinningMode*/, cloud.KeyLocal, true /*allowNonLocal*/, WebdRoutingOpts{}, wizard, false /*defaults*/, "" /*imagePullSecret*/, false /*mirrorDeps*/, false /*createRegistry*/, false /*noIdp*/, false, /*noMonitoring*/ - settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept) + settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept, nil /*overlays*/) } // --wizard dispatches to the orchestrator exactly once; the linear phases run not at all. @@ -164,7 +164,7 @@ func TestRunInit_WizardPathSkipsPreflightRoutingRefusal(t *testing.T) { false /*skipBuild*/, false /*skipInstall*/, 2*time.Minute, tags, "" /*pinningMode*/, cloud.KeyGKE, false /*allowNonLocal*/, WebdRoutingOpts{}, wizard, false /*defaults*/, "" /*imagePullSecret*/, false /*mirrorDeps*/, false /*createRegistry*/, false /*noIdp*/, false, /*noMonitoring*/ - settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept) + settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept, nil /*overlays*/) } // --accept-existing on gke with no routing flags reaches the wizard dispatch diff --git a/cmd/oap/internal/installcmd/install.go b/cmd/oap/internal/installcmd/install.go index 4621bfe5..949432cd 100644 --- a/cmd/oap/internal/installcmd/install.go +++ b/cmd/oap/internal/installcmd/install.go @@ -50,6 +50,7 @@ import ( "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" "github.com/authzed/openagentprimitives/pkg/platform/cloud" "github.com/authzed/openagentprimitives/pkg/platform/manifests" + "github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay" apnats "github.com/authzed/openagentprimitives/pkg/platform/nats" ) @@ -114,6 +115,7 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command { timeout := installDefaultTimeout externalToolTimeout := 30 * time.Minute var ( + overlayDirs []string workspaceClass string noWorkspaceRWX bool workspaceRecheck bool @@ -150,6 +152,12 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command { if err := apcmd.ValidateDryRunMode(dryRun); err != nil { return err } + // Same reasoning: a broken or conflicting overlay is refused before + // the cluster is touched. + overlays, err := loadOverlays(cmd.OutOrStdout(), overlay.Base, overlayDirs) + if err != nil { + return err + } // sigCtx is canceled only by SIGINT; it is NOT subject to the install // --timeout. This is the recheckCtx passed into RunInstall so that an // exhausted overall timeout does not make the keep-waiting "Y" a no-op. @@ -256,6 +264,7 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command { WithoutBuilder: withoutBuilder, BuilderStarters: builderStarters, Rail: nil, // no wizard rail on a standalone `oap install` + Overlays: overlays, }) // Part 2: On SIGINT, decide what to do about what was applied. // sigCtx.Err() != nil means SIGINT fired; the timeout only cancels ctx, @@ -295,6 +304,7 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command { cmd.Flags().StringVar(&tags.Operator, "operator-image", "", "Override operator image (default: spicebox-operator:dev)") cmd.Flags().StringVar(&tags.Runner, "runner-image", "", "Override runner image (default: agentprimitives-runner:dev)") cmd.Flags().StringVar(&tags.Registry, "image-registry", "", "Rewrite first-party images to /: (remote clusters; build+push via `oap init`/`oap build --image-registry`)") + cmd.Flags().StringArrayVar(&overlayDirs, "overlay", nil, overlayFlagUsage) apcmd.DryRunModeFlag(cmd, &dryRun) cmd.Flags().DurationVar(&timeout, "timeout", timeout, "Timeout for K8s-side install work (does not bound external CLI calls; see --external-tool-timeout)") cmd.Flags().DurationVar(&externalToolTimeout, "external-tool-timeout", externalToolTimeout, "Timeout for a single external CLI call (gcloud); generous because cloud LROs like GKE node-pool recreation run for many minutes") @@ -725,6 +735,10 @@ type InstallConfig struct { // Only the unified init wizard passes a non-nil rail, to draw its step // gutter beside the install checklist. Rail progress.RailProvider + // Overlays are the validated --overlay Components, laid over every + // first-party document before install's own per-document rewrites. Nil + // (the zero value, and every caller without --overlay) applies nothing. + Overlays *overlay.Set } func RunInstall(ctx, recheckCtx context.Context, cfg InstallConfig) error { @@ -880,6 +894,11 @@ func RunInstall(ctx, recheckCtx context.Context, cfg InstallConfig) error { if err != nil { return err } + for _, d := range docs { + if err := cfg.Overlays.Apply(d); err != nil { + return err + } + } if dryRun == apcmd.DryRunClient { for _, d := range docs { @@ -1417,7 +1436,7 @@ func RunInstall(ctx, recheckCtx context.Context, cfg InstallConfig) error { Dynamic: bundle.Dynamic, Typed: bundle.Typed, Apply: func(ctx context.Context, groups [][]byte) error { - return applyManifestGroups(ctx, bundle.Dynamic, groups, imagePullSecret, firstPartyRefs, mirrorMap, func(id string) { + return applyManifestGroups(ctx, bundle.Dynamic, groups, cfg.Overlays, imagePullSecret, firstPartyRefs, mirrorMap, func(id string) { rep.Info("applied %s", id) }) }, @@ -2669,18 +2688,23 @@ func spicedbClusterServing(typed kubernetes.Interface, ns, deployName string) pr // applyManifestGroups splits each manifest in groups into its documents, mirrors // dependency image refs (air-gapped installs), injects an imagePullSecret into // first-party Deployment docs when secret is set, and server-side-applies every -// document via kube.Apply under the "ap-install" field manager. It is the shared +// document via kube.Apply under the "ap-install" field manager. overlays are laid +// over each document first, so install's own rewrites still win on the fields +// they set. It is the shared // apply body used by the install pipeline's Apply closure; emit, when non-nil, is // called with a human-readable id per applied doc. injectImagePullSecret is a // no-op for non-first-party (dependency) images, so passing firstPartyRefs for // every component is safe — only channelsd's first-party image is affected. -func applyManifestGroups(ctx context.Context, dyn dynamic.Interface, groups [][]byte, secret string, firstPartyRefs map[string]bool, mirrorMap map[string]string, emit func(id string)) error { +func applyManifestGroups(ctx context.Context, dyn dynamic.Interface, groups [][]byte, overlays *overlay.Set, secret string, firstPartyRefs map[string]bool, mirrorMap map[string]string, emit func(id string)) error { for _, m := range groups { docs, err := manifests.Split(m) if err != nil { return fmt.Errorf("split manifest: %w", err) } for _, d := range docs { + if err := overlays.Apply(d); err != nil { + return err + } if err := mirrorImageRefs(d, mirrorMap); err != nil { return fmt.Errorf("mirror image refs: %w", err) } diff --git a/cmd/oap/internal/installcmd/overlays.go b/cmd/oap/internal/installcmd/overlays.go new file mode 100644 index 00000000..c3d7f9fb --- /dev/null +++ b/cmd/oap/internal/installcmd/overlays.go @@ -0,0 +1,39 @@ +package installcmd + +import ( + "io" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/cliout" + "github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay" +) + +const overlayFlagUsage = "Layer a local kustomize Component directory over the installed manifests (repeatable; applied in order). " + + "Use it for changes that must survive re-installs and upgrades, such as a raised memory limit. " + + "Overlays that set the same field to different values are refused before anything is installed" + +// loadOverlays validates the --overlay directories against base and says which +// resources they change. It runs before any cluster call, so a broken or +// conflicting overlay costs nothing. No dirs returns a nil Set, which applies +// nothing. +func loadOverlays(out io.Writer, base func() ([][]byte, error), dirs []string) (*overlay.Set, error) { + if len(dirs) == 0 { + return nil, nil + } + b, err := base() + if err != nil { + return nil, err + } + s, err := overlay.Load(b, dirs) + if err != nil { + return nil, err + } + patched := s.Patched() + if len(patched) == 0 { + cliout.Warn(out, "overlays %v change nothing oap installs", dirs) + return s, nil + } + for _, id := range patched { + cliout.Info(out, "overlay patches %s", id) + } + return s, nil +} diff --git a/cmd/oap/internal/installcmd/overlays_test.go b/cmd/oap/internal/installcmd/overlays_test.go new file mode 100644 index 00000000..920eb83a --- /dev/null +++ b/cmd/oap/internal/installcmd/overlays_test.go @@ -0,0 +1,204 @@ +package installcmd + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/apcmd" + "github.com/authzed/openagentprimitives/cmd/oap/internal/kube" + "github.com/authzed/openagentprimitives/pkg/platform/apimage" + "github.com/authzed/openagentprimitives/pkg/platform/cloud" + "github.com/authzed/openagentprimitives/pkg/platform/manifests" + "github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay" +) + +// memoryOverlayDir writes a kustomize Component raising container's memory +// limit on the named Deployment. +func memoryOverlayDir(t *testing.T, deployment, container, limit string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte( + "apiVersion: kustomize.config.k8s.io/v1alpha1\nkind: Component\npatches:\n- path: memory.yaml\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "memory.yaml"), []byte( + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: "+deployment+"\n namespace: agentprimitives-system\n"+ + "spec:\n template:\n spec:\n containers:\n - name: "+container+"\n resources:\n limits:\n memory: "+limit+"\n"), 0o644)) + return dir +} + +func containerMemoryLimit(t *testing.T, obj map[string]any, container string) string { + t.Helper() + cs, _, err := unstructured.NestedSlice(obj, "spec", "template", "spec", "containers") + require.NoError(t, err) + for _, c := range cs { + m := c.(map[string]any) + if m["name"] == container { + v, _, err := unstructured.NestedString(m, "resources", "limits", "memory") + require.NoError(t, err) + return v + } + } + require.FailNowf(t, "missing container", "%s", container) + return "" +} + +func TestLoadOverlays_NoDirsIsNilAndSilent(t *testing.T) { + var out bytes.Buffer + s, err := loadOverlays(&out, overlay.Base, nil) + require.NoError(t, err) + assert.Nil(t, s) + assert.Empty(t, out.String()) +} + +func TestLoadOverlays_NarratesPatchedResources(t *testing.T) { + var out bytes.Buffer + s, err := loadOverlays(&out, overlay.Base, []string{memoryOverlayDir(t, "spicebox-operator", "operator", "2Gi")}) + require.NoError(t, err) + require.NotNil(t, s) + assert.Contains(t, out.String(), "overlay patches Deployment.apps agentprimitives-system/spicebox-operator") +} + +func TestApplyManifestGroups_OverlayAppliedBeforeInstallRewrites(t *testing.T) { + ov, err := loadOverlays(io.Discard, overlay.Base, []string{memoryOverlayDir(t, "spicebox-channelsd", "channelsd", "1Gi")}) + require.NoError(t, err) + groups, err := manifests.ChannelsD() + require.NoError(t, err) + + s := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(s)) + dyn := dynfake.NewSimpleDynamicClient(s) + firstParty := map[string]bool{apimage.Channelsd.Name + ":dev": true} + require.NoError(t, applyManifestGroups(context.Background(), dyn, groups, ov, "pull-secret", firstParty, nil, nil)) + + got, err := dyn.Resource(schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}). + Namespace("agentprimitives-system").Get(context.Background(), "spicebox-channelsd", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "1Gi", containerMemoryLimit(t, got.Object, "channelsd")) + secrets, _, err := unstructured.NestedSlice(got.Object, "spec", "template", "spec", "imagePullSecrets") + require.NoError(t, err) + assert.Equal(t, []any{map[string]any{"name": "pull-secret"}}, secrets, "install's own rewrite survives the overlay") +} + +func TestRender_Overlays(t *testing.T) { + offline := &apcmd.Globals{BundleFn: func() (*kube.Bundle, error) { + panic("render must not touch the cluster") + }} + gke := func(out string, overlays ...string) RenderConfig { + return RenderConfig{OutputDir: out, ClusterKind: cloud.KeyGKE, NoDigestPin: true, Overlays: overlays} + } + + t.Run("overlay is copied, listed as a component, and wins in the build", func(t *testing.T) { + out := t.TempDir() + ov := memoryOverlayDir(t, "spicebox-operator", "operator", "2Gi") + require.NoError(t, runRender(context.Background(), io.Discard, offline, gke(out, ov), false)) + + rel := userOverlayPath(0, ov) + assert.FileExists(t, filepath.Join(out, rel, "kustomization.yaml")) + k, err := os.ReadFile(filepath.Join(out, "kustomization.yaml")) + require.NoError(t, err) + assert.Contains(t, string(k), "components:\n - "+rel+"\n") + + built, err := buildOverlay(out) + require.NoError(t, err) + docs, err := manifests.Split(built) + require.NoError(t, err) + var found bool + for _, d := range docs { + if d.GetKind() == "Deployment" && d.GetName() == "spicebox-operator" { + found = true + assert.Equal(t, "2Gi", containerMemoryLimit(t, d.Object, "operator")) + } + } + assert.True(t, found, "operator Deployment in the built output") + }) + + t.Run("re-render without the overlay: copy removed, not left behind", func(t *testing.T) { + out := t.TempDir() + ov := memoryOverlayDir(t, "spicebox-operator", "operator", "2Gi") + require.NoError(t, runRender(context.Background(), io.Discard, offline, gke(out, ov), false)) + require.NoError(t, runRender(context.Background(), io.Discard, offline, gke(out), false)) + assert.NoDirExists(t, filepath.Join(out, userOverlaysDir)) + }) + + t.Run("conflicting overlays: refused before anything is written", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "deploy") + err := runRender(context.Background(), io.Discard, offline, gke(out, + memoryOverlayDir(t, "spicebox-operator", "operator", "2Gi"), + memoryOverlayDir(t, "spicebox-operator", "operator", "4Gi")), false) + require.Error(t, err) + assert.Contains(t, err.Error(), "overlays conflict") + assert.NoDirExists(t, out) + }) + + t.Run("overlay for a component render does not emit: refused up front, naming the flag", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "deploy") + pg := memoryOverlayDir(t, "spicebox-postgres", "postgres", "2Gi") + err := runRender(context.Background(), io.Discard, offline, gke(out, pg), false) + require.Error(t, err) + assert.Contains(t, err.Error(), "spicebox-postgres") + assert.Contains(t, err.Error(), "--with-postgres") + assert.NoDirExists(t, out) + + cfg := gke(out, pg) + cfg.WithPostgres = true + require.NoError(t, runRender(context.Background(), io.Discard, offline, cfg, false)) + }) + + t.Run("render and install produce the same overlaid fields", func(t *testing.T) { + out := t.TempDir() + ov := memoryOverlayDir(t, "spicebox-operator", "operator", "2Gi") + require.NoError(t, runRender(context.Background(), io.Discard, offline, gke(out, ov), false)) + built, err := buildOverlay(out) + require.NoError(t, err) + rendered := findDoc(t, built, "Deployment", "spicebox-operator") + + set, err := loadOverlays(io.Discard, overlay.Base, []string{ov}) + require.NoError(t, err) + installed := findDoc(t, manifests.Install, "Deployment", "spicebox-operator") + require.NoError(t, set.Apply(installed)) + + resources := func(d *unstructured.Unstructured) any { + cs, _, err := unstructured.NestedSlice(d.Object, "spec", "template", "spec", "containers") + require.NoError(t, err) + return cs[0].(map[string]any)["resources"] + } + assert.Equal(t, resources(installed), resources(rendered)) + }) + + t.Run("overlay inside the output's overlays dir: refused", func(t *testing.T) { + out := t.TempDir() + inside := filepath.Join(out, userOverlaysDir, "mine") + require.NoError(t, os.MkdirAll(inside, 0o755)) + src := memoryOverlayDir(t, "spicebox-operator", "operator", "2Gi") + require.NoError(t, os.CopyFS(inside, os.DirFS(src))) + err := runRender(context.Background(), io.Discard, offline, gke(out, inside), false) + require.Error(t, err) + assert.Contains(t, err.Error(), "keep overlays outside the output directory") + assert.DirExists(t, inside, "the source overlay must not be deleted") + }) +} + +func findDoc(t *testing.T, yaml []byte, kind, name string) *unstructured.Unstructured { + t.Helper() + docs, err := manifests.Split(yaml) + require.NoError(t, err) + for _, d := range docs { + if d.GetKind() == kind && d.GetName() == name { + return d + } + } + require.FailNowf(t, "missing doc", "%s %s", kind, name) + return nil +} diff --git a/cmd/oap/internal/installcmd/render.go b/cmd/oap/internal/installcmd/render.go index cff95e32..de4cc487 100644 --- a/cmd/oap/internal/installcmd/render.go +++ b/cmd/oap/internal/installcmd/render.go @@ -44,7 +44,11 @@ type RenderConfig struct { // operator webhook serving cert; you provide spicebox-webhook-tls + the VWC // caBundle yourself (for clusters without cert-manager). ExternalWebhookTLS bool - Strat cloud.Strategy + // Overlays are user kustomize Component directories, copied into the + // output and listed under `components:` in the order given — the same + // directories `oap install --overlay` accepts. + Overlays []string + Strat cloud.Strategy } func NewRenderCmd(g *apcmd.Globals) *cobra.Command { @@ -84,6 +88,7 @@ func NewRenderCmd(g *apcmd.Globals) *cobra.Command { cmd.Flags().StringVar(&cfg.Routing.trustedHostname, "trusted-hostname", "", "Bare hostname for webd's trusted origin; emits a Gateway + HTTPRoute (opt-in). Requires --gateway-class. Omit to bring your own ingress.") cmd.Flags().StringVar(&cfg.Routing.sandboxHostname, "sandbox-hostname", "", "Bare hostname for webd's artifact-viewer sandbox origin (adds a second HTTPS listener + HTTPRoute). Requires --trusted-hostname.") cmd.Flags().StringVar(&cfg.Routing.gatewayClass, "gateway-class", "", "GatewayClass name for the emitted Gateway (required with --trusted-hostname; render cannot auto-detect it).") + cmd.Flags().StringArrayVar(&cfg.Overlays, "overlay", nil, overlayFlagUsage) cmd.Flags().BoolVar(&validate, "validate", false, "After rendering, lint the overlay for completeness and consistency") return cmd } @@ -115,6 +120,12 @@ func runRender(ctx context.Context, out io.Writer, g *apcmd.Globals, cfg RenderC if err := validateRenderFlags(cfg); err != nil { return err } + // The same checks `oap install --overlay` runs, but against what this + // render emits: a patch for a component render leaves out would otherwise + // pass here and fail opaquely in the final kustomize build. + if _, err := loadOverlays(out, func() ([][]byte, error) { return renderOverlayBase(cfg) }, cfg.Overlays); err != nil { + return fmt.Errorf("%w (render emits the core install, channelsd and NATS, plus only the dependency components enabled by flags, such as --with-postgres)", err) + } // Registry digest resolution is the ONLY remote call render makes, and it is // read-only. Skipped entirely under --no-digest-pin or a local (no-registry) diff --git a/cmd/oap/internal/installcmd/render_components.go b/cmd/oap/internal/installcmd/render_components.go index 2b884306..1e26f92e 100644 --- a/cmd/oap/internal/installcmd/render_components.go +++ b/cmd/oap/internal/installcmd/render_components.go @@ -82,3 +82,25 @@ func writeDependencyComponents(cfg RenderConfig) error { } return nil } + +// renderOverlayBase returns the bundled manifests a render with cfg emits — the +// core install, channelsd, NATS and each enabled dependency component — for +// validating --overlay against. Unsubstituted: validation only needs the +// resources' identities and shapes. +func renderOverlayBase(cfg RenderConfig) ([][]byte, error) { + out := [][]byte{manifests.Install} + loads := []func() ([][]byte, error){manifests.ChannelsD, manifests.NATS} + for _, d := range dependencyComponents { + if d.enabled(cfg) { + loads = append(loads, d.load) + } + } + for _, load := range loads { + docs, err := load() + if err != nil { + return nil, err + } + out = append(out, docs...) + } + return out, nil +} diff --git a/cmd/oap/internal/installcmd/render_overlay.go b/cmd/oap/internal/installcmd/render_overlay.go index 5ccb7f5b..2685bc20 100644 --- a/cmd/oap/internal/installcmd/render_overlay.go +++ b/cmd/oap/internal/installcmd/render_overlay.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sigs.k8s.io/kustomize/api/krusty" "sigs.k8s.io/kustomize/kyaml/filesys" @@ -165,6 +166,9 @@ func WriteOverlay(cfg RenderConfig) error { if err := writeDependencyComponents(cfg); err != nil { return err } + if err := writeUserOverlays(cfg); err != nil { + return err + } k, err := renderKustomization(cfg) if err != nil { @@ -215,10 +219,19 @@ func renderKustomization(cfg RenderConfig) ([]byte, error) { "kind: Kustomization\n" + resources + patches - if names := enabledComponentNames(cfg); len(names) > 0 { + var components []string + for _, n := range enabledComponentNames(cfg) { + components = append(components, "components/"+n) + } + // User overlays come after oap's own components so they can patch those + // too; oap's `patches:` above still apply last, as on `oap install`. + for i, dir := range cfg.Overlays { + components = append(components, userOverlayPath(i, dir)) + } + if len(components) > 0 { k += "components:\n" - for _, n := range names { - k += " - components/" + n + "\n" + for _, c := range components { + k += " - " + c + "\n" } } return []byte(k), nil @@ -236,3 +249,46 @@ func buildOverlay(dir string) ([]byte, error) { } return res.AsYaml() } + +// userOverlaysDir holds copies of the --overlay directories inside the render +// output, so the output stays self-contained for a GitOps controller that only +// sees the committed directory. +const userOverlaysDir = "overlays" + +// userOverlayPath is overlay i's directory relative to the output root. The +// index keeps two overlays with the same basename apart and records the +// application order in the path itself. +func userOverlayPath(i int, dir string) string { + return fmt.Sprintf("%s/%02d-%s", userOverlaysDir, i, filepath.Base(filepath.Clean(dir))) +} + +// writeUserOverlays replaces OutputDir/overlays with copies of cfg.Overlays. +// Replacing rather than merging is what keeps the output a pure function of the +// flags: an overlay dropped from the command line must not linger on disk. An +// overlay that lives inside that directory is refused, since replacing it would +// delete the source. +func writeUserOverlays(cfg RenderConfig) error { + root := filepath.Join(cfg.OutputDir, userOverlaysDir) + absRoot, err := filepath.Abs(root) + if err != nil { + return err + } + for _, dir := range cfg.Overlays { + abs, err := filepath.Abs(dir) + if err != nil { + return err + } + if rel, err := filepath.Rel(absRoot, abs); err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("overlay %s is inside %s, which render regenerates; keep overlays outside the output directory", dir, root) + } + } + if err := os.RemoveAll(root); err != nil { + return err + } + for i, dir := range cfg.Overlays { + if err := os.CopyFS(filepath.Join(cfg.OutputDir, userOverlayPath(i, dir)), os.DirFS(dir)); err != nil { + return fmt.Errorf("copy overlay %s: %w", dir, err) + } + } + return nil +} diff --git a/pkg/platform/manifests/overlay/overlay.go b/pkg/platform/manifests/overlay/overlay.go new file mode 100644 index 00000000..7aba0825 --- /dev/null +++ b/pkg/platform/manifests/overlay/overlay.go @@ -0,0 +1,551 @@ +// Package overlay layers user-supplied kustomize Components over the manifests +// oap installs, so a change an operator needs on their cluster (a raised memory +// limit, an extra env var) lives in files they own and survives every re-install +// and upgrade instead of being reset to the bundled value. +// +// An overlay is a directory holding a kustomization.yaml of `kind: Component`. +// A Component, unlike a classic overlay, does not name its base: oap supplies +// it. That is what lets one overlay directory serve both install paths — `oap +// install` layers it over the documents it is about to apply, and `oap render` +// lists it under the rendered kustomization's `components:`. +// +// Load builds every overlay against a base — Base, every first-party manifest +// `oap install` can apply, or the narrower set a render emits — and refuses, +// before anything touches a cluster: +// +// - a patch that matches no resource (kustomize's own check — a typo'd +// Deployment name fails instead of being silently ignored); +// - an overlay that adds, removes or renames a resource — overlays modify what +// oap installs, they do not change which resources it installs; +// - two overlays that set the same field to different values. Each overlay is +// built on its own and the fields it changes are compared; agreeing values +// are not a conflict. +// +// Apply then lays the overlays over a document at the moment install writes it, +// by running kustomize again with that document — as install holds it, after +// its own image substitution — swapped into the base. Kustomize therefore +// patches exactly what install applies. A patch derived once from the bundled +// base could not: a list with no merge key (container args) is carried whole, +// so the bundle's copy of it would undo install's rewrite of an image ref +// inside it. +package overlay + +import ( + "bytes" + "encoding/json" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "reflect" + "slices" + "sort" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/yaml" + + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +// Set is a validated set of overlays. A nil *Set is valid and applies nothing, +// so callers without overlays need no branch. +type Set struct { + dirs []string + // patched holds the resources the overlays change; Apply rebuilds only + // those, and passes every other document through untouched. + patched map[resID]bool + + // mu serializes Apply: every call rewrites the shared base in fsys. The + // install pipeline applies components concurrently. + mu sync.Mutex + fsys filesys.FileSystem + baseDocs []*unstructured.Unstructured +} + +// resID identifies a resource across the base and the built output. Version is +// deliberately absent: install may apply the same resource at a different +// apiVersion than the bundle names, and it is still the same object. +type resID struct { + Group, Kind, Namespace, Name string +} + +func (id resID) String() string { + k := id.Kind + if id.Group != "" { + k += "." + id.Group + } + if id.Namespace != "" { + return k + " " + id.Namespace + "/" + id.Name + } + return k + " " + id.Name +} + +func idOf(u *unstructured.Unstructured) resID { + gvk := u.GroupVersionKind() + return resID{Group: gvk.Group, Kind: gvk.Kind, Namespace: u.GetNamespace(), Name: u.GetName()} +} + +// Base returns the documents an overlay is validated against: every first-party +// manifest `oap install` can apply. It is the same set whether or not a given +// optional component is enabled on this install, so whether an overlay is valid +// never depends on install flags. +func Base() ([][]byte, error) { + out := [][]byte{manifests.Install} + for _, load := range []func() ([][]byte, error){ + manifests.ChannelsD, manifests.NATS, manifests.Postgres, manifests.Neo4j, manifests.Graphiti, + } { + docs, err := load() + if err != nil { + return nil, err + } + out = append(out, docs...) + } + return out, nil +} + +// Load reads each overlay directory and validates the set against base. Directories apply in the order given. An +// empty dirs returns a nil Set. +func Load(base [][]byte, dirs []string) (*Set, error) { + if len(dirs) == 0 { + return nil, nil + } + fsys := filesys.MakeFsInMemory() + baseDocs, err := splitAll(base) + if err != nil { + return nil, fmt.Errorf("overlay base: %w", err) + } + if err := writeBase(fsys, baseDocs); err != nil { + return nil, err + } + for i, dir := range dirs { + if err := copyOverlay(fsys, dir, overlayRoot(i)); err != nil { + return nil, err + } + } + + baseByID := index(baseDocs) + + // Each overlay alone: catch a bad patch with the right directory named, and + // collect the fields it changes for the conflict check. + changes := make([]map[resID]map[string]any, len(dirs)) + for i, dir := range dirs { + built, err := build(fsys, []int{i}) + if err != nil { + return nil, fmt.Errorf("overlay %s: %w", dir, err) + } + if err := sameResources(baseByID, built); err != nil { + return nil, fmt.Errorf("overlay %s: %w", dir, err) + } + changes[i] = map[resID]map[string]any{} + for id, doc := range built { + if c := changedLeaves(baseByID[id].Object, doc.Object); len(c) > 0 { + changes[i][id] = c + } + } + } + if err := conflicts(dirs, changes); err != nil { + return nil, err + } + + s := &Set{dirs: slices.Clone(dirs), patched: map[resID]bool{}, fsys: fsys, baseDocs: baseDocs} + built, err := s.buildAll() + if err != nil { + return nil, fmt.Errorf("overlays: %w", err) + } + for id, doc := range built { + if !reflect.DeepEqual(baseByID[id].Object, doc.Object) { + s.patched[id] = true + } + } + return s, nil +} + +func (s *Set) buildAll() (map[resID]*unstructured.Unstructured, error) { + all := make([]int, len(s.dirs)) + for i := range all { + all[i] = i + } + return build(s.fsys, all) +} + +// Dirs returns the overlay directories in application order. +func (s *Set) Dirs() []string { + if s == nil { + return nil + } + return slices.Clone(s.dirs) +} + +// Patched returns the resources the overlays change, sorted, for narration. +func (s *Set) Patched() []string { + if s == nil { + return nil + } + out := make([]string, 0, len(s.patched)) + for id := range s.patched { + out = append(out, id.String()) + } + sort.Strings(out) + return out +} + +// Apply lays the overlays over d in place. A resource no overlay changes, and +// any call on a nil Set, leaves d unchanged. For one they change, the overlays +// are rebuilt with d in its bundled counterpart's place, so they patch d itself +// and every field they leave alone keeps the value install gave it. +func (s *Set) Apply(d *unstructured.Unstructured) error { + if s == nil { + return nil + } + id := idOf(d) + if !s.patched[id] { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + docs := make([]*unstructured.Unstructured, len(s.baseDocs)) + for i, b := range s.baseDocs { + if idOf(b) == id { + docs[i] = d + } else { + docs[i] = b + } + } + if err := writeBase(s.fsys, docs); err != nil { + return fmt.Errorf("apply overlays to %s: %w", id, err) + } + built, err := s.buildAll() + if err != nil { + return fmt.Errorf("apply overlays to %s: %w", id, err) + } + out, ok := built[id] + if !ok { + return fmt.Errorf("apply overlays to %s: missing from the build", id) + } + d.Object = out.Object + return nil +} + +const ( + fsBase = "/oap/base" + fsOverlays = "/oap/overlays" + fsBuild = "/oap/build" +) + +func overlayRoot(i int) string { return path.Join(fsOverlays, fmt.Sprint(i)) } + +func writeBase(fsys filesys.FileSystem, docs []*unstructured.Unstructured) error { + var buf bytes.Buffer + for _, d := range docs { + b, err := yaml.Marshal(d.Object) + if err != nil { + return err + } + buf.WriteString("---\n") + buf.Write(b) + } + if err := fsys.MkdirAll(fsBase); err != nil { + return err + } + if err := fsys.WriteFile(path.Join(fsBase, "base.yaml"), buf.Bytes()); err != nil { + return err + } + return fsys.WriteFile(path.Join(fsBase, "kustomization.yaml"), []byte("resources:\n- base.yaml\n")) +} + +// copyOverlay copies dir into the in-memory filesystem at dst, refusing anything +// that is not a kustomize Component. Copying, rather than building from disk, +// keeps the base off the user's filesystem and means a build reads exactly the +// files Load validated. +func copyOverlay(fsys filesys.FileSystem, dir, dst string) error { + if err := CheckComponent(dir); err != nil { + return err + } + return filepath.WalkDir(dir, func(p string, e fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + target := path.Join(dst, filepath.ToSlash(rel)) + if e.IsDir() { + return fsys.MkdirAll(target) + } + if !e.Type().IsRegular() { + return fmt.Errorf("overlay %s: %s is not a regular file", dir, rel) + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + return fsys.WriteFile(target, b) + }) +} + +// kustomizationFiles are the names kustomize accepts for a kustomization, in its +// own lookup order. +var kustomizationFiles = []string{"kustomization.yaml", "kustomization.yml", "Kustomization"} + +// CheckComponent reports whether dir holds a kustomization of `kind: +// Component`. A classic overlay names its own base by path, which cannot point +// at the documents install holds in memory, so it is refused with a message +// saying what to change. +func CheckComponent(dir string) error { + for _, name := range kustomizationFiles { + b, err := os.ReadFile(filepath.Join(dir, name)) + if os.IsNotExist(err) { + continue + } + if err != nil { + return fmt.Errorf("overlay %s: %w", dir, err) + } + var k struct { + Kind string `json:"kind"` + } + if err := yaml.Unmarshal(b, &k); err != nil { + return fmt.Errorf("overlay %s: parse %s: %w", dir, name, err) + } + if k.Kind != "Component" { + return fmt.Errorf("overlay %s: %s must be a kustomize Component (apiVersion: kustomize.config.k8s.io/v1alpha1, kind: Component), not %q — oap supplies the base, so the overlay must not name one", dir, name, k.Kind) + } + return nil + } + return fmt.Errorf("overlay %s: no kustomization.yaml found", dir) +} + +// build runs kustomize over the base plus the overlays at idx, in order. +func build(fsys filesys.FileSystem, idx []int) (map[resID]*unstructured.Unstructured, error) { + dir := path.Join(fsBuild, strings.Trim(strings.ReplaceAll(fmt.Sprint(idx), " ", "-"), "[]")) + var k strings.Builder + // kustomize refuses absolute roots, so every reference is relative to dir + // (one level under fsBuild, itself a sibling of fsBase and fsOverlays). + k.WriteString("resources:\n- ../../base\ncomponents:\n") + for _, i := range idx { + k.WriteString("- ../../overlays/" + fmt.Sprint(i) + "\n") + } + if err := fsys.MkdirAll(dir); err != nil { + return nil, err + } + if err := fsys.WriteFile(path.Join(dir, "kustomization.yaml"), []byte(k.String())); err != nil { + return nil, err + } + res, err := krusty.MakeKustomizer(krusty.MakeDefaultOptions()).Run(fsys, dir) + if err != nil { + return nil, err + } + out, err := res.AsYaml() + if err != nil { + return nil, err + } + docs, err := manifests.Split(out) + if err != nil { + return nil, err + } + return index(docs), nil +} + +func splitAll(base [][]byte) ([]*unstructured.Unstructured, error) { + var out []*unstructured.Unstructured + for _, b := range base { + docs, err := manifests.Split(b) + if err != nil { + return nil, err + } + out = append(out, docs...) + } + return out, nil +} + +func index(docs []*unstructured.Unstructured) map[resID]*unstructured.Unstructured { + m := make(map[resID]*unstructured.Unstructured, len(docs)) + for _, d := range docs { + m[idOf(d)] = d + } + return m +} + +// sameResources refuses an overlay that changes which resources exist. Install +// applies resources from several bundles at different moments; an added +// resource would belong to none of them, and a removed or renamed one would be +// applied anyway under its bundled identity. +func sameResources(base, built map[resID]*unstructured.Unstructured) error { + var added, removed []string + for id := range built { + if _, ok := base[id]; !ok { + added = append(added, id.String()) + } + } + for id := range base { + if _, ok := built[id]; !ok { + removed = append(removed, id.String()) + } + } + if len(added) == 0 && len(removed) == 0 { + return nil + } + sort.Strings(added) + sort.Strings(removed) + var parts []string + if len(added) > 0 { + parts = append(parts, "adds "+strings.Join(added, ", ")) + } + if len(removed) > 0 { + parts = append(parts, "removes "+strings.Join(removed, ", ")) + } + return fmt.Errorf("%s; overlays may only modify the resources oap installs (a rename counts as both)", strings.Join(parts, " and ")) +} + +// removed marks a field an overlay deleted, so a deletion conflicts with a set. +type removedMarker struct{} + +// changedLeaves returns every leaf path whose value differs between a and b, +// mapped to b's value. A list of objects is keyed by the first of listKeys +// that every item in both lists carries uniquely, matching how a +// strategic-merge patch addresses containers, env vars, ports and mounts, so +// two overlays touching different items never collide on a shared index. A +// list no key fits is compared whole, which can report a conflict between +// overlays that touched different items but never misses a real one. +func changedLeaves(a, b any) map[string]any { + out := map[string]any{} + walkDiff("", a, b, out) + return out +} + +func walkDiff(p string, a, b any, out map[string]any) { + if reflect.DeepEqual(a, b) { + return + } + am, aok := a.(map[string]any) + bm, bok := b.(map[string]any) + if aok && bok { + for k := range union(am, bm) { + av, ain := am[k] + bv, bin := bm[k] + switch { + case !bin: + out[p+"/"+k] = removedMarker{} + case !ain: + walkDiff(p+"/"+k, nil, bv, out) + default: + walkDiff(p+"/"+k, av, bv, out) + } + } + return + } + al, aok := a.([]any) + bl, bok := b.([]any) + if aok && bok { + if ak, bk, ok := keyedLists(al, bl); ok { + for k := range union(ak, bk) { + bv, bin := bk[k] + if !bin { + out[p+"/["+k+"]"] = removedMarker{} + continue + } + walkDiff(p+"/["+k+"]", ak[k], bv, out) + } + return + } + } + if bm, ok := b.(map[string]any); ok && a == nil { + for k, v := range bm { + walkDiff(p+"/"+k, nil, v, out) + } + return + } + out[p] = b +} + +// listKeys are the strategic-merge keys of the lists in the manifests oap +// ships: "name" for containers, env, volumes and named ports; mountPath for +// volumeMounts; containerPort for unnamed container ports. +var listKeys = []string{"name", "mountPath", "containerPort"} + +func keyedLists(a, b []any) (map[string]any, map[string]any, bool) { + for _, key := range listKeys { + ak, aok := byKey(a, key) + bk, bok := byKey(b, key) + if aok && bok { + return ak, bk, true + } + } + return nil, nil, false +} + +func byKey(l []any, key string) (map[string]any, bool) { + m := make(map[string]any, len(l)) + for _, item := range l { + obj, ok := item.(map[string]any) + if !ok { + return nil, false + } + v, ok := obj[key] + if !ok { + return nil, false + } + k := fmt.Sprint(v) + if _, dup := m[k]; dup { + return nil, false + } + m[k] = obj + } + return m, true +} + +func union[V any](a, b map[string]V) map[string]struct{} { + out := make(map[string]struct{}, len(a)+len(b)) + for k := range a { + out[k] = struct{}{} + } + for k := range b { + out[k] = struct{}{} + } + return out +} + +// conflicts reports every field two overlays set to different values. All +// conflicts are reported at once so a user fixing them is not sent round the +// loop once per field. +func conflicts(dirs []string, changes []map[resID]map[string]any) error { + var found []string + for i := range changes { + for j := i + 1; j < len(changes); j++ { + for id, ci := range changes[i] { + cj, ok := changes[j][id] + if !ok { + continue + } + for field, vi := range ci { + vj, ok := cj[field] + if !ok || reflect.DeepEqual(vi, vj) { + continue + } + found = append(found, fmt.Sprintf(" %s %s: %s sets %s, %s sets %s", id, field, dirs[i], describe(vi), dirs[j], describe(vj))) + } + } + } + } + if len(found) == 0 { + return nil + } + sort.Strings(found) + return fmt.Errorf("overlays conflict:\n%s", strings.Join(found, "\n")) +} + +func describe(v any) string { + if _, ok := v.(removedMarker); ok { + return "(removed)" + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Sprint(v) + } + return string(b) +} diff --git a/pkg/platform/manifests/overlay/overlay_test.go b/pkg/platform/manifests/overlay/overlay_test.go new file mode 100644 index 00000000..c293cf7b --- /dev/null +++ b/pkg/platform/manifests/overlay/overlay_test.go @@ -0,0 +1,295 @@ +package overlay + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +const componentHeader = "apiVersion: kustomize.config.k8s.io/v1alpha1\nkind: Component\n" + +// writeOverlay materializes an overlay directory holding kustomization (the +// full kustomization.yaml text) and each named file. +func writeOverlay(t *testing.T, kustomization string, files map[string]string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte(kustomization), 0o644)) + for name, body := range files { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644)) + } + return dir +} + +// operatorMemoryOverlay raises the operator container's memory limit to limit. +func operatorMemoryOverlay(t *testing.T, limit string) string { + t.Helper() + return writeOverlay(t, componentHeader+"patches:\n- path: memory.yaml\n", map[string]string{ + "memory.yaml": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: spicebox-operator\n namespace: agentprimitives-system\n" + + "spec:\n template:\n spec:\n containers:\n - name: operator\n resources:\n limits:\n memory: " + limit + "\n", + }) +} + +func webdMemoryOverlay(t *testing.T, limit string) string { + t.Helper() + return writeOverlay(t, componentHeader+"patches:\n- path: memory.yaml\n", map[string]string{ + "memory.yaml": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: spicebox-webd\n namespace: agentprimitives-system\n" + + "spec:\n template:\n spec:\n containers:\n - name: webd\n resources:\n limits:\n memory: " + limit + "\n", + }) +} + +func testBase(t *testing.T) [][]byte { + t.Helper() + base, err := Base() + require.NoError(t, err) + return base +} + +// bundledDoc returns the named document from the embedded install bundle, as +// install would hold it just before applying. +func bundledDoc(t *testing.T, kind, name string) *unstructured.Unstructured { + t.Helper() + docs, err := manifests.Split(manifests.Install) + require.NoError(t, err) + for _, d := range docs { + if d.GetKind() == kind && d.GetName() == name { + return d + } + } + require.FailNowf(t, "missing bundled doc", "%s %s", kind, name) + return nil +} + +func container(t *testing.T, d *unstructured.Unstructured, name string) map[string]any { + t.Helper() + cs, _, err := unstructured.NestedSlice(d.Object, "spec", "template", "spec", "containers") + require.NoError(t, err) + for _, c := range cs { + if m := c.(map[string]any); m["name"] == name { + return m + } + } + require.FailNowf(t, "missing container", "%s", name) + return nil +} + +func memory(t *testing.T, c map[string]any, field string) string { + t.Helper() + v, _, err := unstructured.NestedString(c, "resources", field, "memory") + require.NoError(t, err) + return v +} + +func TestLoad_NoOverlaysIsNilAndApplyIsNoOp(t *testing.T) { + s, err := Load(testBase(t), nil) + require.NoError(t, err) + assert.Nil(t, s) + + d := bundledDoc(t, "Deployment", "spicebox-operator") + before := d.DeepCopy() + require.NoError(t, s.Apply(d)) + assert.Equal(t, before, d) +} + +func TestApply_RaisesOperatorMemoryAndKeepsInstallRewrites(t *testing.T) { + s, err := Load(testBase(t), []string{operatorMemoryOverlay(t, "2Gi")}) + require.NoError(t, err) + assert.Equal(t, []string{"Deployment.apps agentprimitives-system/spicebox-operator"}, s.Patched()) + + // Install rewrites the image before applying; the overlay must not undo it. + d := bundledDoc(t, "Deployment", "spicebox-operator") + c := container(t, d, "operator") + c["image"] = "registry.example/spicebox-operator@sha256:abc" + cs, _, _ := unstructured.NestedSlice(d.Object, "spec", "template", "spec", "containers") + for i := range cs { + if cs[i].(map[string]any)["name"] == "operator" { + cs[i] = c + } + } + require.NoError(t, unstructured.SetNestedSlice(d.Object, cs, "spec", "template", "spec", "containers")) + + require.NoError(t, s.Apply(d)) + got := container(t, d, "operator") + assert.Equal(t, "2Gi", memory(t, got, "limits")) + assert.Equal(t, "256Mi", memory(t, got, "requests"), "a field the overlay did not name keeps its bundled value") + assert.Equal(t, "registry.example/spicebox-operator@sha256:abc", got["image"]) +} + +func TestApply_UntouchedResourceIsUnchanged(t *testing.T) { + s, err := Load(testBase(t), []string{operatorMemoryOverlay(t, "2Gi")}) + require.NoError(t, err) + d := bundledDoc(t, "Deployment", "spicebox-webd") + before := d.DeepCopy() + require.NoError(t, s.Apply(d)) + assert.Equal(t, before, d) +} + +func TestLoad_Validation(t *testing.T) { + cases := []struct { + name string + dirs func(t *testing.T) []string + wantErr []string // substrings; empty means Load succeeds + }{ + { + name: "two overlays on different resources: accepted", + dirs: func(t *testing.T) []string { + return []string{operatorMemoryOverlay(t, "2Gi"), webdMemoryOverlay(t, "1Gi")} + }, + }, + { + name: "two overlays agreeing on a field: accepted", + dirs: func(t *testing.T) []string { + return []string{operatorMemoryOverlay(t, "2Gi"), operatorMemoryOverlay(t, "2Gi")} + }, + }, + { + name: "two overlays setting one field differently: conflict naming both and the field", + dirs: func(t *testing.T) []string { + return []string{operatorMemoryOverlay(t, "2Gi"), operatorMemoryOverlay(t, "4Gi")} + }, + wantErr: []string{"overlays conflict", "spicebox-operator", "/[operator]/resources/limits/memory", `"2Gi"`, `"4Gi"`}, + }, + { + name: "patch naming a resource oap does not ship: refused", + dirs: func(t *testing.T) []string { + return []string{writeOverlay(t, componentHeader+"patches:\n- path: p.yaml\n", map[string]string{ + "p.yaml": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: spicebox-operatr\n namespace: agentprimitives-system\nspec:\n replicas: 2\n", + })} + }, + wantErr: []string{"spicebox-operatr"}, + }, + { + name: "overlay adding a resource: refused", + dirs: func(t *testing.T) []string { + return []string{writeOverlay(t, componentHeader+"resources:\n- cm.yaml\n", map[string]string{ + "cm.yaml": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: extra\n namespace: agentprimitives-system\n", + })} + }, + wantErr: []string{"adds ConfigMap agentprimitives-system/extra", "may only modify"}, + }, + { + name: "classic overlay (kind: Kustomization): refused with how to fix", + dirs: func(t *testing.T) []string { + return []string{writeOverlay(t, "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n- ../base\n", nil)} + }, + wantErr: []string{"must be a kustomize Component"}, + }, + { + name: "directory without a kustomization: refused", + dirs: func(t *testing.T) []string { + return []string{t.TempDir()} + }, + wantErr: []string{"no kustomization.yaml found"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := Load(testBase(t), tc.dirs(t)) + if len(tc.wantErr) == 0 { + assert.NoError(t, err) + return + } + require.Error(t, err) + for _, want := range tc.wantErr { + assert.Contains(t, err.Error(), want) + } + }) + } +} + +func TestChangedLeaves_KeysNamedListsByName(t *testing.T) { + a := map[string]any{"cs": []any{ + map[string]any{"name": "x", "v": "1"}, + map[string]any{"name": "y", "v": "1"}, + }} + b := map[string]any{"cs": []any{ + map[string]any{"name": "x", "v": "1"}, + map[string]any{"name": "y", "v": "2"}, + map[string]any{"name": "z", "v": "3"}, + }} + got := changedLeaves(a, b) + assert.Equal(t, map[string]any{"/cs/[y]/v": "2", "/cs/[z]/name": "z", "/cs/[z]/v": "3"}, got) +} + +// An overlay that changes a list with no merge key (container args) must not +// carry the bundle's copy of that list along with it: install rewrites the +// runner image ref inside those args before applying, and the overlay is laid +// over that rewritten document, not over the bundle. +func TestApply_ListWithoutMergeKeyKeepsInstallRewrites(t *testing.T) { + dir := writeOverlay(t, componentHeader+"patches:\n- target:\n kind: Deployment\n name: spicebox-operator\n patch: |-\n - op: add\n path: /spec/template/spec/containers/0/args/-\n value: --extra-flag\n", nil) + s, err := Load(testBase(t), []string{dir}) + require.NoError(t, err) + + d := bundledDoc(t, "Deployment", "spicebox-operator") + raw, err := json.Marshal(d.Object) + require.NoError(t, err) + rewritten := strings.ReplaceAll(string(raw), "agentprimitives-runner:dev", "registry.example/agentprimitives-runner@sha256:abc") + require.NoError(t, json.Unmarshal([]byte(rewritten), &d.Object)) + + require.NoError(t, s.Apply(d)) + args, _, err := unstructured.NestedStringSlice(container(t, d, "operator"), "args") + require.NoError(t, err) + assert.Contains(t, args, "--extra-flag") + assert.Contains(t, args, "--runner-image=registry.example/agentprimitives-runner@sha256:abc") + assert.NotContains(t, args, "--runner-image=agentprimitives-runner:dev") +} + +// A kind outside the client-go scheme (oap's own CRs) goes through the same +// rebuild: the overlay's change lands and install's substitution of an image +// ref elsewhere in the resource survives. +func TestApply_CustomResourceKeepsInstallRewrites(t *testing.T) { + dir := writeOverlay(t, componentHeader+"patches:\n- target:\n kind: SpiceboxToolchain\n name: claude\n patch: |-\n - op: add\n path: /spec/provides/-\n value: claude-extra\n", nil) + s, err := Load(testBase(t), []string{dir}) + require.NoError(t, err) + assert.Equal(t, []string{"SpiceboxToolchain.agentprimitives.authzed.com claude"}, s.Patched()) + + d := bundledDoc(t, "SpiceboxToolchain", "claude") + require.NoError(t, unstructured.SetNestedField(d.Object, "registry.example/ap-toolchain-claude@sha256:abc", "spec", "source", "image")) + require.NoError(t, s.Apply(d)) + + provides, _, err := unstructured.NestedStringSlice(d.Object, "spec", "provides") + require.NoError(t, err) + assert.Equal(t, []string{"claude", "claude-code", "claude-extra"}, provides) + image, _, err := unstructured.NestedString(d.Object, "spec", "source", "image") + require.NoError(t, err) + assert.Equal(t, "registry.example/ap-toolchain-claude@sha256:abc", image) +} + +func TestLoad_Refusals(t *testing.T) { + t.Run("symlink inside an overlay: refused as not a regular file", func(t *testing.T) { + dir := operatorMemoryOverlay(t, "2Gi") + require.NoError(t, os.Symlink(filepath.Join(dir, "memory.yaml"), filepath.Join(dir, "link.yaml"))) + _, err := Load(testBase(t), []string{dir}) + require.Error(t, err) + assert.Contains(t, err.Error(), "link.yaml is not a regular file") + }) + + t.Run("one overlay removes a field another sets: conflict shows the removal", func(t *testing.T) { + remove := writeOverlay(t, componentHeader+"patches:\n- target:\n kind: Deployment\n name: spicebox-operator\n patch: |-\n - op: remove\n path: /spec/template/spec/containers/0/resources/limits/memory\n", nil) + _, err := Load(testBase(t), []string{remove, operatorMemoryOverlay(t, "4Gi")}) + require.Error(t, err) + assert.Contains(t, err.Error(), "/[operator]/resources/limits/memory") + assert.Contains(t, err.Error(), "(removed)") + assert.Contains(t, err.Error(), `"4Gi"`) + }) +} + +func TestChangedLeaves_KeysListsByFirstFittingMergeKey(t *testing.T) { + a := map[string]any{"mounts": []any{ + map[string]any{"mountPath": "/a", "readOnly": true}, + map[string]any{"mountPath": "/b", "readOnly": true}, + }} + b := map[string]any{"mounts": []any{ + map[string]any{"mountPath": "/a", "readOnly": true}, + map[string]any{"mountPath": "/b", "readOnly": false}, + }} + assert.Equal(t, map[string]any{"/mounts/[/b]/readOnly": false}, changedLeaves(a, b)) +} diff --git a/site/content/docs/gitops.mdx b/site/content/docs/gitops.mdx index 2e25f372..7e70005f 100644 --- a/site/content/docs/gitops.mdx +++ b/site/content/docs/gitops.mdx @@ -102,6 +102,13 @@ prints the app-internal secrets you still need to provide. It cannot verify that a *linked* or externally-provided Secret actually exists in the cluster — that is a cluster-state fact `oap check` reports after the sync. +## Customize with overlays + +`--overlay ` (repeatable) layers your own kustomize Components over the +render: each is copied into `overlays/` and listed under `components:`. They are +the same overlays `oap install --overlay` takes, checked the same way; see +[Customizing the install](/docs/install-overlays). + ## Verify the synced cluster After Argo CD / Flux reports synced, confirm the install is healthy with the diff --git a/site/content/docs/health-upgrades.mdx b/site/content/docs/health-upgrades.mdx index bb644a0b..2a201bd3 100644 --- a/site/content/docs/health-upgrades.mdx +++ b/site/content/docs/health-upgrades.mdx @@ -39,6 +39,9 @@ oap install --builder-starters user:$(oap identity canonical-id you@example.com) [Agent Builder](/docs/agent-builder-install). The starter list replaces what was there, so pass the full list on every upgrade. +A re-install resets anything changed by hand on the cluster to the bundled value. Keep cluster-specific changes, such +as a raised memory limit, in [overlays](/docs/install-overlays) and pass them with `--overlay` on every upgrade. + The platform is deliberately fail-closed about its own configuration — a missing artifact store, an unrecognized cluster kind, or an unset required backend crash-loops the component rather than starting in a diff --git a/site/content/docs/install-overlays.mdx b/site/content/docs/install-overlays.mdx new file mode 100644 index 00000000..892d7bdf --- /dev/null +++ b/site/content/docs/install-overlays.mdx @@ -0,0 +1,90 @@ +export const meta = { + title: 'Customizing the install', + section: 'Guides', + group: 'Operations', + order: 202, + description: + 'Keep cluster-specific changes, like a raised memory limit, in kustomize overlays you own, so every re-install and upgrade applies them instead of resetting them.', +} + +# Customizing the install + +
+ oap install applies the bundled platform manifests, and every re-install or upgrade applies them + again. A change made by hand on the cluster, such as a memory limit raised with kubectl, is reset + the next time. To keep a change, put it in an overlay and pass it on every install. +
+ +## Write an overlay + +An overlay is a directory holding a [kustomize Component](https://kubectl.docs.kubernetes.io/guides/config_management/components/): +a `kustomization.yaml` of `kind: Component` plus the patches it lists. It does not name a base. OAP supplies the +base: the manifests it is about to install. + +This one raises the operator's memory limit: + +```yaml +# overlays/operator-memory/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - path: memory.yaml +``` + +```yaml +# overlays/operator-memory/memory.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spicebox-operator + namespace: agentprimitives-system +spec: + template: + spec: + containers: + - name: operator + resources: + limits: + memory: 2Gi +``` + +Any kustomize patch works, including strategic-merge and JSON 6902 patches. A patch can target any resource OAP +installs: the operator, webd, authzd, extractord, channelsd, NATS, Postgres, Neo4j and Graphiti. + +## Apply it + +Pass `--overlay` to `oap init` or `oap install`. Repeat the flag to layer several, applied in the order given: + +```bash +oap install --overlay overlays/operator-memory --overlay overlays/webd-env ... +``` + +The install lists each resource the overlays change before it starts. Pass the same overlays on every later +install and upgrade. The overlay is an input like any other flag: an install run without it puts the bundled values +back. + +## What is refused + +Overlays are checked against every manifest OAP ships before anything is built or applied. The install stops +with an error when: + +- **Two overlays conflict.** Two overlays that set the same field to different values are refused, and the error + names the resource, the field and both values. Overlays that agree on a value, or change different fields, are + fine. +- **A patch matches nothing.** A misspelled resource name fails rather than being silently ignored. +- **An overlay adds, removes or renames a resource.** Overlays change the resources OAP installs; they do not + change which resources it installs. +- **The directory is not a Component.** A classic overlay, which names its own base with `resources: [../base]`, + is refused with a pointer to `kind: Component`. + +Overlays patch the manifests as the install will apply them, with image references already rewritten for your +registry, so every field an overlay does not name keeps the value the install gave it. Image pull secrets and registry +mirrors are applied after the overlays and take precedence on the fields they set. + +## With GitOps + +[`oap render`](/docs/gitops) accepts the same `--overlay` flags. It copies each overlay into the output's +`overlays/` directory and lists it under `components:`, so an overlay written for direct installs carries over +unchanged. It runs the same checks, against the resources the render emits: the core install, channelsd, NATS and +the dependency components enabled by flags. Render does not emit Neo4j or Graphiti, and emits Postgres only with +`--with-postgres`, so an overlay that patches one of those is refused unless the render includes it.