Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions cmd/oap/internal/installcmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/authzed/openagentprimitives/pkg/platform/cloud"
idpregistry "github.com/authzed/openagentprimitives/pkg/platform/identity/idp/registry"
"github.com/authzed/openagentprimitives/pkg/platform/manifests"
"github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay"
)

// wizardMode controls how (or whether) the settings wizard runs after a
Expand Down Expand Up @@ -99,6 +100,7 @@ func NewInitCmd(g *apcmd.Globals) *cobra.Command {
var externalSpiceDBToken string
var externalSpiceDBInsecure bool
var acceptExisting bool
var overlayDirs []string
// dm is populated below via settingscmd.RegisterDefaultModelFlags(cmd), after cmd
// exists. The RunE closure captures the variable (not its zero value at
// closure-literal time) — by the time cobra invokes RunE, dm has already
Expand All @@ -112,6 +114,12 @@ func NewInitCmd(g *apcmd.Globals) *cobra.Command {
if err := validateInitFlags(localMode, clusterKindFlag, developMode, externalSpiceDBEndpoint, externalSpiceDBToken, acceptExisting, skipInstall); err != nil {
return err
}
// Before the build step: a broken or conflicting overlay must not
// cost the user a full image build to discover.
overlays, err := loadOverlays(cmd.OutOrStdout(), overlay.Base, overlayDirs)
if err != nil {
return err
}
// --local is sugar for --cluster-kind=local; validateInitFlags has
// already refused a conflicting explicit --cluster-kind alongside it.
clusterKind := clusterKindFlag
Expand All @@ -138,11 +146,12 @@ func NewInitCmd(g *apcmd.Globals) *cobra.Command {
// check are done, on every kind, instead of blocking on a
// foreground tunnel whose death left that ConfigMap advertising a
// URL that forwarded nothing.
return runInit(cmd.Context(), cmd.OutOrStdout(), g, skipBuild, skipInstall, timeout, tags, pinningMode, clusterKind, allowNonLocal, r, wizardFlag, defaultsFlag, imagePullSecret, mirrorDeps, createRegistry, noIdp, noMonitoring, *dm, artifactStoreURL, externalSpiceDBEndpoint, externalSpiceDBToken, externalSpiceDBInsecure, acceptExisting)
return runInit(cmd.Context(), cmd.OutOrStdout(), g, skipBuild, skipInstall, timeout, tags, pinningMode, clusterKind, allowNonLocal, r, wizardFlag, defaultsFlag, imagePullSecret, mirrorDeps, createRegistry, noIdp, noMonitoring, *dm, artifactStoreURL, externalSpiceDBEndpoint, externalSpiceDBToken, externalSpiceDBInsecure, acceptExisting, overlays)
},
}
cmd.Flags().BoolVar(&skipBuild, "skip-build", false, "Skip oap build all")
cmd.Flags().BoolVar(&skipInstall, "skip-install", false, "Skip oap install")
cmd.Flags().StringArrayVar(&overlayDirs, "overlay", nil, overlayFlagUsage)
cmd.Flags().DurationVar(&timeout, "timeout", timeout, "Overall deadline for the install + check loop")
cmd.Flags().StringVar(&tags.Operator, "operator-image", "", "Override operator image")
cmd.Flags().StringVar(&tags.Runner, "runner-image", "", "Override runner image")
Expand Down Expand Up @@ -298,7 +307,7 @@ func validateInitFlags(localMode bool, clusterKind string, developMode bool, ext
// prompts for it instead (see there). A non-empty result skips the bundled
// SpiceDB install (RunInstall) and, after install completes, best-effort
// applies the base schema to the caller's endpoint (applySchemaToEndpoint).
func runInit(ctx context.Context, out io.Writer, g *apcmd.Globals, skipBuild, skipInstall bool, timeout time.Duration, tags manifests.Tags, pinningMode string, clusterKind string, allowNonLocal bool, r WebdRoutingOpts, wizardFlag, defaultsFlag bool, imagePullSecret string, mirrorDeps, createRegistry bool, noIdp bool, noMonitoring bool, dm settingscmd.DefaultModelOpts, artifactStoreURL string, externalSpiceDBEndpoint, externalSpiceDBToken string, externalSpiceDBInsecure bool, acceptExisting bool) error {
func runInit(ctx context.Context, out io.Writer, g *apcmd.Globals, skipBuild, skipInstall bool, timeout time.Duration, tags manifests.Tags, pinningMode string, clusterKind string, allowNonLocal bool, r WebdRoutingOpts, wizardFlag, defaultsFlag bool, imagePullSecret string, mirrorDeps, createRegistry bool, noIdp bool, noMonitoring bool, dm settingscmd.DefaultModelOpts, artifactStoreURL string, externalSpiceDBEndpoint, externalSpiceDBToken string, externalSpiceDBInsecure bool, acceptExisting bool, overlays *overlay.Set) error {
if err := validPinningMode(pinningMode); err != nil {
return err
}
Expand Down Expand Up @@ -492,6 +501,7 @@ func runInit(ctx context.Context, out io.Writer, g *apcmd.Globals, skipBuild, sk
WithoutBuilder: true,
BuilderStarters: nil,
Rail: nil, // today's linear `oap init` draws no wizard rail
Overlays: overlays,
}); err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oap/internal/installcmd/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ func TestRunInit_DispatchesToWizard(t *testing.T) {
false /*skipBuild*/, false /*skipInstall*/, 2*time.Minute, manifests.Tags{},
"" /*pinningMode*/, cloud.KeyLocal, true /*allowNonLocal*/, WebdRoutingOpts{},
wizard, false /*defaults*/, "" /*imagePullSecret*/, false /*mirrorDeps*/, false /*createRegistry*/, false /*noIdp*/, false, /*noMonitoring*/
settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept)
settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept, nil /*overlays*/)
}

// --wizard dispatches to the orchestrator exactly once; the linear phases run not at all.
Expand Down Expand Up @@ -164,7 +164,7 @@ func TestRunInit_WizardPathSkipsPreflightRoutingRefusal(t *testing.T) {
false /*skipBuild*/, false /*skipInstall*/, 2*time.Minute, tags,
"" /*pinningMode*/, cloud.KeyGKE, false /*allowNonLocal*/, WebdRoutingOpts{},
wizard, false /*defaults*/, "" /*imagePullSecret*/, false /*mirrorDeps*/, false /*createRegistry*/, false /*noIdp*/, false, /*noMonitoring*/
settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept)
settingscmd.DefaultModelOpts{}, "" /*artifactStoreURL*/, "" /*extEndpoint*/, "" /*extToken*/, false /*extInsecure*/, accept, nil /*overlays*/)
}

// --accept-existing on gke with no routing flags reaches the wizard dispatch
Expand Down
30 changes: 27 additions & 3 deletions cmd/oap/internal/installcmd/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import (
"github.com/authzed/openagentprimitives/pkg/platform/bootstrap"
"github.com/authzed/openagentprimitives/pkg/platform/cloud"
"github.com/authzed/openagentprimitives/pkg/platform/manifests"
"github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay"
apnats "github.com/authzed/openagentprimitives/pkg/platform/nats"
)

Expand Down Expand Up @@ -114,6 +115,7 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command {
timeout := installDefaultTimeout
externalToolTimeout := 30 * time.Minute
var (
overlayDirs []string
workspaceClass string
noWorkspaceRWX bool
workspaceRecheck bool
Expand Down Expand Up @@ -150,6 +152,12 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command {
if err := apcmd.ValidateDryRunMode(dryRun); err != nil {
return err
}
// Same reasoning: a broken or conflicting overlay is refused before
// the cluster is touched.
overlays, err := loadOverlays(cmd.OutOrStdout(), overlay.Base, overlayDirs)
if err != nil {
return err
}
// sigCtx is canceled only by SIGINT; it is NOT subject to the install
// --timeout. This is the recheckCtx passed into RunInstall so that an
// exhausted overall timeout does not make the keep-waiting "Y" a no-op.
Expand Down Expand Up @@ -256,6 +264,7 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command {
WithoutBuilder: withoutBuilder,
BuilderStarters: builderStarters,
Rail: nil, // no wizard rail on a standalone `oap install`
Overlays: overlays,
})
// Part 2: On SIGINT, decide what to do about what was applied.
// sigCtx.Err() != nil means SIGINT fired; the timeout only cancels ctx,
Expand Down Expand Up @@ -295,6 +304,7 @@ func NewInstallCmd(g *apcmd.Globals) *cobra.Command {
cmd.Flags().StringVar(&tags.Operator, "operator-image", "", "Override operator image (default: spicebox-operator:dev)")
cmd.Flags().StringVar(&tags.Runner, "runner-image", "", "Override runner image (default: agentprimitives-runner:dev)")
cmd.Flags().StringVar(&tags.Registry, "image-registry", "", "Rewrite first-party images to <registry>/<name>:<tag> (remote clusters; build+push via `oap init`/`oap build --image-registry`)")
cmd.Flags().StringArrayVar(&overlayDirs, "overlay", nil, overlayFlagUsage)
apcmd.DryRunModeFlag(cmd, &dryRun)
cmd.Flags().DurationVar(&timeout, "timeout", timeout, "Timeout for K8s-side install work (does not bound external CLI calls; see --external-tool-timeout)")
cmd.Flags().DurationVar(&externalToolTimeout, "external-tool-timeout", externalToolTimeout, "Timeout for a single external CLI call (gcloud); generous because cloud LROs like GKE node-pool recreation run for many minutes")
Expand Down Expand Up @@ -725,6 +735,10 @@ type InstallConfig struct {
// Only the unified init wizard passes a non-nil rail, to draw its step
// gutter beside the install checklist.
Rail progress.RailProvider
// Overlays are the validated --overlay Components, laid over every
// first-party document before install's own per-document rewrites. Nil
// (the zero value, and every caller without --overlay) applies nothing.
Overlays *overlay.Set
}

func RunInstall(ctx, recheckCtx context.Context, cfg InstallConfig) error {
Expand Down Expand Up @@ -880,6 +894,11 @@ func RunInstall(ctx, recheckCtx context.Context, cfg InstallConfig) error {
if err != nil {
return err
}
for _, d := range docs {
if err := cfg.Overlays.Apply(d); err != nil {
return err
}
}

if dryRun == apcmd.DryRunClient {
for _, d := range docs {
Expand Down Expand Up @@ -1417,7 +1436,7 @@ func RunInstall(ctx, recheckCtx context.Context, cfg InstallConfig) error {
Dynamic: bundle.Dynamic,
Typed: bundle.Typed,
Apply: func(ctx context.Context, groups [][]byte) error {
return applyManifestGroups(ctx, bundle.Dynamic, groups, imagePullSecret, firstPartyRefs, mirrorMap, func(id string) {
return applyManifestGroups(ctx, bundle.Dynamic, groups, cfg.Overlays, imagePullSecret, firstPartyRefs, mirrorMap, func(id string) {
rep.Info("applied %s", id)
})
},
Expand Down Expand Up @@ -2669,18 +2688,23 @@ func spicedbClusterServing(typed kubernetes.Interface, ns, deployName string) pr
// applyManifestGroups splits each manifest in groups into its documents, mirrors
// dependency image refs (air-gapped installs), injects an imagePullSecret into
// first-party Deployment docs when secret is set, and server-side-applies every
// document via kube.Apply under the "ap-install" field manager. It is the shared
// document via kube.Apply under the "ap-install" field manager. overlays are laid
// over each document first, so install's own rewrites still win on the fields
// they set. It is the shared
// apply body used by the install pipeline's Apply closure; emit, when non-nil, is
// called with a human-readable id per applied doc. injectImagePullSecret is a
// no-op for non-first-party (dependency) images, so passing firstPartyRefs for
// every component is safe — only channelsd's first-party image is affected.
func applyManifestGroups(ctx context.Context, dyn dynamic.Interface, groups [][]byte, secret string, firstPartyRefs map[string]bool, mirrorMap map[string]string, emit func(id string)) error {
func applyManifestGroups(ctx context.Context, dyn dynamic.Interface, groups [][]byte, overlays *overlay.Set, secret string, firstPartyRefs map[string]bool, mirrorMap map[string]string, emit func(id string)) error {
for _, m := range groups {
docs, err := manifests.Split(m)
if err != nil {
return fmt.Errorf("split manifest: %w", err)
}
for _, d := range docs {
if err := overlays.Apply(d); err != nil {
return err
}
if err := mirrorImageRefs(d, mirrorMap); err != nil {
return fmt.Errorf("mirror image refs: %w", err)
}
Expand Down
39 changes: 39 additions & 0 deletions cmd/oap/internal/installcmd/overlays.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package installcmd

import (
"io"

"github.com/authzed/openagentprimitives/cmd/oap/internal/cliout"
"github.com/authzed/openagentprimitives/pkg/platform/manifests/overlay"
)

const overlayFlagUsage = "Layer a local kustomize Component directory over the installed manifests (repeatable; applied in order). " +
"Use it for changes that must survive re-installs and upgrades, such as a raised memory limit. " +
"Overlays that set the same field to different values are refused before anything is installed"

// loadOverlays validates the --overlay directories against base and says which
// resources they change. It runs before any cluster call, so a broken or
// conflicting overlay costs nothing. No dirs returns a nil Set, which applies
// nothing.
func loadOverlays(out io.Writer, base func() ([][]byte, error), dirs []string) (*overlay.Set, error) {
if len(dirs) == 0 {
return nil, nil
}
b, err := base()
if err != nil {
return nil, err
}
s, err := overlay.Load(b, dirs)
if err != nil {
return nil, err
}
patched := s.Patched()
if len(patched) == 0 {
cliout.Warn(out, "overlays %v change nothing oap installs", dirs)
return s, nil
}
for _, id := range patched {
cliout.Info(out, "overlay patches %s", id)
}
return s, nil
}
Loading
Loading