From 1350684a5cbcb8dedc8253f6ed875159ea573b42 Mon Sep 17 00:00:00 2001 From: Mikhail Savin Date: Sat, 18 Jul 2026 20:48:08 +0300 Subject: [PATCH] fix: warn when config pre-loading times out and make the timeout configurable Config pre-loading (used to register custom commands and dynamic pipeline flags) was cancelled after a hardcoded 10s timeout. When variable resolution took longer than that, the error was swallowed and commands later failed with unrelated errors like 'unknown flag: --apps'. - print a clear warning when pre-loading exceeds the timeout, explaining that custom commands, pipeline flags and variables from the config might be unavailable for this run - defer pre-load log messages until the log level flags are parsed so --silent suppresses them, shell completions stay silent, and on fatal errors such as 'unknown flag' the warning is flushed right before the error - keep the underlying load error (Unwrap) and log it at debug level so --debug reveals the real cause, e.g. which variable timed out - allow overriding the timeout via the DEVSPACE_PRELOAD_TIMEOUT environment variable (a duration such as 30s or plain seconds; 0 disables the timeout, and the plain-seconds path guards against int64 overflow) - skip the variable resolver in RawConfig.GetEnv when its context is already cancelled to avoid doomed command executions after a timeout - document the pre-loading timeout in the variables docs Fixes #3213 Signed-off-by: Mikhail Savin --- cmd/root.go | 122 +++++++++++++++++++++++-- docs/pages/configuration/variables.mdx | 4 + 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 62307446ee..cdb7eecc3b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,7 +5,9 @@ import ( "flag" "fmt" "io" + "math" "os" + "strconv" "strings" "sync" "time" @@ -67,6 +69,12 @@ func NewRootCmd(f factory.Factory) *cobra.Command { log.SetLevel(logrus.DebugLevel) } + // print log messages deferred during config pre-loading, now that the + // log level from the flags is known; skip them for shell completions + if cobraCmd.Name() != cobra.ShellCompRequestCmd && cobraCmd.Name() != cobra.ShellCompNoDescRequestCmd && cobraCmd.Name() != "completion" { + flushPreloadLogs(log) + } + ansi.DisableColors(globalFlags.NoColors) if globalFlags.KubeConfig != "" { @@ -159,6 +167,10 @@ func Execute() { os.Exit(retCode.ExitCode) } + // print any log messages deferred during config pre-loading, so e.g. an + // "unknown flag" error caused by a pre-load timeout is explained + flushPreloadLogs(f.GetLog()) + // error hooks pluginErr := hook.ExecuteHooks(nil, map[string]interface{}{"error": err}, "root.errorExecution", "command:error") if pluginErr != nil { @@ -200,8 +212,16 @@ func BuildRoot(f factory.Factory, excludePlugins bool) *cobra.Command { if os.Getenv(expression.DevSpaceSkipPreloadEnv) == "" { rawConfig, err = parseConfig(f) if err != nil { - f.GetLog().Debugf("error parsing raw config: %v", err) - } else { + // defer the messages until the log level from the flags is known + var timeoutErr *preloadTimeoutError + if errors.As(err, &timeoutErr) { + deferPreloadLog(logrus.WarnLevel, timeoutErr.Error()) + deferPreloadLog(logrus.DebugLevel, fmt.Sprintf("error pre-loading config: %v", timeoutErr.Unwrap())) + } else { + deferPreloadLog(logrus.DebugLevel, fmt.Sprintf("error parsing raw config: %v", err)) + } + } + if rawConfig != nil { env.GlobalGetEnv = rawConfig.GetEnv } } @@ -318,6 +338,84 @@ func disableKlog() { klogv2.SetOutput(io.Discard) } +// DevSpacePreloadTimeoutEnv can be used to change the maximum time DevSpace waits +// for pre-loading the config (e.g. resolving variables from commands) before +// executing a command. Accepts a duration such as 30s or 1m as well as plain +// seconds; 0 disables the timeout. Must be set in the process environment, as it +// is read before .env files and config variables are loaded. +const DevSpacePreloadTimeoutEnv = "DEVSPACE_PRELOAD_TIMEOUT" + +// defaultPreloadTimeout is the default timeout for pre-loading the config +const defaultPreloadTimeout = time.Second * 10 + +// preloadLogs collects log messages produced while pre-loading the config in +// BuildRoot, which runs before cobra has parsed the log-level flags; they are +// flushed once the final log level is known (or before a fatal error) +type preloadLog struct { + level logrus.Level + message string +} + +var preloadLogs []preloadLog + +func deferPreloadLog(level logrus.Level, message string) { + preloadLogs = append(preloadLogs, preloadLog{level: level, message: message}) +} + +func flushPreloadLogs(logger log.Logger) { + for _, l := range preloadLogs { + if l.level == logrus.WarnLevel { + logger.Warnf("%s", l.message) + } else { + logger.Debugf("%s", l.message) + } + } + preloadLogs = nil +} + +// preloadTimeoutError is returned by parseConfig if pre-loading the config was +// cancelled because it took longer than the configured timeout +type preloadTimeoutError struct { + timeout time.Duration + err error +} + +func (e *preloadTimeoutError) Error() string { + return fmt.Sprintf("pre-loading the config took longer than %s and was cancelled, which usually means resolving a variable or expression took too long. Custom commands, pipeline flags and variables defined in the config might be unavailable for this run. You can increase this timeout via the %s environment variable (0 disables it), e.g. %s=30s", e.timeout, DevSpacePreloadTimeoutEnv, DevSpacePreloadTimeoutEnv) +} + +func (e *preloadTimeoutError) Unwrap() error { + return e.err +} + +// preloadTimeout returns the timeout for pre-loading the config, 0 meaning no timeout +func preloadTimeout() time.Duration { + value := os.Getenv(DevSpacePreloadTimeoutEnv) + if value == "" { + return defaultPreloadTimeout + } + + timeout, err := time.ParseDuration(value) + if err != nil { + // also allow specifying plain seconds, e.g. DEVSPACE_PRELOAD_TIMEOUT=30 + seconds, convErr := strconv.Atoi(value) + if convErr != nil { + deferPreloadLog(logrus.WarnLevel, fmt.Sprintf("Invalid value %q for %s, falling back to %s: %v", value, DevSpacePreloadTimeoutEnv, defaultPreloadTimeout, err)) + return defaultPreloadTimeout + } + if int64(seconds) > math.MaxInt64/int64(time.Second) { + // converting to a duration would overflow; treat as no timeout + return 0 + } + timeout = time.Duration(seconds) * time.Second + } + if timeout < 0 { + deferPreloadLog(logrus.WarnLevel, fmt.Sprintf("Invalid value %q for %s, falling back to %s: timeout must not be negative", value, DevSpacePreloadTimeoutEnv, defaultPreloadTimeout)) + return defaultPreloadTimeout + } + return timeout +} + func parseConfig(f factory.Factory) (*RawConfig, error) { // get current working dir cwd, err := os.Getwd() @@ -341,8 +439,13 @@ func parseConfig(f factory.Factory) (*RawConfig, error) { } // Parse commands - timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() + timeoutCtx := context.Background() + timeout := preloadTimeout() + if timeout > 0 { + var cancel context.CancelFunc + timeoutCtx, cancel = context.WithTimeout(timeoutCtx, timeout) + defer cancel() + } r := &RawConfig{ resolved: map[string]string{}, @@ -350,6 +453,12 @@ func parseConfig(f factory.Factory) (*RawConfig, error) { _, err = configLoader.LoadWithParser(timeoutCtx, nil, nil, r, &loader.ConfigOptions{ Dry: true, }, log.Discard) + if err != nil && timeoutCtx.Err() != nil { + // pre-loading was cancelled by the timeout, so custom commands and pipeline + // flags from the config might be missing; return a dedicated error to avoid + // confusing follow-up errors such as "unknown flag" + return r, &preloadTimeoutError{timeout: timeout, err: err} + } if r.Resolver != nil { return r, nil } @@ -394,8 +503,9 @@ func (r *RawConfig) GetEnv(name string) string { return value } - // try to find devspace variable - if r.Resolver != nil { + // try to find devspace variable; skip the resolver if its context is already + // cancelled (e.g. after a pre-load timeout), as resolving could never succeed + if r.Resolver != nil && r.Ctx != nil && r.Ctx.Err() == nil { r.resolvedMutex.Lock() defer r.resolvedMutex.Unlock() diff --git a/docs/pages/configuration/variables.mdx b/docs/pages/configuration/variables.mdx index 12b6e887cd..6146ae5a44 100644 --- a/docs/pages/configuration/variables.mdx +++ b/docs/pages/configuration/variables.mdx @@ -73,6 +73,10 @@ vars: args: ["rev-parse", "HEAD"] ``` +:::warning Pre-Loading Timeout +Before executing a command, DevSpace pre-loads the config (including resolving variables from commands) to make custom commands and pipeline flags available. This pre-loading is limited to 10 seconds by default. If resolving all variables takes longer than that, DevSpace prints a warning and custom commands and pipeline flags defined in the config will be unavailable for this run. You can change this timeout via the `DEVSPACE_PRELOAD_TIMEOUT` environment variable, e.g. `DEVSPACE_PRELOAD_TIMEOUT=30s` (plain seconds such as `30` also work, and `0` disables the timeout entirely). Note that `DEVSPACE_PRELOAD_TIMEOUT` must be set in the process environment: it is read before `.env` files (loaded via `DEVSPACE_ENV_FILE`) and config variables are processed, so setting it there has no effect. +::: + ### From User Input (Question) DevSpace can also ask the user to provide a value for a variable and you can provide a custom question and configure other input attributes for the question: