diff --git a/config/crds/agentprimitives.authzed.com_agentclasses.yaml b/config/crds/agentprimitives.authzed.com_agentclasses.yaml index d5f1ab3b..2c2f4b46 100644 --- a/config/crds/agentprimitives.authzed.com_agentclasses.yaml +++ b/config/crds/agentprimitives.authzed.com_agentclasses.yaml @@ -1542,6 +1542,14 @@ spec: required: - prompt type: object + maxConcurrentSessions: + description: |- + MaxConcurrentSessions: at most this many of this agent's sessions hold + capacity at once. Delegated child sessions are not counted and are + never blocked by it. Unset means no limit. + format: int32 + minimum: 1 + type: integer mcpServers: description: |- MCPServers references MCPServer CRs in the same namespace whose diff --git a/config/crds/agentprimitives.authzed.com_agentsessions.yaml b/config/crds/agentprimitives.authzed.com_agentsessions.yaml index 4348063b..45b16e32 100644 --- a/config/crds/agentprimitives.authzed.com_agentsessions.yaml +++ b/config/crds/agentprimitives.authzed.com_agentsessions.yaml @@ -432,6 +432,33 @@ spec: - artifactID type: object type: array + admission: + description: |- + Admission is where this session stands in its AgentClass's admission + line while it waits for its pod set to be admitted. Written by the + operator's admitter only when the position changes, and cleared when the + session is admitted. Nil when the session is not waiting. + properties: + joinedAt: + description: |- + JoinedAt is when the session took its place in line: its creation, or + the wake that re-queued it. + format: date-time + type: string + lineLength: + description: LineLength is how many sessions are waiting in the + line. + format: int32 + type: integer + position: + description: Position is the session's 1-based place in its line. + format: int32 + type: integer + required: + - joinedAt + - lineLength + - position + type: object agentWakeCredit: description: |- AgentWakeCredit is how many further AGENT-DRIVEN wakes this session will diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index f34e9444..0f33313e 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -11,10 +11,12 @@ import ( "github.com/google/uuid" "io" "log/slog" + "maps" "net" "net/http" "os" runtimedebug "runtime/debug" + "slices" "strings" "sync/atomic" "time" @@ -102,6 +104,8 @@ import ( _ "github.com/authzed/openagentprimitives/pkg/channels/channelkinds/onepassword" // register onepassword relsync.Kind for the relationshipsource controller (relsync.Get); also reachable transitively via the relsource/imports blank import above, but this is the explicit wiring site for THIS aspect, not an accident of that one _ "github.com/authzed/openagentprimitives/pkg/channels/channelkinds/slack" // register slack kind for channel-controller validation "github.com/authzed/openagentprimitives/pkg/cli/clikit" + sessionadmission "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/admission/admitter" agentclassctrl "github.com/authzed/openagentprimitives/pkg/controllers/agentclass" "github.com/authzed/openagentprimitives/pkg/controllers/agentidentity" agentsessionctrl "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" @@ -757,6 +761,13 @@ type config struct { idleStorageReclaimAfter time.Duration failedSandboxReapGrace time.Duration capacityHoldTimeout time.Duration + // sessionAdmission gates every booting AgentSession's pods on the + // admitter's whole-set, fair-share decision; false admits at once. + sessionAdmission bool + // admissionMinWarm is how long an idle session keeps its admission (and + // warm pods) before the admitter may reclaim it early for a session of its + // AgentClass waiting in line. + admissionMinWarm time.Duration oauthRefreshThreshold time.Duration workspaceStorageClass string workspaceSize string @@ -882,6 +893,10 @@ func newCommand() *cobra.Command { "How long a Failed session's sandbox pods (bundle SpiceboxSessions + runner pod) are kept for debugging before teardown. The AgentSession itself is kept either way. Set to 0 (or negative) to disable reaping entirely (keep everything, for debugging).") fs.DurationVar(&cfg.capacityHoldTimeout, "capacity-hold-timeout", time.Hour, "How long a booting AgentSession whose sandbox/detector pods are Pending on a cluster capacity shortfall (Insufficient cpu/memory, too many pods) is HELD — queued with a notice in its channel, resuming by itself once the scheduler places the pod — instead of failing at the ordinary readiness deadline. Measured from provisioning start. Pods that can never fit, or are blocked by a taint or volume affinity, still fail at the ordinary deadline. Set to a negative value to disable holding.") + fs.BoolVar(&cfg.sessionAdmission, "session-admission", true, + "Admit each booting AgentSession's whole pod set against node capacity before any of its pods is created, in join order with fair share between AgentClasses; sessions that do not fit wait in line. Set to false to admit every session at once (pods then queue in the scheduler as before).") + fs.DurationVar(&cfg.admissionMinWarm, "admission-min-warm", 5*time.Minute, + "Minimum time an idle AgentSession keeps its admission (and warm pods) before the admitter may put it to sleep early, ahead of its sleepAfter grace, to make room for a session of the same AgentClass waiting in line. A held (SessionHold) session is never reclaimed. Only applies with --session-admission.") fs.DurationVar(&cfg.oauthRefreshThreshold, "oauth-refresh-threshold", 5*time.Minute, "Default refresh threshold for type=oauth AgentIdentity credentials. AgentIdentities with spec.refreshThreshold override this per-identity.") fs.StringVar(&cfg.workspaceStorageClass, "workspace-storage-class", "", @@ -2668,6 +2683,7 @@ func run(cfg *config) { DefaultSessionSleepAfter: cfg.defaultSessionSleepAfter, FailedSandboxReapGrace: cfg.failedSandboxReapGrace, CapacityHoldTimeout: cfg.capacityHoldTimeout, + AdmissionEnabled: cfg.sessionAdmission, DefaultSessionStorageRetention: cfg.sessionStorageRetention, SessionGCAfter: cfg.sessionGCAfter, NodePinnedStorageReclaimGrace: cfg.nodePinnedStorageReclaimGrace, @@ -2748,6 +2764,35 @@ func run(cfg *config) { } log.Info("registered controller", "name", "AgentSession", "operatorURL", operatorURL) + // The admitter decides which booting sessions the reconciler above may + // create pods for. Leader-only (the controller-runtime default), so one + // operator admits against the free space at a time. Capacity is declared + // as the interface and assigned a concrete value, never a typed-nil + // pointer (see the repo conventions on nil interfaces). + // + // The capacity snapshot reads nodes and pods from the cached client while + // sessions are listed through the APIReader. Only the session list must + // see the admitter's own last writes (a lagging cache would show an + // admitted session still waiting and admit a second into its space); + // node and pod state the admitter never writes, so a cache a moment + // behind costs at most one pass's accuracy, and an uncached list of every + // pod on each pass would be a full apiserver read per wake. + var admissionCapacity sessionadmission.CapacitySource = sessionadmission.KubeCapacitySource{Reader: mgr.GetClient()} + if err := (&admitter.Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + // The same namespace set the manager cache is scoped to, so the + // uncached session list stays inside the RBAC a watch set grants. + Namespaces: slices.Sorted(maps.Keys(cacheNamespaces(cfg.watchNamespaces, operatorNamespace(cfg.podNamespace)))), + Capacity: admissionCapacity, + Enabled: cfg.sessionAdmission, + MinWarm: cfg.admissionMinWarm, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to register admission controller") + os.Exit(1) + } + log.Info("registered controller", "name", "Admission", "enabled", cfg.sessionAdmission) + // Workshop provisions/tears down a builder session's isolated namespace, // exact RBAC, SpiceDB tuple and bearer token (spec §1.2/§1.3). spiceDBClient // is a real, non-nil *spicedb.Client here (construction failure exits diff --git a/internal/cmd/operator/main_controllers_test.go b/internal/cmd/operator/main_controllers_test.go index 88fb1b31..dfbf84c5 100644 --- a/internal/cmd/operator/main_controllers_test.go +++ b/internal/cmd/operator/main_controllers_test.go @@ -61,6 +61,7 @@ func TestOperatorRegistersExpectedControllers(t *testing.T) { "AgentClass": true, "Channel": true, "AgentSession": true, + "Admission": true, "GuardianAgentSessionGrants": true, "MonitoringWatchers": true, "UserIdentity": true, diff --git a/pkg/apis/v1alpha1/agentclass_types.go b/pkg/apis/v1alpha1/agentclass_types.go index 61cf1a0d..e2b424ce 100644 --- a/pkg/apis/v1alpha1/agentclass_types.go +++ b/pkg/apis/v1alpha1/agentclass_types.go @@ -175,6 +175,13 @@ type AgentClassSpec struct { // +optional Budget *BudgetConfig `json:"budget,omitempty"` + // MaxConcurrentSessions: at most this many of this agent's sessions hold + // capacity at once. Delegated child sessions are not counted and are + // never blocked by it. Unset means no limit. + // +optional + // +kubebuilder:validation:Minimum=1 + MaxConcurrentSessions *int32 `json:"maxConcurrentSessions,omitempty"` + // BoundEntities is the PRE-RENAME spelling of spec.authz.slots. // // Still deserialized on purpose. This is an AUTHZ field: dropping it from diff --git a/pkg/apis/v1alpha1/agentsession_ownership.go b/pkg/apis/v1alpha1/agentsession_ownership.go index 41e391d8..8cd9f3fd 100644 --- a/pkg/apis/v1alpha1/agentsession_ownership.go +++ b/pkg/apis/v1alpha1/agentsession_ownership.go @@ -48,9 +48,11 @@ var operatorOwnedStatusFields = map[string]bool{ // Stamped by the operator when it acts on a UI-serve request, exactly as // lastWakeAt is for a conversation wake. Separate fields because the two // requests must not consume each other — see AnnotationUIServeRequestedAt. - "lastUIServeAt": true, - "lastIdleAt": true, - "sleptAt": true, + "lastUIServeAt": true, + "lastIdleAt": true, + "sleptAt": true, + // admission is the admitter's view of a waiting session's place in line. + "admission": true, "awaitingUserInputSince": true, // parentExchange is written by the RUNNER of a delegated child (the // ask_parent meta tool, and the loop's two answer-arrived clears), like diff --git a/pkg/apis/v1alpha1/agentsession_types.go b/pkg/apis/v1alpha1/agentsession_types.go index bd5b12cc..a739112b 100644 --- a/pkg/apis/v1alpha1/agentsession_types.go +++ b/pkg/apis/v1alpha1/agentsession_types.go @@ -399,6 +399,13 @@ type AgentSessionStatus struct { // +optional EstimatedCost *EstimatedSessionCost `json:"estimatedCost,omitempty"` + // Admission is where this session stands in its AgentClass's admission + // line while it waits for its pod set to be admitted. Written by the + // operator's admitter only when the position changes, and cleared when the + // session is admitted. Nil when the session is not waiting. + // +optional + Admission *AdmissionStatus `json:"admission,omitempty"` + // AwaitingUserInputSince is set when the agent yields via // await_user_message and cleared on resume or terminal. It is the durable // floor under the at-most-once turn_activity yield event: a non-nil value @@ -1598,3 +1605,14 @@ type GoalExecutionReference struct { GoalRevision int64 `json:"goalRevision"` ConsentDigest string `json:"consentDigest"` } + +// AdmissionStatus is a waiting session's place in its admission line. +type AdmissionStatus struct { + // Position is the session's 1-based place in its line. + Position int32 `json:"position"` + // LineLength is how many sessions are waiting in the line. + LineLength int32 `json:"lineLength"` + // JoinedAt is when the session took its place in line: its creation, or + // the wake that re-queued it. + JoinedAt metav1.Time `json:"joinedAt"` +} diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index 39ac3b1e..71651939 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -511,6 +511,45 @@ const ( ReasonCapacityHoldExpired = "HoldExpired" ) +// AgentSession admission. +const ( + // AgentSessionConditionAdmitted is True while the operator has admitted the + // session's whole pod set against its AgentClass's line, and False once the + // session has been released back to the line (reclaimed, or a placement + // misfit). Absent until the session first joins a line. + AgentSessionConditionAdmitted = "Admitted" +) + +// AgentSessionConditionAdmitted reasons. +const ( + ReasonAdmitted = "Admitted" + // ReasonAdmissionQueued marks a session that has reached the admission + // gate and is waiting in line. Its transition time is the moment it + // arrived, which the waiting timeout is measured from; the admitter + // considers only sessions carrying it, so a session parked at an earlier + // gate never reserves anything. + ReasonAdmissionQueued = "Queued" + // ReasonAdmissionAdopted marks a session whose pods were created before + // admission was switched on: the admitter adopts it as admitted, so it is + // counted as a holder rather than gated with its pods already up. + ReasonAdmissionAdopted = "Adopted" + // ReasonAdmittedUnknownCapacity marks a session admitted while the + // admitter had no reliable view of cluster capacity (no node data, a + // failed snapshot, or a namespace-scoped operator that cannot see other + // namespaces' pods). Nothing proved its set fits, so a stall is not + // treated as a misfit: its pods keep the ordinary pod-level hold. + ReasonAdmittedUnknownCapacity = "AdmittedUnknownCapacity" + // ReasonAdmissionReclaimed marks a session released so a session ahead of + // it in line could be admitted. + ReasonAdmissionReclaimed = "Reclaimed" + // ReasonAdmissionMisfit marks a session released because its pods did not + // fit where they were placed. + ReasonAdmissionMisfit = "PlacementMismatch" + // ReasonCapacityHoldWaitingAdmission is the CapacityHold reason while a + // session waits in line to be admitted. + ReasonCapacityHoldWaitingAdmission = "WaitingForAdmission" +) + // Settings reasons (shared by AgentClass + AgentSession SettingsAccepted and the // Settings CR SelfConsistent conditions). const ( @@ -1028,6 +1067,15 @@ const ( // coalesces rather than duplicating. AnnotationUIServeRequestedAt = "agentprimitives.authzed.com/ui-serve-requested-at" + // AnnotationReclaimRequested asks the operator to put a warm Idle session + // to sleep now, ahead of its sleepAfter grace, because a session of the + // same AgentClass is waiting in the admission line and freeing this one's + // pods lets it in. Written by the admitter (RFC3339 of the pass that asked; + // left alone if already present); the AgentSession reconciler's sleep path + // honors it — still behind the SessionHold guard — and removes it once the + // session is slept, or as stale once the session has left Idle. + AnnotationReclaimRequested = "agentprimitives.authzed.com/reclaim-requested" + // AnnotationSidecarFailureReported carries, as a JSON object, the // sidecar-name → failure-fingerprint map the runner has already reported to // the user — the durable half of its once-per-failure contract, since a diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 6d61f6b1..bd1aa892 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdmissionStatus) DeepCopyInto(out *AdmissionStatus) { + *out = *in + in.JoinedAt.DeepCopyInto(&out.JoinedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdmissionStatus. +func (in *AdmissionStatus) DeepCopy() *AdmissionStatus { + if in == nil { + return nil + } + out := new(AdmissionStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentClass) DeepCopyInto(out *AgentClass) { *out = *in @@ -167,6 +183,11 @@ func (in *AgentClassSpec) DeepCopyInto(out *AgentClassSpec) { *out = new(BudgetConfig) **out = **in } + if in.MaxConcurrentSessions != nil { + in, out := &in.MaxConcurrentSessions, &out.MaxConcurrentSessions + *out = new(int32) + **out = **in + } if in.BoundEntities != nil { in, out := &in.BoundEntities, &out.BoundEntities *out = make([]AuthzSlot, len(*in)) @@ -995,6 +1016,11 @@ func (in *AgentSessionStatus) DeepCopyInto(out *AgentSessionStatus) { *out = new(EstimatedSessionCost) (*in).DeepCopyInto(*out) } + if in.Admission != nil { + in, out := &in.Admission, &out.Admission + *out = new(AdmissionStatus) + (*in).DeepCopyInto(*out) + } if in.AwaitingUserInputSince != nil { in, out := &in.AwaitingUserInputSince, &out.AwaitingUserInputSince *out = (*in).DeepCopy() diff --git a/pkg/bronzethread/bundle.go b/pkg/bronzethread/bundle.go index 7771004a..a323d583 100644 --- a/pkg/bronzethread/bundle.go +++ b/pkg/bronzethread/bundle.go @@ -275,6 +275,11 @@ type Bundle struct { // raises a warning naming what stopped running. MetaToolReplies map[string]MetaToolReply `json:"metaToolReplies,omitempty"` + // Admission runs the bundle with whole-set session admission on, against + // a fixed capacity. Absent — every bundle but the admission scenarios — + // sessions are ungated, as before. See sessions.go. + Admission *Admission `json:"admission,omitempty"` + // LLM is the model's side, POSITIONAL: the nth Send gets the nth entry. LLM []LLMStep `json:"llm"` @@ -361,6 +366,10 @@ func (b Bundle) Validate() error { return err } + if err := b.validateSessions(); err != nil { + return err + } + return b.ValidateToolOutputs() } @@ -632,6 +641,12 @@ type LLMStep struct { // position. Steps for one class stay positional among themselves. AgentClass string `json:"agentClass,omitempty"` + // Thread scopes the step to the session a named userTurns thread opened, + // for the case AgentClass cannot tell apart: several sessions of ONE class + // at once. A request belongs to the thread whose opening text is its first + // user message. Steps for one thread stay positional among themselves. + Thread string `json:"thread,omitempty"` + // Expect is the divergence check. It is verified against the actual request // BEFORE the reply is returned, so a system that took a different path // fails loudly instead of being handed an answer to a question it did not @@ -883,6 +898,10 @@ type Assertions struct { // Channel asserts on the posts the bound output channel received. Channel *ChannelAssertions `json:"channel,omitempty"` + // Admission claims the order the line admitted and finished the run's + // sessions in. Requires Bundle.Admission. See sessions.go. + Admission *AdmissionAssertions `json:"admission,omitempty"` + // ToolsCalled names tools the run must have DISPATCHED, read back off the // replayed session's own transcript. // diff --git a/pkg/bronzethread/sessions.go b/pkg/bronzethread/sessions.go new file mode 100644 index 00000000..54bded5c --- /dev/null +++ b/pkg/bronzethread/sessions.go @@ -0,0 +1,164 @@ +package bronzethread + +import ( + "fmt" + "strings" +) + +// A bundle normally drives ONE session: every user turn lands on the same +// thread of the fixture's one conversational Channel, and the transcript is a +// single positional stream. A scenario about how several sessions share the +// cluster — the admission line — cannot be written that way: it needs several +// sessions, of more than one AgentClass, alive at once. +// +// Three optional fields open that up, and a bundle that sets none of them is +// exactly the single-session bundle it always was: +// +// - UserTurn.Thread (and Channel) start or continue a NAMED session. Each +// thread is its own channel key, so its own AgentSession. +// - LLMStep.Thread scopes a transcript step to one thread's session, the +// way AgentClass scopes it to a class. Several sessions of one class +// cannot be told apart by class, and an unscoped step is answered by +// whichever session asks next — which is a different transcript every +// run once sessions overlap. +// - UserTurn.WaitFor holds a turn until other sessions reach a state, so a +// scenario can say "while A is still working" without guessing at timing. + +// Barrier states a TurnBarrier can wait for. +const ( + // UntilAwaitingInput: the session is parked in await_user_message, still + // Running. The only state a follow-up turn can be delivered INTO rather + // than starting a fresh turn. + UntilAwaitingInput = "awaitingInput" + // UntilIdle: the session's turn ended (phase Idle). + UntilIdle = "idle" +) + +// TurnBarrier holds a user turn until the session on Thread reaches Until. +type TurnBarrier struct { + Thread string `json:"thread"` + Until string `json:"until"` +} + +// Admission turns on whole-set session admission for the run, against a +// fixed capacity, so a bundle can exercise the line itself. Absent, the run is +// ungated exactly as before — the harness then has no capacity to measure. +type Admission struct { + // CapacitySessions is how many sessions' whole pod sets the cluster can + // hold at once. The harness has no nodes, so this stands in for them. + CapacitySessions int `json:"capacitySessions"` +} + +// AdmissionAssertions are claims about the order the line admitted sessions +// in, read from what the run actually did. Threads are named as in userTurns. +// +// Ordering is by FIRST OBSERVATION: the driver watches every session for the +// whole run and records when each was first seen admitted and first seen +// finished. Two sessions admitted in one pass may be seen together, so only +// STRICT orderings are claimable — which is all a line promises. +type AdmissionAssertions struct { + // AdmittedBefore: each pair's First was admitted strictly before Then. + AdmittedBefore []ThreadOrder `json:"admittedBefore,omitempty"` + // CompletedBefore: each pair's First finished its work (reached Idle) + // strictly before Then did. + CompletedBefore []ThreadOrder `json:"completedBefore,omitempty"` + // PeakAdmitted is the EXACT largest number of sessions the run held + // admitted at once. Equal to the capacity proves the line actually + // queued; above it is the over-admission the line exists to prevent. + PeakAdmitted int `json:"peakAdmitted,omitempty"` +} + +// ThreadOrder is one strict ordering claim between two threads. +type ThreadOrder struct { + First string `json:"first"` + Then string `json:"then"` +} + +// Threads lists every named thread the bundle's user turns open, in the order +// they first appear. +func (b Bundle) Threads() []string { + var out []string + seen := map[string]bool{} + for _, ut := range b.UserTurns { + if ut.Thread != "" && !seen[ut.Thread] { + seen[ut.Thread] = true + out = append(out, ut.Thread) + } + } + return out +} + +// OpeningText is the text of the first user turn on thread: the message that +// opened that thread's session, and so the first user message of every model +// request the session makes. It is how a thread-scoped step recognizes its +// session's requests. The unnamed thread has none: an unscoped step must match +// every session's requests, including a delegated child's. +func (b Bundle) OpeningText(thread string) string { + if thread == "" { + return "" + } + for _, ut := range b.UserTurns { + if ut.Thread == thread { + return ut.Text + } + } + return "" +} + +// validateSessions refuses a multi-session bundle the driver could not route. +func (b Bundle) validateSessions() error { + threads := map[string]bool{} + for _, th := range b.Threads() { + threads[th] = true + } + if len(threads) > 0 && b.Trigger != nil { + return fmt.Errorf("named threads start sessions from userTurns; a triggered bundle has one session") + } + for i, ut := range b.UserTurns { + if len(threads) > 0 && ut.Thread == "" { + return fmt.Errorf("userTurns[%d] names no thread, but others do; once one turn names its session every turn must", i) + } + if ut.Channel != "" && ut.Thread == "" { + return fmt.Errorf("userTurns[%d] names a channel but no thread", i) + } + for j, w := range ut.WaitFor { + if !threads[w.Thread] { + return fmt.Errorf("userTurns[%d].waitFor[%d] names thread %q, which no user turn opens", i, j, w.Thread) + } + if w.Until != UntilAwaitingInput && w.Until != UntilIdle { + return fmt.Errorf("userTurns[%d].waitFor[%d].until is %q; want %q or %q", + i, j, w.Until, UntilAwaitingInput, UntilIdle) + } + } + } + // Each thread's opening text is how its steps find its requests, so no + // opening may contain another's: a request would then match both scopes. + for a := range threads { + for c := range threads { + if a != c && strings.Contains(b.OpeningText(a), b.OpeningText(c)) { + return fmt.Errorf("thread %q's opening text contains thread %q's, so their transcript steps cannot be told apart", a, c) + } + } + } + for i, step := range b.LLM { + if step.Thread != "" && !threads[step.Thread] { + return fmt.Errorf("llm[%d] is scoped to thread %q, which no user turn opens", i, step.Thread) + } + } + if b.Admission != nil && b.Admission.CapacitySessions < 1 { + return fmt.Errorf("admission.capacitySessions must be at least 1; a cluster with room for no session admits nothing") + } + if a := b.Assert.Admission; a != nil { + if b.Admission == nil { + return fmt.Errorf("assert.admission makes claims about a line, but the bundle turns admission on nowhere (admission)") + } + for _, list := range [][]ThreadOrder{a.AdmittedBefore, a.CompletedBefore} { + for _, o := range list { + if !threads[o.First] || !threads[o.Then] { + return fmt.Errorf("assert.admission orders %q before %q, but each must be a thread a user turn opens", o.First, o.Then) + } + } + } + } + return nil +} diff --git a/pkg/bronzethread/sessions_test.go b/pkg/bronzethread/sessions_test.go new file mode 100644 index 00000000..6b623675 --- /dev/null +++ b/pkg/bronzethread/sessions_test.go @@ -0,0 +1,129 @@ +package bronzethread_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + bt "github.com/authzed/openagentprimitives/pkg/bronzethread" +) + +// multiSessionBundle is the minimum runnable bundle with two named threads, a +// step scoped to one of them, admission on and one ordering claim. +func multiSessionBundle() bt.Bundle { + b := runnableBundle() + b.UserTurns = []bt.UserTurn{ + {Text: "review pull request 1", Thread: "a1"}, + {Text: "summarize the notes", Thread: "b", Channel: "notes-fake"}, + {Text: "wrap up", Thread: "a1", WaitFor: []bt.TurnBarrier{{Thread: "a1", Until: bt.UntilAwaitingInput}}}, + } + b.LLM = []bt.LLMStep{{Thread: "a1", Reply: []bt.ReplyPart{{Text: "none"}}}} + b.Admission = &bt.Admission{CapacitySessions: 1} + b.Assert.Admission = &bt.AdmissionAssertions{ + AdmittedBefore: []bt.ThreadOrder{{First: "a1", Then: "b"}}, + } + return b +} + +// TestBundle_ValidateSessions covers the preconditions a multi-session bundle +// adds: every turn routed, every scope and barrier naming a real thread, and +// admission claims only where admission is on. +func TestBundle_ValidateSessions(t *testing.T) { + cases := []struct { + name string + doctor func(b bt.Bundle) bt.Bundle + wantErr string // empty means the bundle must validate + }{ + { + name: "a multi-session admission bundle validates", + doctor: func(b bt.Bundle) bt.Bundle { return b }, + }, + { + name: "one turn names no thread while others do: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.UserTurns[1].Thread = ""; return b }, + wantErr: "every turn must", + }, + { + name: "a channel with no thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns = []bt.UserTurn{{Text: "hi", Channel: "notes-fake"}} + b.LLM[0].Thread = "" + b.Assert.Admission = nil + return b + }, + wantErr: "names a channel but no thread", + }, + { + name: "a barrier naming an unknown thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[2].WaitFor[0].Thread = "zz" + return b + }, + wantErr: "which no user turn opens", + }, + { + name: "a barrier with an unknown state: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[2].WaitFor[0].Until = "sleeping" + return b + }, + wantErr: "until is \"sleeping\"", + }, + { + name: "a step scoped to an unknown thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.LLM[0].Thread = "zz"; return b }, + wantErr: "llm[0] is scoped to thread \"zz\"", + }, + { + name: "one thread's opening text contains another's: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[1].Text = "please review pull request 1 too" + return b + }, + wantErr: "cannot be told apart", + }, + { + name: "admission with no capacity: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.Admission.CapacitySessions = 0; return b }, + wantErr: "capacitySessions must be at least 1", + }, + { + name: "admission claims with admission off: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.Admission = nil; return b }, + wantErr: "turns admission on nowhere", + }, + { + name: "an admission claim naming an unknown thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.Assert.Admission.CompletedBefore = []bt.ThreadOrder{{First: "a1", Then: "zz"}} + return b + }, + wantErr: "each must be a thread a user turn opens", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.doctor(multiSessionBundle()).Validate() + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err, "this bundle must not be runnable") + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// Threads lists named threads once each, in first-appearance order, and +// OpeningText is the thread's FIRST turn — the message every request of that +// session starts from. +func TestBundle_ThreadsAndOpeningText(t *testing.T) { + b := multiSessionBundle() + assert.Equal(t, []string{"a1", "b"}, b.Threads()) + assert.Equal(t, "review pull request 1", b.OpeningText("a1"), "the first turn, not the follow-up") + assert.Empty(t, b.OpeningText("zz")) + assert.Empty(t, runnableBundle().Threads(), "a single-session bundle names no threads") + assert.Empty(t, runnableBundle().OpeningText(""), + "the unnamed thread has no opening text, so an unscoped step stays unscoped") +} diff --git a/pkg/bronzethread/userturn.go b/pkg/bronzethread/userturn.go index 4f1b8c08..d28a6510 100644 --- a/pkg/bronzethread/userturn.go +++ b/pkg/bronzethread/userturn.go @@ -45,6 +45,15 @@ type BundleAttachment struct { type UserTurn struct { Text string `json:"text"` Attachments []BundleAttachment `json:"attachments,omitempty"` + + // Thread names the session this turn lands on; Channel, the fake Channel + // it is sent through (empty: the fixture's only one). Both empty is the + // single-session bundle. See sessions.go. + Thread string `json:"thread,omitempty"` + Channel string `json:"channel,omitempty"` + + // WaitFor holds this turn until each named session reaches its state. + WaitFor []TurnBarrier `json:"waitFor,omitempty"` } // UnmarshalJSON accepts the bare-string and object forms. diff --git a/pkg/channels/sessionnotice/sessionnotice.go b/pkg/channels/sessionnotice/sessionnotice.go index 09e27a53..6dc10fcf 100644 --- a/pkg/channels/sessionnotice/sessionnotice.go +++ b/pkg/channels/sessionnotice/sessionnotice.go @@ -15,6 +15,9 @@ package sessionnotice import ( + "fmt" + + "github.com/dustin/go-humanize" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -209,10 +212,18 @@ func CapacityHold(sess *spiceboxv1alpha1.AgentSession) (string, bool) { if c == nil || c.Status != metav1.ConditionTrue { return "", false } - if c.Message == "" { - return SchedulingLead, true + text := c.Message + if text == "" { + text = SchedulingLead + } + // The admitter records where the session stands in its class's line. The + // notice is posted once per hold episode, so this is the position at the + // time it posts. The class name stands in for a display name: the session + // status carries none, and this package stays a pure derivation (no lookup). + if a := sess.Status.Admission; a != nil && a.Position > 0 { + text += fmt.Sprintf(" You're %s in line for %s.", humanize.Ordinal(int(a.Position)), sess.Spec.Class) } - return c.Message, true + return text, true } // CapacityHoldExpired reports whether the session failed because its capacity diff --git a/pkg/channels/sessionnotice/sessionnotice_test.go b/pkg/channels/sessionnotice/sessionnotice_test.go index 3c98bb19..754d3139 100644 --- a/pkg/channels/sessionnotice/sessionnotice_test.go +++ b/pkg/channels/sessionnotice/sessionnotice_test.go @@ -175,3 +175,34 @@ func TestDerive_HealthySessionIsSilent(t *testing.T) { assert.Empty(t, sessionnotice.Derive(sess(spiceboxv1alpha1.AgentSessionPhaseRunning))) assert.Empty(t, sessionnotice.Derive(nil), "a nil session is not a panic") } + +func TestCapacityHoldPosition(t *testing.T) { + const holdMsg = "The cluster is out of room right now; this will start on its own once capacity frees up (gives up at 17:23 UTC)." + hold := cond(spiceboxv1alpha1.AgentSessionConditionCapacityHold, metav1.ConditionTrue, + spiceboxv1alpha1.ReasonCapacityHoldWaiting, holdMsg) + + cases := []struct { + name string + admission *spiceboxv1alpha1.AdmissionStatus + want string + }{ + {name: "position 3 of 5: hold text plus ordinal sentence naming the class", + admission: &spiceboxv1alpha1.AdmissionStatus{Position: 3, LineLength: 5}, + want: holdMsg + " You're 3rd in line for demo-agent."}, + {name: "no admission status: hold text unchanged", + want: holdMsg}, + {name: "position 0: hold text unchanged", + admission: &spiceboxv1alpha1.AdmissionStatus{Position: 0, LineLength: 5}, + want: holdMsg}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := sess(spiceboxv1alpha1.AgentSessionPhasePending, hold) + s.Spec.Class = "demo-agent" + s.Status.Admission = tc.admission + got, held := sessionnotice.CapacityHold(s) + require.True(t, held) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go new file mode 100644 index 00000000..a2614a63 --- /dev/null +++ b/pkg/controllers/admission/admitter/controller.go @@ -0,0 +1,884 @@ +// Package admitter is the one controller that decides which booting +// AgentSessions may create their pods. Each pass rebuilds every admission line +// from the AgentSession objects themselves, prices each session's whole pod +// set, and hands the result to admission.Decide; the AgentSession reconciler +// then creates pods only for a session it marked Admitted=True. +// +// The pass keeps no queue state in memory: the line is derived from +// AgentSession status on every pass, so an operator restart or leader change +// resumes exactly where the last leader stopped. It runs on the leader only +// (controller-runtime's default for a controller; nothing here overrides it), +// so two operators never admit against the same free space. +package admitter + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" + "github.com/authzed/openagentprimitives/pkg/controllers/agentstatus" +) + +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=subagentrequests,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch + +// Key is the single request every event maps to: a pass always decides every +// line at once, so there is nothing per-object to reconcile. +var Key = types.NamespacedName{Namespace: "admission", Name: "cluster"} + +// resyncInterval re-runs the pass with no event, so capacity freed by pods the +// admitter does not watch (other workloads finishing, nodes joining) is seen. +const resyncInterval = 15 * time.Second + +// Session pods carry one of two labels: the runner, detector and sidecar pods +// the AgentSession reconciler builds carry the session's name under +// sessionPodLabel; sandbox pods are built for a bundle SpiceboxSession +// (podspec.Build) and carry that SpiceboxSession's name under +// sandboxPodLabel. A pod event with either can change a holder's unplaced +// demand. +const ( + sessionPodLabel = "agentprimitives.authzed.com/agentsession" + sandboxPodLabel = "agentprimitives.authzed.com/session" +) + +// Reconciler runs the admission pass. +type Reconciler struct { + Client client.Client + // APIReader lists AgentSessions uncached. The pass must see its own last + // Admitted=True writes: a cache that lags them would show an admitted + // session as still waiting, free its capacity, and let the pass admit a + // second session into the same space. Nil falls back to Client. + APIReader client.Reader + // Namespaces is the operator's watch set (--watch-namespaces). When set, + // sessions are listed one namespace at a time: the APIReader list is + // uncached, so a cluster-wide one would 403 under the per-namespace RBAC a + // watch set exists for, and would reach sessions in namespaces this + // operator does not own. Empty means cluster-wide. + Namespaces []string + // Capacity snapshots what the cluster can still place. Nil, or a snapshot + // error, means unknown capacity: every waiting session fits. + Capacity admission.CapacitySource + // Enabled false admits every waiting session at once, with no capacity + // check (--session-admission=false). + Enabled bool + // MinWarm is how long an Idle session keeps its warm pods before a pass may + // ask for them back early for a session of its line that does not fit + // (--admission-min-warm). + MinWarm time.Duration + // capacityUnknown is whether the previous pass found capacity unknown, so + // the reason is logged once on a known→unknown transition rather than on + // every pass (the admitter is woken by every session and pod event). + capacityUnknown atomic.Bool + // Now is the clock; nil means time.Now. + Now func() time.Time + // Demand prices a session's whole pod set. Nil means the default: get the + // session's AgentClass and call agentsession.DemandFor. A test seam, for + // a harness where no AgentSession reconciler has resolved settings. + Demand func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) +} + +func (r *Reconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +func (r *Reconciler) sessionReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +// lineOf is the admission line a session draws capacity from: its AgentClass, +// which lives in the session's namespace. +func lineOf(sess *spiceboxv1alpha1.AgentSession) string { + return sess.Namespace + "/" + sess.Spec.Class +} + +func isTerminal(sess *spiceboxv1alpha1.AgentSession) bool { + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseSucceeded || + sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseFailed +} + +// isWaiting matches a session the AgentSession reconciler's gate has put in +// line: booting ("" or Pending) and marked Admitted=False/Queued. The marker +// is required, not just the phase: a session's settings are persisted, and +// its phase set to Pending, before the start-approval, identity and +// credential gates park it and before a fatal-settings or image-pin block +// stops it — a session in any of those states never reached the line and must +// never be admitted, or it would reserve capacity it cannot use. +func isWaiting(sess *spiceboxv1alpha1.AgentSession) bool { + return (sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending) && + admission.IsQueued(sess) +} + +func (r *Reconciler) demand(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if r.Demand != nil { + return r.Demand(ctx, sess) + } + var ac spiceboxv1alpha1.AgentClass + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: sess.Namespace, Name: sess.Spec.Class}, &ac); err != nil { + return admission.Demand{}, fmt.Errorf("get AgentClass %s: %w", lineOf(sess), err) + } + return agentsession.DemandFor(ctx, r.Client, sess, &ac) +} + +// Reconcile runs one admission pass over every line. +func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + requeue := ctrl.Result{RequeueAfter: resyncInterval} + + sessions, err := r.listSessions(ctx) + if err != nil { + return ctrl.Result{}, err + } + var waiting, holding, preAdmission []*spiceboxv1alpha1.AgentSession + var errs []error + for i := range sessions { + sess := &sessions[i] + if !isWaiting(sess) { + if err := r.clearPosition(ctx, sess); err != nil { + errs = append(errs, err) + } + } + switch { + case isTerminal(sess): + // Never a holder, even while still Admitted=True with pods: any + // pods it has placed already reduce node headroom in the snapshot. + case admission.IsAdmitted(sess): + holding = append(holding, sess) + case admission.PreAdmission(sess): + preAdmission = append(preAdmission, sess) + case isWaiting(sess): + waiting = append(waiting, sess) + } + } + if len(waiting) == 0 && (len(preAdmission) == 0 || !r.Enabled) { + return requeue, errors.Join(errs...) + } + now := r.now() + + if !r.Enabled { + for _, sess := range waiting { + if err := r.admit(ctx, sess, now, spiceboxv1alpha1.ReasonAdmitted, "Session admission is disabled; admitted at once."); err != nil { + errs = append(errs, err) + } + } + return requeue, errors.Join(errs...) + } + + delegated, err := r.listDelegations(ctx) + if err != nil { + // Defer the whole pass rather than decide on a view known to be wrong: + // with no delegation index every genuine child would read as a + // non-child, losing its cap exemption and, if warm idle, becoming a + // cap-reclaim candidate during an API blip. + return requeue, errors.Join(append(errs, err)...) + } + pods, podsKnown := r.listPods(ctx) + adopted, adoptErrs := r.adoptPreAdmission(ctx, preAdmission, pods, podsKnown, now) + errs = append(errs, adoptErrs...) + holding = append(holding, adopted...) + if len(waiting) == 0 { + return requeue, errors.Join(errs...) + } + + in := admission.Input{} + byKey := make(map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, len(waiting)) + for _, sess := range waiting { + d, err := r.demand(ctx, sess) + if err != nil { + logger.Info("admission: cannot price session's pod set; it keeps waiting", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + continue + } + key := client.ObjectKeyFromObject(sess) + byKey[key] = sess + in.Waiting = append(in.Waiting, admission.Candidate{ + Key: key, + Line: lineOf(sess), + JoinedAt: admission.JoinedAt(sess), + Child: delegated.isChild(ctx, sess), + Demand: d, + }) + } + in.Caps, in.Waiting = r.capLines(ctx, in.Waiting, byKey) + + pool := reclaimPool{ + sessions: map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{}, + inFlight: map[string][]admission.PlacedPod{}, + requested: map[string]int{}, + } + for _, sess := range holding { + d, err := r.demand(ctx, sess) + if err != nil { + // Still a holder: it counts toward its line's cap. With no + // demand it adds nothing to the line's share, and its unplaced + // pods go unreserved (its placed ones are in the snapshot), which + // is the accepted cost of an unpriceable holder. + logger.Info("admission: cannot price an admitted session's pod set; counting it with no unplaced demand", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + } + child := delegated.isChild(ctx, sess) + in.Holders = append(in.Holders, admission.Holder{ + Key: client.ObjectKeyFromObject(sess), + Line: lineOf(sess), + Child: child, + Demand: d, + Unplaced: unplaced(sess.Namespace, d, pods, podsKnown), + }) + if podsKnown && isWarmIdle(sess) { + pool.addWarm(sess, child, boundPods(sess.Namespace, d, pods)) + } + } + if podsKnown { + r.addDrainingReclaims(ctx, &pool, sessions, pods) + } + + in.Model = r.snapshot(ctx, len(in.Waiting)) + in.TotalCPUMilli, in.TotalMemBytes = in.Model.Totals() + + dec := admission.Decide(in) + + // Under an unknown model nothing proved the set fits, so the reconciler + // must not read a later stall as the model being wrong: the reason tells + // its misfit release to leave the session to the pod-level hold. + reason, message := spiceboxv1alpha1.ReasonAdmitted, "The session's whole pod set fits the cluster." + if !in.Model.Known() { + reason, message = spiceboxv1alpha1.ReasonAdmittedUnknownCapacity, + "Cluster capacity is unknown to the operator; admitted in line order without a fit check." + } + for _, key := range dec.Admit { + if err := r.admit(ctx, byKey[key], now, reason, message); err != nil { + errs = append(errs, err) + } + } + for key, pos := range dec.Positions { + if err := r.writePosition(ctx, byKey[key], pos); err != nil { + errs = append(errs, err) + } + } + // Capacity reclaims plan only on a known model (PlanReclaim returns nil + // otherwise); cap reclaims need no capacity data, so they run either way. + errs = append(errs, r.requestReclaims(ctx, in.Waiting, dec, &pool, in.Model, now)...) + return requeue, errors.Join(errs...) +} + +// capLines reads spec.maxConcurrentSessions from the AgentClass of every line +// with a waiting candidate, keyed by line as Decide expects. A class that does +// not exist has no cap (pricing has already refused its sessions when it +// prices from the class). A class that cannot be read fails closed: that +// line's non-child candidates sit this pass out, since admitting them could +// exceed a cap nobody could check; its children stay, as a cap never blocks +// them. +func (r *Reconciler) capLines(ctx context.Context, waiting []admission.Candidate, + sessions map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, +) (map[string]int32, []admission.Candidate) { + caps := map[string]int32{} + unreadable := map[string]bool{} + checked := map[string]bool{} + for _, c := range waiting { + if checked[c.Line] { + continue + } + checked[c.Line] = true + sess := sessions[c.Key] + var ac spiceboxv1alpha1.AgentClass + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: sess.Namespace, Name: sess.Spec.Class}, &ac); err != nil { + if !apierrors.IsNotFound(err) { + log.FromContext(ctx).Info("admission: cannot read the class's session cap; its non-delegated sessions keep waiting this pass", + "class", c.Line, "err", err.Error()) + unreadable[c.Line] = true + } + continue + } + if limit := ac.Spec.MaxConcurrentSessions; limit != nil { + caps[c.Line] = *limit + } + } + if len(unreadable) == 0 { + return caps, waiting + } + kept := waiting[:0:0] + for _, c := range waiting { + if c.Child || !unreadable[c.Line] { + kept = append(kept, c) + } + } + return caps, kept +} + +// isWarmIdle matches a holder whose pods could be given back early: Idle, +// channel-attached (the AgentSession reconciler's sleep path, which honors the +// reclaim request, runs only for those), not yet slept, with a known idle +// start for PlanReclaim's minWarm check. +func isWarmIdle(sess *spiceboxv1alpha1.AgentSession) bool { + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseIdle && + sess.Spec.InputChannel != nil && + sess.Status.SleptAt == nil && + sess.Status.LastIdleAt != nil +} + +// boundPods is the part of a session's pod set the capacity snapshot counts +// against a node: its demand's pods (matched by podIndex.lookup, as unplaced +// does) that are bound and not finished, priced from the live pod spec. A pod +// already marked for deletion is included — the snapshot still counts it +// until it is gone — so it is exactly what a reclaim gives back. +func boundPods(namespace string, d admission.Demand, pods podIndex) []admission.PlacedPod { + var placed []admission.PlacedPod + for _, p := range d.Pods { + pod := pods.lookup(namespace, p) + if pod == nil || !isBound(pod) { + continue + } + placed = append(placed, admission.PlacedPod{ + Node: pod.Spec.NodeName, + PodDemand: admission.PodDemandFor(pod.Name, &pod.Spec), + }) + } + return placed +} + +// reclaimPool is what a pass may plan reclaims from, per line. +// +// inFlight is the capacity reclaims requested on an earlier pass are already +// giving back: the bound pods of a session carrying the reclaim request but +// not yet slept, and of a slept session released for a reclaim whose pods are +// still being torn down. The snapshot still counts those pods, so without +// crediting them every pass between the request and the last pod's deletion — +// and the admitter is woken by each of those status writes and pod events — +// would see the head still not fit and reclaim the next-oldest warm session +// too, cascading through the line for one head. +// +// requested counts, per line, the non-child holders already carrying a +// reclaim request and not yet slept: each is already giving a cap slot back, +// so a cap-blocked head waits for it rather than asking for another. +type reclaimPool struct { + warm []admission.WarmIdle + sessions map[types.NamespacedName]*spiceboxv1alpha1.AgentSession + inFlight map[string][]admission.PlacedPod + requested map[string]int +} + +// addWarm files a warm Idle holder's bound pods: as in flight when a reclaim +// is already requested (so it is not chosen again), else as a candidate. A +// session with no bound pods has nothing to give back and is skipped. child is +// the holder's proven delegation (delegations.isChild). +func (p *reclaimPool) addWarm(sess *spiceboxv1alpha1.AgentSession, child bool, placed []admission.PlacedPod) { + line := lineOf(sess) + if _, requested := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]; requested { + p.inFlight[line] = append(p.inFlight[line], placed...) + if !child { + p.requested[line]++ + } + return + } + if len(placed) == 0 { + return + } + key := client.ObjectKeyFromObject(sess) + p.warm = append(p.warm, admission.WarmIdle{Key: key, Line: line, IdleSince: sess.Status.LastIdleAt.Time, Pods: placed, Child: child}) + p.sessions[key] = sess +} + +// isDrainingReclaim matches a session the sleep path has already released for +// a reclaim (slept, Admitted=False/Reclaimed). It is neither a holder nor +// waiting, but its pods may still be terminating. +func isDrainingReclaim(sess *spiceboxv1alpha1.AgentSession) bool { + if sess.Status.SleptAt == nil { + return false + } + cond := meta.FindStatusCondition(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + return cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == spiceboxv1alpha1.ReasonAdmissionReclaimed +} + +// sessionsWithPods is every session (by namespace/name) that has at least one +// listed pod: its runner, detector or sidecar pods by the session label, its +// sandbox pods by their bundle SpiceboxSession's label — which a pod adopted +// from a SandboxWarmPool also carries, under a pool-minted name. +type sessionsWithPods map[types.NamespacedName]bool + +func indexSessionsWithPods(pods podIndex) sessionsWithPods { + labelled := sessionsWithPods{} + for _, pod := range pods.byName { + for _, l := range []string{sessionPodLabel, sandboxPodLabel} { + if v := pod.Labels[l]; v != "" { + labelled[types.NamespacedName{Namespace: pod.Namespace, Name: v}] = true + } + } + } + return labelled +} + +// has reports whether sess has a listed pod. +func (l sessionsWithPods) has(sess *spiceboxv1alpha1.AgentSession) bool { + if l[types.NamespacedName{Namespace: sess.Namespace, Name: sess.Name}] { + return true + } + for _, b := range sess.Status.BundleSessions { + if l[types.NamespacedName{Namespace: sess.Namespace, Name: b.SpiceboxSessionName}] { + return true + } + } + return false +} + +// adoptPreAdmission writes Admitted=True/Adopted on every pre-admission +// session (admission.PreAdmission) that still has a listed pod, and returns +// them so this pass counts them as holders. One with no listed pod holds +// nothing and is left alone; so is every one when the pod list failed, since +// adopting a session with no pods would reserve capacity for nothing. +func (r *Reconciler) adoptPreAdmission(ctx context.Context, sessions []*spiceboxv1alpha1.AgentSession, + pods podIndex, podsKnown bool, now time.Time, +) ([]*spiceboxv1alpha1.AgentSession, []error) { + if len(sessions) == 0 || !podsKnown { + return nil, nil + } + withPods := indexSessionsWithPods(pods) + var adopted []*spiceboxv1alpha1.AgentSession + var errs []error + for _, sess := range sessions { + if !withPods.has(sess) { + continue + } + log.FromContext(ctx).Info("admission: adopting a session whose pods predate admission", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "phase", sess.Status.Phase) + if err := r.admit(ctx, sess, now, spiceboxv1alpha1.ReasonAdmissionAdopted, + "The session's pods were created before admission was enabled; adopted as admitted."); err != nil { + errs = append(errs, err) + } + // Counted as a holder either way: its pods are up whether or not the + // write landed, and the next pass retries the write. + adopted = append(adopted, sess) + } + return adopted, errs +} + +// addDrainingReclaims credits the still-listed bound pods of every draining +// reclaim to its line. A slept session stays Admitted=False/Reclaimed for as +// long as it sleeps, so pricing is skipped unless one of its pods (runner pod +// by session label, sandbox pod by its bundle SpiceboxSession's label) is +// still listed — the common case is a long-gone pod set. +func (r *Reconciler) addDrainingReclaims(ctx context.Context, pool *reclaimPool, sessions []spiceboxv1alpha1.AgentSession, pods podIndex) { + withPods := indexSessionsWithPods(pods) + for i := range sessions { + sess := &sessions[i] + if !isDrainingReclaim(sess) || !withPods.has(sess) { + continue + } + d, err := r.demand(ctx, sess) + if err != nil { + log.FromContext(ctx).Info("admission: cannot price a reclaimed session's terminating pods; not crediting them to its line", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + continue + } + line := lineOf(sess) + pool.inFlight[line] = append(pool.inFlight[line], boundPods(sess.Namespace, d, pods)...) + } +} + +// requestReclaims asks for the warm Idle sessions picked for each line's head +// (its still-waiting position 1) to be slept early. model is the one Decide +// reserved into, so this pass's admissions are respected. Lines go in name +// order; each line's frees and its head's demand are applied to model before +// the next line plans, so two lines never count the same free space. +// +// A head whose line is at its cap (dec.CapBlocked) needs a slot, not space: +// PlanCapReclaim picks one warm non-child holder of the line, unless one +// already requested is still in flight (it is already giving a slot back). +// Any other head gets PlanReclaim's capacity plan; before it plans, the +// line's in-flight reclaims are freed on model, so only the shortfall they do +// not already cover is reclaimed. +func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Candidate, dec admission.Decision, + pool *reclaimPool, model admission.CapacityModel, now time.Time, +) []error { + var heads []admission.Candidate + for _, c := range waiting { + if pos, ok := dec.Positions[c.Key]; ok && pos.Index == 1 { + heads = append(heads, c) + } + } + slices.SortFunc(heads, func(a, b admission.Candidate) int { return strings.Compare(a.Line, b.Line) }) + + pods := make(map[types.NamespacedName][]admission.PlacedPod, len(pool.warm)) + for _, w := range pool.warm { + pods[w.Key] = w.Pods + } + var errs []error + for _, head := range heads { + inFlight := pool.inFlight[head.Line] + model.Free(inFlight) + var plan []types.NamespacedName + if dec.CapBlocked[head.Line] { + if key, ok := admission.PlanCapReclaim(head, pool.warm, now, r.MinWarm); ok && pool.requested[head.Line] == 0 { + plan = []types.NamespacedName{key} + } + } else { + plan = admission.PlanReclaim(head, pool.warm, model, now, r.MinWarm) + } + for _, key := range plan { + model.Free(pods[key]) + if err := r.requestReclaim(ctx, pool.sessions[key], head.Key, now); err != nil { + errs = append(errs, err) + } + } + if len(plan) == 0 && len(inFlight) == 0 { + continue + } + // Hold the freed space for this head so a later line cannot plan on it. + // With a capacity plan, PlanReclaim proved the head fits; a failure + // means that invariant broke. With only in-flight frees, or a cap + // reclaim (which frees a slot, not necessarily enough space), it may + // simply not fit yet. + if !model.Reserve(head.Demand) && len(plan) > 0 && !dec.CapBlocked[head.Line] { + log.FromContext(ctx).Info("admission: head does not fit after its planned reclaims were freed; the plan's fit check and the model disagree", + "class", head.Line, "head", head.Key.String(), "reclaims", len(plan)) + } + } + return errs +} + +// requestReclaim stamps AnnotationReclaimRequested on sess with a +// metadata-only merge patch, unless a request is already there: a later pass +// re-planning the same reclaim must leave the first request's time alone. +func (r *Reconciler) requestReclaim(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, head types.NamespacedName, now time.Time) error { + if _, ok := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]; ok { + return nil + } + logger := log.FromContext(ctx) + logger.Info("admission: requesting early reclaim of a warm idle session for its line's waiting head", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "head", head.String()) + patch := fmt.Appendf(nil, `{"metadata":{"annotations":{%q:%q}}}`, + spiceboxv1alpha1.AnnotationReclaimRequested, now.UTC().Format(time.RFC3339)) + target := &spiceboxv1alpha1.AgentSession{ObjectMeta: metav1.ObjectMeta{Name: sess.Name, Namespace: sess.Namespace}} + if err := r.Client.Patch(ctx, target, client.RawPatch(types.MergePatchType, patch)); err != nil { + logger.Info("admission: writing the reclaim request failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + return fmt.Errorf("request reclaim of %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + +// delegation is what a SubagentRequest's status records about the child it +// created: the request (its UID, which the child's controller owner reference +// names) and the parent it delegated for. +type delegation struct { + request types.UID + parent spiceboxv1alpha1.NamespacedRef +} + +// delegations indexes, by child session, every delegation the SubagentRequest +// controller has recorded. +// +// A session's own spec.parent is not proof that it is a delegated child: +// anyone who may create an AgentSession can set it, and a child skips its +// class's cap and goes first in its line. A request's status.childRef is +// written only by the operator (no namespace principal is granted +// subagentrequests/status), and only once it has created that child itself, +// so the exemption is read from there. +type delegations map[types.NamespacedName]delegation + +// listDelegations indexes every SubagentRequest that names its child. A list +// error is returned, and the caller defers the whole pass on it: nothing is +// decided, admitted or reclaimed until a later pass can read the delegation +// records, since without them no session could be proven a delegated child. +func (r *Reconciler) listDelegations(ctx context.Context) (delegations, error) { + var list spiceboxv1alpha1.SubagentRequestList + if err := r.Client.List(ctx, &list); err != nil { + log.FromContext(ctx).Info("admission: list SubagentRequests failed; deferring this admission pass", + "err", err.Error()) + return delegations{}, fmt.Errorf("admission pass: list SubagentRequests: %w", err) + } + out := make(delegations, len(list.Items)) + for i := range list.Items { + sr := &list.Items[i] + if ref := sr.Status.ChildRef; ref != nil { + out[types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}] = delegation{request: sr.UID, parent: sr.Spec.Parent} + } + } + return out, nil +} + +// isChild reports whether sess is a delegated child: it names a parent, and a +// SubagentRequest that owns it records creating it for that same parent. A +// session claiming a parent with no such record is treated as non-delegated +// and logged. That includes a genuine child in the moment between its +// creation and its request's status write; it waits as an ordinary session +// until the write lands, which wakes the pass (SetupWithManager). +func (d delegations) isChild(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) bool { + claimed := sess.Spec.Parent + if claimed == nil { + return false + } + if got, ok := d[client.ObjectKeyFromObject(sess)]; ok && got.parent == *claimed { + if owner := metav1.GetControllerOf(sess); owner != nil && owner.UID == got.request { + return true + } + } + log.FromContext(ctx).Info("admission: session names a parent but no SubagentRequest records delegating it; treating it as a non-delegated session", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "claimedParent", claimed.Namespace+"/"+claimed.Name) + return false +} + +// listSessions lists every AgentSession the operator owns: cluster-wide with +// no watch set, else one namespaced list per watched namespace. Any failed +// list fails the pass (logged by the caller's error return) rather than +// deciding lines from a partial view. +func (r *Reconciler) listSessions(ctx context.Context) ([]spiceboxv1alpha1.AgentSession, error) { + if len(r.Namespaces) == 0 { + var list spiceboxv1alpha1.AgentSessionList + if err := r.sessionReader().List(ctx, &list); err != nil { + return nil, fmt.Errorf("admission pass: list AgentSessions: %w", err) + } + return list.Items, nil + } + var out []spiceboxv1alpha1.AgentSession + for _, ns := range r.Namespaces { + var list spiceboxv1alpha1.AgentSessionList + if err := r.sessionReader().List(ctx, &list, client.InNamespace(ns)); err != nil { + return nil, fmt.Errorf("admission pass: list AgentSessions in namespace %q: %w", ns, err) + } + out = append(out, list.Items...) + } + return out, nil +} + +// snapshot returns the capacity model, or an unknown one when there is no +// source, when the snapshot fails, or when the operator is namespace-scoped: +// missing data must not block sessions. The reason is logged once per +// known→unknown transition (see capacityUnknown). +// +// A namespace-scoped operator (a watch set) sees only its own namespaces' +// pods, so node headroom computed from them overstates what is free by every +// pod in every other namespace. An optimistic model admits sets the scheduler +// cannot place, and the misfit release then reaps and re-admits them in a +// loop; an unknown one admits in line order and leaves stalls to the +// pod-level hold. +func (r *Reconciler) snapshot(ctx context.Context, waiting int) admission.CapacityModel { + unknown := func(why string, kv ...any) admission.CapacityModel { + if !r.capacityUnknown.Swap(true) { + log.FromContext(ctx).Info("admission: cluster capacity unknown; admitting in line order without a fit check", + append([]any{"reason", why, "waiting", waiting}, kv...)...) + } + return admission.NewNodeModel(nil, nil) + } + if len(r.Namespaces) > 0 { + return unknown("the operator is namespace-scoped (--watch-namespaces) and cannot see other namespaces' pods", + "namespaces", r.Namespaces) + } + if r.Capacity == nil { + return unknown("no capacity source is configured") + } + m, err := r.Capacity.Snapshot(ctx) + if err != nil { + return unknown("the capacity snapshot failed", "err", err.Error()) + } + if !m.Known() { + return unknown("no node reports allocatable resources") + } + if r.capacityUnknown.Swap(false) { + log.FromContext(ctx).Info("admission: cluster capacity known again; admitting by fit") + } + return m +} + +// podIndex is the cluster's pods, looked up for a session's demand entries by +// role: a sandbox pod by its bundle SpiceboxSession's label (sandboxPodLabel), +// every other pod by name. +// +// The split is because a pod adopted from a SandboxWarmPool keeps the +// pool-minted name it was created under; tryAdopt injects the +// SpiceboxSession label onto it, and that label is the only stable link. The +// runner, detector and separate-pod sidecar pods are always created by this +// operator under the deterministic name their PodDemand carries. +type podIndex struct { + byName map[types.NamespacedName]*corev1.Pod + bySandbox map[types.NamespacedName][]*corev1.Pod +} + +func newPodIndex(pods []corev1.Pod) podIndex { + idx := podIndex{ + byName: make(map[types.NamespacedName]*corev1.Pod, len(pods)), + bySandbox: map[types.NamespacedName][]*corev1.Pod{}, + } + for i := range pods { + p := &pods[i] + idx.byName[client.ObjectKeyFromObject(p)] = p + if v := p.Labels[sandboxPodLabel]; v != "" { + k := types.NamespacedName{Namespace: p.Namespace, Name: v} + idx.bySandbox[k] = append(idx.bySandbox[k], p) + } + } + return idx +} + +// lookup finds the pod a demand entry in namespace describes, or nil. Of +// several pods labelled for one sandbox (a replacement beside a terminating +// predecessor), a bound, unfinished one is preferred: that is the pod the +// snapshot is counting against a node. +func (idx podIndex) lookup(namespace string, p admission.PodDemand) *corev1.Pod { + if p.SandboxSession == "" { + return idx.byName[types.NamespacedName{Namespace: namespace, Name: p.Name}] + } + cands := idx.bySandbox[types.NamespacedName{Namespace: namespace, Name: p.SandboxSession}] + for _, pod := range cands { + if isBound(pod) { + return pod + } + } + if len(cands) > 0 { + return cands[0] + } + return nil +} + +// isBound reports whether the snapshot counts pod against a node: bound and +// not finished. +func isBound(pod *corev1.Pod) bool { + return pod.Spec.NodeName != "" && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed +} + +// listPods indexes every pod for the unplaced-demand check. A list error is +// logged and reported as not known, so every holder pod counts as unplaced. +func (r *Reconciler) listPods(ctx context.Context) (podIndex, bool) { + var pods corev1.PodList + if err := r.Client.List(ctx, &pods); err != nil { + log.FromContext(ctx).Info("admission: list pods failed; counting every admitted session's pods as unplaced", "err", err.Error()) + return podIndex{}, false + } + return newPodIndex(pods.Items), true +} + +// unplaced is the part of a holder's demand the capacity snapshot does not +// already account for, each entry matched to its pod in the session's +// namespace by podIndex.lookup: +// +// - a pod bound to a node (spec.nodeName set) is in the snapshot: placed; +// - a pod that exists but is not yet bound, or does not exist yet, is not: +// unplaced, so Decide reserves it before admitting anyone else. +// +// When the pod list failed (known=false), every pod is unplaced: reserving too +// much under-admits for a pass, never over-admits. +func unplaced(namespace string, d admission.Demand, pods podIndex, known bool) admission.Demand { + var out admission.Demand + for _, p := range d.Pods { + if known { + if pod := pods.lookup(namespace, p); pod != nil && pod.Spec.NodeName != "" { + continue + } + } + out.Pods = append(out.Pods, p) + } + return out +} + +// admit writes Admitted=True with reason at now and clears the session's line +// position. +func (r *Reconciler) admit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, now time.Time, reason, message string) error { + original := sess.DeepCopy() + meta.SetStatusCondition(&sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + ObservedGeneration: sess.Generation, + LastTransitionTime: metav1.NewTime(now), + }) + sess.Status.Admission = nil + if err := agentstatus.WriteOwned(ctx, r.Client, sess, original, spiceboxv1alpha1.OwnerOperator); err != nil { + log.FromContext(ctx).Info("admission: writing Admitted=True failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + return fmt.Errorf("admit %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + +// clearPosition removes a line position left on a session that is no longer +// waiting — it went terminal, was parked, or was admitted by a write that did +// not clear it — so its thread never shows a place in line it does not have. +// A no-op, with no write, when there is nothing to clear. +func (r *Reconciler) clearPosition(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) error { + if sess.Status.Admission == nil { + return nil + } + original := sess.DeepCopy() + sess.Status.Admission = nil + if err := agentstatus.WriteOwned(ctx, r.Client, sess, original, spiceboxv1alpha1.OwnerOperator); err != nil { + log.FromContext(ctx).Info("admission: clearing a stale line position failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "phase", sess.Status.Phase, "err", err.Error()) + return fmt.Errorf("clear admission position for %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + +// writePosition records a waiting session's place in line, only when its +// position, line length or join time changed. +func (r *Reconciler) writePosition(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, pos admission.Position) error { + want := &spiceboxv1alpha1.AdmissionStatus{ + Position: pos.Index, + LineLength: pos.LineLength, + // The apiserver stores times to the second; compare at that precision + // so an unchanged join time never reads as a change. + JoinedAt: metav1.NewTime(admission.JoinedAt(sess)).Rfc3339Copy(), + } + if cur := sess.Status.Admission; cur != nil && cur.Position == want.Position && + cur.LineLength == want.LineLength && cur.JoinedAt.Time.Equal(want.JoinedAt.Time) { + return nil + } + original := sess.DeepCopy() + sess.Status.Admission = want + if err := agentstatus.WriteOwned(ctx, r.Client, sess, original, spiceboxv1alpha1.OwnerOperator); err != nil { + log.FromContext(ctx).Info("admission: writing line position failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + return fmt.Errorf("write admission position for %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + +// SetupWithManager maps every AgentSession event, every SubagentRequest event +// (its status.childRef write is what proves a waiting session a delegated +// child), and every event on a pod carrying a session label, to the single +// Key. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + toKey := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: Key}} + }) + sessionPod := predicate.NewPredicateFuncs(func(obj client.Object) bool { + labels := obj.GetLabels() + _, runner := labels[sessionPodLabel] + _, sandbox := labels[sandboxPodLabel] + return runner || sandbox + }) + return ctrl.NewControllerManagedBy(mgr). + Named("admission"). + Watches(&spiceboxv1alpha1.AgentSession{}, toKey). + Watches(&spiceboxv1alpha1.SubagentRequest{}, toKey). + Watches(&corev1.Pod{}, toKey, builder.WithPredicates(sessionPod)). + Complete(r) +} diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go new file mode 100644 index 00000000..e63d571b --- /dev/null +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -0,0 +1,1056 @@ +package admitter + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" +) + +const testClass = "demo-agent" + +var ( + baseTime = time.Date(2026, 11, 10, 9, 0, 0, 0, time.UTC) + passTime = baseTime.Add(time.Hour) +) + +// oneCPUDemand is the pod set every test session needs: one 1-CPU pod. +func oneCPUDemand(_ context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + return admission.Demand{Pods: []admission.PodDemand{{Name: sess.Name + "-runner", CPUMilli: 1000, MemBytes: 1 << 20}}}, nil +} + +// roomFor is a CapacitySource with one node that fits n oneCPUDemand pod sets. +type roomFor int + +func (n roomFor) Snapshot(context.Context) (admission.CapacityModel, error) { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, + Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(int64(n)*1000, resource.DecimalSI), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("110"), + }}, + } + return admission.NewNodeModel([]corev1.Node{node}, nil), nil +} + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + sch := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(sch), "core scheme") + require.NoError(t, spiceboxv1alpha1.AddToScheme(sch), "spicebox scheme") + return sch +} + +// waitingSession is a booting, never-admitted session created at +// baseTime+offset that reached the admission gate at once +// (Admitted=False/Queued at creation). +func waitingSession(name string, offset time.Duration) *spiceboxv1alpha1.AgentSession { + return &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "default", + CreationTimestamp: metav1.NewTime(baseTime.Add(offset)), + }, + Spec: spiceboxv1alpha1.AgentSessionSpec{Class: testClass}, + Status: spiceboxv1alpha1.AgentSessionStatus{ + Phase: spiceboxv1alpha1.AgentSessionPhasePending, + Conditions: []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, + LastTransitionTime: metav1.NewTime(baseTime.Add(offset)), + }}, + }, + } +} + +// newReconciler builds an admitter over a fake client holding objs, counting +// every status patch it sends. +func newReconciler(t *testing.T, enabled bool, capacity admission.CapacitySource, objs ...client.Object) (*Reconciler, client.Client, *atomic.Int32) { + t.Helper() + var statusPatches atomic.Int32 + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + statusPatches.Add(1) + return cl.SubResource(sub).Patch(ctx, obj, p, opts...) + }, + }). + Build() + r := &Reconciler{ + Client: c, + Capacity: capacity, + Enabled: enabled, + Now: func() time.Time { return passTime }, + Demand: oneCPUDemand, + } + return r, c, &statusPatches +} + +func runPass(t *testing.T, r *Reconciler) reconcile.Result { + t.Helper() + res, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: Key}) + require.NoError(t, err, "Reconcile") + return res +} + +func getSession(t *testing.T, c client.Client, name string) *spiceboxv1alpha1.AgentSession { + t.Helper() + var s spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: name}, &s), "Get %s", name) + return &s +} + +func assertAdmitted(t *testing.T, s *spiceboxv1alpha1.AgentSession) { + t.Helper() + cond := meta.FindStatusCondition(s.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if !assert.NotNil(t, cond, "%s: Admitted condition", s.Name) { + return + } + assert.Equal(t, metav1.ConditionTrue, cond.Status, "%s: Admitted status", s.Name) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmitted, cond.Reason, "%s: Admitted reason", s.Name) + assert.True(t, cond.LastTransitionTime.Time.Equal(passTime), "%s: Admitted transition at the pass's Now, got %v", s.Name, cond.LastTransitionTime) + assert.Nil(t, s.Status.Admission, "%s: an admitted session holds no line position", s.Name) +} + +func assertPosition(t *testing.T, s *spiceboxv1alpha1.AgentSession, index, length int32) { + t.Helper() + assert.False(t, admission.IsAdmitted(s), "%s: not admitted", s.Name) + if !assert.NotNil(t, s.Status.Admission, "%s: status.admission", s.Name) { + return + } + assert.Equal(t, index, s.Status.Admission.Position, "%s: position", s.Name) + assert.Equal(t, length, s.Status.Admission.LineLength, "%s: line length", s.Name) + assert.True(t, s.Status.Admission.JoinedAt.Time.Equal(s.CreationTimestamp.Time), "%s: joinedAt is creation, got %v", s.Name, s.Status.Admission.JoinedAt) +} + +func TestAdmitterPass(t *testing.T) { + t.Run("three waiting demo-agent sessions, room for one: oldest gets Admitted=True, others get Position 1 and 2 of 2", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + waitingSession("sess-c", 3*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + ) + res := runPass(t, r) + assert.Equal(t, 15*time.Second, res.RequeueAfter, "periodic requeue") + + assertAdmitted(t, getSession(t, c, "sess-a")) + assertPosition(t, getSession(t, c, "sess-b"), 1, 2) + assertPosition(t, getSession(t, c, "sess-c"), 2, 2) + }) + + t.Run("Enabled=false: every waiting session admitted, with no room at all", func(t *testing.T) { + r, c, _ := newReconciler(t, false, roomFor(0), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + waitingSession("sess-c", 3*time.Minute), + ) + runPass(t, r) + for _, name := range []string{"sess-a", "sess-b", "sess-c"} { + assertAdmitted(t, getSession(t, c, name)) + } + }) + + t.Run("Pending without the Queued marker (parked at an earlier gate, or fatal settings): not admitted, no position", func(t *testing.T) { + parked := waitingSession("sess-parked", 0) + parked.Status.Conditions = nil + r, c, _ := newReconciler(t, true, roomFor(5), parked, waitingSession("sess-a", time.Minute)) + runPass(t, r) + + got := getSession(t, c, "sess-parked") + assert.Nil(t, meta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + "a session that never reached the gate is never admitted") + assert.Nil(t, got.Status.Admission, "and holds no line position") + assertAdmitted(t, getSession(t, c, "sess-a")) + }) + + t.Run("a position write is skipped when unchanged: second pass sends no status patch", func(t *testing.T) { + r, c, patches := newReconciler(t, true, roomFor(1), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + waitingSession("sess-c", 3*time.Minute), + ) + runPass(t, r) + require.Positive(t, patches.Load(), "first pass writes admission and positions") + patches.Store(0) + + runPass(t, r) + assert.Zero(t, patches.Load(), "nothing changed, so nothing is written") + assertPosition(t, getSession(t, c, "sess-b"), 1, 2) + }) + + t.Run("a holder still occupying the room: nobody is admitted, the line starts at 1", func(t *testing.T) { + holder := waitingSession("sess-holder", 0) + holder.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + holder.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + r, c, _ := newReconciler(t, true, roomFor(1), holder, + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + ) + runPass(t, r) + assertPosition(t, getSession(t, c, "sess-a"), 1, 2) + assertPosition(t, getSession(t, c, "sess-b"), 2, 2) + }) + + t.Run("a terminal session still Admitted=True: not a holder, so the head is admitted", func(t *testing.T) { + done := waitingSession("sess-done", 0) + done.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + done.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + r, c, _ := newReconciler(t, true, roomFor(1), done, waitingSession("sess-a", time.Minute)) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "sess-a")) + }) + + t.Run("a session parked at a pre-start gate: not waiting, never admitted or positioned", func(t *testing.T) { + parked := waitingSession("sess-parked", 0) + parked.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingDecision + r, c, _ := newReconciler(t, true, roomFor(5), parked) + runPass(t, r) + got := getSession(t, c, "sess-parked") + assert.False(t, admission.IsAdmitted(got), "parked session not admitted") + assert.Nil(t, got.Status.Admission, "parked session has no line position") + }) + + t.Run("demand fails for the oldest: it keeps waiting unpositioned, the next is admitted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + ) + r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Name == "sess-a" { + return admission.Demand{}, fmt.Errorf("settings not resolved") + } + return oneCPUDemand(ctx, sess) + } + runPass(t, r) + a := getSession(t, c, "sess-a") + assert.False(t, admission.IsAdmitted(a), "sess-a not admitted") + assert.Nil(t, a.Status.Admission, "sess-a not positioned") + assertAdmitted(t, getSession(t, c, "sess-b")) + }) + + t.Run("capacity snapshot fails: capacity unknown, every waiting session admitted as AdmittedUnknownCapacity", func(t *testing.T) { + r, c, _ := newReconciler(t, true, failingSource{}, + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + ) + runPass(t, r) + assertAdmittedUnknown(t, getSession(t, c, "sess-a")) + assertAdmittedUnknown(t, getSession(t, c, "sess-b")) + }) + + t.Run("namespace-scoped operator (watch set): capacity treated as unknown even with no room, admitted as AdmittedUnknownCapacity", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(0), + waitingSession("sess-a", time.Minute), + ) + r.Namespaces = []string{"default"} + runPass(t, r) + assertAdmittedUnknown(t, getSession(t, c, "sess-a")) + }) + + t.Run("known capacity: admitted with reason Admitted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), waitingSession("sess-a", time.Minute)) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "sess-a")) + }) +} + +// assertAdmittedUnknown asserts s was admitted under an unknown capacity +// model, which the reconciler's misfit release must leave alone. +func assertAdmittedUnknown(t *testing.T, s *spiceboxv1alpha1.AgentSession) { + t.Helper() + cond := meta.FindStatusCondition(s.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if !assert.NotNil(t, cond, "%s: Admitted condition", s.Name) { + return + } + assert.Equal(t, metav1.ConditionTrue, cond.Status, "%s: Admitted status", s.Name) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmittedUnknownCapacity, cond.Reason, "%s: Admitted reason", s.Name) +} + +type failingSource struct{} + +func (failingSource) Snapshot(context.Context) (admission.CapacityModel, error) { + return nil, fmt.Errorf("nodes is forbidden") +} + +// TestUnplacedDemand pins how much of a holder's pod set Decide must still +// reserve: a pod bound to a node is already in the snapshot; a pending or +// missing one is not. +func TestUnplacedDemand(t *testing.T) { + d := admission.Demand{Pods: []admission.PodDemand{ + {Name: "bound", CPUMilli: 1}, {Name: "pending", CPUMilli: 2}, {Name: "missing", CPUMilli: 4}, + // A sandbox pod adopted from a SandboxWarmPool: bound under a + // pool-minted name, found by its bundle SpiceboxSession's label. + {Name: "s1-code-pod", SandboxSession: "s1-code", CPUMilli: 8}, + }} + pods := newPodIndex([]corev1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "bound", Namespace: "default"}, Spec: corev1.PodSpec{NodeName: "node-a"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "pending", Namespace: "default"}}, + { + ObjectMeta: metav1.ObjectMeta{Name: "warmpool-x7k2p", Namespace: "default", Labels: map[string]string{sandboxPodLabel: "s1-code"}}, + Spec: corev1.PodSpec{NodeName: "node-a"}, + }, + }) + cases := []struct { + name string + pods podIndex + known bool + want []string + }{ + {name: "pods listed: pending and missing pods are unplaced, the pool-named sandbox pod is placed", pods: pods, known: true, want: []string{"pending", "missing"}}, + {name: "pod list failed: every pod counted unplaced", known: false, want: []string{"bound", "pending", "missing", "s1-code-pod"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := unplaced("default", d, tc.pods, tc.known) + var names []string + for _, p := range got.Pods { + names = append(names, p.Name) + } + assert.Equal(t, tc.want, names) + }) + } +} + +// TestAdmitterWatchNamespaces pins the --watch-namespaces contract: under +// namespaced RBAC a cluster-wide list 403s, so with a watch set the pass lists +// sessions one namespace at a time and never touches any other namespace. +func TestAdmitterWatchNamespaces(t *testing.T) { + inNamespace := func(s *spiceboxv1alpha1.AgentSession, ns string) *spiceboxv1alpha1.AgentSession { + s.Namespace = ns + return s + } + cases := []struct { + name string + namespaces []string + wantLists []string + wantAdmitted []string + }{ + { + name: "watch set: one namespaced list per watched namespace, unwatched sessions untouched", + namespaces: []string{"team-a", "team-b"}, + wantLists: []string{"team-a", "team-b"}, wantAdmitted: []string{"team-a/sess-a", "team-b/sess-b"}, + }, + { + name: "no watch set: one cluster-wide list, every namespace admitted", + wantLists: []string{""}, wantAdmitted: []string{"other/sess-c", "team-a/sess-a", "team-b/sess-b"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var lists []string + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects( + inNamespace(waitingSession("sess-a", time.Minute), "team-a"), + inNamespace(waitingSession("sess-b", 2*time.Minute), "team-b"), + inNamespace(waitingSession("sess-c", 3*time.Minute), "other"), + ). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*spiceboxv1alpha1.AgentSessionList); ok { + lo := (&client.ListOptions{}).ApplyOptions(opts) + lists = append(lists, lo.Namespace) + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + r := &Reconciler{ + Client: c, Capacity: roomFor(5), Enabled: true, Namespaces: tc.namespaces, + Now: func() time.Time { return passTime }, Demand: oneCPUDemand, + } + runPass(t, r) + assert.Equal(t, tc.wantLists, lists, "AgentSession list scopes") + + var all spiceboxv1alpha1.AgentSessionList + require.NoError(t, c.List(context.Background(), &all), "list all") + var admitted []string + for i := range all.Items { + if admission.IsAdmitted(&all.Items[i]) { + admitted = append(admitted, all.Items[i].Namespace+"/"+all.Items[i].Name) + } + } + assert.ElementsMatch(t, tc.wantAdmitted, admitted, "admitted sessions") + }) + } +} + +// warmIdleSession is an admitted, channel-attached Idle session that went idle +// idleFor before the pass and has not slept: its pods still hold capacity. +func warmIdleSession(name string, idleFor time.Duration) *spiceboxv1alpha1.AgentSession { + s := waitingSession(name, 0) + s.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: name} + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + idleAt := metav1.NewTime(passTime.Add(-idleFor)) + s.Status.LastIdleAt = &idleAt + s.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + return s +} + +// boundRunnerPod is the session's oneCPUDemand pod, bound to node-a and +// requesting what oneCPUDemand prices. +func boundRunnerPod(sessName string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: sessName + "-runner", Namespace: "default", + Labels: map[string]string{sessionPodLabel: sessName}, + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{{ + Name: "runner", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Mi"), + }}, + }}, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } +} + +// cpuNode is node-a with cpus whole CPUs allocatable. +func cpuNode(cpus int64) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, + Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(cpus, resource.DecimalSI), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("110"), + }}, + } +} + +// TestAdmitterEarlyReclaim covers the reclaim request a pass makes for a line +// whose head does not fit: it annotates the warm Idle sessions of that line +// PlanReclaim picks, priced from the live pods the cluster snapshot also sees. +func TestAdmitterEarlyReclaim(t *testing.T) { + const minWarm = 5 * time.Minute + cases := []struct { + name string + objs []client.Object + wantAnnotated []string + wantClean []string + }{ + { + name: "line has a waiting head, one warm idle past minWarm: reclaim annotation set on it only", + objs: []client.Object{ + cpuNode(2), + warmIdleSession("sess-old", 10*time.Minute), boundRunnerPod("sess-old"), + warmIdleSession("sess-young", time.Minute), boundRunnerPod("sess-young"), + waitingSession("sess-wait", time.Minute), + }, + wantAnnotated: []string{"sess-old"}, + wantClean: []string{"sess-young", "sess-wait"}, + }, + { + name: "no waiting head: no annotation", + objs: []client.Object{ + cpuNode(1), + warmIdleSession("sess-old", 10*time.Minute), boundRunnerPod("sess-old"), + }, + wantClean: []string{"sess-old"}, + }, + { + name: "the head fits without a reclaim: admitted, no annotation", + objs: []client.Object{ + cpuNode(2), + warmIdleSession("sess-old", 10*time.Minute), boundRunnerPod("sess-old"), + waitingSession("sess-wait", time.Minute), + }, + wantClean: []string{"sess-old", "sess-wait"}, + }, + { + name: "only warm idle is younger than minWarm: no annotation", + objs: []client.Object{ + cpuNode(1), + warmIdleSession("sess-young", time.Minute), boundRunnerPod("sess-young"), + waitingSession("sess-wait", time.Minute), + }, + wantClean: []string{"sess-young", "sess-wait"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, c, _ := newReconciler(t, true, nil, tc.objs...) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + runPass(t, r) + + for _, name := range tc.wantAnnotated { + got := getSession(t, c, name) + assert.Equal(t, passTime.Format(time.RFC3339), got.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + "%s: reclaim requested at the pass's Now", name) + } + for _, name := range tc.wantClean { + assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "%s: no reclaim requested", name) + } + }) + } + + t.Run("a reclaim already requested: the annotation is not rewritten", func(t *testing.T) { + old := warmIdleSession("sess-old", 10*time.Minute) + earlier := baseTime.Format(time.RFC3339) + old.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: earlier} + r, c, _ := newReconciler(t, true, nil, cpuNode(1), old, boundRunnerPod("sess-old"), waitingSession("sess-wait", time.Minute)) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + runPass(t, r) + assert.Equal(t, earlier, getSession(t, c, "sess-old").Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + "the first request's time stands") + }) +} + +// TestAdmitterReclaimInFlightIsCredited pins that one head gets one reclaim: +// between the request and the reclaimed pods actually leaving the node, every +// pass (each woken by the sleep's status write or a pod event) still sees the +// head not fit in the snapshot. Crediting the in-flight reclaim's bound pods +// keeps those passes from reclaiming the next-oldest warm session too. +func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { + const minWarm = 5 * time.Minute + cases := []struct { + name string + advance func(t *testing.T, c client.Client) // what happened to sess-1 between passes + }{ + { + name: "reclaim requested but not yet slept: second pass annotates nothing new", + advance: func(*testing.T, client.Client) {}, + }, + { + name: "slept and Admitted=False/Reclaimed, pod still bound: second pass annotates nothing new", + advance: func(t *testing.T, c client.Client) { + markReclaimed(t, c, "sess-1") + }, + }, + { + name: "slept and Admitted=False/Reclaimed, pod terminating (DeletionTimestamp): second pass annotates nothing new", + advance: func(t *testing.T, c client.Client) { + markReclaimed(t, c, "sess-1") + var pod corev1.Pod + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "sess-1-runner"}, &pod)) + require.NoError(t, c.Delete(context.Background(), &pod), "delete held by a finalizer: pod lingers terminating") + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(&pod), &pod)) + require.NotNil(t, pod.DeletionTimestamp, "pod is terminating") + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pod1 := boundRunnerPod("sess-1") + pod1.Finalizers = []string{"test/hold"} + r, c, _ := newReconciler(t, true, nil, + cpuNode(3), + warmIdleSession("sess-1", 30*time.Minute), pod1, + warmIdleSession("sess-2", 20*time.Minute), boundRunnerPod("sess-2"), + warmIdleSession("sess-3", 10*time.Minute), boundRunnerPod("sess-3"), + waitingSession("sess-wait", time.Minute), + ) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + + runPass(t, r) + require.Contains(t, getSession(t, c, "sess-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") + require.NotContains(t, getSession(t, c, "sess-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims only one") + + tc.advance(t, c) + runPass(t, r) + for _, name := range []string{"sess-2", "sess-3"} { + assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "%s: the in-flight reclaim already covers the head", name) + } + }) + } +} + +// A draining reclaim whose pod set cannot be priced is not credited to its +// line: its terminating pods still fill the snapshot, so the head still does +// not fit and the next pass reclaims the next-oldest warm session. +func TestAdmitterDrainingReclaimUnpriceable(t *testing.T) { + r, c, _ := newReconciler(t, true, nil, + cpuNode(3), + warmIdleSession("sess-1", 30*time.Minute), boundRunnerPod("sess-1"), + warmIdleSession("sess-2", 20*time.Minute), boundRunnerPod("sess-2"), + warmIdleSession("sess-3", 10*time.Minute), boundRunnerPod("sess-3"), + waitingSession("sess-wait", time.Minute), + ) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = 5 * time.Minute + + runPass(t, r) + require.Contains(t, getSession(t, c, "sess-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") + + markReclaimed(t, c, "sess-1") + r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Name == "sess-1" { + return admission.Demand{}, fmt.Errorf("AgentClass gone") + } + return oneCPUDemand(ctx, sess) + } + runPass(t, r) + assert.Contains(t, getSession(t, c, "sess-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "sess-1's pods are uncredited, so the next-oldest is reclaimed") + assert.NotContains(t, getSession(t, c, "sess-3").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "one more reclaim covers the head") +} + +// markReclaimed does what the sleep path's status write does: SleptAt and +// Admitted=False/Reclaimed. The reclaim annotation stays (it is removed only +// after the reap) and the pods are left for the test to dispose of. +func markReclaimed(t *testing.T, c client.Client, name string) { + t.Helper() + s := getSession(t, c, name) + slept := metav1.NewTime(passTime) + s.Status.SleptAt = &slept + meta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionReclaimed, LastTransitionTime: metav1.NewTime(passTime), + }) + require.NoError(t, c.Status().Update(context.Background(), s), "mark %s reclaimed", name) +} + +// cappedClass is testClass with spec.maxConcurrentSessions set. +func cappedClass(limit int32) *spiceboxv1alpha1.AgentClass { + return &spiceboxv1alpha1.AgentClass{ + ObjectMeta: metav1.ObjectMeta{Name: testClass, Namespace: "default"}, + Spec: spiceboxv1alpha1.AgentClassSpec{MaxConcurrentSessions: &limit}, + } +} + +// admittedRunning is a non-child session already holding its admission. +func admittedRunning(name string) *spiceboxv1alpha1.AgentSession { + s := waitingSession(name, 0) + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + s.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + return s +} + +// childOf makes s a session delegated by parent the way the SubagentRequest +// controller builds one: spec.parent set and controlled by its request. +// requestFor is that request, with the child recorded on its status; a test +// leaves it out to make s a session that only claims a parent. +func childOf(s *spiceboxv1alpha1.AgentSession, parent string) *spiceboxv1alpha1.AgentSession { + s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: parent} + s.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(requestFor(s), + spiceboxv1alpha1.SchemeGroupVersion.WithKind("SubagentRequest"))} + return s +} + +// requestFor is the SubagentRequest that created child (see childOf). +func requestFor(child *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.SubagentRequest { + sr := &spiceboxv1alpha1.SubagentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: child.Name + "-req", Namespace: child.Namespace, UID: types.UID(child.Name + "-req-uid")}, + Spec: spiceboxv1alpha1.SubagentRequestSpec{Class: testClass, Task: "do the delegated work"}, + Status: spiceboxv1alpha1.SubagentRequestStatus{ + ChildRef: &spiceboxv1alpha1.NamespacedRef{Namespace: child.Namespace, Name: child.Name}, + }, + } + if child.Spec.Parent != nil { + sr.Spec.Parent = *child.Spec.Parent + } + return sr +} + +func TestAdmitterConcurrencyCap(t *testing.T) { + t.Run("class cap 1 with one admitted parent: its waiting child is admitted; a waiting non-child is not", func(t *testing.T) { + child := childOf(waitingSession("sess-child", 2*time.Minute), "sess-parent") + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(1), + admittedRunning("sess-parent"), + waitingSession("sess-other", time.Minute), + child, requestFor(child), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "sess-child")) + assertPosition(t, getSession(t, c, "sess-other"), 1, 1) + }) + + t.Run("class cap 2 with one admitted parent: the oldest waiting non-child takes the second slot, the next waits", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(2), + admittedRunning("sess-parent"), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "sess-a")) + assertPosition(t, getSession(t, c, "sess-b"), 1, 1) + }) + + t.Run("no AgentClass to read a cap from: the line is uncapped, every waiting session that fits is admitted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + admittedRunning("sess-parent"), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "sess-a")) + assertAdmitted(t, getSession(t, c, "sess-b")) + }) + + t.Run("an unpriceable holder still takes its cap slot: the waiting session is cap-blocked despite room", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(1), + admittedRunning("sess-holder"), + waitingSession("sess-wait", time.Minute), + ) + r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Name == "sess-holder" { + return admission.Demand{}, fmt.Errorf("settings not resolved") + } + return oneCPUDemand(ctx, sess) + } + runPass(t, r) + assert.True(t, admission.IsAdmitted(getSession(t, c, "sess-holder")), "the unpriceable holder stays admitted") + assertPosition(t, getSession(t, c, "sess-wait"), 1, 1) + }) + + t.Run("AgentClass read fails: the line's non-children keep waiting unpositioned, its child is still admitted", func(t *testing.T) { + child := childOf(waitingSession("sess-child", 2*time.Minute), "sess-parent") + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects( + cappedClass(1), + admittedRunning("sess-parent"), + waitingSession("sess-other", time.Minute), + child, requestFor(child), + ). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*spiceboxv1alpha1.AgentClass); ok { + return fmt.Errorf("apiserver unavailable") + } + return cl.Get(ctx, key, obj, opts...) + }, + }). + Build() + r := &Reconciler{ + Client: c, Capacity: roomFor(5), Enabled: true, + Now: func() time.Time { return passTime }, Demand: oneCPUDemand, + } + runPass(t, r) + assertAdmitted(t, getSession(t, c, "sess-child")) + other := getSession(t, c, "sess-other") + assert.False(t, admission.IsAdmitted(other), "sess-other not admitted past a cap it could not read") + assert.Nil(t, other.Status.Admission, "sess-other not positioned") + }) +} + +// TestAdmitterChildNeedsProvenance pins that spec.parent alone does not make +// a session a delegated child: anyone who may create an AgentSession can set +// it. Without a SubagentRequest that created it for that parent, the session +// is capped and waits in arrival order like any other. A failed request list +// defers the pass instead of deciding without the index. +func TestAdmitterChildNeedsProvenance(t *testing.T) { + cases := []struct { + name string + // objs is the claimed child's own objects: the session, and any request. + objs func(claimed *spiceboxv1alpha1.AgentSession) []client.Object + // listFails fails every SubagentRequest list. + listFails bool + }{ + { + name: "spec.parent with no SubagentRequest: capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "sess-parent"} + return []client.Object{s} + }, + }, + { + name: "the request records a different parent: capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + sr := requestFor(childOf(s, "sess-parent")) + s.Spec.Parent.Name = "sess-elsewhere" + return []client.Object{s, sr} + }, + }, + { + name: "the request records this child but does not own it: capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + sr := requestFor(childOf(s, "sess-parent")) + s.OwnerReferences = nil + return []client.Object{s, sr} + }, + }, + { + name: "SubagentRequest list fails: the pass is deferred, nobody is admitted or positioned", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + s = childOf(s, "sess-parent") + return []client.Object{s, requestFor(s)} + }, + listFails: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + objs := append([]client.Object{ + cappedClass(1), + admittedRunning("sess-parent"), + waitingSession("sess-other", time.Minute), + }, tc.objs(waitingSession("sess-claimed", 2*time.Minute))...) + var funcs interceptor.Funcs + if tc.listFails { + funcs.List = func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*spiceboxv1alpha1.SubagentRequestList); ok { + return fmt.Errorf("apiserver unavailable") + } + return cl.List(ctx, list, opts...) + } + } + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(funcs). + Build() + r := &Reconciler{ + Client: c, Capacity: roomFor(5), Enabled: true, + Now: func() time.Time { return passTime }, Demand: oneCPUDemand, + } + _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: Key}) + if tc.listFails { + assert.ErrorContains(t, err, "list SubagentRequests", "the failed list is reported so the pass retries") + for _, name := range []string{"sess-other", "sess-claimed"} { + s := getSession(t, c, name) + assert.False(t, admission.IsAdmitted(s), "%s not admitted on a deferred pass", name) + assert.Nil(t, s.Status.Admission, "%s not positioned on a deferred pass", name) + } + return + } + require.NoError(t, err, "Reconcile") + assertPosition(t, getSession(t, c, "sess-other"), 1, 2) + assertPosition(t, getSession(t, c, "sess-claimed"), 2, 2) + }) + } +} + +// preAdmissionSession is a session that started its pods before admission was +// switched on: in phase, with its runner pod recorded on status, and no +// Admitted condition at all. +func preAdmissionSession(name string, phase string) *spiceboxv1alpha1.AgentSession { + s := waitingSession(name, 0) + s.Status.Phase = phase + s.Status.Conditions = nil + s.Status.RunnerPodName = name + "-runner" + return s +} + +// TestAdmitterAdoptsPreAdmissionSessions pins the rollout: a session whose +// pods predate admission is adopted (Admitted=True/Adopted) and counted as a +// holder — toward its line's cap, and its unplaced pods reserved — rather +// than gated with its pods up or left uncounted. +func TestAdmitterAdoptsPreAdmissionSessions(t *testing.T) { + assertAdopted := func(t *testing.T, s *spiceboxv1alpha1.AgentSession) { + t.Helper() + cond := meta.FindStatusCondition(s.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if assert.NotNil(t, cond, "%s: Admitted condition", s.Name) { + assert.Equal(t, metav1.ConditionTrue, cond.Status, "%s: adopted as admitted", s.Name) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionAdopted, cond.Reason, "%s: reason", s.Name) + } + } + + t.Run("Running pre-admission session with a bound pod, class cap 1: adopted, and the cap holds the waiting session back", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(1), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("sess-old"), + waitingSession("sess-a", time.Minute), + ) + runPass(t, r) + assertAdopted(t, getSession(t, c, "sess-old")) + assertPosition(t, getSession(t, c, "sess-a"), 1, 1) + }) + + t.Run("Pending pre-admission session with an unbound pod, room for one: adopted, its unplaced pod reserved, the waiting session waits", func(t *testing.T) { + pod := boundRunnerPod("sess-old") + pod.Spec.NodeName = "" + pod.Status.Phase = corev1.PodPending + r, c, _ := newReconciler(t, true, roomFor(1), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhasePending), pod, + waitingSession("sess-a", time.Minute), + ) + runPass(t, r) + assertAdopted(t, getSession(t, c, "sess-old")) + assertPosition(t, getSession(t, c, "sess-a"), 1, 1) + }) + + t.Run("pre-admission session with nothing waiting: still adopted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseIdle), boundRunnerPod("sess-old"), + ) + runPass(t, r) + assertAdopted(t, getSession(t, c, "sess-old")) + }) + + t.Run("pre-admission status but no pod listed: not adopted (nothing to hold)", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseRunning), + waitingSession("sess-a", time.Minute), + ) + runPass(t, r) + assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "sess-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + "no pods: not a holder") + assertAdmitted(t, getSession(t, c, "sess-a")) + }) + + t.Run("Enabled=false: nothing adopted", func(t *testing.T) { + r, c, _ := newReconciler(t, false, roomFor(1), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("sess-old"), + ) + runPass(t, r) + assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "sess-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted)) + }) +} + +// TestAdmitterCapReclaim pins early reclaim for a line blocked by its cap +// rather than by capacity: warm idle holders count toward the cap, so the +// oldest one past minWarm is asked back, one per head, and not while an +// earlier request for the line is still in flight. +func TestAdmitterCapReclaim(t *testing.T) { + const minWarm = 5 * time.Minute + requested := func(s *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.AgentSession { + s.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: baseTime.Format(time.RFC3339)} + return s + } + cases := []struct { + name string + objs []client.Object + wantAnnotated []string + wantClean []string + }{ + { + name: "cap 2 held by two warm idle sessions, plenty of room: the oldest is reclaimed, only it", + objs: []client.Object{ + cpuNode(8), cappedClass(2), + warmIdleSession("sess-old", 30*time.Minute), boundRunnerPod("sess-old"), + warmIdleSession("sess-mid", 20*time.Minute), boundRunnerPod("sess-mid"), + waitingSession("sess-wait", time.Minute), + }, + wantAnnotated: []string{"sess-old"}, + wantClean: []string{"sess-mid", "sess-wait"}, + }, + { + name: "a cap reclaim already in flight for the line: no second one", + objs: []client.Object{ + cpuNode(8), cappedClass(2), + requested(warmIdleSession("sess-old", 30*time.Minute)), boundRunnerPod("sess-old"), + warmIdleSession("sess-mid", 20*time.Minute), boundRunnerPod("sess-mid"), + waitingSession("sess-wait", time.Minute), + }, + wantClean: []string{"sess-mid", "sess-wait"}, + }, + { + name: "the cap is held by a running session, the warm one is too young: nothing reclaimed", + objs: []client.Object{ + cpuNode(8), cappedClass(2), + admittedRunning("sess-run"), + warmIdleSession("sess-young", time.Minute), boundRunnerPod("sess-young"), + waitingSession("sess-wait", time.Minute), + }, + wantClean: []string{"sess-young", "sess-wait"}, + }, + { + name: "under the cap and the head fits: admitted, nothing reclaimed", + objs: []client.Object{ + cpuNode(8), cappedClass(3), + warmIdleSession("sess-old", 30*time.Minute), boundRunnerPod("sess-old"), + waitingSession("sess-wait", time.Minute), + }, + wantClean: []string{"sess-old"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, c, _ := newReconciler(t, true, nil, tc.objs...) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + runPass(t, r) + for _, name := range tc.wantAnnotated { + assert.Equal(t, passTime.Format(time.RFC3339), getSession(t, c, name).Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + "%s: reclaim requested at the pass's Now", name) + } + for _, name := range tc.wantClean { + assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "%s: no reclaim requested", name) + } + }) + } +} + +// A warm idle holder whose sandbox pod was adopted from a SandboxWarmPool has +// that pod under a pool-minted name. Matched by name, it was never counted as +// bound: its demand stayed "unplaced" (reserved on top of the snapshot, which +// already counts it — a double count for life), and an early reclaim could +// not credit what it gives back. Matched by its bundle SpiceboxSession's +// label, it is the bundle entry's pod. +func TestBoundPodsMatchesPoolNamedSandboxPod(t *testing.T) { + d := admission.Demand{Pods: []admission.PodDemand{ + {Name: "sess-runner", CPUMilli: 1000}, + {Name: "sess-code-pod", SandboxSession: "sess-code", CPUMilli: 500}, + }} + runner := boundRunnerPod("sess") + sandbox := boundRunnerPod("ignored") + sandbox.Name = "warmpool-x7k2p" + sandbox.Labels = map[string]string{sandboxPodLabel: "sess-code"} + pods := newPodIndex([]corev1.Pod{*runner, *sandbox}) + + got := boundPods("default", d, pods) + var names []string + for _, p := range got { + names = append(names, p.Name) + } + assert.ElementsMatch(t, []string{"sess-runner", "warmpool-x7k2p"}, names, "both the runner and the pool-named sandbox pod are bound") + assert.Empty(t, unplaced("default", d, pods, true).Pods, "nothing left unplaced") +} + +// A session that is no longer waiting must not keep showing a line position +// ("3rd in line") it no longer has: the pass clears a stale status.admission. +func TestAdmitterClearsStalePosition(t *testing.T) { + stale := func(s *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.AgentSession { + s.Status.Admission = &spiceboxv1alpha1.AdmissionStatus{Position: 3, LineLength: 5, JoinedAt: s.CreationTimestamp} + return s + } + failed := stale(waitingSession("sess-failed", 0)) + failed.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseFailed + parked := stale(waitingSession("sess-parked", 0)) + parked.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingCredentials + + r, c, _ := newReconciler(t, true, roomFor(1), failed, parked) + runPass(t, r) + assert.Nil(t, getSession(t, c, "sess-failed").Status.Admission, "a Failed session holds no line position") + assert.Nil(t, getSession(t, c, "sess-parked").Status.Admission, "a parked session holds no line position") +} diff --git a/pkg/controllers/admission/admitter/restart_integration_test.go b/pkg/controllers/admission/admitter/restart_integration_test.go new file mode 100644 index 00000000..8bcd5bc7 --- /dev/null +++ b/pkg/controllers/admission/admitter/restart_integration_test.go @@ -0,0 +1,293 @@ +//go:build integration + +package admitter + +import ( + "context" + "fmt" + "os" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/testenv" +) + +func TestMain(m *testing.M) { os.Exit(testenv.RunPackage(m)) } + +const restartNamespace = "admission-restart" + +// demandIn prices sessions in namespace ns with oneCPUDemand and every other +// session as needing nothing. The admitter lists cluster-wide, and envtest is +// shared: another test's sessions must not take this test's room. +func demandIn(ns string) func(context.Context, *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + return func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Namespace != ns { + return admission.Demand{}, nil + } + return oneCPUDemand(ctx, sess) + } +} + +// startAdmitter runs one admitter in its own manager, as a fresh operator +// process would, and returns a stop func that blocks until the manager exits. +// Only sessions in ns are priced (demandIn). +func startAdmitter(t *testing.T, env *testenv.Env, ns string) (stop func()) { + t.Helper() + mgr, err := ctrl.NewManager(env.Cfg, ctrl.Options{ + Scheme: env.Scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + Controller: ctrlconfig.Controller{SkipNameValidation: ptr.To(true)}, + LeaderElection: false, + }) + require.NoError(t, err, "ctrl.NewManager") + require.NoError(t, (&Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Capacity: roomFor(2), + Enabled: true, + // envtest runs no AgentSession reconciler, so settings are never + // resolved; every session needs the same one-CPU pod set. + Demand: demandIn(ns), + }).SetupWithManager(mgr), "admitter SetupWithManager") + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + if err := mgr.Start(ctx); err != nil { + t.Errorf("manager exited: %v", err) + } + }() + require.True(t, mgr.GetCache().WaitForCacheSync(ctx), "cache sync") + var once bool + stop = func() { + if once { + return + } + once = true + cancel() + <-done + } + t.Cleanup(stop) + return stop +} + +// admittedLive lists the names of non-terminal sessions with Admitted=True, +// and every session ever admitted (terminal ones included), sorted. +func admittedLive(t *testing.T, c client.Client) (live, ever []string) { + t.Helper() + var list spiceboxv1alpha1.AgentSessionList + require.NoError(t, c.List(context.Background(), &list, client.InNamespace(restartNamespace)), "list sessions") + for i := range list.Items { + s := &list.Items[i] + if !admission.IsAdmitted(s) { + continue + } + ever = append(ever, s.Name) + if !isTerminal(s) { + live = append(live, s.Name) + } + } + slices.Sort(live) + slices.Sort(ever) + return live, ever +} + +// waitForAdmitted polls until exactly want sessions have ever been admitted. +func waitForAdmitted(t *testing.T, c client.Client, want int) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var ever []string + for time.Now().Before(deadline) { + _, ever = admittedLive(t, c) + if len(ever) >= want { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("want %d sessions admitted within 30s, got %v", want, ever) +} + +// assertStaysAdmitted asserts over a window that the admitted set never grows +// past wantEver and the live holders never exceed the room. +func assertStaysAdmitted(t *testing.T, c client.Client, wantLive, wantEver []string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + live, ever := admittedLive(t, c) + if !assert.Equal(t, wantLive, live, "live holders") || !assert.Equal(t, wantEver, ever, "ever admitted") { + return + } + time.Sleep(100 * time.Millisecond) + } +} + +// TestAdmitterRestartRebuildsLine is the restart property: the line lives in +// AgentSession status, not in the admitter's memory, so a new operator +// process continues it exactly and never double-admits. +// +// Sessions are created s1..s5 in order. creationTimestamp is to the second, so +// several share a join time and the tie breaks on namespace/name, which is +// the same order. +func TestAdmitterRestartRebuildsLine(t *testing.T) { + env := testenv.Shared(t) + c := env.Client + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: restartNamespace}}); !apierrors.IsAlreadyExists(err) { + require.NoError(t, err, "create namespace") + } + + stop := startAdmitter(t, env, restartNamespace) + for i := 1; i <= 5; i++ { + sess := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("s%d", i), Namespace: restartNamespace}, + Spec: spiceboxv1alpha1.AgentSessionSpec{ + Class: testClass, + Prompt: spiceboxv1alpha1.PromptSource{Inline: "review this"}, + }, + } + require.NoError(t, c.Create(ctx, sess), "create s%d", i) + // envtest runs no AgentSession reconciler: put the session in line the + // way its admission gate does, Admitted=False/Queued at arrival. + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + meta.SetStatusCondition(&sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, Message: "waiting in line", + LastTransitionTime: sess.CreationTimestamp, + }) + require.NoError(t, c.Status().Update(ctx, sess), "queue s%d", i) + } + + waitForAdmitted(t, c, 2) + assertStaysAdmitted(t, c, []string{"s1", "s2"}, []string{"s1", "s2"}) + stop() + + startAdmitter(t, env, restartNamespace) + // Release a holder the way a finished session does: it goes terminal. + var s1 spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: restartNamespace, Name: "s1"}, &s1), "get s1") + s1.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + require.NoError(t, c.Status().Update(ctx, &s1), "mark s1 Succeeded") + + waitForAdmitted(t, c, 3) + assertStaysAdmitted(t, c, []string{"s2", "s3"}, []string{"s1", "s2", "s3"}) + + var s4 spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: restartNamespace, Name: "s4"}, &s4), "get s4") + require.NotNil(t, s4.Status.Admission, "s4 is positioned in line") + assert.Equal(t, int32(1), s4.Status.Admission.Position, "s4 heads the line") + assert.Equal(t, int32(2), s4.Status.Admission.LineLength, "two still waiting") +} + +const adoptNamespace = "admission-adopt" + +// TestAdmitterAdoptsPreAdmissionSessionsEnvtest is the rollout property +// against a real apiserver: sessions whose pods were created before admission +// was switched on — a Running one with a bound runner pod, a Pending one +// mid-boot with an unbound sandbox pod under a warm-pool name — carry no +// Admitted condition. The admitter adopts both (Admitted=True/Adopted) and +// counts them as holders: the Pending one's unplaced pod takes one of the two +// CPUs, so of two queued sessions exactly one is admitted. +func TestAdmitterAdoptsPreAdmissionSessionsEnvtest(t *testing.T) { + env := testenv.Shared(t) + c := env.Client + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: adoptNamespace}}); !apierrors.IsAlreadyExists(err) { + require.NoError(t, err, "create namespace") + } + + create := func(name string, mutate func(s *spiceboxv1alpha1.AgentSession)) { + t.Helper() + sess := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: adoptNamespace}, + Spec: spiceboxv1alpha1.AgentSessionSpec{ + Class: testClass, + Prompt: spiceboxv1alpha1.PromptSource{Inline: "review this"}, + }, + } + require.NoError(t, c.Create(ctx, sess), "create %s", name) + mutate(sess) + require.NoError(t, c.Status().Update(ctx, sess), "status %s", name) + } + pod := func(name string, labels map[string]string, node string) { + t.Helper() + require.NoError(t, c.Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: adoptNamespace, Labels: labels}, + Spec: corev1.PodSpec{ + NodeName: node, + Containers: []corev1.Container{{Name: "c", Image: "busybox"}}, + }, + }), "create pod %s", name) + } + + create("old-running", func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + s.Status.RunnerPodName = "old-running-runner" + }) + pod("old-running-runner", map[string]string{sessionPodLabel: "old-running"}, "node-a") + create("old-booting", func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + s.Status.BundleSessions = []spiceboxv1alpha1.ResolvedBundle{{Name: "code", SpiceboxSessionName: "old-booting-code"}} + }) + pod("warmpool-x7k2p", map[string]string{sandboxPodLabel: "old-booting-code"}, "") + for _, name := range []string{"new-a", "new-b"} { + create(name, func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + meta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, Message: "waiting in line", + LastTransitionTime: s.CreationTimestamp, + }) + }) + } + + startAdmitter(t, env, adoptNamespace) + + reasons := func() map[string]string { + var list spiceboxv1alpha1.AgentSessionList + require.NoError(t, c.List(ctx, &list, client.InNamespace(adoptNamespace)), "list sessions") + out := map[string]string{} + for i := range list.Items { + if cond := meta.FindStatusCondition(list.Items[i].Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted); cond != nil && cond.Status == metav1.ConditionTrue { + out[list.Items[i].Name] = cond.Reason + } + } + return out + } + deadline := time.Now().Add(30 * time.Second) + var got map[string]string + for time.Now().Before(deadline) { + got = reasons() + if len(got) >= 3 { + break + } + time.Sleep(100 * time.Millisecond) + } + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionAdopted, got["old-running"], "running pre-admission session adopted") + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionAdopted, got["old-booting"], "booting pre-admission session (pool-named pod) adopted") + assert.Equal(t, spiceboxv1alpha1.ReasonAdmitted, got["new-a"], "the oldest queued session takes the free CPU") + + // Over a window, the adopted holders keep new-b out. + window := time.Now().Add(2 * time.Second) + for time.Now().Before(window) { + if !assert.NotContains(t, reasons(), "new-b", "new-b stays in line: the adopted holders count") { + return + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/pkg/controllers/admission/capacity.go b/pkg/controllers/admission/capacity.go new file mode 100644 index 00000000..cc0dbe10 --- /dev/null +++ b/pkg/controllers/admission/capacity.go @@ -0,0 +1,220 @@ +package admission + +import ( + "cmp" + "context" + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/platform/schedfit" +) + +// PlacedPod is a pod reserved on a specific node, so it can be freed later. +type PlacedPod struct { + Node string + PodDemand +} + +// CapacityModel is a mutable view of what a cluster can still place. +type CapacityModel interface { + // Known reports whether the model has real capacity data. An unknown model + // admits everything: missing data must not block sessions. + Known() bool + // Reserve places every pod of d or none of them, committing only on + // success. + Reserve(d Demand) bool + // Free returns placed pods' requests to their nodes. + Free(pods []PlacedPod) + // Clone returns an independent copy. + Clone() CapacityModel + // Totals is the cluster's schedulable size — allocatable summed over the + // nodes the model places on — which Decide's dominant share is measured + // against. Zero when unknown; it does not shrink as Reserve places pods. + Totals() (cpuMilli, memBytes int64) +} + +// CapacitySource produces a CapacityModel snapshot. It is the seam a future +// non-Kubernetes backend plugs into. +type CapacitySource interface { + Snapshot(ctx context.Context) (CapacityModel, error) +} + +// KubeCapacitySource snapshots node headroom from the Kubernetes API. +type KubeCapacitySource struct { + Reader client.Reader +} + +// Snapshot lists nodes and all pods and builds a node model. A list error is +// returned for the caller to treat as unknown capacity. +func (s KubeCapacitySource) Snapshot(ctx context.Context) (CapacityModel, error) { + var nodes corev1.NodeList + if err := s.Reader.List(ctx, &nodes); err != nil { + return nil, fmt.Errorf("list nodes: %w", err) + } + var pods corev1.PodList + if err := s.Reader.List(ctx, &pods); err != nil { + return nil, fmt.Errorf("list pods: %w", err) + } + return NewNodeModel(nodes.Items, pods.Items), nil +} + +// PodDemandFor prices one pod from its spec the way the capacity snapshot +// counts it (schedfit.EffectiveRequests), carrying its tolerations so a node +// model places it only where the scheduler could. +func PodDemandFor(name string, spec *corev1.PodSpec) PodDemand { + cpu, mem, eph := schedfit.EffectiveRequests(spec) + return PodDemand{Name: name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph, Tolerations: spec.Tolerations} +} + +type nodeSlot struct { + name string + cpuMilli, memBytes, ephemeralBytes int64 + // taints are the node's NoSchedule and NoExecute taints: a pod is placed + // here only if it tolerates every one. PreferNoSchedule is a preference + // the scheduler may override, so it is not recorded. + taints []corev1.Taint + // pods is the remaining pod slots; unlimitedPods is set when the node + // reports no `pods` allocatable (fake or unusual nodes), so it is not + // read as zero. + pods int64 + unlimitedPods bool +} + +func (n *nodeSlot) fits(p PodDemand) bool { + return p.CPUMilli <= n.cpuMilli && p.MemBytes <= n.memBytes && + p.EphemeralBytes <= n.ephemeralBytes && (n.unlimitedPods || n.pods >= 1) && + n.tolerates(p.Tolerations) +} + +// tolerates reports whether tols tolerate every one of the node's recorded +// taints, matched by the upstream Toleration.ToleratesTaint. Comparison +// operators (Lt/Gt) are matched as disabled, the scheduler's default: a pod +// relying on them is kept off the node rather than admitted onto it. +func (n *nodeSlot) tolerates(tols []corev1.Toleration) bool { + logger := klog.Background() + for i := range n.taints { + if !slices.ContainsFunc(tols, func(t corev1.Toleration) bool { + return t.ToleratesTaint(logger, &n.taints[i], false) + }) { + return false + } + } + return true +} + +// blockingTaints is the subset of a node's taints that keep an intolerant pod +// off it. +func blockingTaints(node corev1.Node) []corev1.Taint { + var out []corev1.Taint + for _, t := range node.Spec.Taints { + if t.Effect == corev1.TaintEffectNoSchedule || t.Effect == corev1.TaintEffectNoExecute { + out = append(out, t) + } + } + return out +} + +func (n *nodeSlot) take(p PodDemand) { + n.cpuMilli -= p.CPUMilli + n.memBytes -= p.MemBytes + n.ephemeralBytes -= p.EphemeralBytes + n.pods-- +} + +type nodeModel struct { + known bool + nodes []nodeSlot + // totalCPUMilli and totalMemBytes are allocatable summed over the + // schedulable nodes, fixed at construction. + totalCPUMilli, totalMemBytes int64 +} + +// NewNodeModel builds a model from node headroom. Unschedulable nodes are +// skipped, and a node's NoSchedule/NoExecute taints are recorded so Reserve +// places only pods that tolerate them. With no nodes, or none reporting allocatable, the model is unknown. +func NewNodeModel(nodes []corev1.Node, pods []corev1.Pod) CapacityModel { + m := &nodeModel{} + if !schedfit.CeilingFromNodes(nodes).Known { + return m + } + m.known = true + for i := range nodes { + if nodes[i].Spec.Unschedulable { + continue + } + h := schedfit.HeadroomOn(nodes[i], pods) + alloc := nodes[i].Status.Allocatable + m.totalCPUMilli += alloc.Cpu().MilliValue() + m.totalMemBytes += alloc.Memory().Value() + _, hasPods := nodes[i].Status.Allocatable[corev1.ResourcePods] + m.nodes = append(m.nodes, nodeSlot{ + name: nodes[i].Name, + cpuMilli: h.CPUMilli, + memBytes: h.MemBytes, + ephemeralBytes: h.EphemeralBytes, + pods: h.Pods, + unlimitedPods: !hasPods, + taints: blockingTaints(nodes[i]), + }) + } + return m +} + +func (m *nodeModel) Known() bool { return m.known } + +// Reserve places pods first-fit-decreasing by cpu then memory, on a scratch +// copy so a failure leaves the model untouched. +func (m *nodeModel) Reserve(d Demand) bool { + if !m.known { + return true + } + order := slices.Clone(d.Pods) + slices.SortStableFunc(order, func(a, b PodDemand) int { + return cmp.Or(cmp.Compare(b.CPUMilli, a.CPUMilli), cmp.Compare(b.MemBytes, a.MemBytes)) + }) + scratch := slices.Clone(m.nodes) + for _, p := range order { + placed := false + for i := range scratch { + if scratch[i].fits(p) { + scratch[i].take(p) + placed = true + break + } + } + if !placed { + return false + } + } + m.nodes = scratch + return true +} + +func (m *nodeModel) Free(pods []PlacedPod) { + if !m.known { + return + } + for _, p := range pods { + for i := range m.nodes { + if m.nodes[i].name != p.Node { + continue + } + m.nodes[i].cpuMilli += p.CPUMilli + m.nodes[i].memBytes += p.MemBytes + m.nodes[i].ephemeralBytes += p.EphemeralBytes + m.nodes[i].pods++ + } + } +} + +func (m *nodeModel) Clone() CapacityModel { + return &nodeModel{known: m.known, nodes: slices.Clone(m.nodes), totalCPUMilli: m.totalCPUMilli, totalMemBytes: m.totalMemBytes} +} + +func (m *nodeModel) Totals() (cpuMilli, memBytes int64) { + return m.totalCPUMilli, m.totalMemBytes +} diff --git a/pkg/controllers/admission/capacity_test.go b/pkg/controllers/admission/capacity_test.go new file mode 100644 index 00000000..73cc7335 --- /dev/null +++ b/pkg/controllers/admission/capacity_test.go @@ -0,0 +1,266 @@ +package admission + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func testNode(name, cpu, mem, pods string) corev1.Node { + rl := corev1.ResourceList{} + if cpu != "" { + rl[corev1.ResourceCPU] = resource.MustParse(cpu) + } + if mem != "" { + rl[corev1.ResourceMemory] = resource.MustParse(mem) + } + if pods != "" { + rl[corev1.ResourcePods] = resource.MustParse(pods) + } + return corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}, Status: corev1.NodeStatus{Allocatable: rl}} +} + +func testPod(node, cpu string) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p-" + node + "-" + cpu}, + Spec: corev1.PodSpec{NodeName: node, Containers: []corev1.Container{{ + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(cpu)}}, + }}}, + } +} + +func pod(name string, cpuMilli int64) PodDemand { + return PodDemand{Name: name, CPUMilli: cpuMilli, MemBytes: 1 << 20} +} + +func TestNodeModelReserve(t *testing.T) { + sixNodes := func() ([]corev1.Node, []corev1.Pod) { + var nodes []corev1.Node + var pods []corev1.Pod + for _, n := range []string{"a", "b", "c", "d", "e", "f"} { + nodes = append(nodes, testNode(n, "1", "1Gi", "10")) + pods = append(pods, testPod(n, "600m")) + } + return nodes, pods + } + cases := []struct { + name string + build func() ([]corev1.Node, []corev1.Pod) + demand Demand + then *Demand // second Reserve, if any + want bool + wantTo *bool + known bool + }{ + { + name: "aggregate free but no single node fits: false", + build: sixNodes, + demand: Demand{Pods: []PodDemand{pod("big", 1000)}}, + want: false, known: true, + }, + { + name: "two pods split across two nodes: true, both committed; identical second Reserve fails", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "1", "1Gi", "10"), testNode("b", "1", "1Gi", "10")}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 800), pod("y", 800)}}, + then: &Demand{Pods: []PodDemand{pod("x", 800), pod("y", 800)}}, + want: true, wantTo: ptr(false), known: true, + }, + { + name: "pod-count exhausted on the only roomy node: false", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "8", "8Gi", "1")}, []corev1.Pod{testPod("a", "100m")} + }, + demand: Demand{Pods: []PodDemand{pod("x", 100)}}, + want: false, known: true, + }, + { + name: "missing pods allocatable is unlimited slots: true", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "8", "8Gi", "")}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 100), pod("y", 100)}}, + want: true, known: true, + }, + { + name: "unschedulable node is skipped: false", + build: func() ([]corev1.Node, []corev1.Pod) { + n := testNode("a", "8", "8Gi", "10") + n.Spec.Unschedulable = true + return []corev1.Node{n, testNode("b", "100m", "8Gi", "10")}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 1000)}}, + want: false, known: true, + }, + { + name: "unknown (no allocatable reported): Known()==false and Reserve true", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{{ObjectMeta: metav1.ObjectMeta{Name: "a"}}}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 1<<40)}}, + want: true, known: false, + }, + { + name: "failed Reserve commits nothing: a later fitting Reserve succeeds", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "1", "1Gi", "10")}, nil + }, + // first pod would fit, second cannot; nothing may stay reserved. + demand: Demand{Pods: []PodDemand{pod("x", 900), pod("y", 900)}}, + then: &Demand{Pods: []PodDemand{pod("z", 900)}}, + want: false, wantTo: ptr(true), known: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + nodes, pods := tc.build() + m := NewNodeModel(nodes, pods) + assert.Equal(t, tc.known, m.Known()) + assert.Equal(t, tc.want, m.Reserve(tc.demand)) + if tc.then != nil { + assert.Equal(t, *tc.wantTo, m.Reserve(*tc.then)) + } + }) + } +} + +func TestNodeModelReserveEphemeralStorage(t *testing.T) { + n := testNode("a", "8", "8Gi", "10") + n.Status.Allocatable[corev1.ResourceEphemeralStorage] = resource.MustParse("1Gi") + m := NewNodeModel([]corev1.Node{n}, nil) + assert.False(t, m.Reserve(Demand{Pods: []PodDemand{{Name: "x", EphemeralBytes: 2 << 30}}}), + "ephemeral storage beyond the node's allocatable must not fit") +} + +func TestNodeModelFree(t *testing.T) { + m := NewNodeModel([]corev1.Node{testNode("a", "1", "1Gi", "10")}, nil) + d := Demand{Pods: []PodDemand{pod("x", 800)}} + require.True(t, m.Reserve(d)) + require.False(t, m.Reserve(d), "node is full") + + m.Free([]PlacedPod{{Node: "a", PodDemand: pod("x", 800)}}) + assert.True(t, m.Reserve(d), "freeing the placed pod makes the Reserve succeed") +} + +func TestNodeModelCloneIsIndependent(t *testing.T) { + m := NewNodeModel([]corev1.Node{testNode("a", "1", "1Gi", "10")}, nil) + c := m.Clone() + d := Demand{Pods: []PodDemand{pod("x", 800)}} + require.True(t, c.Reserve(d)) + assert.True(t, m.Reserve(d), "reserving on the clone must not affect the original") +} + +func TestUnknownModelCloneStaysUnknown(t *testing.T) { + m := NewNodeModel(nil, nil) + c := m.Clone() + assert.False(t, c.Known()) + c.Free([]PlacedPod{{Node: "a"}}) + assert.True(t, c.Reserve(Demand{Pods: []PodDemand{pod("x", 1<<40)}})) +} + +func ptr[T any](v T) *T { return &v } + +func TestKubeCapacitySourceSnapshot(t *testing.T) { + n := testNode("a", "1", "1Gi", "10") + p := testPod("a", "600m") + c := fake.NewClientBuilder().WithObjects(&n, &p).Build() + + m, err := KubeCapacitySource{Reader: c}.Snapshot(t.Context()) + require.NoError(t, err) + assert.True(t, m.Known()) + assert.False(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 500)}}), "the listed pod's request is booked") + assert.True(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 300)}})) +} + +func TestNodeModelTotals(t *testing.T) { + cordoned := testNode("c", "4", "4Gi", "10") + cordoned.Spec.Unschedulable = true + cases := []struct { + name string + nodes []corev1.Node + wantCPU, wantMem int64 + }{ + { + name: "two schedulable nodes: allocatable summed, unaffected by pods placed on them", + nodes: []corev1.Node{testNode("a", "1", "1Gi", "10"), testNode("b", "2", "2Gi", "10")}, + wantCPU: 3000, wantMem: 3 << 30, + }, + { + name: "an unschedulable node: excluded from the totals", + nodes: []corev1.Node{testNode("a", "1", "1Gi", "10"), cordoned}, + wantCPU: 1000, wantMem: 1 << 30, + }, + {name: "no nodes: unknown, zero totals"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := NewNodeModel(tc.nodes, []corev1.Pod{testPod("a", "600m")}) + require.True(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 100)}}), "Reserve") + cpu, mem := m.Totals() + assert.Equal(t, tc.wantCPU, cpu, "cpu total") + assert.Equal(t, tc.wantMem, mem, "memory total") + cpu, mem = m.Clone().Totals() + assert.Equal(t, tc.wantCPU, cpu, "clone cpu total") + assert.Equal(t, tc.wantMem, mem, "clone memory total") + }) + } +} + +// A node's NoSchedule/NoExecute taints keep a pod off it unless the pod +// tolerates them, as the scheduler would; PreferNoSchedule does not. Without +// this the model counts a tainted pool as room, admits a set the scheduler +// then cannot place, and the misfit safety net reaps and re-admits it in a +// loop. +func TestNodeModelReserveRespectsTaints(t *testing.T) { + gpuTaint := corev1.Taint{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule} + tainted := func(effect corev1.TaintEffect) corev1.Node { + n := testNode("gpu", "8", "8Gi", "10") + taint := gpuTaint + taint.Effect = effect + n.Spec.Taints = []corev1.Taint{taint} + return n + } + tolerating := pod("p", 1000) + tolerating.Tolerations = []corev1.Toleration{{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}} + cases := []struct { + name string + nodes []corev1.Node + pod PodDemand + want bool + }{ + {"only a NoSchedule-tainted node, pod does not tolerate: does not fit", []corev1.Node{tainted(corev1.TaintEffectNoSchedule)}, pod("p", 1000), false}, + {"only a NoExecute-tainted node, pod does not tolerate: does not fit", []corev1.Node{tainted(corev1.TaintEffectNoExecute)}, pod("p", 1000), false}, + {"only a NoSchedule-tainted node, pod tolerates it: fits", []corev1.Node{tainted(corev1.TaintEffectNoSchedule)}, tolerating, true}, + {"only a PreferNoSchedule-tainted node: fits (a preference, not a bar)", []corev1.Node{tainted(corev1.TaintEffectPreferNoSchedule)}, pod("p", 1000), true}, + {"a tainted big node beside an untainted small one, pod too big for the small: does not fit", + []corev1.Node{tainted(corev1.TaintEffectNoSchedule), testNode("small", "500m", "1Gi", "10")}, pod("p", 1000), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := NewNodeModel(tc.nodes, nil) + require.True(t, m.Known()) + assert.Equal(t, tc.want, m.Reserve(Demand{Pods: []PodDemand{tc.pod}})) + }) + } +} + +func TestPodDemandFor(t *testing.T) { + tol := corev1.Toleration{Key: "pool", Operator: corev1.TolerationOpEqual, Value: "agents", Effect: corev1.TaintEffectNoSchedule} + spec := corev1.PodSpec{ + Tolerations: []corev1.Toleration{tol}, + Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), corev1.ResourceMemory: resource.MustParse("64Mi"), + }}}}, + } + got := PodDemandFor("runner", &spec) + assert.Equal(t, "runner", got.Name) + assert.Equal(t, int64(250), got.CPUMilli) + assert.Equal(t, int64(64<<20), got.MemBytes) + assert.Equal(t, []corev1.Toleration{tol}, got.Tolerations, "the pod's tolerations ride along for the node model") +} diff --git a/pkg/controllers/admission/decide.go b/pkg/controllers/admission/decide.go new file mode 100644 index 00000000..79daf54a --- /dev/null +++ b/pkg/controllers/admission/decide.go @@ -0,0 +1,192 @@ +package admission + +import ( + "cmp" + "slices" + + "k8s.io/apimachinery/pkg/types" +) + +// Holder is an admitted, not-yet-reclaimed session. Unplaced is the part of +// its pod set not yet bound to a node; Decide reserves it first so two passes +// do not count the same free space. +type Holder struct { + Key types.NamespacedName + Line string + Child bool + Demand Demand + Unplaced Demand +} + +// Input is everything one admission pass needs. +type Input struct { + Waiting []Candidate + Holders []Holder + // Caps is the per-line limit on non-child holders. A line with no entry, + // or a cap <= 0, is uncapped. + Caps map[string]int32 + Model CapacityModel + // TotalCPUMilli and TotalMemBytes normalize usage into a dominant share. + // Zero totals make every share zero. + TotalCPUMilli, TotalMemBytes int64 +} + +// Position is a waiting session's place in its line. Index is 1-based. +type Position struct { + Index, LineLength int32 +} + +// Decision is the outcome of one pass: who to admit now, where everyone +// still waiting stands in their line, and which lines' heads wait on their +// cap rather than on room. +type Decision struct { + Admit []types.NamespacedName + Positions map[types.NamespacedName]Position + // CapBlocked holds every line whose head is still waiting, is not a + // delegated child, and whose non-child holders (counting this pass's + // admissions) have reached the line's cap. Freeing space does nothing for + // such a line; only a holder giving its slot back does. + CapBlocked map[string]bool +} + +// lineState is one line's remaining queue and what it already holds. +type lineState struct { + name string + queue []Candidate + cpuMilli int64 + memBytes int64 + nonChild int32 + capacity int32 // <= 0: uncapped +} + +func (l *lineState) eligible() bool { + if len(l.queue) == 0 { + return false + } + return !l.capBlocked() +} + +// capBlocked reports whether the line's head waits on the cap: a non-child +// head, a capped line, its non-child holders at the cap. +func (l *lineState) capBlocked() bool { + return len(l.queue) > 0 && !l.queue[0].Child && l.capacity > 0 && l.nonChild >= l.capacity +} + +// Decide admits waiting sessions, whole pod set at a time, repeatedly picking +// the eligible line with the lowest dominant share of the cluster and +// admitting its head if the capacity model can place it. A head that does not +// fit blocks only its own line (strict FIFO within a line). Children are +// ordered ahead of others in their line and are never blocked by, nor counted +// toward, the cap. +// +// There is no reservation across lines. A head that does not fit holds no +// space for itself, so other lines' smaller heads keep being admitted into +// whatever frees, and a large head can be starved for as long as they keep +// arriving: work-conserving admission is chosen over a cross-line reservation +// that would idle free capacity while the large head waits for enough to +// free at once. The wait is bounded, not unbounded: a starved head fails +// once it has waited past --capacity-hold-timeout, as any unadmitted session +// does. +// +// Decide mutates in.Model: Unplaced holder demand and every admitted +// candidate's demand are reserved on it. It does not modify in.Waiting or +// in.Holders. +func Decide(in Input) Decision { + for _, h := range in.Holders { + // Each unplaced pod is reserved on its own, best effort: they are not + // yet on any node the snapshot counts, but they will be, so the room + // they take must not go to a waiting head. A pod the model cannot + // place (the model disagrees with the scheduler, or the pod waits on a + // node to free) is simply not reserved; the holder stays admitted + // either way, and its other pods are still reserved. + for _, p := range h.Unplaced.Pods { + in.Model.Reserve(Demand{Pods: []PodDemand{p}}) + } + } + + lines := map[string]*lineState{} + line := func(name string) *lineState { + l, ok := lines[name] + if !ok { + l = &lineState{name: name, capacity: in.Caps[name]} + lines[name] = l + } + return l + } + for _, h := range in.Holders { + l := line(h.Line) + cpu, mem := h.Demand.Total() + l.cpuMilli += cpu + l.memBytes += mem + if !h.Child { + l.nonChild++ + } + } + for _, c := range in.Waiting { + l := line(c.Line) + l.queue = append(l.queue, c) + } + for _, l := range lines { + Order(l.queue) + } + + share := func(l *lineState) float64 { + var s float64 + if in.TotalCPUMilli > 0 { + s = float64(l.cpuMilli) / float64(in.TotalCPUMilli) + } + if in.TotalMemBytes > 0 { + s = max(s, float64(l.memBytes)/float64(in.TotalMemBytes)) + } + return s + } + + var admit []types.NamespacedName + for { + var eligible []*lineState + for _, l := range lines { + if l.eligible() { + eligible = append(eligible, l) + } + } + slices.SortFunc(eligible, func(a, b *lineState) int { + return cmp.Or( + cmp.Compare(share(a), share(b)), + a.queue[0].JoinedAt.Compare(b.queue[0].JoinedAt), + cmp.Compare(a.name, b.name), + ) + }) + admitted := false + for _, l := range eligible { + head := l.queue[0] + if !in.Model.Reserve(head.Demand) { + continue + } + cpu, mem := head.Demand.Total() + l.cpuMilli += cpu + l.memBytes += mem + if !head.Child { + l.nonChild++ + } + l.queue = l.queue[1:] + admit = append(admit, head.Key) + admitted = true + break + } + if !admitted { + break + } + } + + positions := map[types.NamespacedName]Position{} + capBlocked := map[string]bool{} + for _, l := range lines { + for i, c := range l.queue { + positions[c.Key] = Position{Index: int32(i + 1), LineLength: int32(len(l.queue))} + } + if l.capBlocked() { + capBlocked[l.name] = true + } + } + return Decision{Admit: admit, Positions: positions, CapBlocked: capBlocked} +} diff --git a/pkg/controllers/admission/decide_test.go b/pkg/controllers/admission/decide_test.go new file mode 100644 index 00000000..09a95bcb --- /dev/null +++ b/pkg/controllers/admission/decide_test.go @@ -0,0 +1,283 @@ +package admission + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// slotModel is a fake CapacityModel that admits a fixed number of pods. +type slotModel struct { + known bool + slots int +} + +func (m *slotModel) Known() bool { return m.known } +func (m *slotModel) Reserve(d Demand) bool { + if !m.known { + return true + } + if len(d.Pods) > m.slots { + return false + } + m.slots -= len(d.Pods) + return true +} +func (m *slotModel) Free([]PlacedPod) {} +func (m *slotModel) Clone() CapacityModel { c := *m; return &c } +func (m *slotModel) Totals() (int64, int64) { return 0, 0 } + +func key(name string) types.NamespacedName { + return types.NamespacedName{Namespace: "ns", Name: name} +} + +func cand(name, line string, joinedSec int, child bool, cpuMilli ...int64) Candidate { + d := Demand{} + for i, c := range cpuMilli { + d.Pods = append(d.Pods, pod(name+string(rune('0'+i)), c)) + } + return Candidate{Key: key(name), Line: line, JoinedAt: t0.Add(time.Duration(joinedSec) * time.Second), Child: child, Demand: d} +} + +func holder(name, line string, child bool, cpuMilli int64) Holder { + return Holder{Key: key(name), Line: line, Child: child, Demand: Demand{Pods: []PodDemand{pod(name, cpuMilli)}}} +} + +func keys(names ...string) []types.NamespacedName { + var out []types.NamespacedName + for _, n := range names { + out = append(out, key(n)) + } + return out +} + +func TestDecide(t *testing.T) { + oneNodeModel := func(cpu string) CapacityModel { + return NewNodeModel([]corev1.Node{testNode("n1", cpu, "8Gi", "10")}, nil) + } + cases := []struct { + name string + in func() Input + admit []types.NamespacedName + }{ + { + name: "burst on one class, room for two: first two by join time admitted, rest keep order", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("c", "l", 3, false, 100), cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100)}, + Model: &slotModel{known: true, slots: 2}, + } + }, + admit: keys("a", "b"), + }, + { + name: "smaller later session does not jump a larger head in the same line", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("big", "l", 1, false, 100, 100, 100), cand("small", "l", 2, false, 100)}, + Model: &slotModel{known: true, slots: 2}, + } + }, + admit: nil, + }, + { + name: "head of line A does not fit, head of line B does: B admitted", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("a", "A", 1, false, 4000), cand("b", "B", 2, false, 500)}, + Model: oneNodeModel("2"), + } + }, + admit: keys("b"), + }, + { + name: "two lines both fit one more: lower dominant share admitted first", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("a", "A", 1, false, 100), cand("b", "B", 2, false, 100)}, + Holders: []Holder{holder("ha", "A", false, 4000), holder("hb", "B", false, 1000)}, + Model: &slotModel{known: true, slots: 1}, + TotalCPUMilli: 10000, TotalMemBytes: 1 << 40, + } + }, + admit: keys("b"), + }, + { + name: "equal shares across lines: admitted in head JoinedAt order, then by line name", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("c", "C", 2, false, 100), cand("a", "A", 2, false, 100), cand("b", "B", 1, false, 100)}, + Model: &slotModel{known: true, slots: 3}, + } + }, + admit: keys("b", "a", "c"), + }, + { + name: "one free slot: a later child is admitted ahead of an earlier non-child in its line", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("early", "l", 1, false, 100), cand("kid", "l", 2, true, 100)}, + Model: &slotModel{known: true, slots: 1}, + } + }, + admit: keys("kid"), + }, + { + name: "cap 2 with 2 holders: non-child head waits, child head admitted", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100), cand("kid", "l", 2, true, 100)}, + Holders: []Holder{holder("h1", "l", false, 100), holder("h2", "l", false, 100)}, + Caps: map[string]int32{"l": 2}, + Model: &slotModel{known: true, slots: 5}, + } + }, + admit: keys("kid"), + }, + { + name: "cap counts non-child holders admitted in the same pass: cap 1 admits one of two", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100)}, + Caps: map[string]int32{"l": 1}, + Model: &slotModel{known: true, slots: 5}, + } + }, + admit: keys("a"), + }, + { + name: "unplaced holder demand is reserved before waiting heads: nothing new admitted", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 600)}, + Holders: []Holder{{ + Key: key("h"), Line: "l", + Demand: Demand{Pods: []PodDemand{pod("h", 600)}}, + Unplaced: Demand{Pods: []PodDemand{pod("h", 600)}}, + }}, + Model: oneNodeModel("1"), + } + }, + admit: nil, + }, + { + name: "unknown capacity: all admitted in order, cap still applies", + in: func() Input { + return Input{ + Waiting: []Candidate{ + cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100), cand("c", "l", 3, false, 100), + cand("x", "free", 4, false, 100), + }, + Caps: map[string]int32{"l": 2}, + Model: &slotModel{known: false}, + } + }, + admit: keys("a", "b", "x"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Decide(tc.in()) + assert.Equal(t, tc.admit, got.Admit) + }) + } +} + +func TestDecideDoesNotMutateCallerSlices(t *testing.T) { + waiting := []Candidate{cand("b", "l", 2, false, 100), cand("a", "l", 1, false, 100)} + holders := []Holder{holder("h", "l", false, 100)} + Decide(Input{Waiting: waiting, Holders: holders, Model: &slotModel{known: true, slots: 0}}) + assert.Equal(t, key("b"), waiting[0].Key, "Waiting must keep the caller's order") + assert.Len(t, holders, 1) +} + +func TestDecidePositions(t *testing.T) { + got := Decide(Input{ + Waiting: []Candidate{ + cand("a1", "A", 1, false, 100), cand("a2", "A", 2, false, 100), cand("a3", "A", 3, false, 100), + cand("b1", "B", 1, false, 100), cand("b2", "B", 2, false, 100), + }, + Model: &slotModel{known: true, slots: 2}, + }) + require.ElementsMatch(t, keys("a1", "b1"), got.Admit) + assert.Equal(t, map[types.NamespacedName]Position{ + key("a2"): {Index: 1, LineLength: 2}, + key("a3"): {Index: 2, LineLength: 2}, + key("b2"): {Index: 1, LineLength: 1}, + }, got.Positions, "positions are 1-based per line, cover only still-waiting candidates") +} + +// CapBlocked names the lines whose head is still waiting because the line is +// at its cap, not because it does not fit — the lines a cap reclaim can help. +func TestDecideCapBlocked(t *testing.T) { + cases := []struct { + name string + in Input + want map[string]bool + }{ + { + name: "cap 1 with one holder, room to spare: the line is cap-blocked", + in: Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100)}, + Holders: []Holder{holder("h", "l", false, 100)}, + Caps: map[string]int32{"l": 1}, + Model: &slotModel{known: true, slots: 5}, + }, + want: map[string]bool{"l": true}, + }, + { + name: "uncapped line whose head does not fit: not cap-blocked", + in: Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100, 100)}, + Model: &slotModel{known: true, slots: 1}, + }, + want: map[string]bool{}, + }, + { + name: "under its cap, head does not fit: not cap-blocked", + in: Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100, 100)}, + Holders: []Holder{holder("h", "l", false, 100)}, + Caps: map[string]int32{"l": 2}, + Model: &slotModel{known: true, slots: 1}, + }, + want: map[string]bool{}, + }, + { + name: "cap reached by this pass's own admission: cap-blocked", + in: Input{ + Waiting: []Candidate{cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100)}, + Caps: map[string]int32{"l": 1}, + Model: &slotModel{known: true, slots: 5}, + }, + want: map[string]bool{"l": true}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, Decide(tc.in).CapBlocked) + }) + } +} + +// A holder's unplaced pods are reserved one at a time, best effort: one that +// cannot be placed (the model is wrong about it, or it is waiting on a node +// to free) must not stop the others from being reserved, or a waiting head +// would be admitted into space the holder's other pods are about to take. +func TestDecideReservesHolderUnplacedPodsIndividually(t *testing.T) { + in := Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 600)}, + Holders: []Holder{{ + Key: key("h"), Line: "l", + Demand: Demand{Pods: []PodDemand{pod("h-small", 600), pod("h-huge", 4000)}}, + // h-huge fits no node; h-small will land on n1. + Unplaced: Demand{Pods: []PodDemand{pod("h-small", 600), pod("h-huge", 4000)}}, + }}, + Model: NewNodeModel([]corev1.Node{testNode("n1", "1", "8Gi", "10")}, nil), + } + assert.Empty(t, Decide(in).Admit, "the holder's placeable pod is reserved even though its other pod is not") +} diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go new file mode 100644 index 00000000..0e31fe9f --- /dev/null +++ b/pkg/controllers/admission/line.go @@ -0,0 +1,150 @@ +// Package admission models the line of AgentSessions waiting to have their +// whole pod set admitted against an AgentClass's capacity. It is pure: no +// client, no clock, so the ordering rules can be tested in isolation. +package admission + +import ( + "cmp" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" +) + +// PodDemand is the resource request of one pod in a session's pod set. +type PodDemand struct { + Name string + CPUMilli, MemBytes, EphemeralBytes int64 + // Tolerations are the pod's own, from its built spec: a node model places + // the pod only on a node whose NoSchedule/NoExecute taints it tolerates. + Tolerations []corev1.Toleration + // SandboxSession is set on a tool bundle's sandbox pod: the name of the + // bundle SpiceboxSession it runs for. Such a pod is found by that + // SpiceboxSession's label rather than by Name, because one adopted from a + // SandboxWarmPool carries a pool-minted name. + SandboxSession string +} + +// Demand is the whole pod set a session needs admitted together. +type Demand struct { + Pods []PodDemand +} + +// Total sums CPU and memory across the pod set. +func (d Demand) Total() (cpuMilli, memBytes int64) { + for _, p := range d.Pods { + cpuMilli += p.CPUMilli + memBytes += p.MemBytes + } + return cpuMilli, memBytes +} + +// Candidate is one session in a line. Line is the AgentClass +// "namespace/name" the session draws capacity from. +type Candidate struct { + Key types.NamespacedName + Line string + JoinedAt time.Time + Child bool + Demand Demand +} + +func admittedCondition(sess *v1alpha1.AgentSession) *metav1.Condition { + return meta.FindStatusCondition(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) +} + +// JoinedAt is when the session took its place in line, which orders it there: +// its last wake if it has one, else its creation. A cold wake re-joins the +// back of the line, so it counts; a warm wake never left the line, and a +// release (a placement misfit) keeps the place the session already had. The +// waiting timeout is not measured from here but from gate arrival +// (WaitingSince). +func JoinedAt(sess *v1alpha1.AgentSession) time.Time { + if wake := sess.Status.LastWakeAt; wake != nil { + return wake.Time + } + return sess.CreationTimestamp.Time +} + +// WaitingSince is when the session's current wait began: JoinedAt, or the +// later moment its Admitted condition last went False. The gate stamps +// Admitted=False/Queued at the moment a session reaches it, so for a session +// in line this is its gate arrival — not its creation, however long it was +// parked at a start-approval, identity or credential gate first. +func WaitingSince(sess *v1alpha1.AgentSession) time.Time { + joined := JoinedAt(sess) + if c := admittedCondition(sess); c != nil && c.Status == metav1.ConditionFalse && + c.LastTransitionTime.Time.After(joined) { + return c.LastTransitionTime.Time + } + return joined +} + +// IsQueued reports whether the session has reached the admission gate and is +// waiting in line (Admitted=False/Queued). Only such a session is a candidate +// for admission: one parked at an earlier gate, or blocked before it, has not +// reached the line. +func IsQueued(sess *v1alpha1.AgentSession) bool { + c := admittedCondition(sess) + return c != nil && c.Status == metav1.ConditionFalse && c.Reason == v1alpha1.ReasonAdmissionQueued +} + +// PreAdmission reports whether the session started its pods before admission +// was switched on: it has never had an Admitted condition, is not terminal, +// and its status records pods — a runner pod or a bundle SpiceboxSession. The +// reconciler's gate lets such a session through (it already holds what it +// would have been admitted for, and gating it would freeze it with its pods +// up), and the admitter adopts it (Admitted=True/Adopted) so it is counted as +// a holder. +// +// A session whose pods were reaped while it stays non-terminal — slept, or +// Held — is excluded: its status still names the pods it had, but it holds +// nothing, and its next wake must queue like any cold wake. Its reap normally +// records Admitted=False/Reclaimed; one reaped before admission was switched +// on has the condition recorded by its wake, archive wake or hold release +// instead, since those clear the exclusion while the pod names stay. +func PreAdmission(sess *v1alpha1.AgentSession) bool { + switch sess.Status.Phase { + case v1alpha1.AgentSessionPhaseSucceeded, v1alpha1.AgentSessionPhaseFailed, v1alpha1.AgentSessionPhaseHeld: + return false + } + if sess.Status.SleptAt != nil || admittedCondition(sess) != nil { + return false + } + if sess.Status.RunnerPodName != "" { + return true + } + for _, b := range sess.Status.BundleSessions { + if b.SpiceboxSessionName != "" { + return true + } + } + return false +} + +// IsAdmitted reports whether the session's Admitted condition is True. +func IsAdmitted(sess *v1alpha1.AgentSession) bool { + return meta.IsStatusConditionTrue(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) +} + +// Order sorts cands in place into admission order: children first, then +// earliest JoinedAt, then Key string as a deterministic tiebreak. +func Order(cands []Candidate) { + slices.SortStableFunc(cands, func(a, b Candidate) int { + if a.Child != b.Child { + if a.Child { + return -1 + } + return 1 + } + if c := a.JoinedAt.Compare(b.JoinedAt); c != 0 { + return c + } + return cmp.Compare(a.Key.String(), b.Key.String()) + }) +} diff --git a/pkg/controllers/admission/line_test.go b/pkg/controllers/admission/line_test.go new file mode 100644 index 00000000..72775cf2 --- /dev/null +++ b/pkg/controllers/admission/line_test.go @@ -0,0 +1,256 @@ +package admission + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" +) + +var t0 = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + +func at(d time.Duration) time.Time { return t0.Add(d) } + +func mtime(d time.Duration) *metav1.Time { + t := metav1.NewTime(at(d)) + return &t +} + +func sessionAt(created time.Duration) *v1alpha1.AgentSession { + s := &v1alpha1.AgentSession{} + s.CreationTimestamp = *mtime(created) + return s +} + +func withWake(s *v1alpha1.AgentSession, d time.Duration) *v1alpha1.AgentSession { + s.Status.LastWakeAt = mtime(d) + return s +} + +func withAdmitted(s *v1alpha1.AgentSession, status metav1.ConditionStatus, d time.Duration) *v1alpha1.AgentSession { + s.Status.Conditions = append(s.Status.Conditions, metav1.Condition{ + Type: v1alpha1.AgentSessionConditionAdmitted, + Status: status, + LastTransitionTime: *mtime(d), + }) + return s +} + +func TestJoinedAt(t *testing.T) { + cases := []struct { + name string + sess *v1alpha1.AgentSession + want time.Time + }{ + {"never woke: creationTimestamp", sessionAt(0), at(0)}, + { + "cold wake after release (Admitted=False at t1, lastWakeAt t2>t1): lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, time.Minute), 2*time.Minute), + at(2 * time.Minute), + }, + { + "warm wake while admitted (Admitted=True at t0, lastWakeAt after t0): lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionTrue, time.Minute), 2*time.Minute), + at(2 * time.Minute), + }, + { + "cold wake, admitted, then released by a misfit (lastWakeAt t1, Admitted=False at t2>t1): lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, 2*time.Minute), time.Minute), + at(time.Minute), + }, + { + "wake in the same second as the admission transition: lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionTrue, time.Minute), time.Minute), + at(time.Minute), + }, + { + "woke with no Admitted condition yet: lastWakeAt", + withWake(sessionAt(0), time.Minute), + at(time.Minute), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.True(t, tc.want.Equal(JoinedAt(tc.sess)), "want %s got %s", tc.want, JoinedAt(tc.sess)) + }) + } +} + +func TestWaitingSince(t *testing.T) { + cases := []struct { + name string + sess *v1alpha1.AgentSession + want time.Time + }{ + {"never admitted: JoinedAt", sessionAt(0), at(0)}, + { + "released after joining: release time", + withAdmitted(sessionAt(0), metav1.ConditionFalse, 5*time.Minute), + at(5 * time.Minute), + }, + { + "admitted: JoinedAt, not the admission time", + withAdmitted(sessionAt(0), metav1.ConditionTrue, 5*time.Minute), + at(0), + }, + { + "queued at the gate two hours after creation (a start approval in between): the gate-arrival time", + withAdmitted(sessionAt(0), metav1.ConditionFalse, 2*time.Hour), + at(2 * time.Hour), + }, + { + "cold wake after release: the later wake time", + withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, time.Minute), 2*time.Minute), + at(2 * time.Minute), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.True(t, tc.want.Equal(WaitingSince(tc.sess)), "want %s got %s", tc.want, WaitingSince(tc.sess)) + }) + } +} + +func TestIsAdmitted(t *testing.T) { + assert.False(t, IsAdmitted(sessionAt(0)), "no condition: not admitted") + assert.True(t, IsAdmitted(withAdmitted(sessionAt(0), metav1.ConditionTrue, 0)), "Admitted=True") + assert.False(t, IsAdmitted(withAdmitted(sessionAt(0), metav1.ConditionFalse, 0)), "Admitted=False") +} + +func TestIsQueued(t *testing.T) { + queued := func(status metav1.ConditionStatus, reason string) *v1alpha1.AgentSession { + s := sessionAt(0) + s.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.AgentSessionConditionAdmitted, Status: status, Reason: reason, + LastTransitionTime: *mtime(time.Minute), + }} + return s + } + cases := []struct { + name string + sess *v1alpha1.AgentSession + want bool + }{ + {"no Admitted condition: not queued", sessionAt(0), false}, + {"Admitted=False/Queued: queued", queued(metav1.ConditionFalse, v1alpha1.ReasonAdmissionQueued), true}, + {"Admitted=False/Reclaimed: not queued (released, has not reached the gate again)", queued(metav1.ConditionFalse, v1alpha1.ReasonAdmissionReclaimed), false}, + {"Admitted=False/PlacementMismatch: not queued", queued(metav1.ConditionFalse, v1alpha1.ReasonAdmissionMisfit), false}, + {"Admitted=True: not queued", queued(metav1.ConditionTrue, v1alpha1.ReasonAdmitted), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, IsQueued(tc.sess)) + }) + } +} + +func TestPreAdmission(t *testing.T) { + withPods := func(mut func(s *v1alpha1.AgentSession)) *v1alpha1.AgentSession { + s := sessionAt(0) + s.Status.Phase = v1alpha1.AgentSessionPhasePending + mut(s) + return s + } + runner := func(s *v1alpha1.AgentSession) { s.Status.RunnerPodName = "s-runner" } + bundle := func(s *v1alpha1.AgentSession) { + s.Status.BundleSessions = []v1alpha1.ResolvedBundle{{Name: "code", SpiceboxSessionName: "s-code"}} + } + cases := []struct { + name string + sess *v1alpha1.AgentSession + want bool + }{ + {"Pending, bundle sessions recorded, no Admitted condition: pre-admission", withPods(bundle), true}, + {"Running, runner pod recorded, no Admitted condition: pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseRunning + }), true}, + {"warm Idle, runner pod recorded, no Admitted condition: pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseIdle + }), true}, + {"Pending, nothing recorded: a new session, not pre-admission", withPods(func(*v1alpha1.AgentSession) {}), false}, + {"has an Admitted condition: admission already knows it", withPods(func(s *v1alpha1.AgentSession) { + bundle(s) + s.Status.Conditions = []metav1.Condition{{Type: v1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, Reason: v1alpha1.ReasonAdmissionQueued}} + }), false}, + {"slept (pods reaped): not pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseIdle + s.Status.SleptAt = mtime(time.Minute) + }), false}, + {"Held (pods reaped): not pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseHeld + }), false}, + {"terminal: not pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseSucceeded + }), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, PreAdmission(tc.sess)) + }) + } +} + +func TestDemandTotal(t *testing.T) { + d := Demand{Pods: []PodDemand{ + {Name: "runner", CPUMilli: 500, MemBytes: 1 << 20, EphemeralBytes: 9}, + {Name: "sandbox", CPUMilli: 250, MemBytes: 2 << 20}, + }} + cpu, mem := d.Total() + assert.Equal(t, int64(750), cpu) + assert.Equal(t, int64(3<<20), mem) +} + +func TestOrder(t *testing.T) { + key := func(n string) types.NamespacedName { return types.NamespacedName{Namespace: "ns", Name: n} } + cases := []struct { + name string + in []Candidate + order []string + }{ + { + "children before non-children regardless of time", + []Candidate{ + {Key: key("early"), JoinedAt: at(0)}, + {Key: key("late-child"), JoinedAt: at(time.Hour), Child: true}, + }, + []string{"late-child", "early"}, + }, + { + "equal JoinedAt breaks on Key.String()", + []Candidate{ + {Key: key("b"), JoinedAt: at(0)}, + {Key: key("a"), JoinedAt: at(0)}, + }, + []string{"a", "b"}, + }, + { + "otherwise ascending JoinedAt", + []Candidate{ + {Key: key("a"), JoinedAt: at(2 * time.Minute)}, + {Key: key("b"), JoinedAt: at(time.Minute)}, + {Key: key("c"), JoinedAt: at(3 * time.Minute), Child: true}, + {Key: key("d"), JoinedAt: at(0), Child: true}, + }, + []string{"d", "c", "b", "a"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + Order(tc.in) + var got []string + for _, c := range tc.in { + got = append(got, c.Key.Name) + } + assert.Equal(t, tc.order, got) + }) + } +} diff --git a/pkg/controllers/admission/reclaim.go b/pkg/controllers/admission/reclaim.go new file mode 100644 index 00000000..62d9507f --- /dev/null +++ b/pkg/controllers/admission/reclaim.go @@ -0,0 +1,78 @@ +package admission + +import ( + "cmp" + "slices" + "time" + + "k8s.io/apimachinery/pkg/types" +) + +// WarmIdle is an Idle session whose pods still hold capacity and could be +// reclaimed early. +type WarmIdle struct { + Key types.NamespacedName + Line string + IdleSince time.Time + Pods []PlacedPod + // Child is set for a delegated child session, which never counts toward + // its line's cap and so frees no cap slot. + Child bool +} + +// PlanReclaim picks which warm Idle sessions of the head's own line to +// reclaim so the head fits. Only sessions idle at least minWarm are eligible, +// oldest first (ties by key), freed one at a time on a clone of model until +// the head's demand reserves; the freed prefix is returned. It returns nil +// when the model is unknown, when the head already fits, or when freeing every +// eligible session still does not fit. The caller's model is never mutated. +func PlanReclaim(head Candidate, warm []WarmIdle, model CapacityModel, now time.Time, minWarm time.Duration) []types.NamespacedName { + if !model.Known() || model.Clone().Reserve(head.Demand) { + return nil + } + var eligible []WarmIdle + for _, w := range warm { + if w.Line == head.Line && now.Sub(w.IdleSince) >= minWarm { + eligible = append(eligible, w) + } + } + slices.SortFunc(eligible, func(a, b WarmIdle) int { + return cmp.Or(a.IdleSince.Compare(b.IdleSince), cmp.Compare(a.Key.String(), b.Key.String())) + }) + scratch := model.Clone() + var plan []types.NamespacedName + for _, w := range eligible { + scratch.Free(w.Pods) + plan = append(plan, w.Key) + if scratch.Reserve(head.Demand) { + return plan + } + } + return nil +} + +// PlanCapReclaim picks the warm Idle session to reclaim for a head whose line +// is at its cap (Decision.CapBlocked): the head needs a slot, not space, and +// one reclaimed non-child holder gives it one. Of head's line's non-child +// warm sessions idle at least minWarm, the oldest (ties by key) is returned. +// ok is false for a child head (a cap never blocks it) or when none is +// eligible. +func PlanCapReclaim(head Candidate, warm []WarmIdle, now time.Time, minWarm time.Duration) (types.NamespacedName, bool) { + if head.Child { + return types.NamespacedName{}, false + } + var pick *WarmIdle + for i := range warm { + w := &warm[i] + if w.Line != head.Line || w.Child || now.Sub(w.IdleSince) < minWarm { + continue + } + if pick == nil || cmp.Or(w.IdleSince.Compare(pick.IdleSince), cmp.Compare(w.Key.String(), pick.Key.String())) < 0 { + pick = w + } + } + if pick == nil { + return types.NamespacedName{}, false + } + return pick.Key, true +} diff --git a/pkg/controllers/admission/reclaim_test.go b/pkg/controllers/admission/reclaim_test.go new file mode 100644 index 00000000..c9238c85 --- /dev/null +++ b/pkg/controllers/admission/reclaim_test.go @@ -0,0 +1,180 @@ +package admission + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func warmOn(name, line string, idleFor time.Duration, now time.Time, cpuMilli int64) WarmIdle { + return WarmIdle{ + Key: types.NamespacedName{Namespace: "ns", Name: name}, + Line: line, + IdleSince: now.Add(-idleFor), + Pods: []PlacedPod{{Node: "a", PodDemand: pod(name, cpuMilli)}}, + } +} + +// fullNode is one 1-CPU node already holding two 500m pods, so it has no +// headroom until a warm session's pod is freed. +func fullNode() CapacityModel { + return NewNodeModel( + []corev1.Node{testNode("a", "1", "1Gi", "10")}, + []corev1.Pod{testPod("a", "500m"), testPod("a", "500m")}, + ) +} + +func TestPlanReclaim(t *testing.T) { + now := time.Date(2026, 11, 10, 12, 0, 0, 0, time.UTC) + head := func(cpu int64) Candidate { + return Candidate{Key: types.NamespacedName{Namespace: "ns", Name: "head"}, Line: "agent-a", Demand: Demand{Pods: []PodDemand{pod("h", cpu)}}} + } + key := func(n string) types.NamespacedName { return types.NamespacedName{Namespace: "ns", Name: n} } + + cases := []struct { + name string + head Candidate + warm []WarmIdle + model func() CapacityModel + minWarm time.Duration + want []types.NamespacedName + }{ + { + name: "head fits after freeing the oldest of two eligible: returns only the oldest", + head: head(500), + model: fullNode, + warm: []WarmIdle{ + warmOn("newer", "agent-a", 10*time.Minute, now, 500), + warmOn("oldest", "agent-a", 30*time.Minute, now, 500), + }, + minWarm: 5 * time.Minute, + want: []types.NamespacedName{key("oldest")}, + }, + { + name: "head needs both eligible sessions: returns both, oldest first", + head: head(1000), + model: fullNode, + warm: []WarmIdle{ + warmOn("newer", "agent-a", 10*time.Minute, now, 500), + warmOn("oldest", "agent-a", 30*time.Minute, now, 500), + }, + minWarm: 5 * time.Minute, + want: []types.NamespacedName{key("oldest"), key("newer")}, + }, + { + name: "idle-since tie: breaks on key order", + head: head(500), + model: fullNode, + warm: []WarmIdle{warmOn("b", "agent-a", 10*time.Minute, now, 500), warmOn("a", "agent-a", 10*time.Minute, now, 500)}, + minWarm: time.Minute, + want: []types.NamespacedName{key("a")}, + }, + { + name: "candidate idle 4m with minWarm 5m: not eligible, returns nil", + head: head(500), + model: fullNode, + warm: []WarmIdle{warmOn("young", "agent-a", 4*time.Minute, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "other line's warm session: never chosen", + head: head(500), + model: fullNode, + warm: []WarmIdle{warmOn("other", "agent-b", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "freeing every eligible session still does not fit: nil, nothing reclaimed", + head: head(1500), + model: fullNode, + warm: []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500), warmOn("w2", "agent-a", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "head already fits without freeing: nil", + head: head(500), + model: func() CapacityModel { return NewNodeModel([]corev1.Node{testNode("a", "1", "1Gi", "10")}, nil) }, + warm: []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "unknown capacity: nil, unknown capacity already admits", + head: head(500), + model: func() CapacityModel { return NewNodeModel(nil, nil) }, + warm: []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, PlanReclaim(tc.head, tc.warm, tc.model(), now, tc.minWarm)) + }) + } + + t.Run("plan does not mutate the caller's model", func(t *testing.T) { + m := fullNode() + w := []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500)} + assert.Equal(t, []types.NamespacedName{key("w1")}, PlanReclaim(head(500), w, m, now, time.Minute)) + assert.False(t, m.Reserve(head(500).Demand), "caller's model must still be full after planning") + }) +} + +// A line at its cap needs a SLOT, not space: PlanCapReclaim picks the single +// oldest non-child warm idle session of the head's line past minWarm. +func TestPlanCapReclaim(t *testing.T) { + now := time.Date(2026, 11, 10, 12, 0, 0, 0, time.UTC) + head := Candidate{Key: types.NamespacedName{Namespace: "ns", Name: "head"}, Line: "agent-a"} + childWarm := func(w WarmIdle) WarmIdle { w.Child = true; return w } + cases := []struct { + name string + head Candidate + warm []WarmIdle + want string + wantOK bool + }{ + { + name: "two eligible warm sessions: the oldest only", + head: head, + warm: []WarmIdle{ + warmOn("young", "agent-a", 10*time.Minute, now, 100), + warmOn("old", "agent-a", 30*time.Minute, now, 100), + }, + want: "old", wantOK: true, + }, + { + name: "oldest is younger than minWarm, another line's is older: nothing", + head: head, + warm: []WarmIdle{ + warmOn("young", "agent-a", time.Minute, now, 100), + warmOn("other", "agent-b", time.Hour, now, 100), + }, + }, + { + name: "only a delegated child is warm (children do not count toward the cap): nothing", + head: head, + warm: []WarmIdle{childWarm(warmOn("kid", "agent-a", time.Hour, now, 100))}, + }, + { + name: "child head (never cap-blocked): nothing", + head: Candidate{Key: head.Key, Line: "agent-a", Child: true}, + warm: []WarmIdle{warmOn("old", "agent-a", time.Hour, now, 100)}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := PlanCapReclaim(tc.head, tc.warm, now, 5*time.Minute) + assert.Equal(t, tc.wantOK, ok) + if tc.wantOK { + assert.Equal(t, tc.want, got.Name) + } + }) + } +} diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go new file mode 100644 index 00000000..5fbc87da --- /dev/null +++ b/pkg/controllers/agentsession/admission_gate.go @@ -0,0 +1,272 @@ +// pkg/controllers/agentsession/admission_gate.go +// +// The reconciler's half of whole-set admission. The admitter decides WHO may +// start (Admitted=True); this file makes the reconciler honour that decision — +// a booting session creates none of its pods until admitted — and hands +// admission back (Admitted=False/Reclaimed) at the moment its pods are +// reclaimed, so the admitter never counts capacity a session no longer holds. +// Its safety net hands admission back too (Admitted=False/PlacementMismatch) +// when an admitted set the scheduler cannot place would otherwise hold its +// placed half while the rest waits. +package agentsession + +import ( + "context" + "fmt" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/conditions" +) + +// awaitingAdmission reports whether this pass must create none of the +// session's pods because it is booting and not yet admitted. +// +// Only a BOOTING session (Pending, or not yet phased) waits. Every pod a +// session owns is created on a booting pass — a parked phase never provisions +// (sandboxProvisioningDesired), and a Running or AwaitingDecision session +// reached that phase through a Pending pass that was already admitted. So +// gating boot alone keeps the invariant "no pod without Admitted=True", while +// a session that was already running when admission was switched on (no +// Admitted condition at all) keeps reconciling its live pods instead of +// freezing mid-turn. +// +// A warm wake — Idle back to Pending while still Admitted=True — is not +// gated: the session never gave up its place, so it provisions at once. +// +// Nor is a session that booted before admission was switched on +// (admission.PreAdmission): it already has pods, so gating it would leave +// them up while it waited for a second copy of its set. It is let through, +// with no Queued marker, and the admitter adopts it as a holder. +func (r *Reconciler) awaitingAdmission(sess *spiceboxv1alpha1.AgentSession) bool { + if !r.AdmissionEnabled || admission.IsAdmitted(sess) || admission.PreAdmission(sess) { + return false + } + return sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending +} + +// waitForAdmission is the whole reconcile tail of a booting session that is +// not admitted: mark its arrival in line, give up once it has waited past the +// capacity-hold timeout, otherwise surface the wait and requeue. It creates +// nothing. +// +// The first gated pass of a wait stamps Admitted=False/Queued at now (see +// markQueued), and the wait is measured from admission.WaitingSince — that +// arrival time. A session parked for hours at a start-approval, identity or +// credential gate therefore gets a full window from the moment it actually +// joins the line, and so does one released back to it (a placement misfit or +// a reclaim followed by a cold wake), rather than the remainder of an earlier +// one. +// +// A negative --capacity-hold-timeout (hold disabled) means no give-up time: +// there is no pod whose ordinary readiness deadline could apply instead, so +// the session waits for admission with no CapacityHold notice. +func (r *Reconciler) waitForAdmission(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (ctrl.Result, error) { + r.markQueued(sess) + timeout := r.effectiveCapacityHoldTimeout() + since := admission.WaitingSince(sess) + until := since.Add(timeout) + if timeout > 0 && r.now().After(until) { + log.FromContext(ctx).Info("session waited past the capacity-hold timeout without admission; failing", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, + "waitingSince", since.UTC().Format(time.RFC3339), "timeout", timeout.String()) + r.expireCapacityHold(sess, timeout) + return r.markBootFailed(ctx, sess, spiceboxv1alpha1.ReasonAgentSessionBundleFail, + admissionWaitExpiredMessage(timeout)) + } + requeue := capacityHoldRequeue + if timeout > 0 { + r.syncAdmissionHold(sess, until) + // Wake for the give-up time itself rather than up to a requeue late. + if remaining := until.Sub(r.now()); remaining > 0 && remaining < requeue { + requeue = remaining + time.Second + } + } + return ctrl.Result{RequeueAfter: requeue}, r.applyStatus(ctx, sess) +} + +// markQueued records, on sess, that it has reached the admission gate: +// Admitted=False/Queued with now as the transition time, unless it already +// carries that marker (a later pass of the same wait keeps the original +// arrival). The admitter considers only queued sessions, so this marker is +// what puts a session in line; a session parked or blocked before the gate +// never carries it and never reserves anything. The caller persists it. +// +// The transition time is set explicitly rather than left to +// meta.SetStatusCondition, which keeps the old time on a False→False change: +// a session released earlier (Reclaimed, PlacementMismatch) arrives in line +// again now, not at its release. +func (r *Reconciler) markQueued(sess *spiceboxv1alpha1.AgentSession) { + if admission.IsQueued(sess) { + return + } + now := metav1.NewTime(r.now()) + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, + Message: "waiting in line for cluster capacity", + LastTransitionTime: now, + }) + conditions.Find(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted).LastTransitionTime = now +} + +// admissionWaitExpiredMessage is the Failed condition message when a session +// waited unadmitted past the capacity-hold timeout. No pod was ever created, +// so unlike capacityHoldExpiredMessage there is no scheduler detail to quote. +func admissionWaitExpiredMessage(waited time.Duration) string { + return fmt.Sprintf("session waited %s in line for cluster capacity and none freed up; none of its pods were created", waited) +} + +// releaseAdmissionOnReclaim records, on sess, that the session's pods are being +// reclaimed: Admitted True→False/Reclaimed. The caller persists it in the same +// status write that records the reclaim, and calls it before marking the +// session slept or Held. A session that is neither Admitted=True nor (with +// admission on) pre-admission is left untouched — admission off, or never +// admitted — so no condition appears on sessions that never had one. +// +// A pre-admission session not yet adopted gets the release too: once its +// pods are gone, a wake that clears SleptAt would otherwise make it look +// pre-admission again and let it past the gate with nothing reserved. +// Reports whether it changed sess. +func (r *Reconciler) releaseAdmissionOnReclaim(sess *spiceboxv1alpha1.AgentSession) bool { + if !admission.IsAdmitted(sess) && (!r.AdmissionEnabled || !admission.PreAdmission(sess)) { + return false + } + r.setAdmissionReclaimed(sess) + return true +} + +// releaseUnrecordedAdmission records Admitted=False/Reclaimed on a session +// that is resuming from a reap admission never saw — slept, archived or Held +// before admission was switched on, or while it was off — and so carries no +// Admitted condition. Its status still names the pods it had (RunnerPodName +// and BundleSessions are never cleared), so once the resume puts it back in a +// booting phase it would look pre-admission and pass the gate with nothing +// reserved. With the condition it queues like any cold wake. The caller +// persists it, in the same write as the resume. +// +// Inert with admission off, and on a session that already has an Admitted +// condition (its own reap released it). Reports whether it changed sess. +func (r *Reconciler) releaseUnrecordedAdmission(sess *spiceboxv1alpha1.AgentSession) bool { + if !r.AdmissionEnabled || conditions.Find(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) != nil { + return false + } + r.setAdmissionReclaimed(sess) + return true +} + +// setAdmissionReclaimed writes Admitted=False/Reclaimed onto sess. +func (r *Reconciler) setAdmissionReclaimed(sess *spiceboxv1alpha1.AgentSession) { + // r.now() for the transition time: admission.JoinedAt and WaitingSince + // compare it against status.lastWakeAt, and the admitter orders the line + // by them. + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionReclaimed, + Message: "the session's pods were reclaimed; a later wake rejoins the line", + LastTransitionTime: metav1.NewTime(r.now()), + }) +} + +// admissionMisfitGrace is how long an admitted session's bundle pod may stay +// unschedulable on a capacity shortfall before the session gives its pods back +// and returns to its line. The admitter's model said the whole set fits; a pod +// still unplaced this long later means the model was wrong (a pod it did not +// know about, a node draining), and holding the placed half while the rest +// waits is the hold-and-wait admission exists to end. +const admissionMisfitGrace = 2 * time.Minute + +// placementMisfit reports whether this pass must release an admitted session +// whose set the scheduler could not place. stalled is the bundle stage's own +// scheduling-stall verdict (already false for a bundle that regressed +// mid-session, which the ordinary deadline handles). +// +// Only a capacity SHORTFALL counts, for the same reason only a shortfall is +// held (capacityShortfall): a pod blocked by a taint or a volume-affinity +// conflict would be re-admitted at once, fail to place again, and cycle +// forever with its waiting-timeout clock restarting at every release. Left +// alone, it keeps the ordinary readiness deadline and its specific failure +// message. +// +// A session admitted under an unknown capacity model +// (ReasonAdmittedUnknownCapacity) is skipped as well: no fit check admitted +// it, so a stall is not the model being wrong, and releasing it would only +// re-admit it at once into the same stall. Its pods keep the pod-level +// capacity hold and that hold's deadline. +func (r *Reconciler) placementMisfit(sess *spiceboxv1alpha1.AgentSession, stall schedulingStall, stalled bool) bool { + if !r.AdmissionEnabled || !stalled || !admission.IsAdmitted(sess) { + return false + } + if c := conditions.Find(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted); c.Reason == spiceboxv1alpha1.ReasonAdmittedUnknownCapacity { + return false + } + booting := sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending + return booting && capacityShortfall(stall.reason) && stall.pending >= admissionMisfitGrace +} + +// releaseMisfit reaps an admitted session's pods and records +// Admitted=False/PlacementMismatch, so the admitter re-queues it. The release +// is written only after the reap succeeds — released while its pods still +// stood, the admitter would count their capacity twice. A failed reap or +// status write leaves the session admitted, and the next pass decides afresh +// from whatever still stands. +// +// The session keeps its place: admission.JoinedAt is its last wake (a cold +// wake re-joined the line then) or its creation, never an admission +// transition, so it re-joins at the front of its line (it was admitted +// fairly; the model was wrong, not the session). admission.WaitingSince +// restarts when it is queued again at the gate, so the capacity-hold timeout +// gives it a fresh window rather than the remainder of its first one. +// +// That fresh window is why the release is bounded by JoinedAt instead. A +// stall the admitter's model cannot see — a shortfall on the only nodes the +// pod tolerates, beside tainted nodes the model still counts — re-admits the +// session at once, stalls again, and releases again: WaitingSince and the +// bundle deadline both restart every cycle, so neither would ever fire. Once +// the session has been in line longer than the capacity-hold timeout since it +// joined, the release becomes the same failure the waiting timeout gives. +// +// The bundle stage's CapacityHold=True ("gives up at …") is released in the +// same write: its give-up time described pods that no longer exist, and left +// standing, every later stall would rewrite it into a fresh "queued" notice. +func (r *Reconciler) releaseMisfit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, stall schedulingStall) (ctrl.Result, error) { + logger := log.FromContext(ctx) + timeout := r.effectiveCapacityHoldTimeout() + joined := admission.JoinedAt(sess) + expired := timeout > 0 && r.now().Sub(joined) > timeout + logger.Info("admitted session's pods could not be placed; releasing them", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, + "pod", stall.podName, "unschedulableFor", stall.pending.String(), + "schedulerMessage", stall.reason, "joinedAt", joined.UTC().Format(time.RFC3339), + "failingAfterCapacityHoldTimeout", expired) + if _, err := r.reapSessionPods(ctx, sess); err != nil { + return ctrl.Result{}, err // already logged inside reapSessionPods; retried next pass + } + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionMisfit, + Message: fmt.Sprintf("%s could not be placed for %s (%s); its pods were released", stall.podName, stall.pending.Round(time.Second), strings.TrimSuffix(stall.reason, ".")), + LastTransitionTime: metav1.NewTime(r.now()), + }) + if expired { + r.expireCapacityHold(sess, timeout) + return r.markBootFailed(ctx, sess, spiceboxv1alpha1.ReasonAgentSessionBundleFail, + capacityHoldExpiredMessage("sandbox bundle(s)", timeout, stall)) + } + r.releaseCapacityHold(sess) + if err := r.applyStatus(ctx, sess); err != nil { + logger.Info("misfit: recording the admission release failed; will retry", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, "err", err.Error()) + return ctrl.Result{}, fmt.Errorf("record placement-mismatch release: %w", err) + } + return ctrl.Result{RequeueAfter: capacityHoldRequeue}, nil +} diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go new file mode 100644 index 00000000..cc5bafc7 --- /dev/null +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -0,0 +1,784 @@ +// pkg/controllers/agentsession/admission_gate_test.go +// +// The admission gate over the reconciler: with AdmissionEnabled, a booting +// session creates none of its pods until the admitter sets Admitted=True, waits +// in line under CapacityHold=True/WaitingForAdmission, and fails once it has +// waited unadmitted past the capacity-hold timeout. Reclaiming a session's pods +// (sleep or terminal reap) releases its admission. +package agentsession_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" + "github.com/authzed/openagentprimitives/pkg/memory" +) + +// withAdmittedCondition stamps an Admitted condition onto sess. +func withAdmittedCondition(sess *spiceboxv1alpha1.AgentSession, status metav1.ConditionStatus, reason string, at time.Time) { + sess.Status.Conditions = append(sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: status, + Reason: reason, LastTransitionTime: metav1.NewTime(at), + }) +} + +// exists reports whether obj named name is present in the fake client. +func exists(t *testing.T, c client.Client, name string, obj client.Object) bool { + t.Helper() + err := c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: name}, obj) + if apierrors.IsNotFound(err) { + return false + } + require.NoError(t, err, "get %s", name) + return true +} + +func TestAdmissionGate(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + + cases := []struct { + name string + enabled bool // Reconciler.AdmissionEnabled + bundles bool // class has a tool bundle (else: the runner is the first pod) + createdAt time.Duration // session creation, relative to now + // prep stamps admission state onto the session before it is loaded. + prep func(sess *spiceboxv1alpha1.AgentSession) + // holdTimeout is Reconciler.CapacityHoldTimeout; 0 = the default. + holdTimeout time.Duration + + wantPods bool // the bundle SpiceboxSession (bundles) or runner pod (no bundles) exists + wantFailed bool + wantHoldStatus metav1.ConditionStatus // "" = CapacityHold absent + wantHoldReason string + wantMsg []string // substrings of the CapacityHold (held) or Failed message + // wantQueuedAt, when set, is the expected Admitted=False/Queued + // transition time: the moment the session reached the gate. + wantQueuedAt time.Time + // wantNoAdmittedCnd: the gate wrote no Admitted condition at all. + wantNoAdmittedCnd bool + }{ + { + name: "Pending, queued 2m ago: no SpiceboxSession; CapacityHold=True/WaitingForAdmission after grace", + enabled: true, bundles: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-2*time.Minute)) + }, + wantHoldStatus: metav1.ConditionTrue, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + wantMsg: []string{"out of room", "gives up at 17:58 UTC"}, + }, + { + name: "Pending, queued 2m ago, no bundles: no runner pod", + enabled: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-2*time.Minute)) + }, + wantHoldStatus: metav1.ConditionTrue, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + }, + { + name: "Pending, not admitted, inside the grace: no pods and no hold notice yet", + enabled: true, bundles: true, createdAt: -10 * time.Second, + }, + { + name: "first reaches the gate 3h after creation: queued from now, not failed, no hold notice yet", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + wantQueuedAt: nowT, + }, + { + name: "Pending, admitted: bundle SpiceboxSession created as today, no hold", + enabled: true, bundles: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-time.Minute)) + }, + wantPods: true, + }, + { + name: "Pending, admitted, no bundles: runner pod created as today", + enabled: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-time.Minute)) + }, + wantPods: true, + }, + { + name: "warm wake (Pending again with Admitted still True): provisions with no hold", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-2*time.Hour)) + wake := metav1.NewTime(nowT.Add(-time.Minute)) + s.Status.LastWakeAt = &wake + }, + wantPods: true, + }, + { + name: "admitted after a held wait: the WaitingForAdmission hold is released, pods created", + enabled: true, bundles: true, createdAt: -20 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Conditions = append(s.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionCapacityHold, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + LastTransitionTime: metav1.NewTime(nowT.Add(-19 * time.Minute)), + }) + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-time.Second)) + }, + wantPods: true, + wantHoldStatus: metav1.ConditionFalse, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldReleased, + }, + { + name: "queued past the timeout: Failed, CapacityHold=False/HoldExpired", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-61*time.Minute)) + }, + wantFailed: true, + wantHoldStatus: metav1.ConditionFalse, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldExpired, + wantMsg: []string{"waited 1h0m0s", "cluster capacity"}, + }, + { + name: "hold disabled (negative timeout), queued 3h ago: waits on, not failed, no CapacityHold give-up time posted", + enabled: true, bundles: true, createdAt: -3 * time.Hour, holdTimeout: -1, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-3*time.Hour)) + }, + wantQueuedAt: nowT.Add(-3 * time.Hour), + }, + { + name: "released long after creation: re-queued at the gate, the wait restarts there, not failed", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionMisfit, nowT.Add(-5*time.Minute)) + }, + wantQueuedAt: nowT, + }, + { + name: "pre-admission: booted before admission (bundle recorded, no Admitted condition), long ago: proceeds, not queued, not failed", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + s.Status.BundleSessions = []spiceboxv1alpha1.ResolvedBundle{{ + Name: codeBundle.Name, SpiceboxSessionName: agentsession.BundleSessionName(s, codeBundle), + }} + }, + wantPods: true, + wantNoAdmittedCnd: true, + }, + { + name: "AdmissionEnabled=false: bundle SpiceboxSession created without Admitted", + bundles: true, createdAt: -2 * time.Minute, + wantPods: true, + }, + { + name: "AdmissionEnabled=false, no bundles: runner pod created without Admitted", + createdAt: -2 * time.Minute, + wantPods: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + var ac *spiceboxv1alpha1.AgentClass + if tc.bundles { + ac = classWithBundle("ac1", codeBundle) + } else { + ac = classWithBundle("ac1") + } + sess := sessionCreatedAt("s1", "ac1", nowT.Add(tc.createdAt)) + if tc.prep != nil { + tc.prep(sess) + } + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = tc.enabled + r.CapacityHoldTimeout = tc.holdTimeout + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + if tc.bundles { + assert.Equal(t, tc.wantPods, + exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "bundle SpiceboxSession present") + } else { + assert.Equal(t, tc.wantPods, + exists(t, c, agentsession.RunnerPodName(&got), &corev1.Pod{}), + "runner pod present") + } + + hold := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionCapacityHold) + if tc.wantHoldStatus == "" { + assert.Nil(t, hold, "no CapacityHold condition") + } else if assert.NotNil(t, hold, "CapacityHold condition") { + assert.Equal(t, tc.wantHoldStatus, hold.Status) + assert.Equal(t, tc.wantHoldReason, hold.Reason) + } + + if tc.wantNoAdmittedCnd { + assert.Nil(t, apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + "no Admitted condition: the gate let it through without queueing it") + } + if !tc.wantQueuedAt.IsZero() { + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if assert.NotNil(t, adm, "Admitted condition") { + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason) + assert.Equal(t, tc.wantQueuedAt, adm.LastTransitionTime.Time.UTC(), "queued at gate arrival") + } + } + if !tc.wantFailed { + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") + if hold != nil && hold.Status == metav1.ConditionTrue { + for _, s := range tc.wantMsg { + assert.Contains(t, hold.Message, s) + } + } + return + } + require.Equal(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase) + assert.Equal(t, spiceboxv1alpha1.ReasonAgentSessionBundleFail, got.Status.FailureReason) + fc := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionFailed) + require.NotNil(t, fc, "Failed condition set") + for _, s := range tc.wantMsg { + assert.Contains(t, fc.Message, s) + } + }) + } +} + +// A session that clears its start approval two hours after creation reaches +// the admission gate then, not at creation: it is queued from that moment +// (Admitted=False/Queued at now), is not failed for having "waited" through +// the approval, creates nothing, and provisions normally once admitted. +func TestAdmissionGate_QueuedOnArrivalAfterStartApproval(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + clock := func() time.Time { return nowT } + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + ac := classWithBundle("ac1", codeBundle) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-2*time.Hour)) + // Approved: channelsd removed the park marker; the phase still says parked. + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingStartApproval + parked := metav1.NewTime(nowT.Add(-2 * time.Hour)) + sess.Status.StartApprovalParkedAt = &parked + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = true + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + require.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "an approval wait is not a wait in line") + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, got.Status.Phase) + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, adm, "Admitted condition written at the gate") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "queued at gate arrival") + assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting timeout runs from gate arrival") + assert.False(t, exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "no pods before admission") + + // The admitter admits it. + apimeta.SetStatusCondition(&got.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(nowT), + }) + require.NoError(t, c.Status().Update(context.Background(), &got), "admit") + runReconciles(t, r, "s1", 3) + + got = getSession(t, c, "s1") + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase) + assert.True(t, exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "admitted: the bundle SpiceboxSession is created") +} + +// A terminal reap releases admission in the status write that accompanies it, +// and never invents an Admitted condition on a session that never had one. +func TestAdmissionGate_TerminalReapReleases(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + cases := []struct { + name string + admitted bool + wantAdmitted *metav1.ConditionStatus // nil = condition absent + }{ + {name: "admitted Succeeded session reaped: Admitted=False/Reclaimed", admitted: true, wantAdmitted: ptrTo(metav1.ConditionFalse)}, + {name: "never-admitted Succeeded session reaped: no Admitted condition written"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-time.Hour)) + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + finished := metav1.NewTime(nowT.Add(-time.Minute)) + sess.Status.FinishedAt = &finished + if tc.admitted { + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-50*time.Minute)) + } + r, c := fakeReconciler(t, clock, sess) + r.AdmissionEnabled = true + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if tc.wantAdmitted == nil { + assert.Nil(t, adm, "no Admitted condition") + return + } + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, *tc.wantAdmitted, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") + }) + } +} + +// The sleep reap stamps SleptAt and releases admission in the same status +// write (one WriteOwned): a wake clears SleptAt, so a release written +// separately could be lost between the two, and the session would sit +// admitted with no pods. +func TestAdmissionGate_SleepReleasesInOneWrite(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + clock := func() time.Time { return nowT } + + idle := metav1.NewTime(nowT.Add(-time.Hour)) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-2*time.Hour)) + sess.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + sess.Status.LastIdleAt = &idle + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-90*time.Minute)) + ac := classWithBundle("ac1") + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = true + r.DefaultSessionSleepAfter = 10 * time.Minute + + live := getSession(t, c, "s1") + slept, _, err := r.ReconcileSleepForTest(ctx, &live, ac) + require.NoError(t, err) + require.True(t, slept, "past-grace Idle session sleeps") + + got := getSession(t, c, "s1") + assert.NotNil(t, got.Status.SleptAt, "SleptAt stamped") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") +} + +// A pre-admission session (pods started before admission, no Admitted +// condition) that sleeps gives its place up like an admitted one: +// Admitted=False/Reclaimed in the sleep's write. Without it the session would +// look pre-admission again once a wake clears SleptAt, and bypass the gate +// with no pods. +func TestAdmissionGate_PreAdmissionSleepReleases(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + clock := func() time.Time { return nowT } + + idle := metav1.NewTime(nowT.Add(-time.Hour)) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-2*time.Hour)) + sess.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + sess.Status.LastIdleAt = &idle + sess.Status.RunnerPodName = "s1-runner" + ac := classWithBundle("ac1") + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = true + r.DefaultSessionSleepAfter = 10 * time.Minute + + live := getSession(t, c, "s1") + require.True(t, admission.PreAdmission(&live), "fixture is pre-admission") + slept, _, err := r.ReconcileSleepForTest(ctx, &live, ac) + require.NoError(t, err) + require.True(t, slept, "past-grace Idle session sleeps") + + got := getSession(t, c, "s1") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + got.Status.SleptAt = nil + assert.False(t, admission.PreAdmission(&got), "after a wake clears SleptAt it is a normal session that queues") +} + +// A session slept or held before admission was switched on (an upgrade, or a +// run with admission off) has no Admitted condition, and its status still +// names the pods it had before they were reaped. Resumed with admission on — +// a cold wake, an archive wake, or its hold released — it must queue at the gate like any +// cold wake and create none of its pods, not pass as pre-admission on those +// stale names. With admission off it boots at once, as it always has. +func TestAdmissionGate_ResumeWithoutAdmittedConditionQueues(t *testing.T) { + nowT := time.Now().UTC().Truncate(time.Second) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + + // slept: an Idle channel session whose pods the sleep reaped, with a wake + // requested after it slept. + slept := func(s *spiceboxv1alpha1.AgentSession) []client.Object { + idle := metav1.NewTime(nowT.Add(-2 * time.Hour)) + sleptAt := metav1.NewTime(nowT.Add(-time.Hour)) + s.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + s.Status.LastIdleAt = &idle + s.Status.SleptAt = &sleptAt + s.Annotations = map[string]string{ + spiceboxv1alpha1.AnnotationWakeRequestedAt: nowT.Add(-time.Second).Format(time.RFC3339Nano), + } + return nil + } + // archived: swept to Succeeded with its pods reaped, then woken. + archived := func(s *spiceboxv1alpha1.AgentSession) []client.Object { + finished := metav1.NewTime(nowT.Add(-time.Hour)) + s.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + s.Status.FinishedAt = &finished + s.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionIdle, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAgentSessionArchived, LastTransitionTime: finished, + }} + s.Annotations = map[string]string{ + spiceboxv1alpha1.AnnotationWakeRequestedAt: nowT.Add(-time.Second).Format(time.RFC3339Nano), + } + return nil + } + // held: parked at Held with its pods reaped, and the hold since released. + held := func(s *spiceboxv1alpha1.AgentSession) []client.Object { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseHeld + return []client.Object{&spiceboxv1alpha1.SessionHold{ + ObjectMeta: metav1.ObjectMeta{Name: "h1", Namespace: "default"}, + Spec: spiceboxv1alpha1.SessionHoldSpec{ + SessionRef: spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "s1"}, + Reason: "denial streak", + }, + Status: spiceboxv1alpha1.SessionHoldStatus{ + Phase: spiceboxv1alpha1.SessionHoldPhaseReleased, ReleasedBy: "user:approver-1", + }, + }} + } + + cases := []struct { + name string + enabled bool + prep func(s *spiceboxv1alpha1.AgentSession) []client.Object + }{ + {name: "slept before admission, woken with admission on: Admitted=False/Queued, no pods created", enabled: true, prep: slept}, + {name: "held before admission, released with admission on: Admitted=False/Queued, no pods created", enabled: true, prep: held}, + {name: "archived before admission, woken with admission on: Admitted=False/Queued, no pods created", enabled: true, prep: archived}, + {name: "slept, woken with admission off: boots at once, no Admitted condition", prep: slept}, + {name: "held, released with admission off: boots at once, no Admitted condition", prep: held}, + {name: "archived, woken with admission off: boots at once, no Admitted condition", prep: archived}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + ac := classWithBundle("ac1", codeBundle) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-3*time.Hour)) + extra := tc.prep(sess) + // The pods it had before they were reaped, still named on status. + sess.Status.RunnerPodName = agentsession.RunnerPodName(sess) + sess.Status.BundleSessions = []spiceboxv1alpha1.ResolvedBundle{{ + Name: codeBundle.Name, SpiceboxSessionName: agentsession.BundleSessionName(sess, codeBundle), + }} + require.False(t, admission.PreAdmission(sess), "fixture holds no pods while parked") + + r, c := fakeReconciler(t, clock, append([]client.Object{ac, sess}, extra...)...) + r.AdmissionEnabled = tc.enabled + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, got.Status.Phase, "resumed to Pending") + bundlePresent := exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}) + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if !tc.enabled { + assert.True(t, bundlePresent, "admission off: the bundle SpiceboxSession is created at once") + assert.Nil(t, adm, "admission off: no Admitted condition written") + return + } + assert.False(t, bundlePresent, "no bundle SpiceboxSession before admission") + assert.False(t, exists(t, c, agentsession.RunnerPodName(&got), &corev1.Pod{}), "no runner pod before admission") + if assert.NotNil(t, adm, "Admitted condition") { + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason) + } + assert.True(t, admission.IsQueued(&got), "in line for the admitter") + }) + } +} + +func ptrTo[T any](v T) *T { return &v } + +// An ADMITTED booting session whose bundle pod the scheduler still cannot +// place after admissionMisfitGrace gives its pods back and returns to its line +// (Admitted=False/PlacementMismatch), keeping its place: JoinedAt does not +// move, and the bundle stage's stale CapacityHold goes with the pods. Short of +// the grace, with admission off, or on a stall another session finishing +// cannot cure, nothing is released. Once the session has been in line longer +// than the capacity-hold timeout since it joined, the release fails it +// instead, so a stall the admitter's model cannot see cannot cycle forever. +func TestAdmissionGate_Misfit(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + // A shortfall on the nodes the pod tolerates, beside a tainted node the + // admitter's model still counts as room: capacityShortfall says yes, and + // the model re-admits at once after every release. + const mixedSchedMsg = "0/4 nodes are available: 3 Insufficient cpu, 1 node(s) had untolerated taint(s)." + + cases := []struct { + name string + enabled bool // Reconciler.AdmissionEnabled + admitted bool // the session carries Admitted=True + // admittedReason is the Admitted=True reason; "" = Admitted. + admittedReason string + pending time.Duration // how long the bundle pod has been unschedulable + createdAgo time.Duration // session creation (its JoinedAt); 0 = 20m + schedMsg string + woken bool // the session was cold-woken before this admission + heldBefore bool // the bundle stage already set CapacityHold=True/Waiting + + wantReaped bool + wantMisfit bool // Admitted=False/PlacementMismatch at now + wantFailed bool // Failed/BundleFailed, CapacityHold=False/HoldExpired + wantAdmittedNil bool // no Admitted condition at all + }{ + { + name: "admitted, pods unschedulable 2m: pods reaped, released and re-queued, JoinedAt unchanged, CapacityHold released", + enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, heldBefore: true, + wantReaped: true, wantMisfit: true, + }, + { + name: "admitted after a cold wake 10m ago, session created 3h ago, pods unschedulable 2m: released and re-queued, not failed, JoinedAt is the wake", + enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, woken: true, + createdAgo: 3 * time.Hour, + wantReaped: true, wantMisfit: true, + }, + { + name: "admitted, mixed taint+Insufficient stall, released and re-admitted repeatedly: fails once JoinedAt+timeout passes", + enabled: true, admitted: true, pending: 2 * time.Minute, createdAgo: 70 * time.Minute, + schedMsg: mixedSchedMsg, heldBefore: true, + wantReaped: true, wantMisfit: true, wantFailed: true, + }, + { + name: "admitted under unknown capacity, pods unschedulable 2m: no misfit release, pods kept under the pod-level hold, still admitted", + enabled: true, admitted: true, admittedReason: spiceboxv1alpha1.ReasonAdmittedUnknownCapacity, + pending: 2 * time.Minute, schedMsg: capacitySchedMsg, + }, + { + name: "admitted, pods unschedulable 1m (inside the misfit grace): pods kept, still admitted", + enabled: true, admitted: true, pending: time.Minute, schedMsg: capacitySchedMsg, + }, + { + name: "admitted, taint-only stall 2m: not a capacity misfit, pods kept, still admitted", + enabled: true, admitted: true, pending: 2 * time.Minute, + schedMsg: "0/6 nodes are available: 6 node(s) had untolerated taint(s).", + }, + { + name: "admission off, pods unschedulable 2m: pods kept under the pod-level hold, no Admitted condition", + pending: 2 * time.Minute, + schedMsg: capacitySchedMsg, wantAdmittedNil: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + started := nowT.Add(-tc.pending) + createdAgo := tc.createdAgo + if createdAgo == 0 { + createdAgo = 20 * time.Minute + } + + ac := classWithBundle("ac1", codeBundle) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-createdAgo)) + if tc.woken { + wake := metav1.NewTime(nowT.Add(-10 * time.Minute)) + sess.Status.LastWakeAt = &wake + } + if tc.admitted { + reason := tc.admittedReason + if reason == "" { + reason = spiceboxv1alpha1.ReasonAdmitted + } + withAdmittedCondition(sess, metav1.ConditionTrue, reason, started.Add(-time.Second)) + } + if tc.heldBefore { + sess.Status.Conditions = append(sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionCapacityHold, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonCapacityHoldWaiting, + Message: "The cluster is out of room right now; this will start on its own once capacity frees up (gives up at 17:58 UTC).", + LastTransitionTime: metav1.NewTime(started.Add(30 * time.Second)), + }) + } + joinedBefore := admission.JoinedAt(sess) + bundle := bundleWithPod("s1", "code", "s1-code-pod", started) + pod := pendingUnschedulablePod("s1-code-pod", "uid-1", started, tc.schedMsg) + + r, c := fakeReconciler(t, clock, ac, sess, bundle, pod) + r.AdmissionEnabled = tc.enabled + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + assert.Equal(t, !tc.wantReaped, + exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "bundle SpiceboxSession present") + assert.Equal(t, joinedBefore.UTC(), admission.JoinedAt(&got).UTC(), "JoinedAt must not move: the session keeps its place in line") + + hold := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionCapacityHold) + if tc.wantFailed { + require.Equal(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "a session in line past the timeout fails") + assert.Equal(t, spiceboxv1alpha1.ReasonAgentSessionBundleFail, got.Status.FailureReason) + if assert.NotNil(t, hold, "CapacityHold condition") { + assert.Equal(t, metav1.ConditionFalse, hold.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonCapacityHoldExpired, hold.Reason) + } + fc := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionFailed) + if assert.NotNil(t, fc, "Failed condition") { + assert.Contains(t, fc.Message, "waited 1h0m0s for cluster capacity") + assert.Contains(t, fc.Message, "Insufficient cpu") + } + } else { + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") + if tc.wantMisfit && tc.heldBefore && assert.NotNil(t, hold, "CapacityHold condition") { + assert.Equal(t, metav1.ConditionFalse, hold.Status, "the bundle stage's hold goes with the released pods") + assert.Equal(t, spiceboxv1alpha1.ReasonCapacityHoldReleased, hold.Reason) + } + } + + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + switch { + case tc.wantAdmittedNil: + assert.Nil(t, adm, "no Admitted condition") + case tc.wantMisfit: + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") + if tc.wantFailed { + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionMisfit, adm.Reason) + } else { + // Released (PlacementMismatch), then back at the gate on the + // next pass: queued again, its wait restarting there. + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason, "the released session is back in line") + assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting-timeout clock restarts at the release") + } + default: + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionTrue, adm.Status, "still admitted") + } + }) + } +} + +// releaseMisfit gives admission back only once the session's pods are gone: +// released while they still stood, the admitter would count their capacity +// twice. A failed reap writes no release; a failed release write leaves the +// session admitted (its pods already reaped), and the error is returned so +// the next pass decides afresh. +func TestAdmissionGate_MisfitReleaseFailures(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + errBoom := errors.New("apiserver unavailable") + carriesRelease := func(obj client.Object) bool { + sess, ok := obj.(*spiceboxv1alpha1.AgentSession) + if !ok { + return false + } + c := apimeta.FindStatusCondition(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + return c != nil && c.Reason == spiceboxv1alpha1.ReasonAdmissionMisfit + } + + cases := []struct { + name string + // funcs is built per case so it can record onto releaseWrites. + funcs func(releaseWrites *int) interceptor.Funcs + wantErr string + wantReaped bool + }{ + { + name: "reap fails: error returned, pods kept, still admitted, no release written", + funcs: func(releaseWrites *int) interceptor.Funcs { + return interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*spiceboxv1alpha1.SpiceboxSession); ok { + return errBoom + } + return cl.Delete(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + if carriesRelease(obj) { + *releaseWrites++ + } + return cl.SubResource(sub).Patch(ctx, obj, p, opts...) + }, + } + }, + wantErr: "reap bundle SpiceboxSession", + }, + { + name: "release write fails after the reap: error returned, pods reaped, still admitted", + funcs: func(releaseWrites *int) interceptor.Funcs { + return interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + if carriesRelease(obj) { + *releaseWrites++ + return errBoom + } + return cl.SubResource(sub).Patch(ctx, obj, p, opts...) + }, + } + }, + wantErr: "record placement-mismatch release", + wantReaped: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + started := nowT.Add(-2 * time.Minute) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-20*time.Minute)) + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, started.Add(-time.Second)) + bundle := bundleWithPod("s1", "code", "s1-code-pod", started) + pod := pendingUnschedulablePod("s1-code-pod", "uid-1", started, capacitySchedMsg) + + var releaseWrites int + r, c := fakeReconcilerWithFuncs(t, clock, tc.funcs(&releaseWrites), classWithBundle("ac1", codeBundle), sess, bundle, pod) + r.AdmissionEnabled = true + + ctx := memory.WithSystemApproval(context.Background(), "test") + var err error + for range 3 { + if _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "s1"}}); err != nil { + break + } + } + require.Error(t, err, "the failed step is returned so the reconcile retries") + assert.ErrorContains(t, err, tc.wantErr) + + got := getSession(t, c, "s1") + assert.Equal(t, !tc.wantReaped, + exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "bundle SpiceboxSession present") + assert.True(t, admission.IsAdmitted(&got), "no release persisted: still Admitted=True") + if !tc.wantReaped { + assert.Zero(t, releaseWrites, "no release is written before the reap succeeds") + } else { + assert.Equal(t, 1, releaseWrites, "the release is written once the reap succeeded") + } + }) + } +} diff --git a/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go b/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go index b459b517..447854e5 100644 --- a/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go +++ b/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go @@ -36,6 +36,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" @@ -54,6 +55,13 @@ import ( // reconcile clock pinned to clock(). RunnerFactory is wired (unused before the // bundle/detector gates the tests exercise). func fakeReconciler(t *testing.T, clock func() time.Time, objs ...client.Object) (*agentsession.Reconciler, client.Client) { + t.Helper() + return fakeReconcilerWithFuncs(t, clock, interceptor.Funcs{}, objs...) +} + +// fakeReconcilerWithFuncs is fakeReconciler with funcs intercepting the fake +// client's calls, for a test that fails one write. +func fakeReconcilerWithFuncs(t *testing.T, clock func() time.Time, funcs interceptor.Funcs, objs ...client.Object) (*agentsession.Reconciler, client.Client) { t.Helper() scheme := testfixtures.NewScheme(t, rbacv1.AddToScheme, networkingv1.AddToScheme) c := fake.NewClientBuilder(). @@ -67,6 +75,7 @@ func fakeReconciler(t *testing.T, clock func() time.Time, objs ...client.Object) // the deduced converter handles it. A fake-client limitation only — the // same applies work against a real apiserver (netpol_envtest_test.go). WithTypeConverters(managedfields.NewDeducedTypeConverter()). + WithInterceptorFuncs(funcs). Build() r := &agentsession.Reconciler{ Client: c, diff --git a/pkg/controllers/agentsession/bundles.go b/pkg/controllers/agentsession/bundles.go index 18fb45f6..901421e3 100644 --- a/pkg/controllers/agentsession/bundles.go +++ b/pkg/controllers/agentsession/bundles.go @@ -15,6 +15,31 @@ func BundleSessionName(sess *spiceboxv1alpha1.AgentSession, bundle spiceboxv1alp return sess.Name + "-" + bundle.Name } +// desiredBundleSession is the SpiceboxSession the reconciler applies for +// bundle b: it resolves the bundle's identity (bundle, then session, then +// class) and its tier-resolved sandbox backend, then calls BuildBundleSession. +// Shared by the reconciler's bundle loop and DemandFor so both always describe +// the same sandbox. +func desiredBundleSession(sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass, b spiceboxv1alpha1.ToolBundle, claimName string) *spiceboxv1alpha1.SpiceboxSession { + identity := b.AgentIdentity + if identity == "" { + identity = sess.Spec.AgentIdentity + } + if identity == "" { + identity = ac.Spec.AgentIdentity + } + // The tier-resolved sandbox backend for this bundle, or nil when no tier + // expressed a preference — BuildBundleSession then leaves spec.sandbox + // unset, so the SpiceboxClass's own setting stands. + var resolvedSandbox *spiceboxv1alpha1.SandboxBackend + if eff := sess.Status.EffectiveSettings; eff != nil { + if backend, ok := eff.Sandbox[b.Name]; ok { + resolvedSandbox = &backend + } + } + return BuildBundleSession(sess, b, identity, claimName, sess.Status.ResolvedSkillBundles, resolvedSandbox, ac.Spec.Config) +} + // BuildBundleSession returns the SpiceboxSession spec/object the AgentSession // reconciler will create for one bundle. AgentIdentity binds via spec.agent; // the toolspec subset is mirrored onto spec.toolspecs to scope which toolspecs diff --git a/pkg/controllers/agentsession/capacity_hold.go b/pkg/controllers/agentsession/capacity_hold.go index f85d04ac..da5ecc7b 100644 --- a/pkg/controllers/agentsession/capacity_hold.go +++ b/pkg/controllers/agentsession/capacity_hold.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/conditions" ) @@ -143,6 +144,27 @@ func (r *Reconciler) syncCapacityHold(sess *spiceboxv1alpha1.AgentSession, held r.releaseCapacityHold(sess) } +// syncAdmissionHold records that a booting session is waiting in line to be +// admitted: CapacityHold=True/WaitingForAdmission, whose message gives up at +// until (WaitingSince + the hold timeout). Like syncCapacityHold it waits out +// unschedulableGrace first, measured from admission.WaitingSince, so a session +// admitted a few seconds after joining never flashes a "queued" notice. The +// message is a pure function of until, so the condition is written once per +// wait rather than churned by every requeue. Released by releaseCapacityHoldFor +// once the session is admitted. +func (r *Reconciler) syncAdmissionHold(sess *spiceboxv1alpha1.AgentSession, until time.Time) { + if r.now().Sub(admission.WaitingSince(sess)) < unschedulableGrace { + return + } + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionCapacityHold, + Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + Message: capacityHoldMessage(until), + LastTransitionTime: metav1.NewTime(r.now()), + }) +} + // releaseCapacityHoldFor releases an active hold only when it is the given // stage's, and reports whether it did. The bundle stage calls it once every bundle is Ready: from there on // nothing the bundles wait on is held, and the provisioning stages between the diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index 611ee1c2..9b0f25cf 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -651,6 +651,14 @@ type Reconciler struct { // --capacity-hold-timeout. See capacity_hold.go. CapacityHoldTimeout time.Duration + // AdmissionEnabled gates a booting session's pods on admission: while true, + // a Pending session creates none of its pods (bundle SpiceboxSessions, + // separate-pod sidecars, detectors, runner) until the admitter sets + // Admitted=True, and it gives up once it has waited unadmitted past + // CapacityHoldTimeout. The zero value (false) is ungated — today's + // behaviour. See admission_gate.go. + AdmissionEnabled bool + // SpiceDBDeleter, when non-nil, wipes all agentsession relationships in // SpiceDB during finalization, before the finalizer is removed. Nil skips // SpiceDB cleanup, logged rather than silent. @@ -1456,6 +1464,18 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{RequeueAfter: remaining}, nil case reapActionReap: res, err := r.reapSessionPods(ctx, &sess) + // The reaped pods' capacity is free, so the session's admission + // goes back to its line. Written after the reap succeeds, never + // before: released while its pods still stood, the admitter would + // count their capacity twice. A session not Admitted=True is + // untouched, so this writes nothing on later passes. + if err == nil && r.releaseAdmissionOnReclaim(&sess) { + if serr := r.applyStatus(ctx, &sess); serr != nil { + log.FromContext(ctx).Info("reap: recording admission release failed; will retry", + "session", sess.Namespace+"/"+sess.Name, "err", serr.Error()) + return ctrl.Result{}, fmt.Errorf("record admission release after reap: %w", serr) + } + } if err == nil && reclaimAfter > 0 && (res.RequeueAfter == 0 || reclaimAfter < res.RequeueAfter) { res.RequeueAfter = reclaimAfter } @@ -1730,6 +1750,25 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu "session", sess.Namespace+"/"+sess.Name, "err", err.Error()) } + // 2-pre. Admission. With admission on, a booting session that is not yet + // admitted creates NONE of its pods — bundles, separate-pod sidecars, + // detectors and runner all sit below this line — and waits in line here + // until the admitter sets Admitted=True. Everything above (settings, the + // start/identity/credential gates, owner resolution) has already run: the + // admitter needs status.effectiveSettings to size the session's demand. + if r.awaitingAdmission(&sess) { + return r.waitForAdmission(ctx, &sess) + } + // Admitted (or admission off): a wait-in-line hold ends here, persisted now + // for the same reason the bundle stage persists its own release — so a + // bundle-capacity hold set later this pass is a NEW episode with its own + // give-up time rather than diffed away against the stored True. + if r.releaseCapacityHoldFor(&sess, spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission) { + if err := r.applyStatus(ctx, &sess); err != nil { + return ctrl.Result{}, err + } + } + // 2. Bundle SpiceboxSessions, or the vacuous NoBundles path. Only an // active/pending turn provisions; every parked phase (Idle, AwaitingRetry, // AwaitingDecision, AwaitingCredentials) and every terminal phase skips this @@ -1770,27 +1809,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu var earliestBundleStart time.Time bundleSessionStatuses := make([]spiceboxv1alpha1.ResolvedBundle, 0, len(ac.Spec.ToolBundles)) for _, b := range ac.Spec.ToolBundles { - identity := b.AgentIdentity - if identity == "" { - identity = sess.Spec.AgentIdentity - } - if identity == "" { - identity = ac.Spec.AgentIdentity - } - if err := r.ensureSandboxNetworkPolicy(ctx, &sess, b); err != nil { return ctrl.Result{}, err } - // The tier-resolved sandbox backend for this bundle, or nil when no - // tier expressed a preference — BuildBundleSession then leaves - // spec.sandbox unset, so the SpiceboxClass's own setting stands. - var resolvedSandbox *spiceboxv1alpha1.SandboxBackend - if eff := sess.Status.EffectiveSettings; eff != nil { - if backend, ok := eff.Sandbox[b.Name]; ok { - resolvedSandbox = &backend - } - } - desired := BuildBundleSession(&sess, b, identity, claimName, sess.Status.ResolvedSkillBundles, resolvedSandbox, ac.Spec.Config) + desired := desiredBundleSession(&sess, &ac, b, claimName) + identity := desired.Spec.Agent // Server-side apply (idempotent re-create). if err := r.Client.Patch(ctx, desired, client.Apply, client.ForceOwnership, client.FieldOwner("agentsession-bundle"), @@ -1968,6 +1991,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu bundlesWereReady := meta.FindStatusCondition(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionBundlesReady) midSession := bundlesWereReady != nil && bundlesWereReady.Status == metav1.ConditionTrue stall, stalled := r.firstSchedulingStall(ctx, sess.Namespace, notReadyPods) + // Admission's safety net: an admitted set the scheduler still + // cannot place goes back to its line instead of holding the + // placed half (admission_gate.go). Ahead of the deadline, which + // a re-queued session must not be failed by. + if r.placementMisfit(&sess, stall, stalled && !midSession) { + sess.Status.BundleSessions = bundleSessionStatuses + return r.releaseMisfit(ctx, &sess, stall) + } bundleDeadline, held := r.capacityHoldDeadline(&sess, stall, stalled && !midSession, r.effectiveBundleReadyDeadline(ctx)) if bundleProvisioningTimedOut(capacityHoldAdjustedStart(&sess, earliestBundleStart, held, r.now()), r.now(), bundleDeadline) { sess.Status.BundleSessions = bundleSessionStatuses @@ -2332,6 +2363,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // period sleep again instead of idleSleepDue reading it as already-slept. sess.Status.SleptAt = nil } + // The same catch-all for the admitter's reclaim request: one a session left + // Idle before honoring is stale, and must not sleep it on its next Idle. + if err := r.clearStaleReclaimRequest(ctx, &sess); err != nil { + return ctrl.Result{}, err + } // Forensic hold: an unreleased SessionHold naming this session forces // phase=Held and reaps the session's pods (containment). Must run before @@ -2862,57 +2898,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // clear AwaitingDetector reason set meanwhile so the wait is never silent. // The loop is generic — it never branches on inspector id. { - var resolvedDetectors []spiceboxv1alpha1.ResolvedContentGuardDetector - var detKeys []string // Collect the detector-requiring inspectors and their specs. - if eff := sess.Status.EffectiveSettings; eff != nil { - for _, ci := range eff.ContentInspectors { - insp, ok := contentguardregistry.Get(ci.ID) - if !ok { - // Webhook already rejected unknown IDs; defensive skip. - continue - } - dp, ok := insp.(contentguard.DetectorProvider) - if !ok { - // Inspector does not need a detector sidecar (e.g. url-allowlist). - continue - } - raw := ci.Config.Raw - ds, derr := dp.Detector(raw) - if derr != nil || ds == nil { - // Fail-closed: a detector-requiring inspector that can't resolve - // its image must not silently run unscanned. - return r.markBootFailed(ctx, &sess, - spiceboxv1alpha1.ReasonAgentSessionContentGuardHalt, - fmt.Sprintf("content-guard detector %q: %v", ci.ID, derr)) - } - // Use the image exactly as declared in the inspector config: pinned - // by digest it launches the exact bytes, by tag it launches by tag. - // There is no probe path here as there is for SidecarToolbox. - img := ds.Image - resolvedDetectors = append(resolvedDetectors, spiceboxv1alpha1.ResolvedContentGuardDetector{ - Inspector: ci.ID, - Image: img, - HealthPath: ds.HealthPath, - }) - detKeys = append(detKeys, "det-"+ci.ID) - } + resolvedDetectors, derr := resolveContentGuardDetectors(sess.Status.EffectiveSettings) + if derr != nil { + // Fail-closed: a detector-requiring inspector that can't resolve + // its image must not silently run unscanned. + return r.markBootFailed(ctx, &sess, + spiceboxv1alpha1.ReasonAgentSessionContentGuardHalt, derr.Error()) } if len(resolvedDetectors) > 0 { - // Allocate ports in the SAME key space as sidecar ports so they never - // collide. Derive the sidecar key slice from the resolved snapshot. - sidecarKeySlice := make([]string, 0, len(resolvedSidecars)) + sidecarRefs := make([]string, 0, len(resolvedSidecars)) for _, rt := range resolvedSidecars { - sidecarKeySlice = append(sidecarKeySlice, rt.Ref) - } - allKeys := append(append([]string{}, sidecarKeySlice...), detKeys...) - ports := AllocatePorts(allKeys) - for i := range resolvedDetectors { - det := &resolvedDetectors[i] - det.Port = ports["det-"+det.Inspector] - det.PodName = "sidecar-" + sess.Name + "-" + det.Inspector + sidecarRefs = append(sidecarRefs, rt.Ref) } + placeContentGuardDetectors(&sess, resolvedDetectors, sidecarRefs) // Create each detector pod + its deny-all-egress NetworkPolicy, and // reflect the pod IP. Hold runner-pod creation until every PodIP is known. @@ -2934,22 +2934,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu var terminalDetectorFailures []string for i := range resolvedDetectors { det := &resolvedDetectors[i] - spec := cosidecar.Spec{ - Ref: det.Inspector, - // Pin the NetworkPolicy name to the one - // cleanupOrphanedSidecarPods reaps by. On cosidecar's default - // name, removing an inspector mid-session reaps the detector pod - // by label but orphans its NetworkPolicy until session GC. - // BuildPod names the pod by Ref, so only the NP name is affected. - Name: SidecarNetworkPolicyName(&sess, det.Inspector), - Image: det.Image, - Port: det.Port, - Health: cosidecar.Healthcheck{Path: det.HealthPath}, - ImagePullSecret: r.ImagePullSecret, - // Zero egress: the detector ingests potentially hostile tool text - // and must not be able to exfiltrate it to external endpoints. - Egress: cosidecar.EgressPolicy{Denied: true}, - } + spec := detectorPodSpec(&sess, *det, r.ImagePullSecret) pod, _, perr := cosidecar.BuildPod(&sess, spec) if perr != nil { return r.markBootFailed(ctx, &sess, @@ -3111,6 +3096,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // is the source of truth for real container crashes, and the // maxRunnerRestarts kill-switch must fire only on those. sess.Status.LastIdleAt = nil + // A slept or archived session's pods are gone. One reaped before + // admission knew of it has no Admitted condition, and with SleptAt + // cleared its stale pod names would read as pre-admission: record the + // release here, in the wake's own status write, so it queues instead. + if sess.Status.SleptAt != nil || wasArchivedBySweep { + r.releaseUnrecordedAdmission(&sess) + } // Clear SleptAt explicitly, not just via the LastIdleAt catch-all: this // path nils LastIdleAt in the same pass that applies WakeRequested, and // that catch-all ran earlier in this call while phase was still Idle, so diff --git a/pkg/controllers/agentsession/demand.go b/pkg/controllers/agentsession/demand.go new file mode 100644 index 00000000..27ee0e33 --- /dev/null +++ b/pkg/controllers/agentsession/demand.go @@ -0,0 +1,193 @@ +package agentsession + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + harnessregistry "github.com/authzed/openagentprimitives/pkg/agent/harness/registry" + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" + "github.com/authzed/openagentprimitives/pkg/platform/podspec" + "github.com/authzed/openagentprimitives/pkg/tools/toolchain/resolve" + "github.com/authzed/openagentprimitives/pkg/web/secretoutsrv" +) + +// demandPlaceholderImage stands in for an image a pod's demand cannot depend +// on. The SpiceboxSession reconciler fills an unpinned class image from the +// operator's configured sandbox image, which this package-level function does +// not have; requests come from the class's resources alone, so any image +// prices the same. +const demandPlaceholderImage = "demand-estimate" + +// DemandFor returns the resource requests of every pod sess will create, so +// the whole set can be admitted before any of it exists. It reads only; it +// creates nothing. +// +// Each pod is rendered by the same builder the reconciler uses to create it, +// then priced by admission.PodDemandFor (schedfit.EffectiveRequests, plus the +// pod's tolerations for the node model's taint check): +// +// - the runner (BuildRunnerPod, as PodRunnerFactory calls it), with the +// in-pod sidecars it carries; +// - one sandbox per ac.Spec.ToolBundles entry (BuildBundleSession + +// podspec.Build, against the bundle's SpiceboxClass and its toolchains +// resolved as the SpiceboxSession reconciler resolves them at bind); +// - one pod per content-guard detector (cosidecar.BuildPod); +// - one pod per separate-pod SidecarToolbox (BuildSidecarPod). +// +// Inputs that only exist once provisioning runs — the workspace claim, the +// secret-output token hash, a Workshop identity, the operator's URLs and pull +// secret — are left empty: none of them reaches a container's resources. +// +// A referenced object that is missing (a SpiceboxClass, a SidecarToolbox, a +// toolchain) is an error naming it, never a zero demand, and so is a session +// whose settings are not yet resolved. +func DemandFor(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass) (admission.Demand, error) { + eff := sess.Status.EffectiveSettings + if eff == nil { + return admission.Demand{}, fmt.Errorf("session %s/%s has no effectiveSettings yet: its detectors and sandbox backends derive from them", + sess.Namespace, sess.Name) + } + + var pods []*corev1.Pod + + sidecars, err := demandSidecars(ctx, c, sess, ac) + if err != nil { + return admission.Demand{}, err + } + dets, err := resolveContentGuardDetectors(eff) + if err != nil { + return admission.Demand{}, err + } + sidecarRefs := make([]string, 0, len(sidecars)) + for _, rt := range sidecars { + sidecarRefs = append(sidecarRefs, rt.Ref) + } + placeContentGuardDetectors(sess, dets, sidecarRefs) + + h, err := harnessregistry.Resolve(ac.Spec.Harness) + if err != nil { + return admission.Demand{}, fmt.Errorf("resolve harness for %s/%s: %w", sess.Namespace, sess.Name, err) + } + runner, err := BuildRunnerPod(PodSpecOpts{ + Session: sess, + Class: ac, + Image: demandPlaceholderImage, + ServiceAcct: sess.Name + "-runner-sa", + MemoryToken: MemoryTokenSecretName(sess), + ResolvedSidecars: sidecars, + SidecarSecretName: func(ref string) string { return cosidecar.CredentialSecretName(sess.Name, ref) }, + ResolvedContentGuardDetectors: dets, + Harness: h, + }) + if err != nil { + return admission.Demand{}, fmt.Errorf("build runner pod: %w", err) + } + pods = append(pods, runner) + + // A bundle's sandbox pod is keyed by its SpiceboxSession, not its name: a + // pod adopted from a SandboxWarmPool has a pool-minted one. + sandboxSessions := make(map[string]string, len(ac.Spec.ToolBundles)) + for _, b := range ac.Spec.ToolBundles { + pod, err := demandBundlePod(ctx, c, sess, ac, b) + if err != nil { + return admission.Demand{}, err + } + pods = append(pods, pod) + sandboxSessions[pod.Name] = BundleSessionName(sess, b) + } + + for _, det := range dets { + pod, _, err := cosidecar.BuildPod(sess, detectorPodSpec(sess, det, "")) + if err != nil { + return admission.Demand{}, fmt.Errorf("build detector pod %q: %w", det.Inspector, err) + } + pods = append(pods, pod) + } + + soSecretName := secretoutsrv.SecretOutputSecretName(sess.Name) + for _, rt := range sidecars { + if rt.RunMode != RunModeSeparatePod { + continue + } + pod, err := BuildSidecarPod(sess, rt, soSecretName, cosidecar.CredentialSecretName(sess.Name, rt.Ref), nil, "", "", "") + if err != nil { + return admission.Demand{}, fmt.Errorf("build sidecar pod %q: %w", rt.Ref, err) + } + pods = append(pods, pod) + } + + d := admission.Demand{Pods: make([]admission.PodDemand, 0, len(pods))} + for _, p := range pods { + pd := admission.PodDemandFor(p.Name, &p.Spec) + pd.SandboxSession = sandboxSessions[p.Name] + d.Pods = append(d.Pods, pd) + } + return d, nil +} + +// demandBundlePod renders the sandbox pod one tool bundle will run in. It +// takes the reconciler's own desiredBundleSession (identity and tier-resolved +// backend) and mirrors the SpiceboxSession reconciler's bind (class +// snapshot, image default, toolchain resolution), then calls podspec.Build — +// the builder both registered sandbox backends render through. +func demandBundlePod(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass, b spiceboxv1alpha1.ToolBundle) (*corev1.Pod, error) { + // No claim name: whether the workspace is shared depends on the operator's + // configured StorageClass, and only changes the pod's volumes. + bs := desiredBundleSession(sess, ac, b, "") + + var cls spiceboxv1alpha1.SpiceboxClass + if err := c.Get(ctx, types.NamespacedName{Name: b.Class}, &cls); err != nil { + return nil, fmt.Errorf("get SpiceboxClass %q for bundle %q: %w", b.Class, b.Name, err) + } + classSpec := *cls.Spec.DeepCopy() + if classSpec.Image == "" { + classSpec.Image = demandPlaceholderImage + } + mounts, _, err := resolve.Resolve(ctx, c, classSpec.Toolchains) + if err != nil { + return nil, fmt.Errorf("resolve toolchains of SpiceboxClass %q for bundle %q: %w", b.Class, b.Name, err) + } + bs.Status.ResolvedToolchains = mounts + + pod, err := podspec.Build(bs, classSpec) + if err != nil { + return nil, fmt.Errorf("build sandbox pod for bundle %q (SpiceboxClass %q): %w", b.Name, b.Class, err) + } + return pod, nil +} + +// demandSidecars resolves ac's SidecarToolbox refs to the fields a pod build +// reads: spec, run mode, and port. The reconciler's by-digest image rewrite +// and network merge are skipped — neither reaches a container's resources. +func demandSidecars(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass) ([]spiceboxv1alpha1.ResolvedSidecarToolbox, error) { + if len(ac.Spec.SidecarToolboxes) == 0 { + return nil, nil + } + out := make([]spiceboxv1alpha1.ResolvedSidecarToolbox, 0, len(ac.Spec.SidecarToolboxes)) + refs := make([]string, 0, len(ac.Spec.SidecarToolboxes)) + for _, ref := range ac.Spec.SidecarToolboxes { + var tb spiceboxv1alpha1.SidecarToolbox + if err := c.Get(ctx, types.NamespacedName{Namespace: sess.Namespace, Name: ref.Ref}, &tb); err != nil { + return nil, fmt.Errorf("get SidecarToolbox %q in namespace %q: %w", ref.Ref, sess.Namespace, err) + } + spec := *tb.Spec.DeepCopy() + out = append(out, spiceboxv1alpha1.ResolvedSidecarToolbox{ + Name: ref.Name, + Ref: ref.Ref, + Spec: spec, + RunMode: RunModeFor(spec), + }) + refs = append(refs, ref.Ref) + } + ports := AllocatePorts(refs) + for i := range out { + out[i].Port = ports[out[i].Ref] + } + return out, nil +} diff --git a/pkg/controllers/agentsession/demand_test.go b/pkg/controllers/agentsession/demand_test.go new file mode 100644 index 00000000..b8fdbe03 --- /dev/null +++ b/pkg/controllers/agentsession/demand_test.go @@ -0,0 +1,285 @@ +package agentsession + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + _ "github.com/authzed/openagentprimitives/pkg/agent/harness/apnative" // the default harness the runner pod is built with + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/authz/contentguard" + _ "github.com/authzed/openagentprimitives/pkg/authz/contentguard/kinds/promptinjection" // a DetectorProvider inspector + contentguardregistry "github.com/authzed/openagentprimitives/pkg/authz/contentguard/registry" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" + "github.com/authzed/openagentprimitives/pkg/platform/podspec" + "github.com/authzed/openagentprimitives/pkg/platform/schedfit" +) + +const demandMiB = int64(1024 * 1024) + +// demandSpiceboxClass is a cluster-scoped SpiceboxClass whose sandbox +// container is limited to the given cpu/memory/ephemeral-storage. +func demandSpiceboxClass(name, cpu, mem, eph string) *spiceboxv1alpha1.SpiceboxClass { + return &spiceboxv1alpha1.SpiceboxClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: spiceboxv1alpha1.SpiceboxClassSpec{ + Image: "ghcr.io/example/sandbox:v1", + Resources: spiceboxv1alpha1.SpiceboxResources{ + CPU: resource.MustParse(cpu), + Memory: resource.MustParse(mem), + EphemeralStorage: resource.MustParse(eph), + }, + }, + } +} + +// demandAgentClass declares one tool bundle per SpiceboxClass name, named +// "b-", plus the given sidecar toolbox refs. +func demandAgentClass(bundleClasses []string, sidecars ...spiceboxv1alpha1.AgentClassSidecarToolboxRef) *spiceboxv1alpha1.AgentClass { + ac := &spiceboxv1alpha1.AgentClass{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "demo-class"}} + for _, c := range bundleClasses { + ac.Spec.ToolBundles = append(ac.Spec.ToolBundles, spiceboxv1alpha1.ToolBundle{Name: "b-" + c, Class: c}) + } + ac.Spec.SidecarToolboxes = sidecars + return ac +} + +// demandSession is a session whose settings are resolved, with a +// prompt-injection inspector when withDetector is set. +func demandSession(t *testing.T, withDetector bool) *spiceboxv1alpha1.AgentSession { + t.Helper() + eff := &spiceboxv1alpha1.EffectiveSettings{ + Model: spiceboxv1alpha1.ModelConfig{APIKey: spiceboxv1alpha1.SecretKeyRef{Name: "llm", Key: "token"}}, + } + if withDetector { + eff.ContentInspectors = []spiceboxv1alpha1.ContentInspectorConfig{{ + ID: "prompt-injection", + Config: apiextv1.JSON{Raw: []byte(`{"detectorImage":"ghcr.io/example/detector:v1","port":9080}`)}, + }} + } + return &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "demo-session", UID: "uid-1"}, + Status: spiceboxv1alpha1.AgentSessionStatus{EffectiveSettings: eff}, + } +} + +func demandSidecarToolbox(name string, secretGated bool) *spiceboxv1alpha1.SidecarToolbox { + tb := &spiceboxv1alpha1.SidecarToolbox{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: name}, + Spec: spiceboxv1alpha1.SidecarToolboxSpec{ + Name: name, + Source: spiceboxv1alpha1.SidecarToolboxSource{Image: "ghcr.io/example/" + name + ":v1"}, + }, + } + if secretGated { + tb.Spec.SecretInputs = []spiceboxv1alpha1.SidecarToolboxSecretInput{{Name: "TOKEN", Deliver: "env", From: "tok"}} + } + return tb +} + +func demandClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, spiceboxv1alpha1.AddToScheme(s)) + return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build() +} + +// podDemandOf prices pod the way the admitter will: schedfit.EffectiveRequests +// over the spec the real builder produced. +func podDemandOf(pod *corev1.Pod) admission.PodDemand { + cpu, mem, eph := schedfit.EffectiveRequests(&pod.Spec) + return admission.PodDemand{Name: pod.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph} +} + +func demandByName(d admission.Demand) map[string]admission.PodDemand { + out := make(map[string]admission.PodDemand, len(d.Pods)) + for _, p := range d.Pods { + out[p.Name] = p + } + return out +} + +func TestDemandForMatchesBuiltPods(t *testing.T) { + half := demandSpiceboxClass("half", "500m", "512Mi", "1Gi") + one := demandSpiceboxClass("one", "1", "1Gi", "2Gi") + ac := demandAgentClass([]string{"half", "one"}) + sess := demandSession(t, true) + c := demandClient(t, half, one) + + got, err := DemandFor(context.Background(), c, sess, ac) + require.NoError(t, err, "DemandFor") + byName := demandByName(got) + require.Len(t, got.Pods, 4, "runner + two bundles + one detector, got %v", got.Pods) + + // Each bundle's demand equals what the real sandbox builder produces for it. + for _, tc := range []struct { + class *spiceboxv1alpha1.SpiceboxClass + cpuMilli, memMiB int64 + }{ + {half, 500, 512}, + {one, 1000, 1024}, + } { + bundle := spiceboxv1alpha1.ToolBundle{Name: "b-" + tc.class.Name, Class: tc.class.Name} + bs := BuildBundleSession(sess, bundle, "", "", nil, nil, nil) + pod, berr := podspec.Build(bs, tc.class.Spec) + require.NoError(t, berr, "build bundle pod for %s", tc.class.Name) + want := podDemandOf(pod) + // A sandbox pod is matched to its demand by its bundle SpiceboxSession's + // label, not its name: a pod adopted from a warm pool has a pool name. + want.SandboxSession = bs.Name + + assert.Equal(t, want, byName[pod.Name], "bundle %s demand equals the built pod's effective requests", bundle.Name) + assert.Equal(t, tc.cpuMilli, byName[pod.Name].CPUMilli, + "bundle %s is priced at its class's cpu limit (requests default to limits)", bundle.Name) + assert.Equal(t, tc.memMiB*demandMiB, byName[pod.Name].MemBytes, "bundle %s memory", bundle.Name) + } + + // The detector's demand equals what cosidecar.BuildPod produces for it. + dets, derr := resolveContentGuardDetectors(sess.Status.EffectiveSettings) + require.NoError(t, derr, "resolve detectors") + require.Len(t, dets, 1) + detPod, _, perr := cosidecar.BuildPod(sess, detectorPodSpec(sess, dets[0], "")) + require.NoError(t, perr, "build detector pod") + assert.Equal(t, podDemandOf(detPod), byName[detPod.Name], "detector demand equals the built pod's") + + // The runner pod is present under its deterministic name. + _, ok := byName[RunnerPodName(sess)] + assert.True(t, ok, "runner pod %q is in the demand", RunnerPodName(sess)) +} + +func TestDemandForSidecars(t *testing.T) { + ac := demandAgentClass(nil, + spiceboxv1alpha1.AgentClassSidecarToolboxRef{Name: "inpod", Ref: "inpod"}, + spiceboxv1alpha1.AgentClassSidecarToolboxRef{Name: "gated", Ref: "gated"}, + ) + sess := demandSession(t, false) + c := demandClient(t, demandSidecarToolbox("inpod", false), demandSidecarToolbox("gated", true)) + + got, err := DemandFor(context.Background(), c, sess, ac) + require.NoError(t, err, "DemandFor") + byName := demandByName(got) + + assert.Len(t, got.Pods, 2, "runner + the one separate-pod sidecar; the in-pod sidecar rides in the runner pod") + assert.Contains(t, byName, RunnerPodName(sess), "runner pod") + assert.Contains(t, byName, SidecarPodName(sess, "gated"), "secret-gated sidecar gets its own pod") + assert.NotContains(t, byName, SidecarPodName(sess, "inpod"), "in-pod sidecar has no pod of its own") +} + +func TestDemandForErrors(t *testing.T) { + cases := []struct { + name string + ac *spiceboxv1alpha1.AgentClass + sess func(t *testing.T) *spiceboxv1alpha1.AgentSession + objs []client.Object + contains string + }{ + { + name: "missing SpiceboxClass: error naming the class, not a zero demand", + ac: demandAgentClass([]string{"absent"}), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { return demandSession(t, false) }, + contains: `SpiceboxClass "absent"`, + }, + { + name: "missing SidecarToolbox: error naming the toolbox", + ac: demandAgentClass(nil, spiceboxv1alpha1.AgentClassSidecarToolboxRef{Name: "x", Ref: "ghost"}), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { return demandSession(t, false) }, + contains: `SidecarToolbox "ghost"`, + }, + { + name: "missing toolchain on a bundle's class: error naming the toolchain", + ac: demandAgentClass([]string{"withgo"}), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { return demandSession(t, false) }, + objs: func() []client.Object { + cls := demandSpiceboxClass("withgo", "500m", "512Mi", "1Gi") + cls.Spec.Toolchains = []string{"go"} + return []client.Object{cls} + }(), + contains: `"go"`, + }, + { + name: "settings not yet resolved: error, since detectors and backends derive from them", + ac: demandAgentClass(nil), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { + s := demandSession(t, false) + s.Status.EffectiveSettings = nil + return s + }, + contains: "effectiveSettings", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := DemandFor(context.Background(), demandClient(t, tc.objs...), tc.sess(t), tc.ac) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.contains) + assert.Empty(t, got.Pods, "no partial demand on error") + }) + } +} + +// nilDetectorInspector is a DetectorProvider that resolves no detector. +type nilDetectorInspector struct{} + +func (nilDetectorInspector) ID() string { return "test-nil-detector" } +func (nilDetectorInspector) Configure(json.RawMessage) (contentguard.Instance, error) { + return nil, errors.New("not used") +} +func (nilDetectorInspector) Detector(json.RawMessage) (*contentguard.DetectorSpec, error) { + return nil, nil +} + +var registerNilDetector sync.Once + +// promptInjectionID is the promptinjection kind's registry ID. +const promptInjectionID = "prompt-injection" + +// A detector-requiring inspector that cannot produce its detector fails +// closed either way, but says which way: a config error is wrapped (so a +// caller can inspect it) and a nil spec gets its own message rather than +// "detector X: ". +func TestResolveContentGuardDetectorsErrors(t *testing.T) { + registerNilDetector.Do(func() { contentguardregistry.Register(nilDetectorInspector{}) }) + cases := []struct { + name string + inspector spiceboxv1alpha1.ContentInspectorConfig + wantMsg string + wantWrap bool + }{ + { + name: "config the inspector rejects: error wraps the inspector's", + inspector: spiceboxv1alpha1.ContentInspectorConfig{ID: promptInjectionID, Config: apiextv1.JSON{Raw: []byte(`{not json`)}}, + wantMsg: `content-guard detector "` + promptInjectionID + `"`, + wantWrap: true, + }, + { + name: "inspector resolves no detector spec: its own message, no ", + inspector: spiceboxv1alpha1.ContentInspectorConfig{ID: "test-nil-detector", Config: apiextv1.JSON{Raw: []byte(`{}`)}}, + wantMsg: `content-guard detector "test-nil-detector": the inspector resolved no detector spec`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := resolveContentGuardDetectors(&spiceboxv1alpha1.EffectiveSettings{ + ContentInspectors: []spiceboxv1alpha1.ContentInspectorConfig{tc.inspector}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantMsg) + assert.NotContains(t, err.Error(), "") + assert.Equal(t, tc.wantWrap, errors.Unwrap(err) != nil, "wrapped with %%w") + }) + } +} diff --git a/pkg/controllers/agentsession/export_test.go b/pkg/controllers/agentsession/export_test.go index fb4b4ef4..7fc2ddbc 100644 --- a/pkg/controllers/agentsession/export_test.go +++ b/pkg/controllers/agentsession/export_test.go @@ -5,6 +5,7 @@ package agentsession import ( "context" + "time" corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -68,3 +69,9 @@ func SidecarPodFailureForTest(pod *corev1.Pod) *spiceboxv1alpha1.SidecarPodFailu func (r *Reconciler) EnsureDetectorPodForTest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, spec cosidecar.Spec, pod *corev1.Pod) error { return r.ensureDetectorPod(ctx, sess, spec, pod) } + +// ReconcileSleepForTest exposes the unexported reconcileSleep so +// agentsession_test can drive the idle-sleep reap directly. Test-only. +func (r *Reconciler) ReconcileSleepForTest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, class *spiceboxv1alpha1.AgentClass) (bool, time.Duration, error) { + return r.reconcileSleep(ctx, sess, class) +} diff --git a/pkg/controllers/agentsession/hold.go b/pkg/controllers/agentsession/hold.go index ba25113b..b1b83b72 100644 --- a/pkg/controllers/agentsession/hold.go +++ b/pkg/controllers/agentsession/hold.go @@ -220,6 +220,16 @@ func (r *Reconciler) reconcileHold(ctx context.Context, sess *spiceboxv1alpha1.A } } + // The reap below gives the session's admission back + // (Admitted=False/Reclaimed), recorded in the status write that persists + // the Held projection — a held session reserves nothing for its line, and + // on release it rejoins the back of the line through the gate like any + // cold wake. Decided before the projection: a pre-admission session is + // recognised only while its phase is not yet Held. A session admission + // never knew of is left without a condition. Idempotent on later passes: + // a released session is no longer Admitted=True. + r.releaseAdmissionOnReclaim(sess) + // Project the Held phase through the lifecycle machine, exactly as the // other early-return paths do, so phase stays a projection of the log // rather than a string this file invents. @@ -259,12 +269,15 @@ func (r *Reconciler) reconcileHold(ctx context.Context, sess *spiceboxv1alpha1.A // session off PhaseHeld instead of re-reading a log whose last phase-changing // event is still Held. // -// proceed is always true here — either a Released event was just appended and -// the rest of Reconcile should run so normal provisioning resumes, or there -// is no released hold to attribute the release to (a session manually forced -// into PhaseHeld with no SessionHold ever naming it), in which case there is -// nothing this method can do and the session is left exactly as it already -// was, same as if no hold had ever been observed. +// proceed is true here unless the released session must wait for admission — +// either a Released event was just appended and the rest of Reconcile should +// run so normal provisioning resumes, or there is no released hold to +// attribute the release to (a session manually forced into PhaseHeld with no +// SessionHold ever naming it), in which case there is nothing this method can +// do and the session is left exactly as it already was, same as if no hold +// had ever been observed. With admission on, a released session that is not +// admitted persists its release and requeues (proceed=false) so it reaches +// the admission gate before any pod is created. func (r *Reconciler) reconcileHoldRelease(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (ctrl.Result, bool, error) { released, err := r.releasedHoldFor(ctx, sess) if err != nil { @@ -283,6 +296,21 @@ func (r *Reconciler) reconcileHoldRelease(ctx context.Context, sess *spiceboxv1a }); err != nil { return ctrl.Result{}, false, fmt.Errorf("apply Released for %s/%s: %w", sess.Namespace, sess.Name, err) } + // A session held before admission knew of it was reaped with no Admitted + // condition, and back at Pending its stale pod names would read as + // pre-admission. Record the release so it queues like any cold wake. + r.releaseUnrecordedAdmission(sess) + + // This runs below the admission gate in Reconcile, so a released session + // that must wait in line cannot continue this pass: it would fall through + // to pod creation unadmitted. Persist the Pending projection (and the + // release above) and requeue; the next pass meets the gate first. + if r.awaitingAdmission(sess) { + if err := r.applyStatus(ctx, sess); err != nil { + return ctrl.Result{}, false, fmt.Errorf("record hold release for %s/%s: %w", sess.Namespace, sess.Name, err) + } + return ctrl.Result{RequeueAfter: time.Second}, false, nil + } return ctrl.Result{}, true, nil } diff --git a/pkg/controllers/agentsession/hold_test.go b/pkg/controllers/agentsession/hold_test.go index 857fcc7a..294a6df4 100644 --- a/pkg/controllers/agentsession/hold_test.go +++ b/pkg/controllers/agentsession/hold_test.go @@ -916,3 +916,88 @@ func TestReconcile_activeHoldOnRunningSessionParksAndReaps(t *testing.T) { err = c.Get(ctx, client.ObjectKey{Namespace: "default", Name: RunnerPodName(sess)}, &gotRunner) assert.True(t, apierrors.IsNotFound(err), "the runner pod must still be reaped, got err=%v", err) } + +// TestReconcileHold_releasesAdmission pins that a forensic hold gives the +// session's admission back with its pods: Admitted=False/Reclaimed, persisted +// with the Held projection, so a held session holds no phantom reservation. +// A session admission never knew of is left without a condition; one that +// was admitted rejoins the line through the gate once its hold is released. +func TestReconcileHold_releasesAdmission(t *testing.T) { + newHold := func(phase string) *spiceboxv1alpha1.SessionHold { + return &spiceboxv1alpha1.SessionHold{ + ObjectMeta: metav1.ObjectMeta{Name: "h1", Namespace: "demo"}, + Spec: spiceboxv1alpha1.SessionHoldSpec{ + SessionRef: spiceboxv1alpha1.NamespacedRef{Namespace: "demo", Name: "s1"}, + Reason: "denial streak", + }, + Status: spiceboxv1alpha1.SessionHoldStatus{Phase: phase}, + } + } + cases := []struct { + name string + admitted bool + wantReason string // "" = no Admitted condition + }{ + {name: "admitted Running session held: Admitted=False/Reclaimed persisted", admitted: true, wantReason: spiceboxv1alpha1.ReasonAdmissionReclaimed}, + {name: "session with no Admitted condition and no pods recorded held: no condition written"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + sess := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: "s1", Namespace: "demo"}, + Status: spiceboxv1alpha1.AgentSessionStatus{Phase: spiceboxv1alpha1.AgentSessionPhaseRunning}, + } + if tc.admitted { + sess.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Hour)), + }} + } + c, r := newParkTestReconciler(t, sess, newHold(spiceboxv1alpha1.SessionHoldPhaseActive)) + r.AdmissionEnabled = true + + _, proceed, err := r.reconcileHold(ctx, sess) + require.NoError(t, err) + require.False(t, proceed) + + var got spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(sess), &got)) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhaseHeld, got.Status.Phase) + adm := conditionsFind(got.Status.Conditions) + if tc.wantReason == "" { + assert.Nil(t, adm, "no Admitted condition invented") + return + } + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, tc.wantReason, adm.Reason) + + // Released: back to Pending, and through the gate like any arrival. + require.NoError(t, c.Delete(ctx, newHold(spiceboxv1alpha1.SessionHoldPhaseActive))) + require.NoError(t, c.Create(ctx, func() *spiceboxv1alpha1.SessionHold { + h := newHold(spiceboxv1alpha1.SessionHoldPhaseReleased) + h.Status.ReleasedBy = "user:approver-1" + return h + }())) + r.LifecycleMemory = memory.NewLocal(inmem.NewBackend()) + res, proceed, err := r.reconcileHold(ctx, &got) + require.NoError(t, err) + assert.False(t, proceed, "an unadmitted release stops this pass, which runs below the admission gate") + assert.Positive(t, res.RequeueAfter, "requeued so the next pass meets the gate") + var persisted spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(sess), &persisted)) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, persisted.Status.Phase, "Pending projection persisted") + assert.True(t, r.awaitingAdmission(&persisted), "a released hold rejoins the line at the gate") + }) + } +} + +func conditionsFind(conds []metav1.Condition) *metav1.Condition { + for i := range conds { + if conds[i].Type == spiceboxv1alpha1.AgentSessionConditionAdmitted { + return &conds[i] + } + } + return nil +} diff --git a/pkg/controllers/agentsession/sidecarpod.go b/pkg/controllers/agentsession/sidecarpod.go index 4bba0c5c..60aaf0c9 100644 --- a/pkg/controllers/agentsession/sidecarpod.go +++ b/pkg/controllers/agentsession/sidecarpod.go @@ -29,6 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/authz/contentguard" + contentguardregistry "github.com/authzed/openagentprimitives/pkg/authz/contentguard/registry" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" "github.com/authzed/openagentprimitives/pkg/platform/podstatus" ) @@ -61,6 +63,84 @@ func (r *Reconciler) ensureDetectorPod(ctx context.Context, sess *spiceboxv1alph return nil } +// resolveContentGuardDetectors returns one detector per enabled content +// inspector that implements contentguard.DetectorProvider, in settings order. +// Port and PodName are left for placeContentGuardDetectors. A nil eff yields +// none. +// +// An inspector that implements DetectorProvider but cannot produce a detector +// is an error, never a skip: a detector-requiring inspector that can't resolve +// its image must not silently run unscanned. +func resolveContentGuardDetectors(eff *spiceboxv1alpha1.EffectiveSettings) ([]spiceboxv1alpha1.ResolvedContentGuardDetector, error) { + if eff == nil { + return nil, nil + } + var out []spiceboxv1alpha1.ResolvedContentGuardDetector + for _, ci := range eff.ContentInspectors { + insp, ok := contentguardregistry.Get(ci.ID) + if !ok { + // Webhook already rejected unknown IDs; defensive skip. + continue + } + dp, ok := insp.(contentguard.DetectorProvider) + if !ok { + // Inspector does not need a detector sidecar (e.g. url-allowlist). + continue + } + ds, err := dp.Detector(ci.Config.Raw) + if err != nil { + return nil, fmt.Errorf("content-guard detector %q: %w", ci.ID, err) + } + if ds == nil { + return nil, fmt.Errorf("content-guard detector %q: the inspector resolved no detector spec for its config", ci.ID) + } + // Use the image exactly as declared in the inspector config: pinned + // by digest it launches the exact bytes, by tag it launches by tag. + // There is no probe path here as there is for SidecarToolbox. + out = append(out, spiceboxv1alpha1.ResolvedContentGuardDetector{ + Inspector: ci.ID, + Image: ds.Image, + HealthPath: ds.HealthPath, + }) + } + return out, nil +} + +// placeContentGuardDetectors stamps each detector's Port and PodName. Ports +// are allocated in the SAME key space as the session's sidecar refs so a +// detector never collides with a sidecar. +func placeContentGuardDetectors(sess *spiceboxv1alpha1.AgentSession, dets []spiceboxv1alpha1.ResolvedContentGuardDetector, sidecarRefs []string) { + keys := append([]string{}, sidecarRefs...) + for _, d := range dets { + keys = append(keys, "det-"+d.Inspector) + } + ports := AllocatePorts(keys) + for i := range dets { + dets[i].Port = ports["det-"+dets[i].Inspector] + dets[i].PodName = "sidecar-" + sess.Name + "-" + dets[i].Inspector + } +} + +// detectorPodSpec is the cosidecar descriptor a detector pod is built from. +func detectorPodSpec(sess *spiceboxv1alpha1.AgentSession, det spiceboxv1alpha1.ResolvedContentGuardDetector, imagePullSecret string) cosidecar.Spec { + return cosidecar.Spec{ + Ref: det.Inspector, + // Pin the NetworkPolicy name to the one cleanupOrphanedSidecarPods reaps + // by. On cosidecar's default name, removing an inspector mid-session + // reaps the detector pod by label but orphans its NetworkPolicy until + // session GC. BuildPod names the pod by Ref, so only the NP name is + // affected. + Name: SidecarNetworkPolicyName(sess, det.Inspector), + Image: det.Image, + Port: det.Port, + Health: cosidecar.Healthcheck{Path: det.HealthPath}, + ImagePullSecret: imagePullSecret, + // Zero egress: the detector ingests potentially hostile tool text and + // must not be able to exfiltrate it to external endpoints. + Egress: cosidecar.EgressPolicy{Denied: true}, + } +} + // SidecarPodName returns the deterministic name for a session's separate // sidecar pod for the given toolbox ref: "-sidecar-". func SidecarPodName(sess *spiceboxv1alpha1.AgentSession, ref string) string { diff --git a/pkg/controllers/agentsession/sleep.go b/pkg/controllers/agentsession/sleep.go index 8288ad84..0121d36c 100644 --- a/pkg/controllers/agentsession/sleep.go +++ b/pkg/controllers/agentsession/sleep.go @@ -12,6 +12,9 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" lifecyclecore "github.com/authzed/openagentprimitives/pkg/agent/session/lifecycle" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" @@ -56,6 +59,11 @@ func idleSleepDue(sleptAt, lastIdleAt *metav1.Time, sleepAfter time.Duration, no // and wakeable. Returns (slept, requeueAfter, err). Preconditions: caller // confirmed sess.Spec.InputChannel != nil, phase == Idle, and // !shouldWake(sess). +// +// The admitter's reclaim request (AnnotationReclaimRequested) makes the sleep +// due now, whatever SleepAfter says — including when idle sleep is disabled — +// but never past the SessionHold guard. The annotation is removed once the +// session is slept. func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, class *spiceboxv1alpha1.AgentClass) (bool, time.Duration, error) { // Evidence outranks GC. Archiving or reaping a held session would let the // forensic subject expire on a timer, which is exactly the failure mode that @@ -78,7 +86,20 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. if class != nil && class.Spec.Channels != nil && class.Spec.Channels.SleepAfter.Duration > 0 { sleepAfter = class.Spec.Channels.SleepAfter.Duration } + reclaim := reclaimRequested(sess) + if reclaim && sess.Status.SleptAt != nil { + // Already slept, so the request is fulfilled: a previous pass slept the + // session but failed to remove the annotation. Remove it now; sleeping + // again would reap nothing. + return false, 0, r.clearReclaimRequest(ctx, sess) + } due, remaining := idleSleepDue(sess.Status.SleptAt, sess.Status.LastIdleAt, sleepAfter, r.now()) + if reclaim && !due { + log.FromContext(ctx).Info("sleeping idle session early: its AgentClass has a session waiting for the capacity", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, + "reclaimRequested", sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]) + due, remaining = true, 0 + } if !due { return false, remaining, nil // remaining==0 when disabled/already-slept/no-clock } @@ -88,6 +109,12 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. return false, 0, fmt.Errorf("apply Sleep event: %w", err) } patched := sess.DeepCopy() + // The reap below gives the session's place back to its line. Recorded in + // THIS write, beside SleptAt: a wake clears SleptAt, so a release written + // separately could be lost in between, leaving the session admitted with + // no pods and its capacity counted twice. Decided before SleptAt is set, + // which would hide a pre-admission session from the release. + r.releaseAdmissionOnReclaim(patched) now := metav1.NewTime(r.now()) patched.Status.SleptAt = &now if err := agentstatus.WriteOwned(ctx, r.Client, patched, sess, spiceboxv1alpha1.OwnerOperator); err != nil { @@ -96,5 +123,53 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. if _, err := r.reapSessionPods(ctx, sess); err != nil { return false, 0, err // already logged inside; retry } + // The annotation is metadata and SleptAt is status: two subresources, so + // they cannot share the write above. It is removed only AFTER the status + // write and the reap succeed, so a failure anywhere earlier leaves the + // request in place for the retry; a failure here leaves a slept session + // still carrying it, which the already-slept branch above clears on the + // next pass without sleeping again. + if reclaim { + if err := r.clearReclaimRequest(ctx, sess); err != nil { + return true, 0, err + } + } return true, 0, nil } + +// reclaimRequested reports whether the admitter has asked for this session to +// be slept early (AnnotationReclaimRequested). +func reclaimRequested(sess *spiceboxv1alpha1.AgentSession) bool { + _, ok := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested] + return ok +} + +// clearStaleReclaimRequest removes a reclaim request from a session that is no +// longer Idle — it woke, or was held, before the sleep path acted on it. Left +// in place, it would sleep the session the moment it next went Idle, long +// after the admission line that asked for the capacity moved on. An Idle +// session's request is left for reconcileSleep. +func (r *Reconciler) clearStaleReclaimRequest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) error { + if sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseIdle || !reclaimRequested(sess) { + return nil + } + log.FromContext(ctx).Info("clearing a stale reclaim request: the session left Idle before it was slept", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, "phase", sess.Status.Phase) + return r.clearReclaimRequest(ctx, sess) +} + +// clearReclaimRequest removes AnnotationReclaimRequested with a metadata-only +// merge patch, sent on a fresh object carrying just the key so the server's +// response cannot overwrite the caller's in-memory status; the caller's copy +// drops the annotation to match. +func (r *Reconciler) clearReclaimRequest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) error { + patch := fmt.Appendf(nil, `{"metadata":{"annotations":{%q:null}}}`, spiceboxv1alpha1.AnnotationReclaimRequested) + target := &spiceboxv1alpha1.AgentSession{ObjectMeta: metav1.ObjectMeta{Name: sess.Name, Namespace: sess.Namespace}} + if err := r.Client.Patch(ctx, target, client.RawPatch(types.MergePatchType, patch)); err != nil { + log.FromContext(ctx).Info("removing the reclaim-requested annotation failed; retried on the next reconcile", + "session", sess.Namespace+"/"+sess.Name, "err", err.Error()) + return fmt.Errorf("clear reclaim-requested annotation on %s/%s: %w", sess.Namespace, sess.Name, err) + } + delete(sess.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested) + return nil +} diff --git a/pkg/controllers/agentsession/sleep_test.go b/pkg/controllers/agentsession/sleep_test.go index 056fe75b..1efce78a 100644 --- a/pkg/controllers/agentsession/sleep_test.go +++ b/pkg/controllers/agentsession/sleep_test.go @@ -7,13 +7,18 @@ package agentsession import ( + "context" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/memory" ) // TestSandboxProvisioningDesired pins the lazy-provisioning gate: only an @@ -67,3 +72,111 @@ func TestIdleSleepDue(t *testing.T) { }) } } + +// withReclaimRequest seeds the fixture session as Admitted=True and carrying +// the admitter's reclaim-requested annotation, then reloads f.sess from the +// fake client so reconcileSleep sees what a reconcile would read. +func (f *sleepFixture) withReclaimRequest(t *testing.T) { + t.Helper() + ctx := memory.WithSystemApproval(context.Background(), "test") + f.sess.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(f.now.Add(-time.Hour)), + }} + require.NoError(t, f.c.Status().Update(ctx, f.sess), "seed Admitted=True") + live := f.getSession(t) + live.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: f.now.Format(time.RFC3339)} + require.NoError(t, f.c.Update(ctx, live), "seed reclaim annotation") + f.sess = f.getSession(t) +} + +func TestReconcileSleepReclaimRequested(t *testing.T) { + cases := []struct { + name string + sleepAfter time.Duration + }{ + {name: "reclaim annotation on an Idle session within SleepAfter: slept now, Admitted=False/Reclaimed, annotation cleared", sleepAfter: 10 * time.Minute}, + {name: "reclaim annotation with idle sleep disabled: slept now, Admitted=False/Reclaimed, annotation cleared", sleepAfter: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, tc.sleepAfter) // idle 2m: not due on its own + f.withReclaimRequest(t) + + slept, requeueAfter, err := f.r.reconcileSleep(memory.WithSystemApproval(context.Background(), "test"), f.sess, f.class) + require.NoError(t, err) + assert.True(t, slept, "a reclaim request makes the sleep due now") + assert.Zero(t, requeueAfter) + + got := f.getSession(t) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhaseIdle, got.Status.Phase, "stays Idle (wakeable)") + assert.NotNil(t, got.Status.SleptAt, "SleptAt stamped") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if assert.NotNil(t, adm, "Admitted condition") { + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + } + assert.NotContains(t, got.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "annotation cleared") + assert.True(t, f.bundlesGone(t), "pods reaped") + }) + } + + t.Run("reclaim annotation on a held session (SessionHold): not slept", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + require.NoError(t, f.c.Create(ctx, &spiceboxv1alpha1.SessionHold{ + ObjectMeta: metav1.ObjectMeta{Name: "h1", Namespace: "default"}, + Spec: spiceboxv1alpha1.SessionHoldSpec{SessionRef: spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "s1"}}, + Status: spiceboxv1alpha1.SessionHoldStatus{Phase: spiceboxv1alpha1.SessionHoldPhaseActive}, + }), "create SessionHold") + + slept, _, err := f.r.reconcileSleep(ctx, f.sess, f.class) + require.NoError(t, err) + assert.False(t, slept, "the hold guard outranks a reclaim request") + + got := f.getSession(t) + assert.Nil(t, got.Status.SleptAt, "SleptAt not stamped") + assert.True(t, admission.IsAdmitted(got), "still admitted") + assert.False(t, f.bundlesGone(t), "pods kept") + }) + + t.Run("reclaim annotation on an already-slept session: annotation cleared, nothing reaped again", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + sleptAt := metav1.NewTime(f.now.Add(-time.Minute)) + f.sess.Status.SleptAt = &sleptAt + require.NoError(t, f.c.Status().Update(ctx, f.sess), "seed already-slept status") + f.sess = f.getSession(t) + + slept, _, err := f.r.reconcileSleep(ctx, f.sess, f.class) + require.NoError(t, err) + assert.False(t, slept, "already slept: no second sleep") + assert.NotContains(t, f.getSession(t).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "fulfilled request cleared") + assert.False(t, f.bundlesGone(t), "not reaped a second time") + }) + + t.Run("stale reclaim annotation on a session that woke (Running): annotation cleared, not slept", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + f.sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + require.NoError(t, f.c.Status().Update(ctx, f.sess), "seed Running") + f.sess = f.getSession(t) + + require.NoError(t, f.r.clearStaleReclaimRequest(ctx, f.sess)) + got := f.getSession(t) + assert.NotContains(t, got.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "stale request cleared") + assert.NotContains(t, f.sess.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "in-memory copy agrees") + assert.Nil(t, got.Status.SleptAt, "not slept") + assert.False(t, f.bundlesGone(t), "pods kept") + }) + + t.Run("reclaim annotation on an Idle session: clearStaleReclaimRequest leaves it for the sleep path", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + require.NoError(t, f.r.clearStaleReclaimRequest(memory.WithSystemApproval(context.Background(), "test"), f.sess)) + assert.Contains(t, f.getSession(t).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "an Idle session's request is not stale") + }) +} diff --git a/pkg/controllers/agentstatus/statuswrite.go b/pkg/controllers/agentstatus/statuswrite.go index cbcf30ef..3f886e3d 100644 --- a/pkg/controllers/agentstatus/statuswrite.go +++ b/pkg/controllers/agentstatus/statuswrite.go @@ -78,6 +78,7 @@ func WriteOwned(ctx context.Context, c client.Client, sess, original *spiceboxv1 prior := fresh.DeepCopy() for i := range set { meta.SetStatusCondition(&fresh.Status.Conditions, set[i]) + carryRestampedTransition(fresh.Status.Conditions, original, set[i]) } for _, t := range removed { meta.RemoveStatusCondition(&fresh.Status.Conditions, t) @@ -91,6 +92,23 @@ func WriteOwned(ctx context.Context, c client.Client, sess, original *spiceboxv1 }) } +// carryRestampedTransition persists a transition time the writer moved on +// purpose. meta.SetStatusCondition keeps the stored LastTransitionTime +// whenever the status is unchanged, and a writer that builds its conditions +// through it never moves the time on such a change — so a time later than the +// one in original, on an unchanged status, is a deliberate re-stamp (a +// session arriving in line again: Admitted False/Reclaimed → False/Queued at +// now) and is copied onto the stored condition. +func carryRestampedTransition(stored []metav1.Condition, original *spiceboxv1alpha1.AgentSession, cur metav1.Condition) { + old := meta.FindStatusCondition(original.Status.Conditions, cur.Type) + if old == nil || old.Status != cur.Status || !cur.LastTransitionTime.After(old.LastTransitionTime.Time) { + return + } + if c := meta.FindStatusCondition(stored, cur.Type); c != nil && c.Status == cur.Status { + c.LastTransitionTime = cur.LastTransitionTime + } +} + // diffOwnedConditions returns the owner's conditions that were added or changed // (set) and those removed, between original and sess. func diffOwnedConditions(original, sess *spiceboxv1alpha1.AgentSession, owner spiceboxv1alpha1.StatusFieldOwner) (set []metav1.Condition, removed []string) { diff --git a/pkg/controllers/agentstatus/statuswrite_test.go b/pkg/controllers/agentstatus/statuswrite_test.go index 4b1dae38..51853f22 100644 --- a/pkg/controllers/agentstatus/statuswrite_test.go +++ b/pkg/controllers/agentstatus/statuswrite_test.go @@ -3,6 +3,7 @@ package agentstatus import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -131,3 +132,62 @@ func TestWriteOwned_ConditionWrite_DoesNotResendOperatorOwnedFields(t *testing.T assert.True(t, got.Status.EffectiveSettings.ReportSessionCost, "operator-owned effectiveSettings.reportSessionCost must be preserved by a condition write") } + +// A writer that deliberately re-stamps a condition's transition time on a +// reason change with no status change (Admitted False/Reclaimed → +// False/Queued: the session arrives in line now, not at its release) must +// have that time persisted. meta.SetStatusCondition alone keeps the stored +// time whenever the status is unchanged; a writer that did not move the time +// keeps that behaviour. +func TestWriteOwned_CarriesDeliberateTransitionTimeOnSameStatus(t *testing.T) { + released := metav1.NewTime(time.Now().Add(-48 * time.Hour).Truncate(time.Second)) + queued := metav1.NewTime(time.Now().Truncate(time.Second)) + cases := []struct { + name string + wantLTT metav1.Time + mutate func(c *metav1.Condition) + }{ + { + name: "reason changed and transition time moved later: the new time is persisted", + wantLTT: queued, + mutate: func(c *metav1.Condition) { + c.Reason = spiceboxv1alpha1.ReasonAdmissionQueued + c.LastTransitionTime = queued + }, + }, + { + name: "reason changed, transition time untouched: the stored time is kept", + wantLTT: released, + mutate: func(c *metav1.Condition) { + c.Reason = spiceboxv1alpha1.ReasonAdmissionQueued + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + original := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: "sess", Namespace: "default"}, + Status: spiceboxv1alpha1.AgentSessionStatus{Conditions: []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionReclaimed, LastTransitionTime: released, + }}}, + } + c := fake.NewClientBuilder().WithScheme(writeOwnedScheme(t)). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithObjects(original.DeepCopy()).Build() + var base spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(original), &base)) + sess := base.DeepCopy() + tc.mutate(&sess.Status.Conditions[0]) + + require.NoError(t, WriteOwned(context.Background(), c, sess, &base, spiceboxv1alpha1.OwnerOperator)) + + var got spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(original), &got)) + cond := meta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, cond) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, cond.Reason) + assert.True(t, tc.wantLTT.Equal(&cond.LastTransitionTime), "want %v got %v", tc.wantLTT, cond.LastTransitionTime) + }) + } +} diff --git a/pkg/controllers/webhooks/agentsession/webhook.go b/pkg/controllers/webhooks/agentsession/webhook.go index efc3b075..b9d64a95 100644 --- a/pkg/controllers/webhooks/agentsession/webhook.go +++ b/pkg/controllers/webhooks/agentsession/webhook.go @@ -447,6 +447,15 @@ var pinnedStatusFields = []pinnedStatusField{ "fork gate and the recovery mechanism against a runner doing exactly this. channelsd also reads it as the " + "tie-breaker for which session owns a thread (newestActiveSession)", }, + { + name: "admission", + get: func(s *spiceboxv1alpha1.AgentSessionStatus) any { return s.Admission }, + why: "it is the admitter's record of this session's place in its AgentClass's admission line — position, line length " + + "and join time — and only the leader-only admitter writes it (pkg/controllers/admission/admitter). The admitter " + + "compares against the stored value to decide whether to write at all, so a forged value is not corrected until the " + + "real position next changes, and it is the source the 'Nth in line' notice to the waiting human is built from. " + + "A runner has no business in the queue it is waiting in", + }, { name: "effectiveSettings", get: func(s *spiceboxv1alpha1.AgentSessionStatus) any { return s.EffectiveSettings }, @@ -594,10 +603,10 @@ var runnerWritableStatusFields = map[string]string{ "session's own retry of an already-idempotent write, denying its own starter no more than the runner could " + "already withhold by declining to run at all — self-scoped, and any other session sharing that class/subject " + "still attempts the same touch", - "slotPins": "a display-only mirror of slot pin state written by StatusPatcher (pkg/agent/runner/status.go). " + - "Every enforcement decision reads SpiceDB directly (the slot_pin tuple and the MUST_MATCH preconditions on " + - "grant writes), and the approval card derives move state from a live SpiceDB ReadPin rather than this field, " + - "so a forged value only misdescribes the forging session's own `oap session show` output and buys no authority anywhere", + "slotPins": "a display-only mirror of slot pin state written by StatusPatcher (pkg/agent/runner/status.go). " + + "Every enforcement decision reads SpiceDB directly (the slot_pin tuple and the MUST_MATCH preconditions on " + + "grant writes), and the approval card derives move state from a live SpiceDB ReadPin rather than this field, " + + "so a forged value only misdescribes the forging session's own `oap session show` output and buys no authority anywhere", } // Webhook validates AgentSession update admission requests. diff --git a/pkg/controllers/webhooks/agentsession/webhook_test.go b/pkg/controllers/webhooks/agentsession/webhook_test.go index 8e4ddb5d..a00507c1 100644 --- a/pkg/controllers/webhooks/agentsession/webhook_test.go +++ b/pkg/controllers/webhooks/agentsession/webhook_test.go @@ -625,6 +625,13 @@ func TestRunnerCannotForgeSessionIdentity(t *testing.T) { s.Status.AppliedInteractPermissionAt = &t }, }, + { + name: "runner rewrites status.admission to the head of its line: denied (the admitter alone owns the queue position a waiting human is shown)", + sub: "status", + mut: func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Admission = &spiceboxv1alpha1.AdmissionStatus{Position: 1, LineLength: 1} + }, + }, { // The operator turns spiceboxSessionName into BOTH a Role // resourceName and a memory-API token scope, on every reconcile, diff --git a/pkg/platform/manifests/install.yaml b/pkg/platform/manifests/install.yaml index 19defc77..72a348a0 100644 --- a/pkg/platform/manifests/install.yaml +++ b/pkg/platform/manifests/install.yaml @@ -1580,6 +1580,14 @@ spec: required: - prompt type: object + maxConcurrentSessions: + description: |- + MaxConcurrentSessions: at most this many of this agent's sessions hold + capacity at once. Delegated child sessions are not counted and are + never blocked by it. Unset means no limit. + format: int32 + minimum: 1 + type: integer mcpServers: description: |- MCPServers references MCPServer CRs in the same namespace whose @@ -5070,6 +5078,33 @@ spec: - artifactID type: object type: array + admission: + description: |- + Admission is where this session stands in its AgentClass's admission + line while it waits for its pod set to be admitted. Written by the + operator's admitter only when the position changes, and cleared when the + session is admitted. Nil when the session is not waiting. + properties: + joinedAt: + description: |- + JoinedAt is when the session took its place in line: its creation, or + the wake that re-queued it. + format: date-time + type: string + lineLength: + description: LineLength is how many sessions are waiting in the + line. + format: int32 + type: integer + position: + description: Position is the session's 1-based place in its line. + format: int32 + type: integer + required: + - joinedAt + - lineLength + - position + type: object agentWakeCredit: description: |- AgentWakeCredit is how many further AGENT-DRIVEN wakes this session will diff --git a/pkg/platform/schedfit/schedfit.go b/pkg/platform/schedfit/schedfit.go index 0e366648..f8cd200d 100644 --- a/pkg/platform/schedfit/schedfit.go +++ b/pkg/platform/schedfit/schedfit.go @@ -9,6 +9,7 @@ package schedfit import ( "fmt" + "maps" "github.com/dustin/go-humanize" corev1 "k8s.io/api/core/v1" @@ -50,6 +51,11 @@ type Headroom struct { CPUMilli int64 MemBytes int64 EphemeralBytes int64 + + // Pods is the node's allocatable pod count minus its non-terminal pods, + // floored at zero. A node that reports no `pods` allocatable yields 0 here; + // callers that need "unlimited" must check the node's allocatable themselves. + Pods int64 } // CeilingFromNodes computes the per-dimension maximum allocatable across nodes. @@ -103,6 +109,7 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { CPUMilli: milli(alloc, corev1.ResourceCPU), MemBytes: value(alloc, corev1.ResourceMemory), EphemeralBytes: value(alloc, corev1.ResourceEphemeralStorage), + Pods: value(alloc, corev1.ResourcePods), } for i := range pods { p := &pods[i] @@ -112,18 +119,20 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { if p.Status.Phase == corev1.PodSucceeded || p.Status.Phase == corev1.PodFailed { continue } - cpu, mem, eph := effectiveRequests(p) + cpu, mem, eph := EffectiveRequests(&p.Spec) + h.Pods-- h.CPUMilli -= cpu h.MemBytes -= mem h.EphemeralBytes -= eph } + h.Pods = max(h.Pods, 0) h.CPUMilli = max(h.CPUMilli, 0) h.MemBytes = max(h.MemBytes, 0) h.EphemeralBytes = max(h.EphemeralBytes, 0) return h } -// effectiveRequests implements the Kubernetes effective pod request: +// EffectiveRequests implements the Kubernetes effective pod request: // max(largest init-container request, sum of container requests), per // dimension. Init containers run to completion before the containers start, so // the scheduler reserves whichever is larger — not their sum. @@ -131,29 +140,35 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { // Native sidecars are the exception: an init container with restartPolicy // Always keeps running alongside the app containers for the pod's lifetime, so // it ADDS to the sum like an app container instead of merely setting a floor. -func effectiveRequests(p *corev1.Pod) (cpuMilli, memBytes, ephBytes int64) { - for i := range p.Spec.Containers { - req := p.Spec.Containers[i].Resources.Requests +// +// A resource with a limit but no request is priced at its limit, which is the +// default the API server applies at admission. Live pods already carry it; a +// PodSpec a controller has built but not yet created does not (the sandbox +// builder sets Limits alone), and reading Requests alone would price it at +// zero. +func EffectiveRequests(spec *corev1.PodSpec) (cpuMilli, memBytes, ephBytes int64) { + for i := range spec.Containers { + req := requestsOf(&spec.Containers[i]) cpuMilli += milli(req, corev1.ResourceCPU) memBytes += value(req, corev1.ResourceMemory) ephBytes += value(req, corev1.ResourceEphemeralStorage) } // Sidecars first: they belong IN the sum that the init-container floor below // is then measured against, not compared to it. - for i := range p.Spec.InitContainers { - if !isSidecar(&p.Spec.InitContainers[i]) { + for i := range spec.InitContainers { + if !isSidecar(&spec.InitContainers[i]) { continue } - req := p.Spec.InitContainers[i].Resources.Requests + req := requestsOf(&spec.InitContainers[i]) cpuMilli += milli(req, corev1.ResourceCPU) memBytes += value(req, corev1.ResourceMemory) ephBytes += value(req, corev1.ResourceEphemeralStorage) } - for i := range p.Spec.InitContainers { - if isSidecar(&p.Spec.InitContainers[i]) { + for i := range spec.InitContainers { + if isSidecar(&spec.InitContainers[i]) { continue } - req := p.Spec.InitContainers[i].Resources.Requests + req := requestsOf(&spec.InitContainers[i]) cpuMilli = max(cpuMilli, milli(req, corev1.ResourceCPU)) memBytes = max(memBytes, value(req, corev1.ResourceMemory)) ephBytes = max(ephBytes, value(req, corev1.ResourceEphemeralStorage)) @@ -161,6 +176,23 @@ func effectiveRequests(p *corev1.Pod) (cpuMilli, memBytes, ephBytes int64) { return cpuMilli, memBytes, ephBytes } +// requestsOf returns a container's requests as the API server would default +// them: each resource that has a limit and no request is requested at its +// limit. The container itself is not modified. +func requestsOf(c *corev1.Container) corev1.ResourceList { + req := maps.Clone(c.Resources.Requests) + for name, limit := range c.Resources.Limits { + if _, ok := req[name]; ok { + continue + } + if req == nil { + req = corev1.ResourceList{} + } + req[name] = limit + } + return req +} + // isSidecar reports whether an init container is a native sidecar — restartPolicy // Always, meaning it runs for the pod's whole lifetime rather than completing // before the app containers start. @@ -172,15 +204,15 @@ func isSidecar(c *corev1.Container) bool { // // Prefer this over calling Exceeds per container. Kubernetes places a pod // atomically, so what has to fit is the pod's effective request — see -// effectiveRequests — and checking containers one at a time under-detects: three +// EffectiveRequests — and checking containers one at a time under-detects: three // 2Gi containers each clear a 4Gi ceiling while the pod they belong to needs 6Gi -// and can never be placed. Sharing effectiveRequests with HeadroomOn keeps the +// and can never be placed. Sharing EffectiveRequests with HeadroomOn keeps the // "what does this pod actually cost" rule in exactly one place. func ExceedsPod(label string, pod *corev1.Pod, c Ceiling) (string, bool) { if pod == nil || !c.Known { return "", false } - cpuMilli, memBytes, ephBytes := effectiveRequests(pod) + cpuMilli, memBytes, ephBytes := EffectiveRequests(&pod.Spec) req := corev1.ResourceList{} if cpuMilli > 0 { req[corev1.ResourceCPU] = *resource.NewMilliQuantity(cpuMilli, resource.DecimalSI) diff --git a/pkg/platform/schedfit/schedfit_test.go b/pkg/platform/schedfit/schedfit_test.go index 453d26c8..cec18977 100644 --- a/pkg/platform/schedfit/schedfit_test.go +++ b/pkg/platform/schedfit/schedfit_test.go @@ -221,3 +221,49 @@ func TestEffectiveRequests_NativeSidecarAddsRatherThanSettingAFloor(t *testing.T assert.Equal(t, int64(2)*1024*1024*1024, h.MemBytes, "app(1Gi)+sidecar(1Gi)=2Gi booked of 4Gi; treating the sidecar as a floor would leave 3Gi") } + +// TestEffectiveRequests_UnsetRequestDefaultsToLimit covers a PodSpec that has +// not been through the API server yet. The apiserver defaults a container's +// unset request to its limit, per resource, at admission; an in-memory spec a +// controller has only BUILT (podspec.BuildClassSpec sets Limits alone) has not +// had that applied, so reading Requests alone would price it at zero. +func TestEffectiveRequests_UnsetRequestDefaultsToLimit(t *testing.T) { + cases := []struct { + name string + container corev1.ResourceRequirements + init *corev1.ResourceRequirements + cpu, mem, ephBytes int64 + }{ + { + name: "limits only: every dimension is priced at its limit", + container: corev1.ResourceRequirements{Limits: reqList("500m", "1Gi", "2Gi")}, + cpu: 500, mem: gib, ephBytes: 2 * gib, + }, + { + name: "explicit request below the limit: the request wins, limit is not a request", + container: corev1.ResourceRequirements{ + Requests: reqList("100m", "", ""), + Limits: reqList("500m", "1Gi", ""), + }, + cpu: 100, mem: gib, + }, + { + name: "limits-only init container larger than the app: sets the floor", + container: corev1.ResourceRequirements{Limits: reqList("", "1Gi", "")}, + init: &corev1.ResourceRequirements{Limits: reqList("", "3Gi", "")}, + mem: 3 * gib, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + spec := corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Resources: tc.container}}} + if tc.init != nil { + spec.InitContainers = []corev1.Container{{Name: "setup", Resources: *tc.init}} + } + cpu, mem, eph := EffectiveRequests(&spec) + assert.Equal(t, tc.cpu, cpu, "cpu millicores") + assert.Equal(t, tc.mem, mem, "memory bytes") + assert.Equal(t, tc.ephBytes, eph, "ephemeral-storage bytes") + }) + } +} diff --git a/pkg/tools/toolchain/resolve/resolve.go b/pkg/tools/toolchain/resolve/resolve.go index 45bb1294..17e89a74 100644 --- a/pkg/tools/toolchain/resolve/resolve.go +++ b/pkg/tools/toolchain/resolve/resolve.go @@ -42,7 +42,7 @@ var ( // // Names are sorted and deduplicated so the result — and therefore the frozen // status, the set digest, and the PodSpec — is a pure function of the class. -func Resolve(ctx context.Context, c client.Client, names []string) ([]spiceboxv1alpha1.ToolchainMount, string, error) { +func Resolve(ctx context.Context, c client.Reader, names []string) ([]spiceboxv1alpha1.ToolchainMount, string, error) { uniq := make([]string, 0, len(names)) seen := map[string]struct{}{} for _, n := range names { diff --git a/pkg/web/workshopprojectsrv/whitelist_test.go b/pkg/web/workshopprojectsrv/whitelist_test.go index bda91b82..7fa31fcd 100644 --- a/pkg/web/workshopprojectsrv/whitelist_test.go +++ b/pkg/web/workshopprojectsrv/whitelist_test.go @@ -54,6 +54,9 @@ var notProjectedFields = map[string]bool{ "WorkspaceSource": true, "ToolSessionLog": true, "Budget": true, + // MaxConcurrentSessions caps the source agent's own sessions; a stand-in + // is a different class with no line worth limiting. + "MaxConcurrentSessions": true, "BoundEntities": true, "Channels": true, "Authz": true, diff --git a/pkg/web/workshopprojectsrv/workshopprojectsrv.go b/pkg/web/workshopprojectsrv/workshopprojectsrv.go index ac878ce8..f3f6f773 100644 --- a/pkg/web/workshopprojectsrv/workshopprojectsrv.go +++ b/pkg/web/workshopprojectsrv/workshopprojectsrv.go @@ -544,6 +544,8 @@ func reachable(agents []workshopthreadsrv.AgentInThread, ns, class string) bool // Model absent means the tier default applies, Harness absent means // ap-native. // - Budget — the tier default applies. +// - MaxConcurrentSessions — the source agent's session cap; a stand-in is +// a different class, with no line of the source's to limit. // - Subagents, SubagentModes, BoundEntities (deprecated) — DROPPED. A // source class's own roster would almost certainly name classes this // workshop does not have, which the AgentClass reconciler marks diff --git a/test/e2e/admission.go b/test/e2e/admission.go new file mode 100644 index 00000000..c068a3ab --- /dev/null +++ b/test/e2e/admission.go @@ -0,0 +1,116 @@ +//go:build e2e + +package e2e + +import ( + "context" + "strings" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/admission/admitter" +) + +// AdmissionOptions turns on whole-set session admission for one harness: the +// AgentSession reconciler gates every booting session's pods on Admitted=True +// (AdmissionEnabled), and the admitter runs against Capacity. +// +// Opt-in per scenario, like WithTokenAuthz. Nil — every scenario but an +// admission one — leaves sessions ungated, which is the reconciler's zero +// value and what every other test was written against. +type AdmissionOptions struct { + // Capacity is what the admitter places sets against. Required: envtest + // has no nodes, so the production KubeCapacitySource would report unknown + // capacity and admit every session at once — a line that never queues. + Capacity admission.CapacitySource +} + +// registerAdmitter adds the admitter to mgr when admission is on. Called for +// the first manager and again by RestartOperator, so a restarted operator +// resumes the line from AgentSession status as production's does. +func (h *Harness) registerAdmitter(t *testing.T, mgr manager.Manager) { + t.Helper() + if h.opts.Admission == nil { + return + } + if err := (&admitter.Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Capacity: h.opts.Admission.Capacity, + Enabled: true, + }).SetupWithManager(mgr); err != nil { + t.Fatalf("e2e: register admission controller: %v", err) + } +} + +// FixedSetCapacity is a cluster with room for exactly Sets whole session pod +// sets, whatever their size. A set's room is charged to its runner pod, the +// one pod every session's set has exactly one of, so with the harness's pods +// never binding to a node (there is no scheduler), each admitted session holds +// one slot until it is released. +// +// It stands in for nodes, not for the admission logic: the admitter still +// orders lines, applies fair share and reserves holders' unplaced demand +// through admission.Decide exactly as in production. +type FixedSetCapacity struct { + Sets int +} + +// Snapshot returns a fresh model with every set free; the admitter reserves +// current holders on it before deciding. +func (c FixedSetCapacity) Snapshot(context.Context) (admission.CapacityModel, error) { + return &setModel{free: c.Sets, sets: c.Sets}, nil +} + +// setModelSetCPUMilli and setModelSetMemBytes size one set for Totals, the +// denominator of a line's dominant share. Only the RATIO between lines' +// shares matters to Decide, so any positive size will do. +const ( + setModelSetCPUMilli = 4000 + setModelSetMemBytes = 8 << 30 +) + +type setModel struct { + free, sets int +} + +func (m *setModel) Known() bool { return true } + +// Reserve charges one set per runner pod in d. Charging per call instead +// would be wrong: Decide reserves a holder's unplaced pods one at a time, so a +// three-pod holder would take three sets. A pod that is not a runner (sandbox, +// detector, sidecar) rides on its session's set and is free. +func (m *setModel) Reserve(d admission.Demand) bool { + sets := 0 + for _, p := range d.Pods { + if isRunnerPod(p) { + sets++ + } + } + if sets > m.free { + return false + } + m.free -= sets + return true +} + +// isRunnerPod matches the runner pod of a session's demand, named by +// agentsession.RunnerPodName. +func isRunnerPod(p admission.PodDemand) bool { + return p.SandboxSession == "" && strings.HasSuffix(p.Name, "-runner") +} + +// Free is a no-op: a model rebuilt every pass has no placed pods to return, +// and nothing in a scenario frees through the model mid-pass. +func (m *setModel) Free([]admission.PlacedPod) {} + +func (m *setModel) Clone() admission.CapacityModel { + c := *m + return &c +} + +func (m *setModel) Totals() (cpuMilli, memBytes int64) { + return int64(m.sets) * setModelSetCPUMilli, int64(m.sets) * setModelSetMemBytes +} diff --git a/test/e2e/admission_test.go b/test/e2e/admission_test.go new file mode 100644 index 00000000..252aef59 --- /dev/null +++ b/test/e2e/admission_test.go @@ -0,0 +1,51 @@ +//go:build e2e + +package e2e + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/controllers/admission" +) + +// setDemand is one session's pod set as DemandFor prices it: the runner pod +// plus one sandbox pod per tool bundle. +func setDemand(session string, bundles ...string) admission.Demand { + d := admission.Demand{Pods: []admission.PodDemand{{Name: session + "-runner", CPUMilli: 100, MemBytes: 1 << 20}}} + for _, b := range bundles { + d.Pods = append(d.Pods, admission.PodDemand{ + Name: session + "-" + b + "-pod", CPUMilli: 100, MemBytes: 1 << 20, SandboxSession: session + "-" + b, + }) + } + return d +} + +// TestFixedSetCapacityCountsWholeSets pins that the harness model charges one +// set per SESSION, however Decide splits that session's pods across Reserve +// calls. Decide reserves a holder's unplaced pods one at a time; a model that +// charged a set per call let one three-pod holder fill a two-set cluster, so +// the second arrival waited for the first to leave. +func TestFixedSetCapacityCountsWholeSets(t *testing.T) { + holder := setDemand("first", "gitlike", "scratch") + model, err := FixedSetCapacity{Sets: 2}.Snapshot(context.Background()) + require.NoError(t, err) + + key := func(n string) types.NamespacedName { return types.NamespacedName{Namespace: "default", Name: n} } + dec := admission.Decide(admission.Input{ + Holders: []admission.Holder{{Key: key("first"), Line: "default/reviewer", Demand: holder, Unplaced: holder}}, + Waiting: []admission.Candidate{ + {Key: key("second"), Line: "default/reviewer", Demand: setDemand("second", "gitlike", "scratch")}, + {Key: key("third"), Line: "default/reviewer", Demand: setDemand("third", "gitlike", "scratch")}, + }, + Model: model, + }) + + assert.Equal(t, []types.NamespacedName{key("second")}, dec.Admit, + "a two-set cluster with one holder has room for exactly one more session") + assert.Equal(t, admission.Position{Index: 1, LineLength: 1}, dec.Positions[key("third")]) +} diff --git a/test/e2e/bronzethread/testdata/admission-burst/bundle.json b/test/e2e/bronzethread/testdata/admission-burst/bundle.json new file mode 100644 index 00000000..be2f80ef --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-burst/bundle.json @@ -0,0 +1,453 @@ +{ + "name": "admission-burst", + "description": "A burst on a full cluster: three reviewer sessions arrive together, then a session of a second agent, on a cluster with room for exactly two sessions' whole pod sets. The line, not the scheduler, decides who runs.\n\nWhat is claimed, and how each claim is read from the run rather than from the transcript:\n- every session runs to completion and gives its room back (each reaches Idle after being admitted, then sleeps, which releases its admission);\n- the reviewer line is FIFO: a1 and a2 are admitted before a3, and a3 finishes after both;\n- fair share between lines: when a1 frees its room while a2 still holds one, the notes agent's session is admitted ahead of the burst's tail (b before a3);\n- never more than two sessions admitted at once, and exactly two at the peak (so the line actually queued);\n- no session ever holds part of a pod set: a waiting session holds none of its bundle SpiceboxSessions, a Running one holds all of them. The driver watches every session for the whole run to check this.\nEach session's sandbox call is asserted on its TOOL RESULT (lastToolResultContains / IsError), which proves the call reached that session's own bundle sandbox.\n\nWhy a1 and a2 park in await_user_message: a finished session keeps its admission until it sleeps, so WHICH holder frees its room first is an input to this scenario, not a race to leave to timing. a1 parks until every session has arrived, so its release cannot reach the line before b is waiting; a2 parks until b has finished, so it is still holding when a1's room frees, which is the moment fair share has to choose. The notes agent sleeps after 10s rather than 1s for the same reason: a3's room should come from a2 finishing, so a3 finishing last is a fact about the line, not a guess about speed.\n\nCapacity is two, not one, deliberately. With room for a single session no line ever HOLDS capacity at the moment a choice is made (the only holder is the session that just left), both lines tie at zero share, and the earlier arrival wins — so the second agent would be admitted last and fair share could not be observed at all.", + "agentDir": "testdata/admission-burst/manifests", + "agentClass": "burst-reviewer", + "admission": { + "capacitySessions": 2 + }, + "userTurns": [ + { + "text": "review pull request 101", + "thread": "a1", + "channel": "burst-reviewer-fake" + }, + { + "text": "review pull request 202", + "thread": "a2", + "channel": "burst-reviewer-fake" + }, + { + "text": "review pull request 303", + "thread": "a3", + "channel": "burst-reviewer-fake" + }, + { + "text": "check notes for pull request 404", + "thread": "b", + "channel": "burst-notes-fake" + }, + { + "text": "a1 go ahead and finish", + "thread": "a1", + "channel": "burst-reviewer-fake", + "waitFor": [ + { + "thread": "a1", + "until": "awaitingInput" + } + ] + }, + { + "text": "a2 go ahead and finish", + "thread": "a2", + "channel": "burst-reviewer-fake", + "waitFor": [ + { + "thread": "a2", + "until": "awaitingInput" + }, + { + "thread": "b", + "until": "idle" + } + ] + } + ], + "toolOutputs": { + "gitlike_gh": { + "stdout": "title: Update README.md\nstate: OPEN\nmergeable: MERGEABLE\n" + } + }, + "llm": [ + { + "thread": "a1", + "expect": { + "userTextContains": "review pull request 101" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 101" + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_a1": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_a1}}", + "_reason": "reading pull request 101", + "args": [ + "pr", + "view", + "101", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 101 is open and mergeable" + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "await_user_message", + "args": {} + } + } + ] + }, + { + "thread": "a2", + "expect": { + "userTextContains": "review pull request 202" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 202" + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_a2": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_a2}}", + "_reason": "reading pull request 202", + "args": [ + "pr", + "view", + "202", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 202 is open and mergeable" + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "await_user_message", + "args": {} + } + } + ] + }, + { + "thread": "a3", + "expect": { + "userTextContains": "review pull request 303" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 303" + } + } + } + ] + }, + { + "thread": "a3", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_a3": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_a3}}", + "_reason": "reading pull request 303", + "args": [ + "pr", + "view", + "303", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "a3", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 303 is open and mergeable" + } + } + } + ] + }, + { + "thread": "a3", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "looked at pull request 303" + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "userTextContains": "check notes for pull request 404" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 404" + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_b": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_b}}", + "_reason": "reading pull request 404", + "args": [ + "pr", + "view", + "404", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 404 is open and mergeable" + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "looked at pull request 404" + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "userTextContains": "a1 go ahead and finish" + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "reviewed pull request 101" + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "userTextContains": "a2 go ahead and finish" + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "reviewed pull request 202" + } + } + } + ] + } + ], + "assert": { + "admission": { + "peakAdmitted": 2, + "admittedBefore": [ + { + "first": "a1", + "then": "a3" + }, + { + "first": "a2", + "then": "a3" + }, + { + "first": "a1", + "then": "b" + }, + { + "first": "a2", + "then": "b" + }, + { + "first": "b", + "then": "a3" + } + ], + "completedBefore": [ + { + "first": "a1", + "then": "a2" + }, + { + "first": "a1", + "then": "a3" + }, + { + "first": "a2", + "then": "a3" + }, + { + "first": "a1", + "then": "b" + } + ] + } + } +} diff --git a/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml b/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml new file mode 100644 index 00000000..034c6168 --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml @@ -0,0 +1,144 @@ +# Fixture for the admission-burst bundle: two agents sharing one cluster. +# +# burst-reviewer is a code-review agent shape — two sandbox bundles per session, so a +# session's pod set is more than one pod and "all or none" means something. +# burst-notes is a second agent with its own line. Both run with the plan gate +# off and tool calls permissive: what the bundle pins is WHEN each session gets +# to run, and a gate in front of the tool call would only add scenery. +# +# burst-reviewer's channels.sleepAfter is one second, on purpose. A finished +# (Idle) session keeps its admission until it sleeps, so the sleep reap is what +# hands its room to the next session in line; a long grace would only slow the +# scenario. +apiVersion: v1 +kind: Secret +metadata: {name: burst-placeholder, namespace: default} +type: Opaque +stringData: {api-key: "unused-by-the-test-provider"} +--- +apiVersion: v1 +kind: Secret +metadata: {name: burst-fake-creds, namespace: default} +type: Opaque +stringData: {placeholder: "unused-by-the-fake-channel"} +--- +apiVersion: v1 +kind: Secret +metadata: {name: burst-gh-token, namespace: default} +type: Opaque +stringData: {token: "ghp_unusedTheSandboxNeverExecutes"} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: SpiceboxToolspec +metadata: {name: ts-burst-gh, namespace: default} +spec: + name: ts-burst-gh + version: "1" + toolkit: {name: gh, revision: "2026-05-01"} + allowSubcommands: ["pr view"] +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: SpiceboxClass +metadata: {name: cls-burst-gh, namespace: default} +spec: + image: registry.example.com/gh-toolbelt:latest + resources: {cpu: "100m", memory: "64Mi", ephemeralStorage: "16Mi"} + tools: + - {name: gh, command: ["/usr/bin/gh"]} + toolspecs: + - {name: ts-burst-gh} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentIdentity +metadata: {name: burst-identity, namespace: default} +spec: + description: "Fixture identity for the admission burst bundle." + credentials: + # The gh toolkit declares a sensitive GITHUB_TOKEN, so a class is not Valid + # until a credential of this name backs it. Never read: nothing executes. + - name: github-token + type: static + static: {secretRef: {name: burst-gh-token, key: token}} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: burst-reviewer, namespace: default} +spec: + displayName: BurstReviewer + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test # production operator REFUSES this + name: scripted # cosmetic; ignored at runtime + apiKey: {name: burst-placeholder, key: api-key} + systemPrompt: + inline: | + You are a fixture reviewer for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You review pull requests with the gitlike_gh tool. + agentIdentity: burst-identity + authz: + toolCalls: {mode: permissive} + planGate: {mode: disabled} + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + channels: {sleepAfter: 1s} + toolBundles: + - {name: "gitlike", class: "cls-burst-gh", toolspecs: ["ts-burst-gh"]} + - {name: "scratch", class: "cls-burst-gh", toolspecs: ["ts-burst-gh"]} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: burst-notes, namespace: default} +spec: + displayName: BurstNotes + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test # production operator REFUSES this + name: scripted # cosmetic; ignored at runtime + apiKey: {name: burst-placeholder, key: api-key} + systemPrompt: + inline: | + You are a fixture note-taker for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You check pull request status with the gitlike_gh tool. + agentIdentity: burst-identity + authz: + toolCalls: {mode: permissive} + planGate: {mode: disabled} + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + # Longer than the reviewer's: a3's room should come from a2 finishing, not + # from this session sleeping. See the bundle description. + channels: {sleepAfter: 10s} + toolBundles: + - {name: "gitlike", class: "cls-burst-gh", toolspecs: ["ts-burst-gh"]} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: burst-reviewer-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: burst-reviewer + credentialsRef: {secretName: burst-fake-creds} + fake: {echo: false} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: burst-notes-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: burst-notes + credentialsRef: {secretName: burst-fake-creds} + fake: {echo: false} +--- +# The tuple that makes `gh pr view` on demo-org/demo-repo resolve to ALLOW, so +# every session's call succeeds for the session's own reasons. The harness also +# waits on RelationshipsApplied, which a schema-only bootstrap never gets. +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: SpiceDBBootstrap +metadata: {name: burst-repo, namespace: default} +spec: + relationships: + - resource: {type: github_repo_url, id: "aHR0cHM6Ly9naXRodWIuY29tL2RlbW8tb3JnL2RlbW8tcmVwbw"} + relation: owner + subject: {type: user, id: "user@example.com", canonicalize: true} diff --git a/test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json b/test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json new file mode 100644 index 00000000..e466195f --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json @@ -0,0 +1,214 @@ +{ + "name": "admission-child-at-cap", + "description": "A lead delegates to a coder whose class is already at its concurrency cap, and the delegation still completes: the delegate tool result the lead is handed is the child's own answer, not a timeout.\n\ncap-coder sets maxConcurrentSessions: 1. The holder thread is a direct coder session that takes the line's one slot and parks in await_user_message, so it is still holding when the lead delegates. The queued thread is a second direct coder session that arrives while the holder holds, and waits behind the cap. The lead's delegated child then joins the SAME coder line, behind queued in arrival order, at cap. A delegated child is never blocked by its class's cap (and is not counted toward it), and children go first in their line, so the child is admitted at once and its answer reaches the lead.\n\nWhat is claimed, and how each claim is read from the run:\n- the delegation completed: the lead's delegate step asserts the TOOL RESULT is not an error and contains the child's return_result text (lastToolResultContains). A child held behind the cap would leave the parent's delegate call polling until it gave up, and the step would never see that text;\n- the cap was in force while the child ran: queued, which arrived before the lead, is admitted strictly after the lead and after the holder. Without the cap queued would be admitted as soon as it arrived.\n\nThe holder finishes only once the lead is Idle (waitFor), so the coder line is at cap for the lead's whole delegation, and queued's slot comes from the holder sleeping afterwards. Capacity is four session pod sets, more than the run ever holds, so the cap and not cluster room is what queues queued.", + "agentDir": "testdata/admission-child-at-cap/manifests", + "agentClass": "cap-lead", + "defaultUser": "user@example.com", + "admission": { + "capacitySessions": 4 + }, + "userTurns": [ + { + "text": "hold the standing coder slot", + "thread": "holder", + "channel": "cap-coder-fake" + }, + { + "text": "tidy the import blocks when you can", + "thread": "queued", + "channel": "cap-coder-fake", + "waitFor": [ + { + "thread": "holder", + "until": "awaitingInput" + } + ] + }, + { + "text": "have the coder fix the flaky retry test", + "thread": "lead", + "channel": "cap-lead-fake", + "waitFor": [ + { + "thread": "holder", + "until": "awaitingInput" + } + ] + }, + { + "text": "holder may finish now", + "thread": "holder", + "channel": "cap-coder-fake", + "waitFor": [ + { + "thread": "lead", + "until": "idle" + } + ] + } + ], + "llm": [ + { + "thread": "holder", + "expect": { + "userTextContains": "hold the standing coder slot" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "holding the coder slot" + } + } + } + ] + }, + { + "thread": "holder", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "await_user_message", + "args": {} + } + } + ] + }, + { + "thread": "holder", + "expect": { + "userTextContains": "holder may finish now" + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "held the coder slot" + } + } + } + ] + }, + { + "thread": "queued", + "expect": { + "userTextContains": "tidy the import blocks when you can" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "import blocks tidied" + } + } + } + ] + }, + { + "thread": "queued", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "tidied the import blocks" + } + } + } + ] + }, + { + "thread": "lead", + "expect": { + "userTextContains": "have the coder fix the flaky retry test" + }, + "reply": [ + { + "toolUse": { + "name": "delegate", + "args": { + "agent": "cap-coder", + "task": "repair the payments retry test so it stops flaking" + } + } + } + ] + }, + { + "agentClass": "cap-coder", + "expect": { + "userTextContains": "repair the payments retry test so it stops flaking" + }, + "reply": [ + { + "toolUse": { + "name": "return_result", + "args": { + "result": "Payments retry test repaired: the backoff now resets between attempts." + } + } + } + ] + }, + { + "thread": "lead", + "expect": { + "lastToolResult": "delegate", + "lastToolResultIsError": false, + "lastToolResultContains": "the backoff now resets between attempts" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "The coder repaired the retry test." + } + } + } + ] + }, + { + "thread": "lead", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "had the coder repair the retry test" + } + } + } + ] + } + ], + "assert": { + "admission": { + "admittedBefore": [ + { + "first": "holder", + "then": "queued" + }, + { + "first": "lead", + "then": "queued" + } + ] + } + } +} diff --git a/test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml b/test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml new file mode 100644 index 00000000..a69497bc --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml @@ -0,0 +1,85 @@ +# Fixture for the admission-child-at-cap bundle: a lead that delegates to a +# coder whose class is capped at one concurrent session. +# +# The cap is on cap-coder, the CHILD's class, because that is the line the +# delegated child joins: a child session runs under the class it was +# delegated to. A cap on cap-lead would never see the child at all. A class +# cannot delegate to itself (a roster cycle is refused), so parent and child +# are necessarily of different classes, and the coder line is put at its cap +# by a direct (non-delegated) coder session, the "holder" thread. +# +# Both classes are toolless beyond their meta-tool grants and are +# channel-attached with a one-second sleepAfter: a finished session keeps its +# admission until it sleeps, and the sleep is what hands the coder line's one +# slot to the queued session at the end. +apiVersion: v1 +kind: Secret +metadata: {name: capchild-placeholder, namespace: default} +type: Opaque +stringData: {api-key: "unused-by-the-test-provider"} +--- +apiVersion: v1 +kind: Secret +metadata: {name: capchild-fake-creds, namespace: default} +type: Opaque +stringData: {placeholder: "unused-by-the-fake-channel"} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: cap-lead, namespace: default} +spec: + displayName: CapLead + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test # production operator REFUSES this + name: scripted # cosmetic; ignored at runtime + apiKey: {name: capchild-placeholder, key: api-key} + capabilities: + subagents: {} + subagents: ["cap-coder"] + systemPrompt: + inline: | + You are a fixture lead for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You hand bounded coding tasks to the coder with the delegate tool. + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + channels: {sleepAfter: 1s} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: cap-coder, namespace: default} +spec: + displayName: CapCoder + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test + name: scripted + apiKey: {name: capchild-placeholder, key: api-key} + systemPrompt: + inline: | + You are a fixture coder for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You complete coding tasks, whether a person or a lead asked. + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + channels: {sleepAfter: 1s} + maxConcurrentSessions: 1 +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: cap-lead-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: cap-lead + credentialsRef: {secretName: capchild-fake-creds} + fake: {echo: false} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: cap-coder-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: cap-coder + credentialsRef: {secretName: capchild-fake-creds} + fake: {echo: false} diff --git a/test/e2e/conversation.go b/test/e2e/conversation.go index fb5d7284..07821a49 100644 --- a/test/e2e/conversation.go +++ b/test/e2e/conversation.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" "github.com/authzed/openagentprimitives/pkg/channels/channelkinds" fakekind "github.com/authzed/openagentprimitives/pkg/channels/channelkinds/fake" @@ -22,10 +24,15 @@ import ( type SendOption func(*sendCfg) type sendCfg struct { - user, thread string - attachments []channelkinds.InboundAttachment + user, thread, channel string + attachments []channelkinds.InboundAttachment } +// OnChannel sends through the named Channel instead of the fixture's only +// conversational one — for a fixture with one fake Channel per AgentClass, +// where singleChannel would refuse to pick. +func OnChannel(name string) SendOption { return func(c *sendCfg) { c.channel = name } } + // AsUser overrides the requester identity for this inbound — both the // channel-side ExternalID and the email that drives the SpiceDB // canonical-user computation. Pass "alice@example.com" etc; never echo a @@ -74,7 +81,15 @@ func (h *Harness) SendUserMessage(text string, opts ...SendOption) { o(&cfg) } - ch := h.singleChannel("SendUserMessage") + var ch *spiceboxv1alpha1.Channel + if cfg.channel != "" { + ch = &spiceboxv1alpha1.Channel{} + if err := h.K8s.Get(context.Background(), client.ObjectKey{Namespace: h.Namespace(), Name: cfg.channel}, ch); err != nil { + h.t.Fatalf("SendUserMessage: get Channel %q: %v", cfg.channel, err) + } + } else { + ch = h.singleChannel("SendUserMessage") + } // The fake listener registers its Driver lazily on Start (a goroutine), so // a Channel applied shortly before this send races the listener-starter. // Poll for the driver — mirroring ExpectAgentReply's poll — instead of a diff --git a/test/e2e/harness.go b/test/e2e/harness.go index 92d50b43..6c496680 100644 --- a/test/e2e/harness.go +++ b/test/e2e/harness.go @@ -288,6 +288,10 @@ type Options struct { // // nil — every scenario but a replay — leaves assembly untouched. ReplaceAssembledTool func(tool.Tool) (tool.Tool, error) + + // Admission, when non-nil, gates session pods on whole-set admission and + // runs the admitter against its fixed capacity. See AdmissionOptions. + Admission *AdmissionOptions } // delayedSchemaIO wraps a guardian SchemaIO and stalls WriteSchema, widening @@ -1325,9 +1329,12 @@ func (h *Harness) startManager(t *testing.T, env *testenv.Env, spdbCli *spicedb. // mandatory-dependency check. DeniedLister: spdbCli, SlotGrantCopier: spdbCli, + // Zero (ungated) unless the scenario opted into admission. + AdmissionEnabled: h.opts.Admission != nil, }).SetupWithManager(mgr); err != nil { t.Fatalf("e2e: register AgentSession controller: %v", err) } + h.registerAdmitter(t, mgr) // SubagentRequest — authorizes and performs delegation (roster/identity // checks, child creation, lineage tuples) AND propagates the child's @@ -1546,12 +1553,14 @@ func (h *Harness) RestartOperator(t *testing.T) { ForkChecker: h.SpiceDB, // StartChecker: the start gate is enforced in e2e exactly as in // production; StartRefusedNoticePublish is left nil (logged and dropped). - StartChecker: h.SpiceDB, - DeniedLister: h.SpiceDB, - SlotGrantCopier: h.SpiceDB, + StartChecker: h.SpiceDB, + DeniedLister: h.SpiceDB, + SlotGrantCopier: h.SpiceDB, + AdmissionEnabled: h.opts.Admission != nil, }).SetupWithManager(mgr); err != nil { t.Fatalf("e2e: RestartOperator: register AgentSession controller: %v", err) } + h.registerAdmitter(t, mgr) mgrCtx, mgrCancel := context.WithCancel(context.Background()) mgrDone := make(chan struct{}) diff --git a/test/e2e/threadrun/driver.go b/test/e2e/threadrun/driver.go index e4896d0f..39c73de8 100644 --- a/test/e2e/threadrun/driver.go +++ b/test/e2e/threadrun/driver.go @@ -183,6 +183,8 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { HoldToolsFromAssembly: toolCatalog.HeldFromAssembly(), FilterOfferedTools: toolCatalog.Filter(), ReplaceAssembledTool: metaCanner.Replacer(), + Admission: admissionOptionsFor(b), + AwaitIdleTTL: awaitIdleTTLFor(b), }) // Canned outputs for the tools whose disposition is ReplayOutput. Keyed by @@ -426,8 +428,21 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { // function's own doc for why this lives in the general turn loop rather // than a per-bundle hook, and for the no-op it is for every other bundle. workshopSessionBound := false + // The admission watch starts before the first turn: what it guards — no + // partial set, never over capacity — must hold from the first session on. + var watch *admissionWatch + if b.Admission != nil { + watch = startAdmissionWatch(t, h, b) + } + opened := map[string]bool{} for i, turn := range b.UserTurns { - h.SendUserMessage(turn.Text, e2e.InThread("bt-"+b.Name), e2e.WithAttachments(bundleAttachments(t, dir, i, turn)...)) + waitForBarriers(t, h, b, turn) + opts := append(turnSendOptions(b, turn), e2e.WithAttachments(bundleAttachments(t, dir, i, turn)...)) + h.SendUserMessage(turn.Text, opts...) + if b.Admission != nil && turn.Thread != "" && !opened[turn.Thread] { + awaitArrival(t, h, b, turn.Thread) + } + opened[turn.Thread] = true // Contains and NotContains are positional and run on the SAME reply, so a // turn asserting both awaits its reply once and checks presence + absence // together. A turn with neither is not awaited (some turns produce no @@ -445,6 +460,14 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { bindWorkshopSessionOnce(t, h, &workshopSessionBound) } + // An admission bundle's barrier: every session admitted, ran and gave its + // room back. Before the transcript check, which is only meaningful once + // the last queued session has had its turn. + if watch != nil { + waitAllReleased(t, h, b) + checkAdmission(t, watch, b) + } + checkTranscriptConsumed(t, b, tr) // The phase barrier runs FIRST among the state checks: for a bundle whose // run ends without an agent reply (a denied guest start, a forensic hold) @@ -466,7 +489,12 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { checkSpiceDB(t, h, b) checkTranscriptLength(t, h, b) checkToolsCalled(t, h, b) - checkGoldenTrace(t, h, dir) + // A golden trace is ONE session's log. A multi-session bundle has several + // and no defined way to fold them into one file, so it has no golden (yet) + // and asserts on its sessions through its own claims instead. + if len(b.Threads()) == 0 { + checkGoldenTrace(t, h, dir) + } checkMintedIDsConsumed(t, mintedIDs) checkProviderIDsConsumed(t, providerIDs) checkToolCatalogConsulted(t, toolCatalog) @@ -1120,6 +1148,7 @@ func registerTranscript(t *testing.T, h *e2e.Harness, b bt.Bundle) *transcript { parts := parseReplyParts(t, i, step.Reply) captures := compileCaptures(t, i, step.Expect.Capture) scope := scopes[step.AgentClass] + opening := b.OpeningText(step.Thread) // "" for an unscoped step h.LLM.On(func(req llm.Request) bool { // Scope FIRST: an unmatched scope must leave the step pending for // its own session, so a request from the other side falls through @@ -1128,6 +1157,9 @@ func registerTranscript(t *testing.T, h *e2e.Harness, b bt.Bundle) *transcript { if scope != "" && !strings.Contains(systemPromptOf(req), scope) { return false } + if opening != "" && !mentionsText(req, opening) { + return false + } checkExpect(t, i, step.Expect, req) applyCaptures(t, i, captures, req, vars) tr.markConsumed(i) @@ -1177,12 +1209,16 @@ func checkTranscriptConsumed(t *testing.T, b bt.Bundle, tr *transcript) { tr.mu.Lock() defer tr.mu.Unlock() for i, step := range b.LLM { - if step.AgentClass == "" { + scope := step.AgentClass + if step.Thread != "" { + scope = "thread " + step.Thread + } + if scope == "" { continue } assert.True(t, tr.consumed[i], "llm step %d was scoped to %q and never ran: that session did not take the turn the transcript says it did", - i, step.AgentClass) + i, scope) } } diff --git a/test/e2e/threadrun/sessions.go b/test/e2e/threadrun/sessions.go new file mode 100644 index 00000000..82ef7d80 --- /dev/null +++ b/test/e2e/threadrun/sessions.go @@ -0,0 +1,410 @@ +//go:build e2e + +package threadrun + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/agent/llm" + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + bt "github.com/authzed/openagentprimitives/pkg/bronzethread" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/test/e2e" +) + +// Multi-session bundles: named threads, thread-scoped steps, turn barriers, +// and the admission watch. See pkg/bronzethread/sessions.go for the format. +// A bundle that names no thread takes none of these paths. + +// sessionBarrierTimeout bounds every wait on another session's state. Long +// enough for a session to queue behind a whole other session's turn under +// suite contention; short enough that a line that never admits fails by name +// rather than by the suite timeout. +const sessionBarrierTimeout = 2 * time.Minute + +// threadKey is the channel key a turn is sent on. Each bundle gets its own +// thread (see Run), and each NAMED thread inside it its own key, so its own +// AgentSession. +func threadKey(b bt.Bundle, thread string) string { + if thread == "" { + return "bt-" + b.Name + } + return "bt-" + b.Name + "-" + thread +} + +// turnSendOptions routes one user turn: its thread, and its Channel when the +// fixture has more than one. +func turnSendOptions(b bt.Bundle, turn bt.UserTurn) []e2e.SendOption { + opts := []e2e.SendOption{e2e.InThread(threadKey(b, turn.Thread))} + if turn.Channel != "" { + opts = append(opts, e2e.OnChannel(turn.Channel)) + } + return opts +} + +// mentionsText reports whether any user text block of req contains text. A +// thread's opening message is in every request its session makes — and, by +// bt.Bundle.Validate, in no other session's. +func mentionsText(req llm.Request, text string) bool { + for _, m := range req.Messages { + if m.Role != "user" { + continue + } + for _, c := range m.Content { + if c.Type == "text" && strings.Contains(c.Text, text) { + return true + } + } + } + return false +} + +// threadSession returns the AgentSession a named thread opened, if it exists. +func threadSession(ctx context.Context, h *e2e.Harness, b bt.Bundle, thread string) (*spiceboxv1alpha1.AgentSession, error) { + var list spiceboxv1alpha1.AgentSessionList + if err := h.K8s.List(ctx, &list, client.InNamespace(h.Namespace())); err != nil { + return nil, err + } + key := threadKey(b, thread) + for i := range list.Items { + if in := list.Items[i].Spec.InputChannel; in != nil && in.Key == key { + return &list.Items[i], nil + } + } + return nil, nil +} + +// barrierMet reports whether sess has reached the barrier's state. +func barrierMet(sess *spiceboxv1alpha1.AgentSession, until string) bool { + if sess == nil { + return false + } + switch until { + case bt.UntilAwaitingInput: + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseRunning && sess.Status.AwaitingUserInputSince != nil + case bt.UntilIdle: + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseIdle + } + return false +} + +// waitForBarriers blocks until every session the turn waits on has reached +// its state. +func waitForBarriers(t *testing.T, h *e2e.Harness, b bt.Bundle, turn bt.UserTurn) { + t.Helper() + for _, w := range turn.WaitFor { + cutoff := time.Now().Add(sessionBarrierTimeout) + var last *spiceboxv1alpha1.AgentSession + for { + sess, err := threadSession(context.Background(), h, b, w.Thread) + if err == nil { + last = sess + } + if barrierMet(sess, w.Until) { + break + } + if time.Now().After(cutoff) { + phase := "" + if last != nil { + phase = last.Status.Phase + } + t.Fatalf("turn %q waits for thread %q to be %s; it was not within %s (phase %s)", + turn.Text, w.Thread, w.Until, sessionBarrierTimeout, phase) + } + time.Sleep(100 * time.Millisecond) + } + } +} + +// awaitArrival blocks until the session a turn just opened exists, then until +// the wall clock has moved past the second it was created in. +// +// The line orders by creationTimestamp, which the apiserver keeps to the +// second, and breaks a tie on name — which, for a session, is a hash of its +// channel key. Two sessions opened in one second would therefore line up in +// an order unrelated to which arrived first, and a bundle claiming arrival +// order would be claiming a coin toss. Only the bundles with admission on +// pay this. +func awaitArrival(t *testing.T, h *e2e.Harness, b bt.Bundle, thread string) { + t.Helper() + cutoff := time.Now().Add(sessionBarrierTimeout) + for { + sess, err := threadSession(context.Background(), h, b, thread) + if err == nil && sess != nil { + next := sess.CreationTimestamp.Time.Truncate(time.Second).Add(time.Second) + if wait := time.Until(next); wait > 0 { + time.Sleep(wait + 10*time.Millisecond) + } + return + } + if time.Now().After(cutoff) { + t.Fatalf("thread %q: no AgentSession appeared within %s; the turn never reached the pipeline", thread, sessionBarrierTimeout) + } + time.Sleep(50 * time.Millisecond) + } +} + +// admissionWatch observes every named thread's session for the whole run and +// records what the admission scenario asserts on: when each was first seen +// admitted and first seen finished, how many were admitted at once, and every +// moment a session held part of a pod set it should hold all or none of. +// +// It is a property check the run cannot opt out of: the reconciler creates a +// session's bundle SpiceboxSessions one by one, so the claim is not "a set +// appears atomically" but the two states that MUST be whole — a session that +// was never admitted holds none, and a Running session holds every one. +type admissionWatch struct { + b bt.Bundle + h *e2e.Harness + stop context.CancelFunc + done chan struct{} + + mu sync.Mutex + seq int + firstAdmitted map[string]int // thread -> observation number + firstIdle map[string]int + peak int + violations []string + lastErr error +} + +func startAdmissionWatch(t *testing.T, h *e2e.Harness, b bt.Bundle) *admissionWatch { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + w := &admissionWatch{ + b: b, h: h, stop: cancel, done: make(chan struct{}), + firstAdmitted: map[string]int{}, firstIdle: map[string]int{}, + } + go func() { + defer close(w.done) + tick := time.NewTicker(50 * time.Millisecond) + defer tick.Stop() + for { + w.observe(ctx) + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } + }() + t.Cleanup(w.halt) + return w +} + +func (w *admissionWatch) halt() { + w.stop() + <-w.done +} + +// observe takes one snapshot. A list error skips the snapshot (recorded, and +// reported only if nothing was ever observed): a missed snapshot can only +// lose an observation, never invent a violation. +// +// The lists are not one atomic read, so each claim is checked across a pair +// read in the order that makes it sound (h.K8s is the uncached client, so the +// reads are ordered at the apiserver): +// +// - "Running holds every bundle" reads sessions, THEN bundles: a session +// Running at the first read already had its whole set, and nothing +// reclaims a Running session's set. +// - "waiting holds no bundle" reads bundles, THEN sessions: a bundle seen at +// the first read was created after its session was admitted, so the +// session reads admitted at the second. Read the other way round, a +// session admitted and provisioned between the two reads would look like +// a waiting session holding its set. +func (w *admissionWatch) observe(ctx context.Context) { + ns := client.InNamespace(w.h.Namespace()) + var before, after spiceboxv1alpha1.AgentSessionList + var boxes spiceboxv1alpha1.SpiceboxSessionList + var classes spiceboxv1alpha1.AgentClassList + err := w.h.K8s.List(ctx, &classes, ns) + if err == nil { + err = w.h.K8s.List(ctx, &before, ns) + } + if err == nil { + err = w.h.K8s.List(ctx, &boxes, ns) + } + if err == nil { + err = w.h.K8s.List(ctx, &after, ns) + } + w.mu.Lock() + defer w.mu.Unlock() + if err != nil { + w.lastErr = err + return + } + w.seq++ + + bundlesOf := map[string]int{} + for i := range classes.Items { + bundlesOf[classes.Items[i].Name] = len(classes.Items[i].Spec.ToolBundles) + } + held := map[string]int{} + for i := range boxes.Items { + if boxes.Items[i].DeletionTimestamp == nil { + held[boxes.Items[i].Labels["agentprimitives.authzed.com/agentsession"]]++ + } + } + + prefix := threadKey(w.b, "") + "-" + threadOf := func(s *spiceboxv1alpha1.AgentSession) (string, bool) { + if s.Spec.InputChannel == nil || !strings.HasPrefix(s.Spec.InputChannel.Key, prefix) { + return "", false + } + return strings.TrimPrefix(s.Spec.InputChannel.Key, prefix), true + } + + for i := range before.Items { + s := &before.Items[i] + thread, ok := threadOf(s) + if !ok || s.Status.Phase != spiceboxv1alpha1.AgentSessionPhaseRunning { + continue + } + if n, want := held[s.Name], bundlesOf[s.Spec.Class]; n != want { + w.violations = append(w.violations, fmt.Sprintf( + "observation %d: thread %q (%s) is Running with %d of its %d bundle SpiceboxSessions", + w.seq, thread, s.Name, n, want)) + } + } + + admittedNow := 0 + for i := range after.Items { + s := &after.Items[i] + thread, ok := threadOf(s) + if !ok { + continue + } + phase := s.Status.Phase + admitted := admission.IsAdmitted(s) + terminal := phase == spiceboxv1alpha1.AgentSessionPhaseSucceeded || phase == spiceboxv1alpha1.AgentSessionPhaseFailed + if admitted && !terminal { + admittedNow++ + if w.firstAdmitted[thread] == 0 { + w.firstAdmitted[thread] = w.seq + } + } + if phase == spiceboxv1alpha1.AgentSessionPhaseIdle && w.firstAdmitted[thread] != 0 && w.firstIdle[thread] == 0 { + w.firstIdle[thread] = w.seq + } + booting := phase == "" || phase == spiceboxv1alpha1.AgentSessionPhasePending + if n := held[s.Name]; booting && !admitted && n > 0 { + w.violations = append(w.violations, fmt.Sprintf( + "observation %d: thread %q (%s) is waiting unadmitted yet holds %d of its %d bundle SpiceboxSessions", + w.seq, thread, s.Name, n, bundlesOf[s.Spec.Class])) + } + } + w.peak = max(w.peak, admittedNow) +} + +// waitAllReleased blocks until every named thread's session has finished and +// given its admission back through the sleep reap (status.sleptAt). That is +// the end of the line's story: every session admitted, ran, and freed its room. +func waitAllReleased(t *testing.T, h *e2e.Harness, b bt.Bundle) { + t.Helper() + cutoff := time.Now().Add(sessionBarrierTimeout) + for { + var pending []string + for _, th := range b.Threads() { + sess, err := threadSession(context.Background(), h, b, th) + if err != nil || sess == nil || sess.Status.SleptAt == nil { + pending = append(pending, th) + } + } + if len(pending) == 0 { + return + } + if time.Now().After(cutoff) { + t.Fatalf("threads %v never finished and released their admission within %s", pending, sessionBarrierTimeout) + } + time.Sleep(100 * time.Millisecond) + } +} + +// checkAdmission asserts the watch's record against the bundle's claims, and +// the properties every admission bundle holds without claiming them: no +// partial set, never more admitted than the capacity, and every thread +// admitted and finished. +func checkAdmission(t *testing.T, w *admissionWatch, b bt.Bundle) { + t.Helper() + w.halt() + w.mu.Lock() + defer w.mu.Unlock() + require.NotZero(t, w.seq, "the admission watch never observed the cluster (last error: %v)", w.lastErr) + t.Logf("admission record over %d observations: admitted %v, completed %v, peak %d admitted at once", + w.seq, sortedRecord(w.firstAdmitted), sortedRecord(w.firstIdle), w.peak) + + assert.Empty(t, w.violations, "a session held part of a pod set it must hold all or none of") + assert.LessOrEqual(t, w.peak, b.Admission.CapacitySessions, + "more sessions were admitted at once than the cluster has room for") + for _, th := range b.Threads() { + assert.NotZero(t, w.firstAdmitted[th], "thread %q was never admitted", th) + assert.NotZero(t, w.firstIdle[th], "thread %q never finished its work after admission", th) + } + + a := b.Assert.Admission + if a == nil { + return + } + if a.PeakAdmitted > 0 { + assert.Equal(t, a.PeakAdmitted, w.peak, "the most sessions admitted at once") + } + order := func(kind string, first map[string]int, claims []bt.ThreadOrder) { + for _, o := range claims { + f, s := first[o.First], first[o.Then] + assert.True(t, f != 0 && s != 0 && f < s, + "%s: thread %q must come strictly before %q (first observed at %d and %d; full record %v)", + kind, o.First, o.Then, f, s, sortedRecord(first)) + } + } + order("admitted", w.firstAdmitted, a.AdmittedBefore) + order("completed", w.firstIdle, a.CompletedBefore) +} + +// sortedRecord renders a first-observation map in observation order, so a +// failed ordering claim shows the order the run actually took. +func sortedRecord(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.SortFunc(keys, func(a, b string) int { return m[a] - m[b] }) + out := make([]string, 0, len(keys)) + for _, k := range keys { + out = append(out, fmt.Sprintf("%s@%d", k, m[k])) + } + return out +} + +// admissionOptionsFor turns a bundle's admission block into the harness +// option, or nil for every bundle that runs ungated. +func admissionOptionsFor(b bt.Bundle) *e2e.AdmissionOptions { + if b.Admission == nil { + return nil + } + return &e2e.AdmissionOptions{Capacity: e2e.FixedSetCapacity{Sets: b.Admission.CapacitySessions}} +} + +// awaitIdleTTLFor keeps a session parked in await_user_message for as long as +// a barrier may wait on it, for a bundle that waits on one. Every other bundle +// gets zero: await idle-exits at once, the path they were written against. +func awaitIdleTTLFor(b bt.Bundle) time.Duration { + for _, ut := range b.UserTurns { + for _, w := range ut.WaitFor { + if w.Until == bt.UntilAwaitingInput { + return 2 * sessionBarrierTimeout + } + } + } + return 0 +}