From 5c8fa24b2a141b22ff6905ed19a4c80d66b159ef Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 11:58:59 -0500 Subject: [PATCH 01/33] Add the admission line model Pure package describing a session's pod demand, when it joined its class's line, and the order sessions are admitted in. Adds the Admitted condition and its reasons. --- pkg/apis/v1alpha1/conditions.go | 23 ++++ pkg/controllers/admission/line.go | 98 +++++++++++++++ pkg/controllers/admission/line_test.go | 168 +++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 pkg/controllers/admission/line.go create mode 100644 pkg/controllers/admission/line_test.go diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index 39ac3b1e..7c512f22 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -511,6 +511,29 @@ const ( ReasonCapacityHoldExpired = "HoldExpired" ) +// AgentSession admission. +const ( + // AgentSessionConditionAdmitted is True while the operator has admitted the + // session's whole pod set against its AgentClass's line, and False once the + // session has been released back to the line (reclaimed, or a placement + // misfit). Absent until the session first joins a line. + AgentSessionConditionAdmitted = "Admitted" +) + +// AgentSessionConditionAdmitted reasons. +const ( + ReasonAdmitted = "Admitted" + // ReasonAdmissionReclaimed marks a session released so a session ahead of + // it in line could be admitted. + ReasonAdmissionReclaimed = "Reclaimed" + // ReasonAdmissionMisfit marks a session released because its pods did not + // fit where they were placed. + ReasonAdmissionMisfit = "PlacementMismatch" + // ReasonCapacityHoldWaitingAdmission is the CapacityHold reason while a + // session waits in line to be admitted. + ReasonCapacityHoldWaitingAdmission = "WaitingForAdmission" +) + // Settings reasons (shared by AgentClass + AgentSession SettingsAccepted and the // Settings CR SelfConsistent conditions). const ( diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go new file mode 100644 index 00000000..57a19f61 --- /dev/null +++ b/pkg/controllers/admission/line.go @@ -0,0 +1,98 @@ +// Package admission models the line of AgentSessions waiting to have their +// whole pod set admitted against an AgentClass's capacity. It is pure: no +// client, no clock, so the ordering rules can be tested in isolation. +package admission + +import ( + "cmp" + "slices" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" +) + +// PodDemand is the resource request of one pod in a session's pod set. +type PodDemand struct { + Name string + CPUMilli, MemBytes, EphemeralBytes int64 +} + +// Demand is the whole pod set a session needs admitted together. +type Demand struct { + Pods []PodDemand +} + +// Total sums CPU and memory across the pod set. +func (d Demand) Total() (cpuMilli, memBytes int64) { + for _, p := range d.Pods { + cpuMilli += p.CPUMilli + memBytes += p.MemBytes + } + return cpuMilli, memBytes +} + +// Candidate is one session in a line. Line is the AgentClass +// "namespace/name" the session draws capacity from. +type Candidate struct { + Key types.NamespacedName + Line string + JoinedAt time.Time + Child bool + Demand Demand +} + +func admittedCondition(sess *v1alpha1.AgentSession) *metav1.Condition { + return meta.FindStatusCondition(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) +} + +// JoinedAt is when the session took its place in line. A wake that came after +// the last admission transition re-queues the session, so it counts; a wake +// the session was later released after (e.g. a placement misfit) does not, and +// the session keeps its original place at creationTimestamp. +func JoinedAt(sess *v1alpha1.AgentSession) time.Time { + wake := sess.Status.LastWakeAt + if wake == nil { + return sess.CreationTimestamp.Time + } + if c := admittedCondition(sess); c != nil && !wake.Time.After(c.LastTransitionTime.Time) { + return sess.CreationTimestamp.Time + } + return wake.Time +} + +// WaitingSince is when the session's current wait began: JoinedAt, or the +// later moment it was released from admission. +func WaitingSince(sess *v1alpha1.AgentSession) time.Time { + joined := JoinedAt(sess) + if c := admittedCondition(sess); c != nil && c.Status == metav1.ConditionFalse && + c.LastTransitionTime.Time.After(joined) { + return c.LastTransitionTime.Time + } + return joined +} + +// IsAdmitted reports whether the session's Admitted condition is True. +func IsAdmitted(sess *v1alpha1.AgentSession) bool { + return meta.IsStatusConditionTrue(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) +} + +// Order sorts cands in place into admission order: children first, then +// earliest JoinedAt, then Key string as a deterministic tiebreak. +func Order(cands []Candidate) { + slices.SortStableFunc(cands, func(a, b Candidate) int { + if a.Child != b.Child { + if a.Child { + return -1 + } + return 1 + } + if c := a.JoinedAt.Compare(b.JoinedAt); c != 0 { + return c + } + return cmp.Compare(a.Key.String(), b.Key.String()) + }) +} diff --git a/pkg/controllers/admission/line_test.go b/pkg/controllers/admission/line_test.go new file mode 100644 index 00000000..3839a610 --- /dev/null +++ b/pkg/controllers/admission/line_test.go @@ -0,0 +1,168 @@ +package admission + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" +) + +var t0 = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + +func at(d time.Duration) time.Time { return t0.Add(d) } + +func mtime(d time.Duration) *metav1.Time { + t := metav1.NewTime(at(d)) + return &t +} + +func sessionAt(created time.Duration) *v1alpha1.AgentSession { + s := &v1alpha1.AgentSession{} + s.CreationTimestamp = *mtime(created) + return s +} + +func withWake(s *v1alpha1.AgentSession, d time.Duration) *v1alpha1.AgentSession { + s.Status.LastWakeAt = mtime(d) + return s +} + +func withAdmitted(s *v1alpha1.AgentSession, status metav1.ConditionStatus, d time.Duration) *v1alpha1.AgentSession { + s.Status.Conditions = append(s.Status.Conditions, metav1.Condition{ + Type: v1alpha1.AgentSessionConditionAdmitted, + Status: status, + LastTransitionTime: *mtime(d), + }) + return s +} + +func TestJoinedAt(t *testing.T) { + cases := []struct { + name string + sess *v1alpha1.AgentSession + want time.Time + }{ + {"never woke: creationTimestamp", sessionAt(0), at(0)}, + { + "cold wake after release (Admitted=False at t1, lastWakeAt t2>t1): lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, time.Minute), 2*time.Minute), + at(2 * time.Minute), + }, + { + "warm wake while admitted (Admitted=True at t0, lastWakeAt after t0): lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionTrue, time.Minute), 2*time.Minute), + at(2 * time.Minute), + }, + { + "released by misfit after its last wake (lastWakeAt t1, Admitted=False at t2>t1): creationTimestamp", + withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, 2*time.Minute), time.Minute), + at(0), + }, + { + "woke with no Admitted condition yet: lastWakeAt", + withWake(sessionAt(0), time.Minute), + at(time.Minute), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.True(t, tc.want.Equal(JoinedAt(tc.sess)), "want %s got %s", tc.want, JoinedAt(tc.sess)) + }) + } +} + +func TestWaitingSince(t *testing.T) { + cases := []struct { + name string + sess *v1alpha1.AgentSession + want time.Time + }{ + {"never admitted: JoinedAt", sessionAt(0), at(0)}, + { + "released after joining: release time", + withAdmitted(sessionAt(0), metav1.ConditionFalse, 5*time.Minute), + at(5 * time.Minute), + }, + { + "admitted: JoinedAt, not the admission time", + withAdmitted(sessionAt(0), metav1.ConditionTrue, 5*time.Minute), + at(0), + }, + { + "cold wake after release: the later wake time", + withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, time.Minute), 2*time.Minute), + at(2 * time.Minute), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.True(t, tc.want.Equal(WaitingSince(tc.sess)), "want %s got %s", tc.want, WaitingSince(tc.sess)) + }) + } +} + +func TestIsAdmitted(t *testing.T) { + assert.False(t, IsAdmitted(sessionAt(0)), "no condition: not admitted") + assert.True(t, IsAdmitted(withAdmitted(sessionAt(0), metav1.ConditionTrue, 0)), "Admitted=True") + assert.False(t, IsAdmitted(withAdmitted(sessionAt(0), metav1.ConditionFalse, 0)), "Admitted=False") +} + +func TestDemandTotal(t *testing.T) { + d := Demand{Pods: []PodDemand{ + {Name: "runner", CPUMilli: 500, MemBytes: 1 << 20, EphemeralBytes: 9}, + {Name: "sandbox", CPUMilli: 250, MemBytes: 2 << 20}, + }} + cpu, mem := d.Total() + assert.Equal(t, int64(750), cpu) + assert.Equal(t, int64(3<<20), mem) +} + +func TestOrder(t *testing.T) { + key := func(n string) types.NamespacedName { return types.NamespacedName{Namespace: "ns", Name: n} } + cases := []struct { + name string + in []Candidate + order []string + }{ + { + "children before non-children regardless of time", + []Candidate{ + {Key: key("early"), JoinedAt: at(0)}, + {Key: key("late-child"), JoinedAt: at(time.Hour), Child: true}, + }, + []string{"late-child", "early"}, + }, + { + "equal JoinedAt breaks on Key.String()", + []Candidate{ + {Key: key("b"), JoinedAt: at(0)}, + {Key: key("a"), JoinedAt: at(0)}, + }, + []string{"a", "b"}, + }, + { + "otherwise ascending JoinedAt", + []Candidate{ + {Key: key("a"), JoinedAt: at(2 * time.Minute)}, + {Key: key("b"), JoinedAt: at(time.Minute)}, + {Key: key("c"), JoinedAt: at(3 * time.Minute), Child: true}, + {Key: key("d"), JoinedAt: at(0), Child: true}, + }, + []string{"d", "c", "b", "a"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + Order(tc.in) + var got []string + for _, c := range tc.in { + got = append(got, c.Key.Name) + } + assert.Equal(t, tc.order, got) + }) + } +} From c63b761a54bd8c680a9abc9483520bba27047177 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 12:01:56 -0500 Subject: [PATCH 02/33] Model node headroom for whole-set admission Add a node capacity model with all-or-nothing whole-set reservation, and export schedfit.EffectiveRequests plus a pod-slot count on Headroom. --- pkg/controllers/admission/capacity.go | 162 +++++++++++++++++++ pkg/controllers/admission/capacity_test.go | 179 +++++++++++++++++++++ pkg/platform/schedfit/schedfit.go | 36 +++-- 3 files changed, 363 insertions(+), 14 deletions(-) create mode 100644 pkg/controllers/admission/capacity.go create mode 100644 pkg/controllers/admission/capacity_test.go diff --git a/pkg/controllers/admission/capacity.go b/pkg/controllers/admission/capacity.go new file mode 100644 index 00000000..2c17ea00 --- /dev/null +++ b/pkg/controllers/admission/capacity.go @@ -0,0 +1,162 @@ +package admission + +import ( + "cmp" + "context" + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/platform/schedfit" +) + +// PlacedPod is a pod reserved on a specific node, so it can be freed later. +type PlacedPod struct { + Node string + PodDemand +} + +// CapacityModel is a mutable view of what a cluster can still place. +type CapacityModel interface { + // Known reports whether the model has real capacity data. An unknown model + // admits everything: missing data must not block sessions. + Known() bool + // Reserve places every pod of d or none of them, committing only on + // success. + Reserve(d Demand) bool + // Free returns placed pods' requests to their nodes. + Free(pods []PlacedPod) + // Clone returns an independent copy. + Clone() CapacityModel +} + +// CapacitySource produces a CapacityModel snapshot. It is the seam a future +// non-Kubernetes backend plugs into. +type CapacitySource interface { + Snapshot(ctx context.Context) (CapacityModel, error) +} + +// KubeCapacitySource snapshots node headroom from the Kubernetes API. +type KubeCapacitySource struct { + Reader client.Reader +} + +// Snapshot lists nodes and all pods and builds a node model. A list error is +// returned for the caller to treat as unknown capacity. +func (s KubeCapacitySource) Snapshot(ctx context.Context) (CapacityModel, error) { + var nodes corev1.NodeList + if err := s.Reader.List(ctx, &nodes); err != nil { + return nil, fmt.Errorf("list nodes: %w", err) + } + var pods corev1.PodList + if err := s.Reader.List(ctx, &pods); err != nil { + return nil, fmt.Errorf("list pods: %w", err) + } + return NewNodeModel(nodes.Items, pods.Items), nil +} + +type nodeSlot struct { + name string + cpuMilli, memBytes, ephemeralBytes int64 + // pods is the remaining pod slots; unlimitedPods is set when the node + // reports no `pods` allocatable (fake or unusual nodes), so it is not + // read as zero. + pods int64 + unlimitedPods bool +} + +func (n *nodeSlot) fits(p PodDemand) bool { + return p.CPUMilli <= n.cpuMilli && p.MemBytes <= n.memBytes && + p.EphemeralBytes <= n.ephemeralBytes && (n.unlimitedPods || n.pods >= 1) +} + +func (n *nodeSlot) take(p PodDemand) { + n.cpuMilli -= p.CPUMilli + n.memBytes -= p.MemBytes + n.ephemeralBytes -= p.EphemeralBytes + n.pods-- +} + +type nodeModel struct { + known bool + nodes []nodeSlot +} + +// NewNodeModel builds a model from node headroom. Unschedulable nodes are +// skipped. With no nodes, or none reporting allocatable, the model is unknown. +func NewNodeModel(nodes []corev1.Node, pods []corev1.Pod) CapacityModel { + m := &nodeModel{} + if !schedfit.CeilingFromNodes(nodes).Known { + return m + } + m.known = true + for i := range nodes { + if nodes[i].Spec.Unschedulable { + continue + } + h := schedfit.HeadroomOn(nodes[i], pods) + _, hasPods := nodes[i].Status.Allocatable[corev1.ResourcePods] + m.nodes = append(m.nodes, nodeSlot{ + name: nodes[i].Name, + cpuMilli: h.CPUMilli, + memBytes: h.MemBytes, + ephemeralBytes: h.EphemeralBytes, + pods: h.Pods, + unlimitedPods: !hasPods, + }) + } + return m +} + +func (m *nodeModel) Known() bool { return m.known } + +// Reserve places pods first-fit-decreasing by cpu then memory, on a scratch +// copy so a failure leaves the model untouched. +func (m *nodeModel) Reserve(d Demand) bool { + if !m.known { + return true + } + order := slices.Clone(d.Pods) + slices.SortStableFunc(order, func(a, b PodDemand) int { + return cmp.Or(cmp.Compare(b.CPUMilli, a.CPUMilli), cmp.Compare(b.MemBytes, a.MemBytes)) + }) + scratch := slices.Clone(m.nodes) + for _, p := range order { + placed := false + for i := range scratch { + if scratch[i].fits(p) { + scratch[i].take(p) + placed = true + break + } + } + if !placed { + return false + } + } + m.nodes = scratch + return true +} + +func (m *nodeModel) Free(pods []PlacedPod) { + if !m.known { + return + } + for _, p := range pods { + for i := range m.nodes { + if m.nodes[i].name != p.Node { + continue + } + m.nodes[i].cpuMilli += p.CPUMilli + m.nodes[i].memBytes += p.MemBytes + m.nodes[i].ephemeralBytes += p.EphemeralBytes + m.nodes[i].pods++ + } + } +} + +func (m *nodeModel) Clone() CapacityModel { + return &nodeModel{known: m.known, nodes: slices.Clone(m.nodes)} +} diff --git a/pkg/controllers/admission/capacity_test.go b/pkg/controllers/admission/capacity_test.go new file mode 100644 index 00000000..97dbc7ef --- /dev/null +++ b/pkg/controllers/admission/capacity_test.go @@ -0,0 +1,179 @@ +package admission + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func testNode(name, cpu, mem, pods string) corev1.Node { + rl := corev1.ResourceList{} + if cpu != "" { + rl[corev1.ResourceCPU] = resource.MustParse(cpu) + } + if mem != "" { + rl[corev1.ResourceMemory] = resource.MustParse(mem) + } + if pods != "" { + rl[corev1.ResourcePods] = resource.MustParse(pods) + } + return corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}, Status: corev1.NodeStatus{Allocatable: rl}} +} + +func testPod(node, cpu string) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p-" + node + "-" + cpu}, + Spec: corev1.PodSpec{NodeName: node, Containers: []corev1.Container{{ + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(cpu)}}, + }}}, + } +} + +func pod(name string, cpuMilli int64) PodDemand { + return PodDemand{Name: name, CPUMilli: cpuMilli, MemBytes: 1 << 20} +} + +func TestNodeModelReserve(t *testing.T) { + sixNodes := func() ([]corev1.Node, []corev1.Pod) { + var nodes []corev1.Node + var pods []corev1.Pod + for _, n := range []string{"a", "b", "c", "d", "e", "f"} { + nodes = append(nodes, testNode(n, "1", "1Gi", "10")) + pods = append(pods, testPod(n, "600m")) + } + return nodes, pods + } + cases := []struct { + name string + build func() ([]corev1.Node, []corev1.Pod) + demand Demand + then *Demand // second Reserve, if any + want bool + wantTo *bool + known bool + }{ + { + name: "aggregate free but no single node fits: false", + build: sixNodes, + demand: Demand{Pods: []PodDemand{pod("big", 1000)}}, + want: false, known: true, + }, + { + name: "two pods split across two nodes: true, both committed; identical second Reserve fails", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "1", "1Gi", "10"), testNode("b", "1", "1Gi", "10")}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 800), pod("y", 800)}}, + then: &Demand{Pods: []PodDemand{pod("x", 800), pod("y", 800)}}, + want: true, wantTo: ptr(false), known: true, + }, + { + name: "pod-count exhausted on the only roomy node: false", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "8", "8Gi", "1")}, []corev1.Pod{testPod("a", "100m")} + }, + demand: Demand{Pods: []PodDemand{pod("x", 100)}}, + want: false, known: true, + }, + { + name: "missing pods allocatable is unlimited slots: true", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "8", "8Gi", "")}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 100), pod("y", 100)}}, + want: true, known: true, + }, + { + name: "unschedulable node is skipped: false", + build: func() ([]corev1.Node, []corev1.Pod) { + n := testNode("a", "8", "8Gi", "10") + n.Spec.Unschedulable = true + return []corev1.Node{n, testNode("b", "100m", "8Gi", "10")}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 1000)}}, + want: false, known: true, + }, + { + name: "unknown (no allocatable reported): Known()==false and Reserve true", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{{ObjectMeta: metav1.ObjectMeta{Name: "a"}}}, nil + }, + demand: Demand{Pods: []PodDemand{pod("x", 1<<40)}}, + want: true, known: false, + }, + { + name: "failed Reserve commits nothing: a later fitting Reserve succeeds", + build: func() ([]corev1.Node, []corev1.Pod) { + return []corev1.Node{testNode("a", "1", "1Gi", "10")}, nil + }, + // first pod would fit, second cannot; nothing may stay reserved. + demand: Demand{Pods: []PodDemand{pod("x", 900), pod("y", 900)}}, + then: &Demand{Pods: []PodDemand{pod("z", 900)}}, + want: false, wantTo: ptr(true), known: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + nodes, pods := tc.build() + m := NewNodeModel(nodes, pods) + assert.Equal(t, tc.known, m.Known()) + assert.Equal(t, tc.want, m.Reserve(tc.demand)) + if tc.then != nil { + assert.Equal(t, *tc.wantTo, m.Reserve(*tc.then)) + } + }) + } +} + +func TestNodeModelReserveEphemeralStorage(t *testing.T) { + n := testNode("a", "8", "8Gi", "10") + n.Status.Allocatable[corev1.ResourceEphemeralStorage] = resource.MustParse("1Gi") + m := NewNodeModel([]corev1.Node{n}, nil) + assert.False(t, m.Reserve(Demand{Pods: []PodDemand{{Name: "x", EphemeralBytes: 2 << 30}}}), + "ephemeral storage beyond the node's allocatable must not fit") +} + +func TestNodeModelFree(t *testing.T) { + m := NewNodeModel([]corev1.Node{testNode("a", "1", "1Gi", "10")}, nil) + d := Demand{Pods: []PodDemand{pod("x", 800)}} + require.True(t, m.Reserve(d)) + require.False(t, m.Reserve(d), "node is full") + + m.Free([]PlacedPod{{Node: "a", PodDemand: pod("x", 800)}}) + assert.True(t, m.Reserve(d), "freeing the placed pod makes the Reserve succeed") +} + +func TestNodeModelCloneIsIndependent(t *testing.T) { + m := NewNodeModel([]corev1.Node{testNode("a", "1", "1Gi", "10")}, nil) + c := m.Clone() + d := Demand{Pods: []PodDemand{pod("x", 800)}} + require.True(t, c.Reserve(d)) + assert.True(t, m.Reserve(d), "reserving on the clone must not affect the original") +} + +func TestUnknownModelCloneStaysUnknown(t *testing.T) { + m := NewNodeModel(nil, nil) + c := m.Clone() + assert.False(t, c.Known()) + c.Free([]PlacedPod{{Node: "a"}}) + assert.True(t, c.Reserve(Demand{Pods: []PodDemand{pod("x", 1<<40)}})) +} + +func ptr[T any](v T) *T { return &v } + +func TestKubeCapacitySourceSnapshot(t *testing.T) { + n := testNode("a", "1", "1Gi", "10") + p := testPod("a", "600m") + c := fake.NewClientBuilder().WithObjects(&n, &p).Build() + + m, err := KubeCapacitySource{Reader: c}.Snapshot(t.Context()) + require.NoError(t, err) + assert.True(t, m.Known()) + assert.False(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 500)}}), "the listed pod's request is booked") + assert.True(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 300)}})) +} diff --git a/pkg/platform/schedfit/schedfit.go b/pkg/platform/schedfit/schedfit.go index 0e366648..f7c404ac 100644 --- a/pkg/platform/schedfit/schedfit.go +++ b/pkg/platform/schedfit/schedfit.go @@ -50,6 +50,11 @@ type Headroom struct { CPUMilli int64 MemBytes int64 EphemeralBytes int64 + + // Pods is the node's allocatable pod count minus its non-terminal pods, + // floored at zero. A node that reports no `pods` allocatable yields 0 here; + // callers that need "unlimited" must check the node's allocatable themselves. + Pods int64 } // CeilingFromNodes computes the per-dimension maximum allocatable across nodes. @@ -103,6 +108,7 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { CPUMilli: milli(alloc, corev1.ResourceCPU), MemBytes: value(alloc, corev1.ResourceMemory), EphemeralBytes: value(alloc, corev1.ResourceEphemeralStorage), + Pods: value(alloc, corev1.ResourcePods), } for i := range pods { p := &pods[i] @@ -112,18 +118,20 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { if p.Status.Phase == corev1.PodSucceeded || p.Status.Phase == corev1.PodFailed { continue } - cpu, mem, eph := effectiveRequests(p) + cpu, mem, eph := EffectiveRequests(&p.Spec) + h.Pods-- h.CPUMilli -= cpu h.MemBytes -= mem h.EphemeralBytes -= eph } + h.Pods = max(h.Pods, 0) h.CPUMilli = max(h.CPUMilli, 0) h.MemBytes = max(h.MemBytes, 0) h.EphemeralBytes = max(h.EphemeralBytes, 0) return h } -// effectiveRequests implements the Kubernetes effective pod request: +// EffectiveRequests implements the Kubernetes effective pod request: // max(largest init-container request, sum of container requests), per // dimension. Init containers run to completion before the containers start, so // the scheduler reserves whichever is larger — not their sum. @@ -131,29 +139,29 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { // Native sidecars are the exception: an init container with restartPolicy // Always keeps running alongside the app containers for the pod's lifetime, so // it ADDS to the sum like an app container instead of merely setting a floor. -func effectiveRequests(p *corev1.Pod) (cpuMilli, memBytes, ephBytes int64) { - for i := range p.Spec.Containers { - req := p.Spec.Containers[i].Resources.Requests +func EffectiveRequests(spec *corev1.PodSpec) (cpuMilli, memBytes, ephBytes int64) { + for i := range spec.Containers { + req := spec.Containers[i].Resources.Requests cpuMilli += milli(req, corev1.ResourceCPU) memBytes += value(req, corev1.ResourceMemory) ephBytes += value(req, corev1.ResourceEphemeralStorage) } // Sidecars first: they belong IN the sum that the init-container floor below // is then measured against, not compared to it. - for i := range p.Spec.InitContainers { - if !isSidecar(&p.Spec.InitContainers[i]) { + for i := range spec.InitContainers { + if !isSidecar(&spec.InitContainers[i]) { continue } - req := p.Spec.InitContainers[i].Resources.Requests + req := spec.InitContainers[i].Resources.Requests cpuMilli += milli(req, corev1.ResourceCPU) memBytes += value(req, corev1.ResourceMemory) ephBytes += value(req, corev1.ResourceEphemeralStorage) } - for i := range p.Spec.InitContainers { - if isSidecar(&p.Spec.InitContainers[i]) { + for i := range spec.InitContainers { + if isSidecar(&spec.InitContainers[i]) { continue } - req := p.Spec.InitContainers[i].Resources.Requests + req := spec.InitContainers[i].Resources.Requests cpuMilli = max(cpuMilli, milli(req, corev1.ResourceCPU)) memBytes = max(memBytes, value(req, corev1.ResourceMemory)) ephBytes = max(ephBytes, value(req, corev1.ResourceEphemeralStorage)) @@ -172,15 +180,15 @@ func isSidecar(c *corev1.Container) bool { // // Prefer this over calling Exceeds per container. Kubernetes places a pod // atomically, so what has to fit is the pod's effective request — see -// effectiveRequests — and checking containers one at a time under-detects: three +// EffectiveRequests — and checking containers one at a time under-detects: three // 2Gi containers each clear a 4Gi ceiling while the pod they belong to needs 6Gi -// and can never be placed. Sharing effectiveRequests with HeadroomOn keeps the +// and can never be placed. Sharing EffectiveRequests with HeadroomOn keeps the // "what does this pod actually cost" rule in exactly one place. func ExceedsPod(label string, pod *corev1.Pod, c Ceiling) (string, bool) { if pod == nil || !c.Known { return "", false } - cpuMilli, memBytes, ephBytes := effectiveRequests(pod) + cpuMilli, memBytes, ephBytes := EffectiveRequests(&pod.Spec) req := corev1.ResourceList{} if cpuMilli > 0 { req[corev1.ResourceCPU] = *resource.NewMilliQuantity(cpuMilli, resource.DecimalSI) From b60e847331a70069d3bd04f19bea54025190e8ee Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 12:03:58 -0500 Subject: [PATCH 03/33] Decide admissions in order with fair share between agents Admit whole pod sets from per-class lines, picking the line with the lowest dominant share first; a head that does not fit blocks only its own line. --- pkg/controllers/admission/decide.go | 161 +++++++++++++++++++ pkg/controllers/admission/decide_test.go | 191 +++++++++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 pkg/controllers/admission/decide.go create mode 100644 pkg/controllers/admission/decide_test.go diff --git a/pkg/controllers/admission/decide.go b/pkg/controllers/admission/decide.go new file mode 100644 index 00000000..e32bd2a7 --- /dev/null +++ b/pkg/controllers/admission/decide.go @@ -0,0 +1,161 @@ +package admission + +import ( + "cmp" + "slices" + + "k8s.io/apimachinery/pkg/types" +) + +// Holder is an admitted, not-yet-reclaimed session. Unplaced is the part of +// its pod set not yet bound to a node; Decide reserves it first so two passes +// do not count the same free space. +type Holder struct { + Key types.NamespacedName + Line string + Child bool + Demand Demand + Unplaced Demand +} + +// Input is everything one admission pass needs. +type Input struct { + Waiting []Candidate + Holders []Holder + // Caps is the per-line limit on non-child holders. A line with no entry, + // or a cap <= 0, is uncapped. + Caps map[string]int32 + Model CapacityModel + // TotalCPUMilli and TotalMemBytes normalize usage into a dominant share. + // Zero totals make every share zero. + TotalCPUMilli, TotalMemBytes int64 +} + +// Position is a waiting session's place in its line. Index is 1-based. +type Position struct { + Index, LineLength int32 +} + +// Decision is the outcome of one pass: who to admit now, and where everyone +// still waiting stands in their line. +type Decision struct { + Admit []types.NamespacedName + Positions map[types.NamespacedName]Position +} + +// lineState is one line's remaining queue and what it already holds. +type lineState struct { + name string + queue []Candidate + cpuMilli int64 + memBytes int64 + nonChild int32 + capacity int32 // <= 0: uncapped +} + +func (l *lineState) eligible() bool { + if len(l.queue) == 0 { + return false + } + return l.queue[0].Child || l.capacity <= 0 || l.nonChild < l.capacity +} + +// Decide admits waiting sessions, whole pod set at a time, repeatedly picking +// the eligible line with the lowest dominant share of the cluster and +// admitting its head if the capacity model can place it. A head that does not +// fit blocks only its own line (strict FIFO within a line). Children are +// ordered ahead of others in their line and are never blocked by, nor counted +// toward, the cap. +// +// Decide mutates in.Model: Unplaced holder demand and every admitted +// candidate's demand are reserved on it. It does not modify in.Waiting or +// in.Holders. +func Decide(in Input) Decision { + for _, h := range in.Holders { + // A failure means the pods already landed on nodes the snapshot + // accounts for; the holder is admitted either way. + _ = in.Model.Reserve(h.Unplaced) + } + + lines := map[string]*lineState{} + line := func(name string) *lineState { + l, ok := lines[name] + if !ok { + l = &lineState{name: name, capacity: in.Caps[name]} + lines[name] = l + } + return l + } + for _, h := range in.Holders { + l := line(h.Line) + cpu, mem := h.Demand.Total() + l.cpuMilli += cpu + l.memBytes += mem + if !h.Child { + l.nonChild++ + } + } + for _, c := range in.Waiting { + l := line(c.Line) + l.queue = append(l.queue, c) + } + for _, l := range lines { + Order(l.queue) + } + + share := func(l *lineState) float64 { + var s float64 + if in.TotalCPUMilli > 0 { + s = float64(l.cpuMilli) / float64(in.TotalCPUMilli) + } + if in.TotalMemBytes > 0 { + s = max(s, float64(l.memBytes)/float64(in.TotalMemBytes)) + } + return s + } + + var admit []types.NamespacedName + for { + var eligible []*lineState + for _, l := range lines { + if l.eligible() { + eligible = append(eligible, l) + } + } + slices.SortFunc(eligible, func(a, b *lineState) int { + return cmp.Or( + cmp.Compare(share(a), share(b)), + a.queue[0].JoinedAt.Compare(b.queue[0].JoinedAt), + cmp.Compare(a.name, b.name), + ) + }) + admitted := false + for _, l := range eligible { + head := l.queue[0] + if !in.Model.Reserve(head.Demand) { + continue + } + cpu, mem := head.Demand.Total() + l.cpuMilli += cpu + l.memBytes += mem + if !head.Child { + l.nonChild++ + } + l.queue = l.queue[1:] + admit = append(admit, head.Key) + admitted = true + break + } + if !admitted { + break + } + } + + positions := map[types.NamespacedName]Position{} + for _, l := range lines { + for i, c := range l.queue { + positions[c.Key] = Position{Index: int32(i + 1), LineLength: int32(len(l.queue))} + } + } + return Decision{Admit: admit, Positions: positions} +} diff --git a/pkg/controllers/admission/decide_test.go b/pkg/controllers/admission/decide_test.go new file mode 100644 index 00000000..e315f744 --- /dev/null +++ b/pkg/controllers/admission/decide_test.go @@ -0,0 +1,191 @@ +package admission + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// slotModel is a fake CapacityModel that admits a fixed number of pods. +type slotModel struct { + known bool + slots int +} + +func (m *slotModel) Known() bool { return m.known } +func (m *slotModel) Reserve(d Demand) bool { + if !m.known { + return true + } + if len(d.Pods) > m.slots { + return false + } + m.slots -= len(d.Pods) + return true +} +func (m *slotModel) Free([]PlacedPod) {} +func (m *slotModel) Clone() CapacityModel { c := *m; return &c } + +func key(name string) types.NamespacedName { + return types.NamespacedName{Namespace: "ns", Name: name} +} + +func cand(name, line string, joinedSec int, child bool, cpuMilli ...int64) Candidate { + d := Demand{} + for i, c := range cpuMilli { + d.Pods = append(d.Pods, pod(name+string(rune('0'+i)), c)) + } + return Candidate{Key: key(name), Line: line, JoinedAt: t0.Add(time.Duration(joinedSec) * time.Second), Child: child, Demand: d} +} + +func holder(name, line string, child bool, cpuMilli int64) Holder { + return Holder{Key: key(name), Line: line, Child: child, Demand: Demand{Pods: []PodDemand{pod(name, cpuMilli)}}} +} + +func keys(names ...string) []types.NamespacedName { + var out []types.NamespacedName + for _, n := range names { + out = append(out, key(n)) + } + return out +} + +func TestDecide(t *testing.T) { + oneNodeModel := func(cpu string) CapacityModel { + return NewNodeModel([]corev1.Node{testNode("n1", cpu, "8Gi", "10")}, nil) + } + cases := []struct { + name string + in func() Input + admit []types.NamespacedName + }{ + { + name: "burst on one class, room for two: first two by join time admitted, rest keep order", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("c", "l", 3, false, 100), cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100)}, + Model: &slotModel{known: true, slots: 2}, + } + }, + admit: keys("a", "b"), + }, + { + name: "smaller later session does not jump a larger head in the same line", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("big", "l", 1, false, 100, 100, 100), cand("small", "l", 2, false, 100)}, + Model: &slotModel{known: true, slots: 2}, + } + }, + admit: nil, + }, + { + name: "head of line A does not fit, head of line B does: B admitted", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("a", "A", 1, false, 4000), cand("b", "B", 2, false, 500)}, + Model: oneNodeModel("2"), + } + }, + admit: keys("b"), + }, + { + name: "two lines both fit one more: lower dominant share admitted first", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("a", "A", 1, false, 100), cand("b", "B", 2, false, 100)}, + Holders: []Holder{holder("ha", "A", false, 4000), holder("hb", "B", false, 1000)}, + Model: &slotModel{known: true, slots: 1}, + TotalCPUMilli: 10000, TotalMemBytes: 1 << 40, + } + }, + admit: keys("b"), + }, + { + name: "cap 2 with 2 holders: non-child head waits, child head admitted", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100), cand("kid", "l", 2, true, 100)}, + Holders: []Holder{holder("h1", "l", false, 100), holder("h2", "l", false, 100)}, + Caps: map[string]int32{"l": 2}, + Model: &slotModel{known: true, slots: 5}, + } + }, + admit: keys("kid"), + }, + { + name: "cap counts non-child holders admitted in the same pass: cap 1 admits one of two", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100)}, + Caps: map[string]int32{"l": 1}, + Model: &slotModel{known: true, slots: 5}, + } + }, + admit: keys("a"), + }, + { + name: "unplaced holder demand is reserved before waiting heads: nothing new admitted", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 600)}, + Holders: []Holder{{ + Key: key("h"), Line: "l", + Demand: Demand{Pods: []PodDemand{pod("h", 600)}}, + Unplaced: Demand{Pods: []PodDemand{pod("h", 600)}}, + }}, + Model: oneNodeModel("1"), + } + }, + admit: nil, + }, + { + name: "unknown capacity: all admitted in order, cap still applies", + in: func() Input { + return Input{ + Waiting: []Candidate{ + cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100), cand("c", "l", 3, false, 100), + cand("x", "free", 4, false, 100), + }, + Caps: map[string]int32{"l": 2}, + Model: &slotModel{known: false}, + } + }, + admit: keys("a", "b", "x"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Decide(tc.in()) + assert.Equal(t, tc.admit, got.Admit) + }) + } +} + +func TestDecideDoesNotMutateCallerSlices(t *testing.T) { + waiting := []Candidate{cand("b", "l", 2, false, 100), cand("a", "l", 1, false, 100)} + holders := []Holder{holder("h", "l", false, 100)} + Decide(Input{Waiting: waiting, Holders: holders, Model: &slotModel{known: true, slots: 0}}) + assert.Equal(t, key("b"), waiting[0].Key, "Waiting must keep the caller's order") + assert.Len(t, holders, 1) +} + +func TestDecidePositions(t *testing.T) { + got := Decide(Input{ + Waiting: []Candidate{ + cand("a1", "A", 1, false, 100), cand("a2", "A", 2, false, 100), cand("a3", "A", 3, false, 100), + cand("b1", "B", 1, false, 100), cand("b2", "B", 2, false, 100), + }, + Model: &slotModel{known: true, slots: 2}, + }) + require.ElementsMatch(t, keys("a1", "b1"), got.Admit) + assert.Equal(t, map[types.NamespacedName]Position{ + key("a2"): {Index: 1, LineLength: 2}, + key("a3"): {Index: 2, LineLength: 2}, + key("b2"): {Index: 1, LineLength: 1}, + }, got.Positions, "positions are 1-based per line, cover only still-waiting candidates") +} From c088254dce7bd9aed2a6e4783bd18ef251cb447f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 12:19:39 -0500 Subject: [PATCH 04/33] Compute a session's whole pod demand before creating it DemandFor renders every pod a session will create (runner, one sandbox per tool bundle, content-guard detectors, separate-pod sidecars) through the reconciler's own builders and prices each with schedfit.EffectiveRequests, without writing anything. EffectiveRequests now defaults an unset request to the container's limit, as the API server does, so a built-but-not-created sandbox (limits only) is no longer priced at zero. Detector resolution moves into pure helpers shared with the reconciler, and toolchain resolution accepts a client.Reader. --- pkg/controllers/agentsession/controller.go | 71 +------ pkg/controllers/agentsession/demand.go | 199 +++++++++++++++++ pkg/controllers/agentsession/demand_test.go | 223 ++++++++++++++++++++ pkg/controllers/agentsession/sidecarpod.go | 77 +++++++ pkg/platform/schedfit/schedfit.go | 30 ++- pkg/platform/schedfit/schedfit_test.go | 46 ++++ pkg/tools/toolchain/resolve/resolve.go | 2 +- 7 files changed, 583 insertions(+), 65 deletions(-) create mode 100644 pkg/controllers/agentsession/demand.go create mode 100644 pkg/controllers/agentsession/demand_test.go diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index 611ee1c2..c05fdd89 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -2862,57 +2862,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // clear AwaitingDetector reason set meanwhile so the wait is never silent. // The loop is generic — it never branches on inspector id. { - var resolvedDetectors []spiceboxv1alpha1.ResolvedContentGuardDetector - var detKeys []string // Collect the detector-requiring inspectors and their specs. - if eff := sess.Status.EffectiveSettings; eff != nil { - for _, ci := range eff.ContentInspectors { - insp, ok := contentguardregistry.Get(ci.ID) - if !ok { - // Webhook already rejected unknown IDs; defensive skip. - continue - } - dp, ok := insp.(contentguard.DetectorProvider) - if !ok { - // Inspector does not need a detector sidecar (e.g. url-allowlist). - continue - } - raw := ci.Config.Raw - ds, derr := dp.Detector(raw) - if derr != nil || ds == nil { - // Fail-closed: a detector-requiring inspector that can't resolve - // its image must not silently run unscanned. - return r.markBootFailed(ctx, &sess, - spiceboxv1alpha1.ReasonAgentSessionContentGuardHalt, - fmt.Sprintf("content-guard detector %q: %v", ci.ID, derr)) - } - // Use the image exactly as declared in the inspector config: pinned - // by digest it launches the exact bytes, by tag it launches by tag. - // There is no probe path here as there is for SidecarToolbox. - img := ds.Image - resolvedDetectors = append(resolvedDetectors, spiceboxv1alpha1.ResolvedContentGuardDetector{ - Inspector: ci.ID, - Image: img, - HealthPath: ds.HealthPath, - }) - detKeys = append(detKeys, "det-"+ci.ID) - } + resolvedDetectors, derr := resolveContentGuardDetectors(sess.Status.EffectiveSettings) + if derr != nil { + // Fail-closed: a detector-requiring inspector that can't resolve + // its image must not silently run unscanned. + return r.markBootFailed(ctx, &sess, + spiceboxv1alpha1.ReasonAgentSessionContentGuardHalt, derr.Error()) } if len(resolvedDetectors) > 0 { - // Allocate ports in the SAME key space as sidecar ports so they never - // collide. Derive the sidecar key slice from the resolved snapshot. - sidecarKeySlice := make([]string, 0, len(resolvedSidecars)) + sidecarRefs := make([]string, 0, len(resolvedSidecars)) for _, rt := range resolvedSidecars { - sidecarKeySlice = append(sidecarKeySlice, rt.Ref) - } - allKeys := append(append([]string{}, sidecarKeySlice...), detKeys...) - ports := AllocatePorts(allKeys) - for i := range resolvedDetectors { - det := &resolvedDetectors[i] - det.Port = ports["det-"+det.Inspector] - det.PodName = "sidecar-" + sess.Name + "-" + det.Inspector + sidecarRefs = append(sidecarRefs, rt.Ref) } + placeContentGuardDetectors(&sess, resolvedDetectors, sidecarRefs) // Create each detector pod + its deny-all-egress NetworkPolicy, and // reflect the pod IP. Hold runner-pod creation until every PodIP is known. @@ -2934,22 +2898,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu var terminalDetectorFailures []string for i := range resolvedDetectors { det := &resolvedDetectors[i] - spec := cosidecar.Spec{ - Ref: det.Inspector, - // Pin the NetworkPolicy name to the one - // cleanupOrphanedSidecarPods reaps by. On cosidecar's default - // name, removing an inspector mid-session reaps the detector pod - // by label but orphans its NetworkPolicy until session GC. - // BuildPod names the pod by Ref, so only the NP name is affected. - Name: SidecarNetworkPolicyName(&sess, det.Inspector), - Image: det.Image, - Port: det.Port, - Health: cosidecar.Healthcheck{Path: det.HealthPath}, - ImagePullSecret: r.ImagePullSecret, - // Zero egress: the detector ingests potentially hostile tool text - // and must not be able to exfiltrate it to external endpoints. - Egress: cosidecar.EgressPolicy{Denied: true}, - } + spec := detectorPodSpec(&sess, *det, r.ImagePullSecret) pod, _, perr := cosidecar.BuildPod(&sess, spec) if perr != nil { return r.markBootFailed(ctx, &sess, diff --git a/pkg/controllers/agentsession/demand.go b/pkg/controllers/agentsession/demand.go new file mode 100644 index 00000000..4fca8022 --- /dev/null +++ b/pkg/controllers/agentsession/demand.go @@ -0,0 +1,199 @@ +package agentsession + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + harnessregistry "github.com/authzed/openagentprimitives/pkg/agent/harness/registry" + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" + "github.com/authzed/openagentprimitives/pkg/platform/podspec" + "github.com/authzed/openagentprimitives/pkg/platform/schedfit" + "github.com/authzed/openagentprimitives/pkg/tools/toolchain/resolve" + "github.com/authzed/openagentprimitives/pkg/web/secretoutsrv" +) + +// demandPlaceholderImage stands in for an image a pod's demand cannot depend +// on. The SpiceboxSession reconciler fills an unpinned class image from the +// operator's configured sandbox image, which this package-level function does +// not have; requests come from the class's resources alone, so any image +// prices the same. +const demandPlaceholderImage = "demand-estimate" + +// DemandFor returns the resource requests of every pod sess will create, so +// the whole set can be admitted before any of it exists. It reads only; it +// creates nothing. +// +// Each pod is rendered by the same builder the reconciler uses to create it, +// then priced by schedfit.EffectiveRequests: +// +// - the runner (BuildRunnerPod, as PodRunnerFactory calls it), with the +// in-pod sidecars it carries; +// - one sandbox per ac.Spec.ToolBundles entry (BuildBundleSession + +// podspec.Build, against the bundle's SpiceboxClass and its toolchains +// resolved as the SpiceboxSession reconciler resolves them at bind); +// - one pod per content-guard detector (cosidecar.BuildPod); +// - one pod per separate-pod SidecarToolbox (BuildSidecarPod). +// +// Inputs that only exist once provisioning runs — the workspace claim, the +// secret-output token hash, a Workshop identity, the operator's URLs and pull +// secret — are left empty: none of them reaches a container's resources. +// +// A referenced object that is missing (a SpiceboxClass, a SidecarToolbox, a +// toolchain) is an error naming it, never a zero demand, and so is a session +// whose settings are not yet resolved. +func DemandFor(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass) (admission.Demand, error) { + eff := sess.Status.EffectiveSettings + if eff == nil { + return admission.Demand{}, fmt.Errorf("session %s/%s has no effectiveSettings yet: its detectors and sandbox backends derive from them", + sess.Namespace, sess.Name) + } + + var pods []*corev1.Pod + + sidecars, err := demandSidecars(ctx, c, sess, ac) + if err != nil { + return admission.Demand{}, err + } + dets, err := resolveContentGuardDetectors(eff) + if err != nil { + return admission.Demand{}, err + } + sidecarRefs := make([]string, 0, len(sidecars)) + for _, rt := range sidecars { + sidecarRefs = append(sidecarRefs, rt.Ref) + } + placeContentGuardDetectors(sess, dets, sidecarRefs) + + h, err := harnessregistry.Resolve(ac.Spec.Harness) + if err != nil { + return admission.Demand{}, fmt.Errorf("resolve harness for %s/%s: %w", sess.Namespace, sess.Name, err) + } + runner, err := BuildRunnerPod(PodSpecOpts{ + Session: sess, + Class: ac, + Image: demandPlaceholderImage, + ServiceAcct: sess.Name + "-runner-sa", + MemoryToken: MemoryTokenSecretName(sess), + ResolvedSidecars: sidecars, + SidecarSecretName: func(ref string) string { return cosidecar.CredentialSecretName(sess.Name, ref) }, + ResolvedContentGuardDetectors: dets, + Harness: h, + }) + if err != nil { + return admission.Demand{}, fmt.Errorf("build runner pod: %w", err) + } + pods = append(pods, runner) + + for _, b := range ac.Spec.ToolBundles { + pod, err := demandBundlePod(ctx, c, sess, ac, b) + if err != nil { + return admission.Demand{}, err + } + pods = append(pods, pod) + } + + for _, det := range dets { + pod, _, err := cosidecar.BuildPod(sess, detectorPodSpec(sess, det, "")) + if err != nil { + return admission.Demand{}, fmt.Errorf("build detector pod %q: %w", det.Inspector, err) + } + pods = append(pods, pod) + } + + soSecretName := secretoutsrv.SecretOutputSecretName(sess.Name) + for _, rt := range sidecars { + if rt.RunMode != RunModeSeparatePod { + continue + } + pod, err := BuildSidecarPod(sess, rt, soSecretName, cosidecar.CredentialSecretName(sess.Name, rt.Ref), nil, "", "", "") + if err != nil { + return admission.Demand{}, fmt.Errorf("build sidecar pod %q: %w", rt.Ref, err) + } + pods = append(pods, pod) + } + + d := admission.Demand{Pods: make([]admission.PodDemand, 0, len(pods))} + for _, p := range pods { + cpu, mem, eph := schedfit.EffectiveRequests(&p.Spec) + d.Pods = append(d.Pods, admission.PodDemand{Name: p.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph}) + } + return d, nil +} + +// demandBundlePod renders the sandbox pod one tool bundle will run in. It +// mirrors the reconciler's bundle loop (identity and tier-resolved backend +// into BuildBundleSession) and the SpiceboxSession reconciler's bind (class +// snapshot, image default, toolchain resolution), then calls podspec.Build — +// the builder both registered sandbox backends render through. +func demandBundlePod(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass, b spiceboxv1alpha1.ToolBundle) (*corev1.Pod, error) { + identity := b.AgentIdentity + if identity == "" { + identity = sess.Spec.AgentIdentity + } + if identity == "" { + identity = ac.Spec.AgentIdentity + } + var resolvedSandbox *spiceboxv1alpha1.SandboxBackend + if backend, ok := sess.Status.EffectiveSettings.Sandbox[b.Name]; ok { + resolvedSandbox = &backend + } + // No claim name: whether the workspace is shared depends on the operator's + // configured StorageClass, and only changes the pod's volumes. + bs := BuildBundleSession(sess, b, identity, "", sess.Status.ResolvedSkillBundles, resolvedSandbox, ac.Spec.Config) + + var cls spiceboxv1alpha1.SpiceboxClass + if err := c.Get(ctx, types.NamespacedName{Name: b.Class}, &cls); err != nil { + return nil, fmt.Errorf("get SpiceboxClass %q for bundle %q: %w", b.Class, b.Name, err) + } + classSpec := *cls.Spec.DeepCopy() + if classSpec.Image == "" { + classSpec.Image = demandPlaceholderImage + } + mounts, _, err := resolve.Resolve(ctx, c, classSpec.Toolchains) + if err != nil { + return nil, fmt.Errorf("resolve toolchains of SpiceboxClass %q for bundle %q: %w", b.Class, b.Name, err) + } + bs.Status.ResolvedToolchains = mounts + + pod, err := podspec.Build(bs, classSpec) + if err != nil { + return nil, fmt.Errorf("build sandbox pod for bundle %q (SpiceboxClass %q): %w", b.Name, b.Class, err) + } + return pod, nil +} + +// demandSidecars resolves ac's SidecarToolbox refs to the fields a pod build +// reads: spec, run mode, and port. The reconciler's by-digest image rewrite +// and network merge are skipped — neither reaches a container's resources. +func demandSidecars(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass) ([]spiceboxv1alpha1.ResolvedSidecarToolbox, error) { + if len(ac.Spec.SidecarToolboxes) == 0 { + return nil, nil + } + out := make([]spiceboxv1alpha1.ResolvedSidecarToolbox, 0, len(ac.Spec.SidecarToolboxes)) + refs := make([]string, 0, len(ac.Spec.SidecarToolboxes)) + for _, ref := range ac.Spec.SidecarToolboxes { + var tb spiceboxv1alpha1.SidecarToolbox + if err := c.Get(ctx, types.NamespacedName{Namespace: sess.Namespace, Name: ref.Ref}, &tb); err != nil { + return nil, fmt.Errorf("get SidecarToolbox %q in namespace %q: %w", ref.Ref, sess.Namespace, err) + } + spec := *tb.Spec.DeepCopy() + out = append(out, spiceboxv1alpha1.ResolvedSidecarToolbox{ + Name: ref.Name, + Ref: ref.Ref, + Spec: spec, + RunMode: RunModeFor(spec), + }) + refs = append(refs, ref.Ref) + } + ports := AllocatePorts(refs) + for i := range out { + out[i].Port = ports[out[i].Ref] + } + return out, nil +} diff --git a/pkg/controllers/agentsession/demand_test.go b/pkg/controllers/agentsession/demand_test.go new file mode 100644 index 00000000..733ffb92 --- /dev/null +++ b/pkg/controllers/agentsession/demand_test.go @@ -0,0 +1,223 @@ +package agentsession + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + _ "github.com/authzed/openagentprimitives/pkg/agent/harness/apnative" // the default harness the runner pod is built with + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + _ "github.com/authzed/openagentprimitives/pkg/authz/contentguard/kinds/promptinjection" // a DetectorProvider inspector + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" + "github.com/authzed/openagentprimitives/pkg/platform/podspec" + "github.com/authzed/openagentprimitives/pkg/platform/schedfit" +) + +const demandMiB = int64(1024 * 1024) + +// demandSpiceboxClass is a cluster-scoped SpiceboxClass whose sandbox +// container is limited to the given cpu/memory/ephemeral-storage. +func demandSpiceboxClass(name, cpu, mem, eph string) *spiceboxv1alpha1.SpiceboxClass { + return &spiceboxv1alpha1.SpiceboxClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: spiceboxv1alpha1.SpiceboxClassSpec{ + Image: "ghcr.io/example/sandbox:v1", + Resources: spiceboxv1alpha1.SpiceboxResources{ + CPU: resource.MustParse(cpu), + Memory: resource.MustParse(mem), + EphemeralStorage: resource.MustParse(eph), + }, + }, + } +} + +// demandAgentClass declares one tool bundle per SpiceboxClass name, named +// "b-", plus the given sidecar toolbox refs. +func demandAgentClass(bundleClasses []string, sidecars ...spiceboxv1alpha1.AgentClassSidecarToolboxRef) *spiceboxv1alpha1.AgentClass { + ac := &spiceboxv1alpha1.AgentClass{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "demo-class"}} + for _, c := range bundleClasses { + ac.Spec.ToolBundles = append(ac.Spec.ToolBundles, spiceboxv1alpha1.ToolBundle{Name: "b-" + c, Class: c}) + } + ac.Spec.SidecarToolboxes = sidecars + return ac +} + +// demandSession is a session whose settings are resolved, with a +// prompt-injection inspector when withDetector is set. +func demandSession(t *testing.T, withDetector bool) *spiceboxv1alpha1.AgentSession { + t.Helper() + eff := &spiceboxv1alpha1.EffectiveSettings{ + Model: spiceboxv1alpha1.ModelConfig{APIKey: spiceboxv1alpha1.SecretKeyRef{Name: "llm", Key: "token"}}, + } + if withDetector { + eff.ContentInspectors = []spiceboxv1alpha1.ContentInspectorConfig{{ + ID: "prompt-injection", + Config: apiextv1.JSON{Raw: []byte(`{"detectorImage":"ghcr.io/example/detector:v1","port":9080}`)}, + }} + } + return &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "demo-session", UID: "uid-1"}, + Status: spiceboxv1alpha1.AgentSessionStatus{EffectiveSettings: eff}, + } +} + +func demandSidecarToolbox(name string, secretGated bool) *spiceboxv1alpha1.SidecarToolbox { + tb := &spiceboxv1alpha1.SidecarToolbox{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: name}, + Spec: spiceboxv1alpha1.SidecarToolboxSpec{ + Name: name, + Source: spiceboxv1alpha1.SidecarToolboxSource{Image: "ghcr.io/example/" + name + ":v1"}, + }, + } + if secretGated { + tb.Spec.SecretInputs = []spiceboxv1alpha1.SidecarToolboxSecretInput{{Name: "TOKEN", Deliver: "env", From: "tok"}} + } + return tb +} + +func demandClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, spiceboxv1alpha1.AddToScheme(s)) + return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build() +} + +// podDemandOf prices pod the way the admitter will: schedfit.EffectiveRequests +// over the spec the real builder produced. +func podDemandOf(pod *corev1.Pod) admission.PodDemand { + cpu, mem, eph := schedfit.EffectiveRequests(&pod.Spec) + return admission.PodDemand{Name: pod.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph} +} + +func demandByName(d admission.Demand) map[string]admission.PodDemand { + out := make(map[string]admission.PodDemand, len(d.Pods)) + for _, p := range d.Pods { + out[p.Name] = p + } + return out +} + +func TestDemandForMatchesBuiltPods(t *testing.T) { + half := demandSpiceboxClass("half", "500m", "512Mi", "1Gi") + one := demandSpiceboxClass("one", "1", "1Gi", "2Gi") + ac := demandAgentClass([]string{"half", "one"}) + sess := demandSession(t, true) + c := demandClient(t, half, one) + + got, err := DemandFor(context.Background(), c, sess, ac) + require.NoError(t, err, "DemandFor") + byName := demandByName(got) + require.Len(t, got.Pods, 4, "runner + two bundles + one detector, got %v", got.Pods) + + // Each bundle's demand equals what the real sandbox builder produces for it. + for _, tc := range []struct { + class *spiceboxv1alpha1.SpiceboxClass + cpuMilli, memMiB int64 + }{ + {half, 500, 512}, + {one, 1000, 1024}, + } { + bundle := spiceboxv1alpha1.ToolBundle{Name: "b-" + tc.class.Name, Class: tc.class.Name} + pod, berr := podspec.Build(BuildBundleSession(sess, bundle, "", "", nil, nil, nil), tc.class.Spec) + require.NoError(t, berr, "build bundle pod for %s", tc.class.Name) + want := podDemandOf(pod) + + assert.Equal(t, want, byName[pod.Name], "bundle %s demand equals the built pod's effective requests", bundle.Name) + assert.Equal(t, tc.cpuMilli, byName[pod.Name].CPUMilli, + "bundle %s is priced at its class's cpu limit (requests default to limits)", bundle.Name) + assert.Equal(t, tc.memMiB*demandMiB, byName[pod.Name].MemBytes, "bundle %s memory", bundle.Name) + } + + // The detector's demand equals what cosidecar.BuildPod produces for it. + dets, derr := resolveContentGuardDetectors(sess.Status.EffectiveSettings) + require.NoError(t, derr, "resolve detectors") + require.Len(t, dets, 1) + detPod, _, perr := cosidecar.BuildPod(sess, detectorPodSpec(sess, dets[0], "")) + require.NoError(t, perr, "build detector pod") + assert.Equal(t, podDemandOf(detPod), byName[detPod.Name], "detector demand equals the built pod's") + + // The runner pod is present under its deterministic name. + _, ok := byName[RunnerPodName(sess)] + assert.True(t, ok, "runner pod %q is in the demand", RunnerPodName(sess)) +} + +func TestDemandForSidecars(t *testing.T) { + ac := demandAgentClass(nil, + spiceboxv1alpha1.AgentClassSidecarToolboxRef{Name: "inpod", Ref: "inpod"}, + spiceboxv1alpha1.AgentClassSidecarToolboxRef{Name: "gated", Ref: "gated"}, + ) + sess := demandSession(t, false) + c := demandClient(t, demandSidecarToolbox("inpod", false), demandSidecarToolbox("gated", true)) + + got, err := DemandFor(context.Background(), c, sess, ac) + require.NoError(t, err, "DemandFor") + byName := demandByName(got) + + assert.Len(t, got.Pods, 2, "runner + the one separate-pod sidecar; the in-pod sidecar rides in the runner pod") + assert.Contains(t, byName, RunnerPodName(sess), "runner pod") + assert.Contains(t, byName, SidecarPodName(sess, "gated"), "secret-gated sidecar gets its own pod") + assert.NotContains(t, byName, SidecarPodName(sess, "inpod"), "in-pod sidecar has no pod of its own") +} + +func TestDemandForErrors(t *testing.T) { + cases := []struct { + name string + ac *spiceboxv1alpha1.AgentClass + sess func(t *testing.T) *spiceboxv1alpha1.AgentSession + objs []client.Object + contains string + }{ + { + name: "missing SpiceboxClass: error naming the class, not a zero demand", + ac: demandAgentClass([]string{"absent"}), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { return demandSession(t, false) }, + contains: `SpiceboxClass "absent"`, + }, + { + name: "missing SidecarToolbox: error naming the toolbox", + ac: demandAgentClass(nil, spiceboxv1alpha1.AgentClassSidecarToolboxRef{Name: "x", Ref: "ghost"}), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { return demandSession(t, false) }, + contains: `SidecarToolbox "ghost"`, + }, + { + name: "missing toolchain on a bundle's class: error naming the toolchain", + ac: demandAgentClass([]string{"withgo"}), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { return demandSession(t, false) }, + objs: func() []client.Object { + cls := demandSpiceboxClass("withgo", "500m", "512Mi", "1Gi") + cls.Spec.Toolchains = []string{"go"} + return []client.Object{cls} + }(), + contains: `"go"`, + }, + { + name: "settings not yet resolved: error, since detectors and backends derive from them", + ac: demandAgentClass(nil), + sess: func(t *testing.T) *spiceboxv1alpha1.AgentSession { + s := demandSession(t, false) + s.Status.EffectiveSettings = nil + return s + }, + contains: "effectiveSettings", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := DemandFor(context.Background(), demandClient(t, tc.objs...), tc.sess(t), tc.ac) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.contains) + assert.Empty(t, got.Pods, "no partial demand on error") + }) + } +} diff --git a/pkg/controllers/agentsession/sidecarpod.go b/pkg/controllers/agentsession/sidecarpod.go index 4bba0c5c..6c90216f 100644 --- a/pkg/controllers/agentsession/sidecarpod.go +++ b/pkg/controllers/agentsession/sidecarpod.go @@ -29,6 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/authz/contentguard" + contentguardregistry "github.com/authzed/openagentprimitives/pkg/authz/contentguard/registry" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" "github.com/authzed/openagentprimitives/pkg/platform/podstatus" ) @@ -61,6 +63,81 @@ func (r *Reconciler) ensureDetectorPod(ctx context.Context, sess *spiceboxv1alph return nil } +// resolveContentGuardDetectors returns one detector per enabled content +// inspector that implements contentguard.DetectorProvider, in settings order. +// Port and PodName are left for placeContentGuardDetectors. A nil eff yields +// none. +// +// An inspector that implements DetectorProvider but cannot produce a detector +// is an error, never a skip: a detector-requiring inspector that can't resolve +// its image must not silently run unscanned. +func resolveContentGuardDetectors(eff *spiceboxv1alpha1.EffectiveSettings) ([]spiceboxv1alpha1.ResolvedContentGuardDetector, error) { + if eff == nil { + return nil, nil + } + var out []spiceboxv1alpha1.ResolvedContentGuardDetector + for _, ci := range eff.ContentInspectors { + insp, ok := contentguardregistry.Get(ci.ID) + if !ok { + // Webhook already rejected unknown IDs; defensive skip. + continue + } + dp, ok := insp.(contentguard.DetectorProvider) + if !ok { + // Inspector does not need a detector sidecar (e.g. url-allowlist). + continue + } + ds, err := dp.Detector(ci.Config.Raw) + if err != nil || ds == nil { + return nil, fmt.Errorf("content-guard detector %q: %v", ci.ID, err) + } + // Use the image exactly as declared in the inspector config: pinned + // by digest it launches the exact bytes, by tag it launches by tag. + // There is no probe path here as there is for SidecarToolbox. + out = append(out, spiceboxv1alpha1.ResolvedContentGuardDetector{ + Inspector: ci.ID, + Image: ds.Image, + HealthPath: ds.HealthPath, + }) + } + return out, nil +} + +// placeContentGuardDetectors stamps each detector's Port and PodName. Ports +// are allocated in the SAME key space as the session's sidecar refs so a +// detector never collides with a sidecar. +func placeContentGuardDetectors(sess *spiceboxv1alpha1.AgentSession, dets []spiceboxv1alpha1.ResolvedContentGuardDetector, sidecarRefs []string) { + keys := append([]string{}, sidecarRefs...) + for _, d := range dets { + keys = append(keys, "det-"+d.Inspector) + } + ports := AllocatePorts(keys) + for i := range dets { + dets[i].Port = ports["det-"+dets[i].Inspector] + dets[i].PodName = "sidecar-" + sess.Name + "-" + dets[i].Inspector + } +} + +// detectorPodSpec is the cosidecar descriptor a detector pod is built from. +func detectorPodSpec(sess *spiceboxv1alpha1.AgentSession, det spiceboxv1alpha1.ResolvedContentGuardDetector, imagePullSecret string) cosidecar.Spec { + return cosidecar.Spec{ + Ref: det.Inspector, + // Pin the NetworkPolicy name to the one cleanupOrphanedSidecarPods reaps + // by. On cosidecar's default name, removing an inspector mid-session + // reaps the detector pod by label but orphans its NetworkPolicy until + // session GC. BuildPod names the pod by Ref, so only the NP name is + // affected. + Name: SidecarNetworkPolicyName(sess, det.Inspector), + Image: det.Image, + Port: det.Port, + Health: cosidecar.Healthcheck{Path: det.HealthPath}, + ImagePullSecret: imagePullSecret, + // Zero egress: the detector ingests potentially hostile tool text and + // must not be able to exfiltrate it to external endpoints. + Egress: cosidecar.EgressPolicy{Denied: true}, + } +} + // SidecarPodName returns the deterministic name for a session's separate // sidecar pod for the given toolbox ref: "-sidecar-". func SidecarPodName(sess *spiceboxv1alpha1.AgentSession, ref string) string { diff --git a/pkg/platform/schedfit/schedfit.go b/pkg/platform/schedfit/schedfit.go index f7c404ac..f8cd200d 100644 --- a/pkg/platform/schedfit/schedfit.go +++ b/pkg/platform/schedfit/schedfit.go @@ -9,6 +9,7 @@ package schedfit import ( "fmt" + "maps" "github.com/dustin/go-humanize" corev1 "k8s.io/api/core/v1" @@ -139,9 +140,15 @@ func HeadroomOn(node corev1.Node, pods []corev1.Pod) Headroom { // Native sidecars are the exception: an init container with restartPolicy // Always keeps running alongside the app containers for the pod's lifetime, so // it ADDS to the sum like an app container instead of merely setting a floor. +// +// A resource with a limit but no request is priced at its limit, which is the +// default the API server applies at admission. Live pods already carry it; a +// PodSpec a controller has built but not yet created does not (the sandbox +// builder sets Limits alone), and reading Requests alone would price it at +// zero. func EffectiveRequests(spec *corev1.PodSpec) (cpuMilli, memBytes, ephBytes int64) { for i := range spec.Containers { - req := spec.Containers[i].Resources.Requests + req := requestsOf(&spec.Containers[i]) cpuMilli += milli(req, corev1.ResourceCPU) memBytes += value(req, corev1.ResourceMemory) ephBytes += value(req, corev1.ResourceEphemeralStorage) @@ -152,7 +159,7 @@ func EffectiveRequests(spec *corev1.PodSpec) (cpuMilli, memBytes, ephBytes int64 if !isSidecar(&spec.InitContainers[i]) { continue } - req := spec.InitContainers[i].Resources.Requests + req := requestsOf(&spec.InitContainers[i]) cpuMilli += milli(req, corev1.ResourceCPU) memBytes += value(req, corev1.ResourceMemory) ephBytes += value(req, corev1.ResourceEphemeralStorage) @@ -161,7 +168,7 @@ func EffectiveRequests(spec *corev1.PodSpec) (cpuMilli, memBytes, ephBytes int64 if isSidecar(&spec.InitContainers[i]) { continue } - req := spec.InitContainers[i].Resources.Requests + req := requestsOf(&spec.InitContainers[i]) cpuMilli = max(cpuMilli, milli(req, corev1.ResourceCPU)) memBytes = max(memBytes, value(req, corev1.ResourceMemory)) ephBytes = max(ephBytes, value(req, corev1.ResourceEphemeralStorage)) @@ -169,6 +176,23 @@ func EffectiveRequests(spec *corev1.PodSpec) (cpuMilli, memBytes, ephBytes int64 return cpuMilli, memBytes, ephBytes } +// requestsOf returns a container's requests as the API server would default +// them: each resource that has a limit and no request is requested at its +// limit. The container itself is not modified. +func requestsOf(c *corev1.Container) corev1.ResourceList { + req := maps.Clone(c.Resources.Requests) + for name, limit := range c.Resources.Limits { + if _, ok := req[name]; ok { + continue + } + if req == nil { + req = corev1.ResourceList{} + } + req[name] = limit + } + return req +} + // isSidecar reports whether an init container is a native sidecar — restartPolicy // Always, meaning it runs for the pod's whole lifetime rather than completing // before the app containers start. diff --git a/pkg/platform/schedfit/schedfit_test.go b/pkg/platform/schedfit/schedfit_test.go index 453d26c8..cec18977 100644 --- a/pkg/platform/schedfit/schedfit_test.go +++ b/pkg/platform/schedfit/schedfit_test.go @@ -221,3 +221,49 @@ func TestEffectiveRequests_NativeSidecarAddsRatherThanSettingAFloor(t *testing.T assert.Equal(t, int64(2)*1024*1024*1024, h.MemBytes, "app(1Gi)+sidecar(1Gi)=2Gi booked of 4Gi; treating the sidecar as a floor would leave 3Gi") } + +// TestEffectiveRequests_UnsetRequestDefaultsToLimit covers a PodSpec that has +// not been through the API server yet. The apiserver defaults a container's +// unset request to its limit, per resource, at admission; an in-memory spec a +// controller has only BUILT (podspec.BuildClassSpec sets Limits alone) has not +// had that applied, so reading Requests alone would price it at zero. +func TestEffectiveRequests_UnsetRequestDefaultsToLimit(t *testing.T) { + cases := []struct { + name string + container corev1.ResourceRequirements + init *corev1.ResourceRequirements + cpu, mem, ephBytes int64 + }{ + { + name: "limits only: every dimension is priced at its limit", + container: corev1.ResourceRequirements{Limits: reqList("500m", "1Gi", "2Gi")}, + cpu: 500, mem: gib, ephBytes: 2 * gib, + }, + { + name: "explicit request below the limit: the request wins, limit is not a request", + container: corev1.ResourceRequirements{ + Requests: reqList("100m", "", ""), + Limits: reqList("500m", "1Gi", ""), + }, + cpu: 100, mem: gib, + }, + { + name: "limits-only init container larger than the app: sets the floor", + container: corev1.ResourceRequirements{Limits: reqList("", "1Gi", "")}, + init: &corev1.ResourceRequirements{Limits: reqList("", "3Gi", "")}, + mem: 3 * gib, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + spec := corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Resources: tc.container}}} + if tc.init != nil { + spec.InitContainers = []corev1.Container{{Name: "setup", Resources: *tc.init}} + } + cpu, mem, eph := EffectiveRequests(&spec) + assert.Equal(t, tc.cpu, cpu, "cpu millicores") + assert.Equal(t, tc.mem, mem, "memory bytes") + assert.Equal(t, tc.ephBytes, eph, "ephemeral-storage bytes") + }) + } +} diff --git a/pkg/tools/toolchain/resolve/resolve.go b/pkg/tools/toolchain/resolve/resolve.go index 45bb1294..17e89a74 100644 --- a/pkg/tools/toolchain/resolve/resolve.go +++ b/pkg/tools/toolchain/resolve/resolve.go @@ -42,7 +42,7 @@ var ( // // Names are sorted and deduplicated so the result — and therefore the frozen // status, the set digest, and the PodSpec — is a pure function of the class. -func Resolve(ctx context.Context, c client.Client, names []string) ([]spiceboxv1alpha1.ToolchainMount, string, error) { +func Resolve(ctx context.Context, c client.Reader, names []string) ([]spiceboxv1alpha1.ToolchainMount, string, error) { uniq := make([]string, 0, len(names)) seen := map[string]struct{}{} for _, n := range names { From bb11f5d77bff57f929c5cbbf80b3a73c1106bcb7 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 12:24:23 -0500 Subject: [PATCH 05/33] Share the bundle session builder between reconcile and demand Identity precedence and the tier-resolved sandbox backend now live in desiredBundleSession, called by both the reconciler's bundle loop and DemandFor, so the two cannot drift. --- pkg/controllers/agentsession/bundles.go | 25 ++++++++++++++++++++++ pkg/controllers/agentsession/controller.go | 20 ++--------------- pkg/controllers/agentsession/demand.go | 17 +++------------ 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/pkg/controllers/agentsession/bundles.go b/pkg/controllers/agentsession/bundles.go index 18fb45f6..901421e3 100644 --- a/pkg/controllers/agentsession/bundles.go +++ b/pkg/controllers/agentsession/bundles.go @@ -15,6 +15,31 @@ func BundleSessionName(sess *spiceboxv1alpha1.AgentSession, bundle spiceboxv1alp return sess.Name + "-" + bundle.Name } +// desiredBundleSession is the SpiceboxSession the reconciler applies for +// bundle b: it resolves the bundle's identity (bundle, then session, then +// class) and its tier-resolved sandbox backend, then calls BuildBundleSession. +// Shared by the reconciler's bundle loop and DemandFor so both always describe +// the same sandbox. +func desiredBundleSession(sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass, b spiceboxv1alpha1.ToolBundle, claimName string) *spiceboxv1alpha1.SpiceboxSession { + identity := b.AgentIdentity + if identity == "" { + identity = sess.Spec.AgentIdentity + } + if identity == "" { + identity = ac.Spec.AgentIdentity + } + // The tier-resolved sandbox backend for this bundle, or nil when no tier + // expressed a preference — BuildBundleSession then leaves spec.sandbox + // unset, so the SpiceboxClass's own setting stands. + var resolvedSandbox *spiceboxv1alpha1.SandboxBackend + if eff := sess.Status.EffectiveSettings; eff != nil { + if backend, ok := eff.Sandbox[b.Name]; ok { + resolvedSandbox = &backend + } + } + return BuildBundleSession(sess, b, identity, claimName, sess.Status.ResolvedSkillBundles, resolvedSandbox, ac.Spec.Config) +} + // BuildBundleSession returns the SpiceboxSession spec/object the AgentSession // reconciler will create for one bundle. AgentIdentity binds via spec.agent; // the toolspec subset is mirrored onto spec.toolspecs to scope which toolspecs diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index c05fdd89..9c2187de 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -1770,27 +1770,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu var earliestBundleStart time.Time bundleSessionStatuses := make([]spiceboxv1alpha1.ResolvedBundle, 0, len(ac.Spec.ToolBundles)) for _, b := range ac.Spec.ToolBundles { - identity := b.AgentIdentity - if identity == "" { - identity = sess.Spec.AgentIdentity - } - if identity == "" { - identity = ac.Spec.AgentIdentity - } - if err := r.ensureSandboxNetworkPolicy(ctx, &sess, b); err != nil { return ctrl.Result{}, err } - // The tier-resolved sandbox backend for this bundle, or nil when no - // tier expressed a preference — BuildBundleSession then leaves - // spec.sandbox unset, so the SpiceboxClass's own setting stands. - var resolvedSandbox *spiceboxv1alpha1.SandboxBackend - if eff := sess.Status.EffectiveSettings; eff != nil { - if backend, ok := eff.Sandbox[b.Name]; ok { - resolvedSandbox = &backend - } - } - desired := BuildBundleSession(&sess, b, identity, claimName, sess.Status.ResolvedSkillBundles, resolvedSandbox, ac.Spec.Config) + desired := desiredBundleSession(&sess, &ac, b, claimName) + identity := desired.Spec.Agent // Server-side apply (idempotent re-create). if err := r.Client.Patch(ctx, desired, client.Apply, client.ForceOwnership, client.FieldOwner("agentsession-bundle"), diff --git a/pkg/controllers/agentsession/demand.go b/pkg/controllers/agentsession/demand.go index 4fca8022..b1a8089c 100644 --- a/pkg/controllers/agentsession/demand.go +++ b/pkg/controllers/agentsession/demand.go @@ -127,25 +127,14 @@ func DemandFor(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.Agen } // demandBundlePod renders the sandbox pod one tool bundle will run in. It -// mirrors the reconciler's bundle loop (identity and tier-resolved backend -// into BuildBundleSession) and the SpiceboxSession reconciler's bind (class +// takes the reconciler's own desiredBundleSession (identity and tier-resolved +// backend) and mirrors the SpiceboxSession reconciler's bind (class // snapshot, image default, toolchain resolution), then calls podspec.Build — // the builder both registered sandbox backends render through. func demandBundlePod(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.AgentSession, ac *spiceboxv1alpha1.AgentClass, b spiceboxv1alpha1.ToolBundle) (*corev1.Pod, error) { - identity := b.AgentIdentity - if identity == "" { - identity = sess.Spec.AgentIdentity - } - if identity == "" { - identity = ac.Spec.AgentIdentity - } - var resolvedSandbox *spiceboxv1alpha1.SandboxBackend - if backend, ok := sess.Status.EffectiveSettings.Sandbox[b.Name]; ok { - resolvedSandbox = &backend - } // No claim name: whether the workspace is shared depends on the operator's // configured StorageClass, and only changes the pod's volumes. - bs := BuildBundleSession(sess, b, identity, "", sess.Status.ResolvedSkillBundles, resolvedSandbox, ac.Spec.Config) + bs := desiredBundleSession(sess, ac, b, "") var cls spiceboxv1alpha1.SpiceboxClass if err := c.Get(ctx, types.NamespacedName{Name: b.Class}, &cls); err != nil { From 8acf65a14042cb179bd6127e86d3bfc2c748ca32 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 12:39:16 -0500 Subject: [PATCH 06/33] Gate session pods on admission and release it on reclaim With Reconciler.AdmissionEnabled, a booting session that is not yet admitted creates none of its pods: it waits in line under CapacityHold=True/WaitingForAdmission and fails with HoldExpired once it has waited past the capacity-hold timeout, measured from WaitingSince. Settings resolution and the pre-start gates still run so the admitter can size the session's demand. Reclaiming a session's pods records Admitted=False/Reclaimed: in the SleptAt write on idle sleep, and after the terminal reap. The zero value keeps today's ungated behaviour. --- .../agentsession/admission_gate.go | 106 +++++++ .../agentsession/admission_gate_test.go | 285 ++++++++++++++++++ pkg/controllers/agentsession/capacity_hold.go | 22 ++ pkg/controllers/agentsession/controller.go | 39 +++ pkg/controllers/agentsession/export_test.go | 7 + pkg/controllers/agentsession/sleep.go | 5 + 6 files changed, 464 insertions(+) create mode 100644 pkg/controllers/agentsession/admission_gate.go create mode 100644 pkg/controllers/agentsession/admission_gate_test.go diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go new file mode 100644 index 00000000..f3830740 --- /dev/null +++ b/pkg/controllers/agentsession/admission_gate.go @@ -0,0 +1,106 @@ +// pkg/controllers/agentsession/admission_gate.go +// +// The reconciler's half of whole-set admission. The admitter decides WHO may +// start (Admitted=True); this file makes the reconciler honour that decision — +// a booting session creates none of its pods until admitted — and hands +// admission back (Admitted=False/Reclaimed) at the moment its pods are +// reclaimed, so the admitter never counts capacity a session no longer holds. +package agentsession + +import ( + "context" + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/conditions" +) + +// awaitingAdmission reports whether this pass must create none of the +// session's pods because it is booting and not yet admitted. +// +// Only a BOOTING session (Pending, or not yet phased) waits. Every pod a +// session owns is created on a booting pass — a parked phase never provisions +// (sandboxProvisioningDesired), and a Running or AwaitingDecision session +// reached that phase through a Pending pass that was already admitted. So +// gating boot alone keeps the invariant "no pod without Admitted=True", while +// a session that was already running when admission was switched on (no +// Admitted condition at all) keeps reconciling its live pods instead of +// freezing mid-turn. +// +// A warm wake — Idle back to Pending while still Admitted=True — is not +// gated: the session never gave up its place, so it provisions at once. +func (r *Reconciler) awaitingAdmission(sess *spiceboxv1alpha1.AgentSession) bool { + if !r.AdmissionEnabled || admission.IsAdmitted(sess) { + return false + } + return sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending +} + +// waitForAdmission is the whole reconcile tail of a booting session that is +// not admitted: give up once it has waited past the capacity-hold timeout, +// otherwise surface the wait and requeue. It creates nothing. +// +// The wait is measured from admission.WaitingSince — the later of joining the +// line and the last release — so a session released back to the line (a +// placement misfit) gets a full window, not the remainder of its first one. +// +// A negative --capacity-hold-timeout (hold disabled) means no give-up time: +// there is no pod whose ordinary readiness deadline could apply instead, so +// the session waits for admission with no CapacityHold notice. +func (r *Reconciler) waitForAdmission(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (ctrl.Result, error) { + timeout := r.effectiveCapacityHoldTimeout() + since := admission.WaitingSince(sess) + until := since.Add(timeout) + if timeout > 0 && r.now().After(until) { + log.FromContext(ctx).Info("session waited past the capacity-hold timeout without admission; failing", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, + "waitingSince", since.UTC().Format(time.RFC3339), "timeout", timeout.String()) + r.expireCapacityHold(sess, timeout) + return r.markBootFailed(ctx, sess, spiceboxv1alpha1.ReasonAgentSessionBundleFail, + admissionWaitExpiredMessage(timeout)) + } + requeue := capacityHoldRequeue + if timeout > 0 { + r.syncAdmissionHold(sess, until) + // Wake for the give-up time itself rather than up to a requeue late. + if remaining := until.Sub(r.now()); remaining > 0 && remaining < requeue { + requeue = remaining + time.Second + } + } + return ctrl.Result{RequeueAfter: requeue}, r.applyStatus(ctx, sess) +} + +// admissionWaitExpiredMessage is the Failed condition message when a session +// waited unadmitted past the capacity-hold timeout. No pod was ever created, +// so unlike capacityHoldExpiredMessage there is no scheduler detail to quote. +func admissionWaitExpiredMessage(waited time.Duration) string { + return fmt.Sprintf("session waited %s in line for cluster capacity and none freed up; none of its pods were created", waited) +} + +// releaseAdmissionOnReclaim records, on sess, that the session's pods are being +// reclaimed: Admitted True→False/Reclaimed. The caller persists it in the same +// status write that records the reclaim. A session that is not Admitted=True +// is left untouched — admission off, or never admitted — so no condition +// appears on sessions that never had one. Reports whether it changed sess. +func (r *Reconciler) releaseAdmissionOnReclaim(sess *spiceboxv1alpha1.AgentSession) bool { + if !admission.IsAdmitted(sess) { + return false + } + // r.now() for the transition time: admission.JoinedAt and WaitingSince + // compare it against status.lastWakeAt, and the admitter orders the line + // by them. + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionReclaimed, + Message: "the session's pods were reclaimed; a later wake rejoins the line", + LastTransitionTime: metav1.NewTime(r.now()), + }) + return true +} diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go new file mode 100644 index 00000000..762e707c --- /dev/null +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -0,0 +1,285 @@ +// pkg/controllers/agentsession/admission_gate_test.go +// +// The admission gate over the reconciler: with AdmissionEnabled, a booting +// session creates none of its pods until the admitter sets Admitted=True, waits +// in line under CapacityHold=True/WaitingForAdmission, and fails once it has +// waited unadmitted past the capacity-hold timeout. Reclaiming a session's pods +// (sleep or terminal reap) releases its admission. +package agentsession_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" + "github.com/authzed/openagentprimitives/pkg/memory" +) + +// withAdmittedCondition stamps an Admitted condition onto sess. +func withAdmittedCondition(sess *spiceboxv1alpha1.AgentSession, status metav1.ConditionStatus, reason string, at time.Time) { + sess.Status.Conditions = append(sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: status, + Reason: reason, LastTransitionTime: metav1.NewTime(at), + }) +} + +// exists reports whether obj named name is present in the fake client. +func exists(t *testing.T, c client.Client, name string, obj client.Object) bool { + t.Helper() + err := c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: name}, obj) + if apierrors.IsNotFound(err) { + return false + } + require.NoError(t, err, "get %s", name) + return true +} + +func TestAdmissionGate(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + + cases := []struct { + name string + enabled bool // Reconciler.AdmissionEnabled + bundles bool // class has a tool bundle (else: the runner is the first pod) + createdAt time.Duration // session creation, relative to now + // prep stamps admission state onto the session before it is loaded. + prep func(sess *spiceboxv1alpha1.AgentSession) + + wantPods bool // the bundle SpiceboxSession (bundles) or runner pod (no bundles) exists + wantFailed bool + wantHoldStatus metav1.ConditionStatus // "" = CapacityHold absent + wantHoldReason string + wantMsg []string // substrings of the CapacityHold (held) or Failed message + }{ + { + name: "Pending, not admitted: no SpiceboxSession; CapacityHold=True/WaitingForAdmission after grace", + enabled: true, bundles: true, createdAt: -2 * time.Minute, + wantHoldStatus: metav1.ConditionTrue, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + wantMsg: []string{"out of room", "gives up at 17:58 UTC"}, + }, + { + name: "Pending, not admitted, no bundles: no runner pod", + enabled: true, createdAt: -2 * time.Minute, + wantHoldStatus: metav1.ConditionTrue, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + }, + { + name: "Pending, not admitted, inside the grace: no pods and no hold notice yet", + enabled: true, bundles: true, createdAt: -10 * time.Second, + }, + { + name: "Pending, admitted: bundle SpiceboxSession created as today, no hold", + enabled: true, bundles: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-time.Minute)) + }, + wantPods: true, + }, + { + name: "Pending, admitted, no bundles: runner pod created as today", + enabled: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-time.Minute)) + }, + wantPods: true, + }, + { + name: "warm wake (Pending again with Admitted still True): provisions with no hold", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-2*time.Hour)) + wake := metav1.NewTime(nowT.Add(-time.Minute)) + s.Status.LastWakeAt = &wake + }, + wantPods: true, + }, + { + name: "admitted after a held wait: the WaitingForAdmission hold is released, pods created", + enabled: true, bundles: true, createdAt: -20 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Conditions = append(s.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionCapacityHold, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + LastTransitionTime: metav1.NewTime(nowT.Add(-19 * time.Minute)), + }) + withAdmittedCondition(s, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-time.Second)) + }, + wantPods: true, + wantHoldStatus: metav1.ConditionFalse, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldReleased, + }, + { + name: "waiting past the timeout from join time: Failed, CapacityHold=False/HoldExpired", + enabled: true, bundles: true, createdAt: -61 * time.Minute, + wantFailed: true, + wantHoldStatus: metav1.ConditionFalse, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldExpired, + wantMsg: []string{"waited 1h0m0s", "cluster capacity"}, + }, + { + name: "released long after creation: the wait is measured from the release, not failed", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionMisfit, nowT.Add(-5*time.Minute)) + }, + wantHoldStatus: metav1.ConditionTrue, + wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + wantMsg: []string{"gives up at 17:55 UTC"}, + }, + { + name: "AdmissionEnabled=false: bundle SpiceboxSession created without Admitted", + bundles: true, createdAt: -2 * time.Minute, + wantPods: true, + }, + { + name: "AdmissionEnabled=false, no bundles: runner pod created without Admitted", + createdAt: -2 * time.Minute, + wantPods: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + var ac *spiceboxv1alpha1.AgentClass + if tc.bundles { + ac = classWithBundle("ac1", codeBundle) + } else { + ac = classWithBundle("ac1") + } + sess := sessionCreatedAt("s1", "ac1", nowT.Add(tc.createdAt)) + if tc.prep != nil { + tc.prep(sess) + } + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = tc.enabled + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + if tc.bundles { + assert.Equal(t, tc.wantPods, + exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "bundle SpiceboxSession present") + } else { + assert.Equal(t, tc.wantPods, + exists(t, c, agentsession.RunnerPodName(&got), &corev1.Pod{}), + "runner pod present") + } + + hold := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionCapacityHold) + if tc.wantHoldStatus == "" { + assert.Nil(t, hold, "no CapacityHold condition") + } else if assert.NotNil(t, hold, "CapacityHold condition") { + assert.Equal(t, tc.wantHoldStatus, hold.Status) + assert.Equal(t, tc.wantHoldReason, hold.Reason) + } + + if !tc.wantFailed { + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") + if hold != nil && hold.Status == metav1.ConditionTrue { + for _, s := range tc.wantMsg { + assert.Contains(t, hold.Message, s) + } + } + return + } + require.Equal(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase) + assert.Equal(t, spiceboxv1alpha1.ReasonAgentSessionBundleFail, got.Status.FailureReason) + fc := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionFailed) + require.NotNil(t, fc, "Failed condition set") + for _, s := range tc.wantMsg { + assert.Contains(t, fc.Message, s) + } + }) + } +} + +// A terminal reap releases admission in the status write that accompanies it, +// and never invents an Admitted condition on a session that never had one. +func TestAdmissionGate_TerminalReapReleases(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + cases := []struct { + name string + admitted bool + wantAdmitted *metav1.ConditionStatus // nil = condition absent + }{ + {name: "admitted Succeeded session reaped: Admitted=False/Reclaimed", admitted: true, wantAdmitted: ptrTo(metav1.ConditionFalse)}, + {name: "never-admitted Succeeded session reaped: no Admitted condition written"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-time.Hour)) + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + finished := metav1.NewTime(nowT.Add(-time.Minute)) + sess.Status.FinishedAt = &finished + if tc.admitted { + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-50*time.Minute)) + } + r, c := fakeReconciler(t, clock, sess) + r.AdmissionEnabled = true + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if tc.wantAdmitted == nil { + assert.Nil(t, adm, "no Admitted condition") + return + } + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, *tc.wantAdmitted, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") + }) + } +} + +// The sleep reap stamps SleptAt and releases admission in the same status +// write (one WriteOwned): a wake clears SleptAt, so a release written +// separately could be lost between the two, and the session would sit +// admitted with no pods. +func TestAdmissionGate_SleepReleasesInOneWrite(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + clock := func() time.Time { return nowT } + + idle := metav1.NewTime(nowT.Add(-time.Hour)) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-2*time.Hour)) + sess.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + sess.Status.LastIdleAt = &idle + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, nowT.Add(-90*time.Minute)) + ac := classWithBundle("ac1") + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = true + r.DefaultSessionSleepAfter = 10 * time.Minute + + live := getSession(t, c, "s1") + slept, _, err := r.ReconcileSleepForTest(ctx, &live, ac) + require.NoError(t, err) + require.True(t, slept, "past-grace Idle session sleeps") + + got := getSession(t, c, "s1") + assert.NotNil(t, got.Status.SleptAt, "SleptAt stamped") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") +} + +func ptrTo[T any](v T) *T { return &v } diff --git a/pkg/controllers/agentsession/capacity_hold.go b/pkg/controllers/agentsession/capacity_hold.go index f85d04ac..da5ecc7b 100644 --- a/pkg/controllers/agentsession/capacity_hold.go +++ b/pkg/controllers/agentsession/capacity_hold.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/conditions" ) @@ -143,6 +144,27 @@ func (r *Reconciler) syncCapacityHold(sess *spiceboxv1alpha1.AgentSession, held r.releaseCapacityHold(sess) } +// syncAdmissionHold records that a booting session is waiting in line to be +// admitted: CapacityHold=True/WaitingForAdmission, whose message gives up at +// until (WaitingSince + the hold timeout). Like syncCapacityHold it waits out +// unschedulableGrace first, measured from admission.WaitingSince, so a session +// admitted a few seconds after joining never flashes a "queued" notice. The +// message is a pure function of until, so the condition is written once per +// wait rather than churned by every requeue. Released by releaseCapacityHoldFor +// once the session is admitted. +func (r *Reconciler) syncAdmissionHold(sess *spiceboxv1alpha1.AgentSession, until time.Time) { + if r.now().Sub(admission.WaitingSince(sess)) < unschedulableGrace { + return + } + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionCapacityHold, + Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, + Message: capacityHoldMessage(until), + LastTransitionTime: metav1.NewTime(r.now()), + }) +} + // releaseCapacityHoldFor releases an active hold only when it is the given // stage's, and reports whether it did. The bundle stage calls it once every bundle is Ready: from there on // nothing the bundles wait on is held, and the provisioning stages between the diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index 9c2187de..e9f70b75 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -651,6 +651,14 @@ type Reconciler struct { // --capacity-hold-timeout. See capacity_hold.go. CapacityHoldTimeout time.Duration + // AdmissionEnabled gates a booting session's pods on admission: while true, + // a Pending session creates none of its pods (bundle SpiceboxSessions, + // separate-pod sidecars, detectors, runner) until the admitter sets + // Admitted=True, and it gives up once it has waited unadmitted past + // CapacityHoldTimeout. The zero value (false) is ungated — today's + // behaviour. See admission_gate.go. + AdmissionEnabled bool + // SpiceDBDeleter, when non-nil, wipes all agentsession relationships in // SpiceDB during finalization, before the finalizer is removed. Nil skips // SpiceDB cleanup, logged rather than silent. @@ -1456,6 +1464,18 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{RequeueAfter: remaining}, nil case reapActionReap: res, err := r.reapSessionPods(ctx, &sess) + // The reaped pods' capacity is free, so the session's admission + // goes back to its line. Written after the reap succeeds, never + // before: released while its pods still stood, the admitter would + // count their capacity twice. A session not Admitted=True is + // untouched, so this writes nothing on later passes. + if err == nil && r.releaseAdmissionOnReclaim(&sess) { + if serr := r.applyStatus(ctx, &sess); serr != nil { + log.FromContext(ctx).Info("reap: recording admission release failed; will retry", + "session", sess.Namespace+"/"+sess.Name, "err", serr.Error()) + return ctrl.Result{}, fmt.Errorf("record admission release after reap: %w", serr) + } + } if err == nil && reclaimAfter > 0 && (res.RequeueAfter == 0 || reclaimAfter < res.RequeueAfter) { res.RequeueAfter = reclaimAfter } @@ -1730,6 +1750,25 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu "session", sess.Namespace+"/"+sess.Name, "err", err.Error()) } + // 2-pre. Admission. With admission on, a booting session that is not yet + // admitted creates NONE of its pods — bundles, separate-pod sidecars, + // detectors and runner all sit below this line — and waits in line here + // until the admitter sets Admitted=True. Everything above (settings, the + // start/identity/credential gates, owner resolution) has already run: the + // admitter needs status.effectiveSettings to size the session's demand. + if r.awaitingAdmission(&sess) { + return r.waitForAdmission(ctx, &sess) + } + // Admitted (or admission off): a wait-in-line hold ends here, persisted now + // for the same reason the bundle stage persists its own release — so a + // bundle-capacity hold set later this pass is a NEW episode with its own + // give-up time rather than diffed away against the stored True. + if r.releaseCapacityHoldFor(&sess, spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission) { + if err := r.applyStatus(ctx, &sess); err != nil { + return ctrl.Result{}, err + } + } + // 2. Bundle SpiceboxSessions, or the vacuous NoBundles path. Only an // active/pending turn provisions; every parked phase (Idle, AwaitingRetry, // AwaitingDecision, AwaitingCredentials) and every terminal phase skips this diff --git a/pkg/controllers/agentsession/export_test.go b/pkg/controllers/agentsession/export_test.go index fb4b4ef4..7fc2ddbc 100644 --- a/pkg/controllers/agentsession/export_test.go +++ b/pkg/controllers/agentsession/export_test.go @@ -5,6 +5,7 @@ package agentsession import ( "context" + "time" corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -68,3 +69,9 @@ func SidecarPodFailureForTest(pod *corev1.Pod) *spiceboxv1alpha1.SidecarPodFailu func (r *Reconciler) EnsureDetectorPodForTest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, spec cosidecar.Spec, pod *corev1.Pod) error { return r.ensureDetectorPod(ctx, sess, spec, pod) } + +// ReconcileSleepForTest exposes the unexported reconcileSleep so +// agentsession_test can drive the idle-sleep reap directly. Test-only. +func (r *Reconciler) ReconcileSleepForTest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, class *spiceboxv1alpha1.AgentClass) (bool, time.Duration, error) { + return r.reconcileSleep(ctx, sess, class) +} diff --git a/pkg/controllers/agentsession/sleep.go b/pkg/controllers/agentsession/sleep.go index 8288ad84..440af9e5 100644 --- a/pkg/controllers/agentsession/sleep.go +++ b/pkg/controllers/agentsession/sleep.go @@ -90,6 +90,11 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. patched := sess.DeepCopy() now := metav1.NewTime(r.now()) patched.Status.SleptAt = &now + // The reap below gives the session's place back to its line. Recorded in + // THIS write, beside SleptAt: a wake clears SleptAt, so a release written + // separately could be lost in between, leaving the session admitted with + // no pods and its capacity counted twice. + r.releaseAdmissionOnReclaim(patched) if err := agentstatus.WriteOwned(ctx, r.Client, patched, sess, spiceboxv1alpha1.OwnerOperator); err != nil { return false, 0, fmt.Errorf("stamp SleptAt: %w", err) } From b17fe18aa5284db7ffa00c7f6dd2eb9c6eb84576 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 12:53:26 -0500 Subject: [PATCH 07/33] Admit sessions from a single leader-only pass The admitter rebuilds every admission line from AgentSession status on each pass, prices each waiting session's whole pod set, and admits in fair-share order against node headroom. It writes Admitted=True and the waiting sessions' line positions (only when they change) to status, so a restarted operator resumes the same line. The operator gains --session-admission (default true) and --admission-min-warm, and passes the admission switch to the AgentSession reconciler. CapacityModel now reports cluster totals for the dominant-share calculation. --- ...tprimitives.authzed.com_agentsessions.yaml | 27 ++ internal/cmd/operator/main.go | 31 ++ .../cmd/operator/main_controllers_test.go | 1 + pkg/apis/v1alpha1/agentsession_ownership.go | 8 +- pkg/apis/v1alpha1/agentsession_types.go | 18 + pkg/apis/v1alpha1/zz_generated.deepcopy.go | 21 ++ .../admission/admitter/controller.go | 343 ++++++++++++++++++ .../admission/admitter/controller_test.go | 284 +++++++++++++++ .../admitter/restart_integration_test.go | 171 +++++++++ pkg/controllers/admission/capacity.go | 16 +- pkg/controllers/admission/capacity_test.go | 34 ++ pkg/controllers/admission/decide_test.go | 5 +- pkg/platform/manifests/install.yaml | 27 ++ 13 files changed, 980 insertions(+), 6 deletions(-) create mode 100644 pkg/controllers/admission/admitter/controller.go create mode 100644 pkg/controllers/admission/admitter/controller_test.go create mode 100644 pkg/controllers/admission/admitter/restart_integration_test.go diff --git a/config/crds/agentprimitives.authzed.com_agentsessions.yaml b/config/crds/agentprimitives.authzed.com_agentsessions.yaml index 4348063b..45b16e32 100644 --- a/config/crds/agentprimitives.authzed.com_agentsessions.yaml +++ b/config/crds/agentprimitives.authzed.com_agentsessions.yaml @@ -432,6 +432,33 @@ spec: - artifactID type: object type: array + admission: + description: |- + Admission is where this session stands in its AgentClass's admission + line while it waits for its pod set to be admitted. Written by the + operator's admitter only when the position changes, and cleared when the + session is admitted. Nil when the session is not waiting. + properties: + joinedAt: + description: |- + JoinedAt is when the session took its place in line: its creation, or + the wake that re-queued it. + format: date-time + type: string + lineLength: + description: LineLength is how many sessions are waiting in the + line. + format: int32 + type: integer + position: + description: Position is the session's 1-based place in its line. + format: int32 + type: integer + required: + - joinedAt + - lineLength + - position + type: object agentWakeCredit: description: |- AgentWakeCredit is how many further AGENT-DRIVEN wakes this session will diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index f34e9444..2b8452bf 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -102,6 +102,8 @@ import ( _ "github.com/authzed/openagentprimitives/pkg/channels/channelkinds/onepassword" // register onepassword relsync.Kind for the relationshipsource controller (relsync.Get); also reachable transitively via the relsource/imports blank import above, but this is the explicit wiring site for THIS aspect, not an accident of that one _ "github.com/authzed/openagentprimitives/pkg/channels/channelkinds/slack" // register slack kind for channel-controller validation "github.com/authzed/openagentprimitives/pkg/cli/clikit" + sessionadmission "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/admission/admitter" agentclassctrl "github.com/authzed/openagentprimitives/pkg/controllers/agentclass" "github.com/authzed/openagentprimitives/pkg/controllers/agentidentity" agentsessionctrl "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" @@ -757,6 +759,13 @@ type config struct { idleStorageReclaimAfter time.Duration failedSandboxReapGrace time.Duration capacityHoldTimeout time.Duration + // sessionAdmission gates every booting AgentSession's pods on the + // admitter's whole-set, fair-share decision; false admits at once. + sessionAdmission bool + // admissionMinWarm is parsed and stored only; its consumer (how long an + // idle session keeps its admission before it may be reclaimed) is not + // wired yet. + admissionMinWarm time.Duration oauthRefreshThreshold time.Duration workspaceStorageClass string workspaceSize string @@ -882,6 +891,10 @@ func newCommand() *cobra.Command { "How long a Failed session's sandbox pods (bundle SpiceboxSessions + runner pod) are kept for debugging before teardown. The AgentSession itself is kept either way. Set to 0 (or negative) to disable reaping entirely (keep everything, for debugging).") fs.DurationVar(&cfg.capacityHoldTimeout, "capacity-hold-timeout", time.Hour, "How long a booting AgentSession whose sandbox/detector pods are Pending on a cluster capacity shortfall (Insufficient cpu/memory, too many pods) is HELD — queued with a notice in its channel, resuming by itself once the scheduler places the pod — instead of failing at the ordinary readiness deadline. Measured from provisioning start. Pods that can never fit, or are blocked by a taint or volume affinity, still fail at the ordinary deadline. Set to a negative value to disable holding.") + fs.BoolVar(&cfg.sessionAdmission, "session-admission", true, + "Admit each booting AgentSession's whole pod set against node capacity before any of its pods is created, in join order with fair share between AgentClasses; sessions that do not fit wait in line. Set to false to admit every session at once (pods then queue in the scheduler as before).") + fs.DurationVar(&cfg.admissionMinWarm, "admission-min-warm", 5*time.Minute, + "Minimum time an idle AgentSession keeps its admission (and warm pods) before it may be reclaimed for a session waiting in line.") fs.DurationVar(&cfg.oauthRefreshThreshold, "oauth-refresh-threshold", 5*time.Minute, "Default refresh threshold for type=oauth AgentIdentity credentials. AgentIdentities with spec.refreshThreshold override this per-identity.") fs.StringVar(&cfg.workspaceStorageClass, "workspace-storage-class", "", @@ -2668,6 +2681,7 @@ func run(cfg *config) { DefaultSessionSleepAfter: cfg.defaultSessionSleepAfter, FailedSandboxReapGrace: cfg.failedSandboxReapGrace, CapacityHoldTimeout: cfg.capacityHoldTimeout, + AdmissionEnabled: cfg.sessionAdmission, DefaultSessionStorageRetention: cfg.sessionStorageRetention, SessionGCAfter: cfg.sessionGCAfter, NodePinnedStorageReclaimGrace: cfg.nodePinnedStorageReclaimGrace, @@ -2748,6 +2762,23 @@ func run(cfg *config) { } log.Info("registered controller", "name", "AgentSession", "operatorURL", operatorURL) + // The admitter decides which booting sessions the reconciler above may + // create pods for. Leader-only (the controller-runtime default), so one + // operator admits against the free space at a time. Capacity is declared + // as the interface and assigned a concrete value, never a typed-nil + // pointer (CLAUDE.md, nil interfaces). + var admissionCapacity sessionadmission.CapacitySource = sessionadmission.KubeCapacitySource{Reader: mgr.GetClient()} + if err := (&admitter.Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Capacity: admissionCapacity, + Enabled: cfg.sessionAdmission, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "unable to register admission controller") + os.Exit(1) + } + log.Info("registered controller", "name", "Admission", "enabled", cfg.sessionAdmission) + // Workshop provisions/tears down a builder session's isolated namespace, // exact RBAC, SpiceDB tuple and bearer token (spec §1.2/§1.3). spiceDBClient // is a real, non-nil *spicedb.Client here (construction failure exits diff --git a/internal/cmd/operator/main_controllers_test.go b/internal/cmd/operator/main_controllers_test.go index 88fb1b31..dfbf84c5 100644 --- a/internal/cmd/operator/main_controllers_test.go +++ b/internal/cmd/operator/main_controllers_test.go @@ -61,6 +61,7 @@ func TestOperatorRegistersExpectedControllers(t *testing.T) { "AgentClass": true, "Channel": true, "AgentSession": true, + "Admission": true, "GuardianAgentSessionGrants": true, "MonitoringWatchers": true, "UserIdentity": true, diff --git a/pkg/apis/v1alpha1/agentsession_ownership.go b/pkg/apis/v1alpha1/agentsession_ownership.go index 41e391d8..8cd9f3fd 100644 --- a/pkg/apis/v1alpha1/agentsession_ownership.go +++ b/pkg/apis/v1alpha1/agentsession_ownership.go @@ -48,9 +48,11 @@ var operatorOwnedStatusFields = map[string]bool{ // Stamped by the operator when it acts on a UI-serve request, exactly as // lastWakeAt is for a conversation wake. Separate fields because the two // requests must not consume each other — see AnnotationUIServeRequestedAt. - "lastUIServeAt": true, - "lastIdleAt": true, - "sleptAt": true, + "lastUIServeAt": true, + "lastIdleAt": true, + "sleptAt": true, + // admission is the admitter's view of a waiting session's place in line. + "admission": true, "awaitingUserInputSince": true, // parentExchange is written by the RUNNER of a delegated child (the // ask_parent meta tool, and the loop's two answer-arrived clears), like diff --git a/pkg/apis/v1alpha1/agentsession_types.go b/pkg/apis/v1alpha1/agentsession_types.go index bd5b12cc..a739112b 100644 --- a/pkg/apis/v1alpha1/agentsession_types.go +++ b/pkg/apis/v1alpha1/agentsession_types.go @@ -399,6 +399,13 @@ type AgentSessionStatus struct { // +optional EstimatedCost *EstimatedSessionCost `json:"estimatedCost,omitempty"` + // Admission is where this session stands in its AgentClass's admission + // line while it waits for its pod set to be admitted. Written by the + // operator's admitter only when the position changes, and cleared when the + // session is admitted. Nil when the session is not waiting. + // +optional + Admission *AdmissionStatus `json:"admission,omitempty"` + // AwaitingUserInputSince is set when the agent yields via // await_user_message and cleared on resume or terminal. It is the durable // floor under the at-most-once turn_activity yield event: a non-nil value @@ -1598,3 +1605,14 @@ type GoalExecutionReference struct { GoalRevision int64 `json:"goalRevision"` ConsentDigest string `json:"consentDigest"` } + +// AdmissionStatus is a waiting session's place in its admission line. +type AdmissionStatus struct { + // Position is the session's 1-based place in its line. + Position int32 `json:"position"` + // LineLength is how many sessions are waiting in the line. + LineLength int32 `json:"lineLength"` + // JoinedAt is when the session took its place in line: its creation, or + // the wake that re-queued it. + JoinedAt metav1.Time `json:"joinedAt"` +} diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 6d61f6b1..d71f22eb 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdmissionStatus) DeepCopyInto(out *AdmissionStatus) { + *out = *in + in.JoinedAt.DeepCopyInto(&out.JoinedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdmissionStatus. +func (in *AdmissionStatus) DeepCopy() *AdmissionStatus { + if in == nil { + return nil + } + out := new(AdmissionStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentClass) DeepCopyInto(out *AgentClass) { *out = *in @@ -995,6 +1011,11 @@ func (in *AgentSessionStatus) DeepCopyInto(out *AgentSessionStatus) { *out = new(EstimatedSessionCost) (*in).DeepCopyInto(*out) } + if in.Admission != nil { + in, out := &in.Admission, &out.Admission + *out = new(AdmissionStatus) + (*in).DeepCopyInto(*out) + } if in.AwaitingUserInputSince != nil { in, out := &in.AwaitingUserInputSince, &out.AwaitingUserInputSince *out = (*in).DeepCopy() diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go new file mode 100644 index 00000000..a23d2242 --- /dev/null +++ b/pkg/controllers/admission/admitter/controller.go @@ -0,0 +1,343 @@ +// Package admitter is the one controller that decides which booting +// AgentSessions may create their pods. Each pass rebuilds every admission line +// from the AgentSession objects themselves, prices each session's whole pod +// set, and hands the result to admission.Decide; the AgentSession reconciler +// then creates pods only for a session it marked Admitted=True. +// +// The pass keeps no queue state in memory: the line is derived from +// AgentSession status on every pass, so an operator restart or leader change +// resumes exactly where the last leader stopped. It runs on the leader only +// (controller-runtime's default for a controller; nothing here overrides it), +// so two operators never admit against the same free space. +package admitter + +import ( + "context" + "errors" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" + "github.com/authzed/openagentprimitives/pkg/controllers/agentstatus" +) + +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions,verbs=get;list;watch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch + +// Key is the single request every event maps to: a pass always decides every +// line at once, so there is nothing per-object to reconcile. +var Key = types.NamespacedName{Namespace: "admission", Name: "cluster"} + +// resyncInterval re-runs the pass with no event, so capacity freed by pods the +// admitter does not watch (other workloads finishing, nodes joining) is seen. +const resyncInterval = 15 * time.Second + +// Session pods carry one of two labels: the runner, detector and sidecar pods +// the AgentSession reconciler builds carry the session's name under +// sessionPodLabel; sandbox pods are built for a bundle SpiceboxSession +// (podspec.Build) and carry that SpiceboxSession's name under +// sandboxPodLabel. A pod event with either can change a holder's unplaced +// demand. +const ( + sessionPodLabel = "agentprimitives.authzed.com/agentsession" + sandboxPodLabel = "agentprimitives.authzed.com/session" +) + +// Reconciler runs the admission pass. +type Reconciler struct { + Client client.Client + // APIReader lists AgentSessions uncached. The pass must see its own last + // Admitted=True writes: a cache that lags them would show an admitted + // session as still waiting, free its capacity, and let the pass admit a + // second session into the same space. Nil falls back to Client. + APIReader client.Reader + // Capacity snapshots what the cluster can still place. Nil, or a snapshot + // error, means unknown capacity: every waiting session fits. + Capacity admission.CapacitySource + // Enabled false admits every waiting session at once, with no capacity + // check (--session-admission=false). + Enabled bool + // Now is the clock; nil means time.Now. + Now func() time.Time + // Demand prices a session's whole pod set. Nil means the default: get the + // session's AgentClass and call agentsession.DemandFor. A test seam, for + // a harness where no AgentSession reconciler has resolved settings. + Demand func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) +} + +func (r *Reconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +func (r *Reconciler) sessionReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +// lineOf is the admission line a session draws capacity from: its AgentClass, +// which lives in the session's namespace. +func lineOf(sess *spiceboxv1alpha1.AgentSession) string { + return sess.Namespace + "/" + sess.Spec.Class +} + +func isTerminal(sess *spiceboxv1alpha1.AgentSession) bool { + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseSucceeded || + sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseFailed +} + +// isWaiting matches the AgentSession reconciler's gate: a booting session +// ("" or Pending) that is not admitted. A session parked at a pre-start gate +// has another phase and is not in line yet. +func isWaiting(sess *spiceboxv1alpha1.AgentSession) bool { + return (sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending) && + !admission.IsAdmitted(sess) +} + +func (r *Reconciler) demand(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if r.Demand != nil { + return r.Demand(ctx, sess) + } + var ac spiceboxv1alpha1.AgentClass + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: sess.Namespace, Name: sess.Spec.Class}, &ac); err != nil { + return admission.Demand{}, fmt.Errorf("get AgentClass %s: %w", lineOf(sess), err) + } + return agentsession.DemandFor(ctx, r.Client, sess, &ac) +} + +// Reconcile runs one admission pass over every line. +func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + requeue := ctrl.Result{RequeueAfter: resyncInterval} + + var list spiceboxv1alpha1.AgentSessionList + if err := r.sessionReader().List(ctx, &list); err != nil { + return ctrl.Result{}, fmt.Errorf("admission pass: list AgentSessions: %w", err) + } + var waiting, holding []*spiceboxv1alpha1.AgentSession + for i := range list.Items { + sess := &list.Items[i] + switch { + case isTerminal(sess): + // Never a holder, even while still Admitted=True with pods: any + // pods it has placed already reduce node headroom in the snapshot. + case admission.IsAdmitted(sess): + holding = append(holding, sess) + case isWaiting(sess): + waiting = append(waiting, sess) + } + } + if len(waiting) == 0 { + return requeue, nil + } + now := r.now() + + if !r.Enabled { + var errs []error + for _, sess := range waiting { + if err := r.admit(ctx, sess, now, "Session admission is disabled; admitted at once."); err != nil { + errs = append(errs, err) + } + } + return requeue, errors.Join(errs...) + } + + in := admission.Input{} + byKey := make(map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, len(waiting)) + for _, sess := range waiting { + d, err := r.demand(ctx, sess) + if err != nil { + logger.Info("admission: cannot price session's pod set; it keeps waiting", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + continue + } + key := client.ObjectKeyFromObject(sess) + byKey[key] = sess + in.Waiting = append(in.Waiting, admission.Candidate{ + Key: key, + Line: lineOf(sess), + JoinedAt: admission.JoinedAt(sess), + Child: sess.Spec.Parent != nil, + Demand: d, + }) + } + + pods, podsKnown := r.listPods(ctx) + for _, sess := range holding { + d, err := r.demand(ctx, sess) + if err != nil { + // Still a holder: it counts toward its line's share and cap. Its + // placed pods are in the snapshot; only its unplaced ones go + // unreserved, which is the accepted cost of an unpriceable holder. + logger.Info("admission: cannot price an admitted session's pod set; counting it with no unplaced demand", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + } + in.Holders = append(in.Holders, admission.Holder{ + Key: client.ObjectKeyFromObject(sess), + Line: lineOf(sess), + Child: sess.Spec.Parent != nil, + Demand: d, + Unplaced: unplaced(sess.Namespace, d, pods, podsKnown), + }) + } + + in.Model = r.snapshot(ctx) + if !in.Model.Known() { + logger.Info("admission: cluster capacity unknown; admitting in line order", "waiting", len(in.Waiting)) + } + in.TotalCPUMilli, in.TotalMemBytes = in.Model.Totals() + + dec := admission.Decide(in) + + var errs []error + for _, key := range dec.Admit { + if err := r.admit(ctx, byKey[key], now, "The session's whole pod set fits the cluster."); err != nil { + errs = append(errs, err) + } + } + for key, pos := range dec.Positions { + if err := r.writePosition(ctx, byKey[key], pos); err != nil { + errs = append(errs, err) + } + } + return requeue, errors.Join(errs...) +} + +// snapshot returns the capacity model, or an unknown one when there is no +// source or the snapshot fails (logged): missing data must not block sessions. +func (r *Reconciler) snapshot(ctx context.Context) admission.CapacityModel { + if r.Capacity == nil { + return admission.NewNodeModel(nil, nil) + } + m, err := r.Capacity.Snapshot(ctx) + if err != nil { + log.FromContext(ctx).Info("admission: capacity snapshot failed; treating capacity as unknown", "err", err.Error()) + return admission.NewNodeModel(nil, nil) + } + return m +} + +// podIndex is the cluster's pods by namespace/name. +type podIndex map[types.NamespacedName]*corev1.Pod + +// listPods indexes every pod for the unplaced-demand check. A list error is +// logged and reported as not known, so every holder pod counts as unplaced. +func (r *Reconciler) listPods(ctx context.Context) (podIndex, bool) { + var pods corev1.PodList + if err := r.Client.List(ctx, &pods); err != nil { + log.FromContext(ctx).Info("admission: list pods failed; counting every admitted session's pods as unplaced", "err", err.Error()) + return nil, false + } + idx := make(podIndex, len(pods.Items)) + for i := range pods.Items { + idx[client.ObjectKeyFromObject(&pods.Items[i])] = &pods.Items[i] + } + return idx, true +} + +// unplaced is the part of a holder's demand the capacity snapshot does not +// already account for, matched to pods by name in the session's namespace +// (DemandFor names each PodDemand after the pod its builder creates): +// +// - a pod bound to a node (spec.nodeName set) is in the snapshot: placed; +// - a pod that exists but is not yet bound, or does not exist yet, is not: +// unplaced, so Decide reserves it before admitting anyone else. +// +// When the pod list failed (known=false), every pod is unplaced: reserving too +// much under-admits for a pass, never over-admits. +func unplaced(namespace string, d admission.Demand, pods podIndex, known bool) admission.Demand { + var out admission.Demand + for _, p := range d.Pods { + if known { + if pod, ok := pods[types.NamespacedName{Namespace: namespace, Name: p.Name}]; ok && pod.Spec.NodeName != "" { + continue + } + } + out.Pods = append(out.Pods, p) + } + return out +} + +// admit writes Admitted=True at now and clears the session's line position. +func (r *Reconciler) admit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, now time.Time, message string) error { + original := sess.DeepCopy() + meta.SetStatusCondition(&sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, + Message: message, + ObservedGeneration: sess.Generation, + LastTransitionTime: metav1.NewTime(now), + }) + sess.Status.Admission = nil + if err := agentstatus.WriteOwned(ctx, r.Client, sess, original, spiceboxv1alpha1.OwnerOperator); err != nil { + log.FromContext(ctx).Info("admission: writing Admitted=True failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + return fmt.Errorf("admit %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + +// writePosition records a waiting session's place in line, only when its +// position, line length or join time changed. +func (r *Reconciler) writePosition(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, pos admission.Position) error { + want := &spiceboxv1alpha1.AdmissionStatus{ + Position: pos.Index, + LineLength: pos.LineLength, + // The apiserver stores times to the second; compare at that precision + // so an unchanged join time never reads as a change. + JoinedAt: metav1.NewTime(admission.JoinedAt(sess)).Rfc3339Copy(), + } + if cur := sess.Status.Admission; cur != nil && cur.Position == want.Position && + cur.LineLength == want.LineLength && cur.JoinedAt.Time.Equal(want.JoinedAt.Time) { + return nil + } + original := sess.DeepCopy() + sess.Status.Admission = want + if err := agentstatus.WriteOwned(ctx, r.Client, sess, original, spiceboxv1alpha1.OwnerOperator); err != nil { + log.FromContext(ctx).Info("admission: writing line position failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + return fmt.Errorf("write admission position for %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + +// SetupWithManager maps every AgentSession event, and every event on a pod +// carrying a session label, to the single Key. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + toKey := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: Key}} + }) + sessionPod := predicate.NewPredicateFuncs(func(obj client.Object) bool { + labels := obj.GetLabels() + _, runner := labels[sessionPodLabel] + _, sandbox := labels[sandboxPodLabel] + return runner || sandbox + }) + return ctrl.NewControllerManagedBy(mgr). + Named("admission"). + Watches(&spiceboxv1alpha1.AgentSession{}, toKey). + Watches(&corev1.Pod{}, toKey, builder.WithPredicates(sessionPod)). + Complete(r) +} diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go new file mode 100644 index 00000000..499976cb --- /dev/null +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -0,0 +1,284 @@ +package admitter + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" +) + +const testClass = "reviewbot" + +var ( + baseTime = time.Date(2026, 11, 10, 9, 0, 0, 0, time.UTC) + passTime = baseTime.Add(time.Hour) +) + +// oneCPUDemand is the pod set every test session needs: one 1-CPU pod. +func oneCPUDemand(_ context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + return admission.Demand{Pods: []admission.PodDemand{{Name: sess.Name + "-runner", CPUMilli: 1000, MemBytes: 1 << 20}}}, nil +} + +// roomFor is a CapacitySource with one node that fits n oneCPUDemand pod sets. +type roomFor int + +func (n roomFor) Snapshot(context.Context) (admission.CapacityModel, error) { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, + Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(int64(n)*1000, resource.DecimalSI), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("110"), + }}, + } + return admission.NewNodeModel([]corev1.Node{node}, nil), nil +} + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + sch := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(sch), "core scheme") + require.NoError(t, spiceboxv1alpha1.AddToScheme(sch), "spicebox scheme") + return sch +} + +// waitingSession is a booting, never-admitted session created at +// baseTime+offset. +func waitingSession(name string, offset time.Duration) *spiceboxv1alpha1.AgentSession { + return &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "default", + CreationTimestamp: metav1.NewTime(baseTime.Add(offset)), + }, + Spec: spiceboxv1alpha1.AgentSessionSpec{Class: testClass}, + Status: spiceboxv1alpha1.AgentSessionStatus{Phase: spiceboxv1alpha1.AgentSessionPhasePending}, + } +} + +// newReconciler builds an admitter over a fake client holding objs, counting +// every status patch it sends. +func newReconciler(t *testing.T, enabled bool, capacity admission.CapacitySource, objs ...client.Object) (*Reconciler, client.Client, *atomic.Int32) { + t.Helper() + var statusPatches atomic.Int32 + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + statusPatches.Add(1) + return cl.SubResource(sub).Patch(ctx, obj, p, opts...) + }, + }). + Build() + r := &Reconciler{ + Client: c, + Capacity: capacity, + Enabled: enabled, + Now: func() time.Time { return passTime }, + Demand: oneCPUDemand, + } + return r, c, &statusPatches +} + +func runPass(t *testing.T, r *Reconciler) reconcile.Result { + t.Helper() + res, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: Key}) + require.NoError(t, err, "Reconcile") + return res +} + +func getSession(t *testing.T, c client.Client, name string) *spiceboxv1alpha1.AgentSession { + t.Helper() + var s spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: name}, &s), "Get %s", name) + return &s +} + +func assertAdmitted(t *testing.T, s *spiceboxv1alpha1.AgentSession) { + t.Helper() + cond := meta.FindStatusCondition(s.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if !assert.NotNil(t, cond, "%s: Admitted condition", s.Name) { + return + } + assert.Equal(t, metav1.ConditionTrue, cond.Status, "%s: Admitted status", s.Name) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmitted, cond.Reason, "%s: Admitted reason", s.Name) + assert.True(t, cond.LastTransitionTime.Time.Equal(passTime), "%s: Admitted transition at the pass's Now, got %v", s.Name, cond.LastTransitionTime) + assert.Nil(t, s.Status.Admission, "%s: an admitted session holds no line position", s.Name) +} + +func assertPosition(t *testing.T, s *spiceboxv1alpha1.AgentSession, index, length int32) { + t.Helper() + assert.False(t, admission.IsAdmitted(s), "%s: not admitted", s.Name) + if !assert.NotNil(t, s.Status.Admission, "%s: status.admission", s.Name) { + return + } + assert.Equal(t, index, s.Status.Admission.Position, "%s: position", s.Name) + assert.Equal(t, length, s.Status.Admission.LineLength, "%s: line length", s.Name) + assert.True(t, s.Status.Admission.JoinedAt.Time.Equal(s.CreationTimestamp.Time), "%s: joinedAt is creation, got %v", s.Name, s.Status.Admission.JoinedAt) +} + +func TestAdmitterPass(t *testing.T) { + t.Run("three waiting reviewbot sessions, room for one: oldest gets Admitted=True, others get Position 1 and 2 of 2", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + waitingSession("rb-c", 3*time.Minute), + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + ) + res := runPass(t, r) + assert.Equal(t, 15*time.Second, res.RequeueAfter, "periodic requeue") + + assertAdmitted(t, getSession(t, c, "rb-a")) + assertPosition(t, getSession(t, c, "rb-b"), 1, 2) + assertPosition(t, getSession(t, c, "rb-c"), 2, 2) + }) + + t.Run("Enabled=false: every waiting session admitted, with no room at all", func(t *testing.T) { + r, c, _ := newReconciler(t, false, roomFor(0), + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + waitingSession("rb-c", 3*time.Minute), + ) + runPass(t, r) + for _, name := range []string{"rb-a", "rb-b", "rb-c"} { + assertAdmitted(t, getSession(t, c, name)) + } + }) + + t.Run("a position write is skipped when unchanged: second pass sends no status patch", func(t *testing.T) { + r, c, patches := newReconciler(t, true, roomFor(1), + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + waitingSession("rb-c", 3*time.Minute), + ) + runPass(t, r) + require.Positive(t, patches.Load(), "first pass writes admission and positions") + patches.Store(0) + + runPass(t, r) + assert.Zero(t, patches.Load(), "nothing changed, so nothing is written") + assertPosition(t, getSession(t, c, "rb-b"), 1, 2) + }) + + t.Run("a holder still occupying the room: nobody is admitted, the line starts at 1", func(t *testing.T) { + holder := waitingSession("rb-holder", 0) + holder.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + holder.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + r, c, _ := newReconciler(t, true, roomFor(1), holder, + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + ) + runPass(t, r) + assertPosition(t, getSession(t, c, "rb-a"), 1, 2) + assertPosition(t, getSession(t, c, "rb-b"), 2, 2) + }) + + t.Run("a terminal session still Admitted=True: not a holder, so the head is admitted", func(t *testing.T) { + done := waitingSession("rb-done", 0) + done.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + done.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + r, c, _ := newReconciler(t, true, roomFor(1), done, waitingSession("rb-a", time.Minute)) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "rb-a")) + }) + + t.Run("a session parked at a pre-start gate: not waiting, never admitted or positioned", func(t *testing.T) { + parked := waitingSession("rb-parked", 0) + parked.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingDecision + r, c, _ := newReconciler(t, true, roomFor(5), parked) + runPass(t, r) + got := getSession(t, c, "rb-parked") + assert.False(t, admission.IsAdmitted(got), "parked session not admitted") + assert.Nil(t, got.Status.Admission, "parked session has no line position") + }) + + t.Run("demand fails for the oldest: it keeps waiting unpositioned, the next is admitted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + ) + r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Name == "rb-a" { + return admission.Demand{}, fmt.Errorf("settings not resolved") + } + return oneCPUDemand(ctx, sess) + } + runPass(t, r) + a := getSession(t, c, "rb-a") + assert.False(t, admission.IsAdmitted(a), "rb-a not admitted") + assert.Nil(t, a.Status.Admission, "rb-a not positioned") + assertAdmitted(t, getSession(t, c, "rb-b")) + }) + + t.Run("capacity snapshot fails: capacity unknown, every waiting session admitted in line order", func(t *testing.T) { + r, c, _ := newReconciler(t, true, failingSource{}, + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "rb-a")) + assertAdmitted(t, getSession(t, c, "rb-b")) + }) +} + +type failingSource struct{} + +func (failingSource) Snapshot(context.Context) (admission.CapacityModel, error) { + return nil, fmt.Errorf("nodes is forbidden") +} + +// TestUnplacedDemand pins how much of a holder's pod set Decide must still +// reserve: a pod bound to a node is already in the snapshot; a pending or +// missing one is not. +func TestUnplacedDemand(t *testing.T) { + d := admission.Demand{Pods: []admission.PodDemand{ + {Name: "bound", CPUMilli: 1}, {Name: "pending", CPUMilli: 2}, {Name: "missing", CPUMilli: 4}, + }} + pods := podIndex{ + {Namespace: "default", Name: "bound"}: &corev1.Pod{Spec: corev1.PodSpec{NodeName: "node-a"}}, + {Namespace: "default", Name: "pending"}: &corev1.Pod{}, + } + cases := []struct { + name string + pods podIndex + known bool + want []string + }{ + {name: "pods listed: pending and missing pods are unplaced", pods: pods, known: true, want: []string{"pending", "missing"}}, + {name: "pod list failed: every pod counted unplaced", known: false, want: []string{"bound", "pending", "missing"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := unplaced("default", d, tc.pods, tc.known) + var names []string + for _, p := range got.Pods { + names = append(names, p.Name) + } + assert.Equal(t, tc.want, names) + }) + } +} diff --git a/pkg/controllers/admission/admitter/restart_integration_test.go b/pkg/controllers/admission/admitter/restart_integration_test.go new file mode 100644 index 00000000..b8bac3b3 --- /dev/null +++ b/pkg/controllers/admission/admitter/restart_integration_test.go @@ -0,0 +1,171 @@ +//go:build integration + +package admitter + +import ( + "context" + "fmt" + "os" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/testenv" +) + +func TestMain(m *testing.M) { os.Exit(testenv.RunPackage(m)) } + +const restartNamespace = "admission-restart" + +// startAdmitter runs one admitter in its own manager, as a fresh operator +// process would, and returns a stop func that blocks until the manager exits. +func startAdmitter(t *testing.T, env *testenv.Env) (stop func()) { + t.Helper() + mgr, err := ctrl.NewManager(env.Cfg, ctrl.Options{ + Scheme: env.Scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + Controller: ctrlconfig.Controller{SkipNameValidation: ptr.To(true)}, + LeaderElection: false, + }) + require.NoError(t, err, "ctrl.NewManager") + require.NoError(t, (&Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Capacity: roomFor(2), + Enabled: true, + // envtest runs no AgentSession reconciler, so settings are never + // resolved; every session needs the same one-CPU pod set. + Demand: oneCPUDemand, + }).SetupWithManager(mgr), "admitter SetupWithManager") + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + if err := mgr.Start(ctx); err != nil { + t.Errorf("manager exited: %v", err) + } + }() + require.True(t, mgr.GetCache().WaitForCacheSync(ctx), "cache sync") + var once bool + stop = func() { + if once { + return + } + once = true + cancel() + <-done + } + t.Cleanup(stop) + return stop +} + +// admittedLive lists the names of non-terminal sessions with Admitted=True, +// and every session ever admitted (terminal ones included), sorted. +func admittedLive(t *testing.T, c client.Client) (live, ever []string) { + t.Helper() + var list spiceboxv1alpha1.AgentSessionList + require.NoError(t, c.List(context.Background(), &list, client.InNamespace(restartNamespace)), "list sessions") + for i := range list.Items { + s := &list.Items[i] + if !admission.IsAdmitted(s) { + continue + } + ever = append(ever, s.Name) + if !isTerminal(s) { + live = append(live, s.Name) + } + } + slices.Sort(live) + slices.Sort(ever) + return live, ever +} + +// waitForAdmitted polls until exactly want sessions have ever been admitted. +func waitForAdmitted(t *testing.T, c client.Client, want int) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var ever []string + for time.Now().Before(deadline) { + _, ever = admittedLive(t, c) + if len(ever) >= want { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("want %d sessions admitted within 30s, got %v", want, ever) +} + +// assertStaysAdmitted asserts over a window that the admitted set never grows +// past wantEver and the live holders never exceed the room. +func assertStaysAdmitted(t *testing.T, c client.Client, wantLive, wantEver []string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + live, ever := admittedLive(t, c) + if !assert.Equal(t, wantLive, live, "live holders") || !assert.Equal(t, wantEver, ever, "ever admitted") { + return + } + time.Sleep(100 * time.Millisecond) + } +} + +// TestAdmitterRestartRebuildsLine is the restart property: the line lives in +// AgentSession status, not in the admitter's memory, so a new operator +// process continues it exactly and never double-admits. +// +// Sessions are created s1..s5 in order. creationTimestamp is to the second, so +// several share a join time and the tie breaks on namespace/name, which is +// the same order. +func TestAdmitterRestartRebuildsLine(t *testing.T) { + env := testenv.Shared(t) + c := env.Client + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: restartNamespace}}); !apierrors.IsAlreadyExists(err) { + require.NoError(t, err, "create namespace") + } + + stop := startAdmitter(t, env) + for i := 1; i <= 5; i++ { + require.NoError(t, c.Create(ctx, &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("s%d", i), Namespace: restartNamespace}, + Spec: spiceboxv1alpha1.AgentSessionSpec{ + Class: testClass, + Prompt: spiceboxv1alpha1.PromptSource{Inline: "review this"}, + }, + }), "create s%d", i) + } + + waitForAdmitted(t, c, 2) + assertStaysAdmitted(t, c, []string{"s1", "s2"}, []string{"s1", "s2"}) + stop() + + startAdmitter(t, env) + // Release a holder the way a finished session does: it goes terminal. + var s1 spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: restartNamespace, Name: "s1"}, &s1), "get s1") + s1.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + require.NoError(t, c.Status().Update(ctx, &s1), "mark s1 Succeeded") + + waitForAdmitted(t, c, 3) + assertStaysAdmitted(t, c, []string{"s2", "s3"}, []string{"s1", "s2", "s3"}) + + var s4 spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: restartNamespace, Name: "s4"}, &s4), "get s4") + require.NotNil(t, s4.Status.Admission, "s4 is positioned in line") + assert.Equal(t, int32(1), s4.Status.Admission.Position, "s4 heads the line") + assert.Equal(t, int32(2), s4.Status.Admission.LineLength, "two still waiting") +} diff --git a/pkg/controllers/admission/capacity.go b/pkg/controllers/admission/capacity.go index 2c17ea00..0050626f 100644 --- a/pkg/controllers/admission/capacity.go +++ b/pkg/controllers/admission/capacity.go @@ -30,6 +30,10 @@ type CapacityModel interface { Free(pods []PlacedPod) // Clone returns an independent copy. Clone() CapacityModel + // Totals is the cluster's schedulable size — allocatable summed over the + // nodes the model places on — which Decide's dominant share is measured + // against. Zero when unknown; it does not shrink as Reserve places pods. + Totals() (cpuMilli, memBytes int64) } // CapacitySource produces a CapacityModel snapshot. It is the seam a future @@ -82,6 +86,9 @@ func (n *nodeSlot) take(p PodDemand) { type nodeModel struct { known bool nodes []nodeSlot + // totalCPUMilli and totalMemBytes are allocatable summed over the + // schedulable nodes, fixed at construction. + totalCPUMilli, totalMemBytes int64 } // NewNodeModel builds a model from node headroom. Unschedulable nodes are @@ -97,6 +104,9 @@ func NewNodeModel(nodes []corev1.Node, pods []corev1.Pod) CapacityModel { continue } h := schedfit.HeadroomOn(nodes[i], pods) + alloc := nodes[i].Status.Allocatable + m.totalCPUMilli += alloc.Cpu().MilliValue() + m.totalMemBytes += alloc.Memory().Value() _, hasPods := nodes[i].Status.Allocatable[corev1.ResourcePods] m.nodes = append(m.nodes, nodeSlot{ name: nodes[i].Name, @@ -158,5 +168,9 @@ func (m *nodeModel) Free(pods []PlacedPod) { } func (m *nodeModel) Clone() CapacityModel { - return &nodeModel{known: m.known, nodes: slices.Clone(m.nodes)} + return &nodeModel{known: m.known, nodes: slices.Clone(m.nodes), totalCPUMilli: m.totalCPUMilli, totalMemBytes: m.totalMemBytes} +} + +func (m *nodeModel) Totals() (cpuMilli, memBytes int64) { + return m.totalCPUMilli, m.totalMemBytes } diff --git a/pkg/controllers/admission/capacity_test.go b/pkg/controllers/admission/capacity_test.go index 97dbc7ef..439fa72f 100644 --- a/pkg/controllers/admission/capacity_test.go +++ b/pkg/controllers/admission/capacity_test.go @@ -177,3 +177,37 @@ func TestKubeCapacitySourceSnapshot(t *testing.T) { assert.False(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 500)}}), "the listed pod's request is booked") assert.True(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 300)}})) } + +func TestNodeModelTotals(t *testing.T) { + cordoned := testNode("c", "4", "4Gi", "10") + cordoned.Spec.Unschedulable = true + cases := []struct { + name string + nodes []corev1.Node + wantCPU, wantMem int64 + }{ + { + name: "two schedulable nodes: allocatable summed, unaffected by pods placed on them", + nodes: []corev1.Node{testNode("a", "1", "1Gi", "10"), testNode("b", "2", "2Gi", "10")}, + wantCPU: 3000, wantMem: 3 << 30, + }, + { + name: "an unschedulable node: excluded from the totals", + nodes: []corev1.Node{testNode("a", "1", "1Gi", "10"), cordoned}, + wantCPU: 1000, wantMem: 1 << 30, + }, + {name: "no nodes: unknown, zero totals"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := NewNodeModel(tc.nodes, []corev1.Pod{testPod("a", "600m")}) + require.True(t, m.Reserve(Demand{Pods: []PodDemand{pod("x", 100)}}), "Reserve") + cpu, mem := m.Totals() + assert.Equal(t, tc.wantCPU, cpu, "cpu total") + assert.Equal(t, tc.wantMem, mem, "memory total") + cpu, mem = m.Clone().Totals() + assert.Equal(t, tc.wantCPU, cpu, "clone cpu total") + assert.Equal(t, tc.wantMem, mem, "clone memory total") + }) + } +} diff --git a/pkg/controllers/admission/decide_test.go b/pkg/controllers/admission/decide_test.go index e315f744..b32e674f 100644 --- a/pkg/controllers/admission/decide_test.go +++ b/pkg/controllers/admission/decide_test.go @@ -27,8 +27,9 @@ func (m *slotModel) Reserve(d Demand) bool { m.slots -= len(d.Pods) return true } -func (m *slotModel) Free([]PlacedPod) {} -func (m *slotModel) Clone() CapacityModel { c := *m; return &c } +func (m *slotModel) Free([]PlacedPod) {} +func (m *slotModel) Clone() CapacityModel { c := *m; return &c } +func (m *slotModel) Totals() (int64, int64) { return 0, 0 } func key(name string) types.NamespacedName { return types.NamespacedName{Namespace: "ns", Name: name} diff --git a/pkg/platform/manifests/install.yaml b/pkg/platform/manifests/install.yaml index 19defc77..3265860c 100644 --- a/pkg/platform/manifests/install.yaml +++ b/pkg/platform/manifests/install.yaml @@ -5070,6 +5070,33 @@ spec: - artifactID type: object type: array + admission: + description: |- + Admission is where this session stands in its AgentClass's admission + line while it waits for its pod set to be admitted. Written by the + operator's admitter only when the position changes, and cleared when the + session is admitted. Nil when the session is not waiting. + properties: + joinedAt: + description: |- + JoinedAt is when the session took its place in line: its creation, or + the wake that re-queued it. + format: date-time + type: string + lineLength: + description: LineLength is how many sessions are waiting in the + line. + format: int32 + type: integer + position: + description: Position is the session's 1-based place in its line. + format: int32 + type: integer + required: + - joinedAt + - lineLength + - position + type: object agentWakeCredit: description: |- AgentWakeCredit is how many further AGENT-DRIVEN wakes this session will From 108cbe019647ea26e50c894fc4a0cfb567a4b048 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 13:00:45 -0500 Subject: [PATCH 08/33] Scope the admitter's session list to the watched namespaces The uncached session list was cluster-wide, which 403s under the per-namespace RBAC --watch-namespaces exists for and reaches sessions in namespaces the operator does not own. With a watch set, the pass now lists one namespace at a time, using the same set the manager cache is scoped to. --- internal/cmd/operator/main.go | 9 ++- .../admission/admitter/controller.go | 39 +++++++++-- .../admission/admitter/controller_test.go | 65 +++++++++++++++++++ 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index 2b8452bf..37378c8b 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -11,10 +11,12 @@ import ( "github.com/google/uuid" "io" "log/slog" + "maps" "net" "net/http" "os" runtimedebug "runtime/debug" + "slices" "strings" "sync/atomic" "time" @@ -2771,8 +2773,11 @@ func run(cfg *config) { if err := (&admitter.Reconciler{ Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), - Capacity: admissionCapacity, - Enabled: cfg.sessionAdmission, + // The same namespace set the manager cache is scoped to, so the + // uncached session list stays inside the RBAC a watch set grants. + Namespaces: slices.Sorted(maps.Keys(cacheNamespaces(cfg.watchNamespaces, operatorNamespace(cfg.podNamespace)))), + Capacity: admissionCapacity, + Enabled: cfg.sessionAdmission, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to register admission controller") os.Exit(1) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index a23d2242..5fd10859 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -68,6 +68,12 @@ type Reconciler struct { // session as still waiting, free its capacity, and let the pass admit a // second session into the same space. Nil falls back to Client. APIReader client.Reader + // Namespaces is the operator's watch set (--watch-namespaces). When set, + // sessions are listed one namespace at a time: the APIReader list is + // uncached, so a cluster-wide one would 403 under the per-namespace RBAC a + // watch set exists for, and would reach sessions in namespaces this + // operator does not own. Empty means cluster-wide. + Namespaces []string // Capacity snapshots what the cluster can still place. Nil, or a snapshot // error, means unknown capacity: every waiting session fits. Capacity admission.CapacitySource @@ -131,13 +137,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R logger := log.FromContext(ctx) requeue := ctrl.Result{RequeueAfter: resyncInterval} - var list spiceboxv1alpha1.AgentSessionList - if err := r.sessionReader().List(ctx, &list); err != nil { - return ctrl.Result{}, fmt.Errorf("admission pass: list AgentSessions: %w", err) + sessions, err := r.listSessions(ctx) + if err != nil { + return ctrl.Result{}, err } var waiting, holding []*spiceboxv1alpha1.AgentSession - for i := range list.Items { - sess := &list.Items[i] + for i := range sessions { + sess := &sessions[i] switch { case isTerminal(sess): // Never a holder, even while still Admitted=True with pods: any @@ -224,6 +230,29 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R return requeue, errors.Join(errs...) } +// listSessions lists every AgentSession the operator owns: cluster-wide with +// no watch set, else one namespaced list per watched namespace. Any failed +// list fails the pass (logged by the caller's error return) rather than +// deciding lines from a partial view. +func (r *Reconciler) listSessions(ctx context.Context) ([]spiceboxv1alpha1.AgentSession, error) { + if len(r.Namespaces) == 0 { + var list spiceboxv1alpha1.AgentSessionList + if err := r.sessionReader().List(ctx, &list); err != nil { + return nil, fmt.Errorf("admission pass: list AgentSessions: %w", err) + } + return list.Items, nil + } + var out []spiceboxv1alpha1.AgentSession + for _, ns := range r.Namespaces { + var list spiceboxv1alpha1.AgentSessionList + if err := r.sessionReader().List(ctx, &list, client.InNamespace(ns)); err != nil { + return nil, fmt.Errorf("admission pass: list AgentSessions in namespace %q: %w", ns, err) + } + out = append(out, list.Items...) + } + return out, nil +} + // snapshot returns the capacity model, or an unknown one when there is no // source or the snapshot fails (logged): missing data must not block sessions. func (r *Reconciler) snapshot(ctx context.Context) admission.CapacityModel { diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 499976cb..d0473782 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -282,3 +282,68 @@ func TestUnplacedDemand(t *testing.T) { }) } } + +// TestAdmitterWatchNamespaces pins the --watch-namespaces contract: under +// namespaced RBAC a cluster-wide list 403s, so with a watch set the pass lists +// sessions one namespace at a time and never touches any other namespace. +func TestAdmitterWatchNamespaces(t *testing.T) { + inNamespace := func(s *spiceboxv1alpha1.AgentSession, ns string) *spiceboxv1alpha1.AgentSession { + s.Namespace = ns + return s + } + cases := []struct { + name string + namespaces []string + wantLists []string + wantAdmitted []string + }{ + { + name: "watch set: one namespaced list per watched namespace, unwatched sessions untouched", + namespaces: []string{"team-a", "team-b"}, + wantLists: []string{"team-a", "team-b"}, wantAdmitted: []string{"team-a/rb-a", "team-b/rb-b"}, + }, + { + name: "no watch set: one cluster-wide list, every namespace admitted", + wantLists: []string{""}, wantAdmitted: []string{"other/rb-c", "team-a/rb-a", "team-b/rb-b"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var lists []string + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects( + inNamespace(waitingSession("rb-a", time.Minute), "team-a"), + inNamespace(waitingSession("rb-b", 2*time.Minute), "team-b"), + inNamespace(waitingSession("rb-c", 3*time.Minute), "other"), + ). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*spiceboxv1alpha1.AgentSessionList); ok { + lo := (&client.ListOptions{}).ApplyOptions(opts) + lists = append(lists, lo.Namespace) + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + r := &Reconciler{ + Client: c, Capacity: roomFor(5), Enabled: true, Namespaces: tc.namespaces, + Now: func() time.Time { return passTime }, Demand: oneCPUDemand, + } + runPass(t, r) + assert.Equal(t, tc.wantLists, lists, "AgentSession list scopes") + + var all spiceboxv1alpha1.AgentSessionList + require.NoError(t, c.List(context.Background(), &all), "list all") + var admitted []string + for i := range all.Items { + if admission.IsAdmitted(&all.Items[i]) { + admitted = append(admitted, all.Items[i].Namespace+"/"+all.Items[i].Name) + } + } + assert.ElementsMatch(t, tc.wantAdmitted, admitted, "admitted sessions") + }) + } +} From a412bf41373b29fcd59125bc12ad259876703e33 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 13:25:19 -0500 Subject: [PATCH 09/33] Re-queue admitted sessions the scheduler could not place An admitted booting session whose bundle pod stays unschedulable on a capacity shortfall for two minutes reaps its pods and goes back to its line as Admitted=False/PlacementMismatch, written after the reap succeeds. JoinedAt is unchanged, so it keeps its place; the waiting timeout restarts at the release. Only with admission on; the pod-level capacity hold is unchanged otherwise. --- .../agentsession/admission_gate.go | 67 +++++++++++++ .../agentsession/admission_gate_test.go | 94 +++++++++++++++++++ pkg/controllers/agentsession/controller.go | 8 ++ 3 files changed, 169 insertions(+) diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index f3830740..fb6ead21 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -5,11 +5,15 @@ // a booting session creates none of its pods until admitted — and hands // admission back (Admitted=False/Reclaimed) at the moment its pods are // reclaimed, so the admitter never counts capacity a session no longer holds. +// Its safety net hands admission back too (Admitted=False/PlacementMismatch) +// when an admitted set the scheduler cannot place would otherwise hold its +// placed half while the rest waits. package agentsession import ( "context" "fmt" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,3 +108,66 @@ func (r *Reconciler) releaseAdmissionOnReclaim(sess *spiceboxv1alpha1.AgentSessi }) return true } + +// admissionMisfitGrace is how long an admitted session's bundle pod may stay +// unschedulable on a capacity shortfall before the session gives its pods back +// and returns to its line. The admitter's model said the whole set fits; a pod +// still unplaced this long later means the model was wrong (a pod it did not +// know about, a node draining), and holding the placed half while the rest +// waits is the hold-and-wait admission exists to end. +const admissionMisfitGrace = 2 * time.Minute + +// placementMisfit reports whether this pass must release an admitted session +// whose set the scheduler could not place. stalled is the bundle stage's own +// scheduling-stall verdict (already false for a bundle that regressed +// mid-session, which the ordinary deadline handles). +// +// Only a capacity SHORTFALL counts, for the same reason only a shortfall is +// held (capacityShortfall): a pod blocked by a taint or a volume-affinity +// conflict would be re-admitted at once, fail to place again, and cycle +// forever with its waiting-timeout clock restarting at every release. Left +// alone, it keeps the ordinary readiness deadline and its specific failure +// message. +func (r *Reconciler) placementMisfit(sess *spiceboxv1alpha1.AgentSession, stall schedulingStall, stalled bool) bool { + if !r.AdmissionEnabled || !stalled || !admission.IsAdmitted(sess) { + return false + } + booting := sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending + return booting && capacityShortfall(stall.reason) && stall.pending >= admissionMisfitGrace +} + +// releaseMisfit reaps an admitted session's pods and records +// Admitted=False/PlacementMismatch, so the admitter re-queues it. The release +// is written only after the reap succeeds — released while its pods still +// stood, the admitter would count their capacity twice. A failed reap or +// status write leaves the session admitted, and the next pass decides afresh +// from whatever still stands. +// +// The session keeps its place: admission.JoinedAt reads only creation and +// wake times, so it re-joins at the front of its line (it was admitted +// fairly; the model was wrong, not the session). admission.WaitingSince +// restarts at this release, so the capacity-hold timeout gives it a fresh +// window rather than the remainder of its first one. +func (r *Reconciler) releaseMisfit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, stall schedulingStall) (ctrl.Result, error) { + logger := log.FromContext(ctx) + logger.Info("admitted session's pods could not be placed; releasing them and re-queueing it at its place in line", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, + "pod", stall.podName, "unschedulableFor", stall.pending.String(), + "schedulerMessage", stall.reason) + if _, err := r.reapSessionPods(ctx, sess); err != nil { + return ctrl.Result{}, err // already logged inside reapSessionPods; retried next pass + } + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionMisfit, + Message: fmt.Sprintf("%s could not be placed for %s (%s); its pods were released and it waits at its place in line", stall.podName, stall.pending.Round(time.Second), strings.TrimSuffix(stall.reason, ".")), + LastTransitionTime: metav1.NewTime(r.now()), + }) + if err := r.applyStatus(ctx, sess); err != nil { + logger.Info("misfit: recording the admission release failed; will retry", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, "err", err.Error()) + return ctrl.Result{}, fmt.Errorf("record placement-mismatch release: %w", err) + } + return ctrl.Result{RequeueAfter: capacityHoldRequeue}, nil +} diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index 762e707c..a3441c8a 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" "github.com/authzed/openagentprimitives/pkg/memory" ) @@ -283,3 +284,96 @@ func TestAdmissionGate_SleepReleasesInOneWrite(t *testing.T) { } func ptrTo[T any](v T) *T { return &v } + +// An ADMITTED booting session whose bundle pod the scheduler still cannot +// place after admissionMisfitGrace gives its pods back and returns to its line +// (Admitted=False/PlacementMismatch), keeping its place: JoinedAt does not +// move. Short of the grace, with admission off, or on a stall another session +// finishing cannot cure, nothing is released. +func TestAdmissionGate_Misfit(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + + cases := []struct { + name string + enabled bool // Reconciler.AdmissionEnabled + admitted bool // the session carries Admitted=True + pending time.Duration // how long the bundle pod has been unschedulable + schedMsg string + woken bool // the session was cold-woken before this admission + + wantReaped bool + wantMisfit bool // Admitted=False/PlacementMismatch at now + wantAdmittedNil bool // no Admitted condition at all + }{ + { + name: "admitted, pods unschedulable 2m: pods reaped, Admitted=False/PlacementMismatch, JoinedAt unchanged", + enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, + wantReaped: true, wantMisfit: true, + }, + { + name: "admitted after a cold wake, pods unschedulable 2m: released, JoinedAt unchanged", + enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, woken: true, + wantReaped: true, wantMisfit: true, + }, + { + name: "admitted, pods unschedulable 1m (inside the misfit grace): pods kept, still admitted", + enabled: true, admitted: true, pending: time.Minute, schedMsg: capacitySchedMsg, + }, + { + name: "admitted, taint-only stall 2m: not a capacity misfit, pods kept, still admitted", + enabled: true, admitted: true, pending: 2 * time.Minute, + schedMsg: "0/6 nodes are available: 6 node(s) had untolerated taint(s).", + }, + { + name: "admission off, pods unschedulable 2m: pods kept under the pod-level hold, no Admitted condition", + pending: 2 * time.Minute, + schedMsg: capacitySchedMsg, wantAdmittedNil: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + started := nowT.Add(-tc.pending) + + ac := classWithBundle("ac1", codeBundle) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-20*time.Minute)) + if tc.woken { + wake := metav1.NewTime(nowT.Add(-10 * time.Minute)) + sess.Status.LastWakeAt = &wake + } + if tc.admitted { + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, started.Add(-time.Second)) + } + joinedBefore := admission.JoinedAt(sess) + bundle := bundleWithPod("s1", "code", "s1-code-pod", started) + pod := pendingUnschedulablePod("s1-code-pod", "uid-1", started, tc.schedMsg) + + r, c := fakeReconciler(t, clock, ac, sess, bundle, pod) + r.AdmissionEnabled = tc.enabled + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + assert.Equal(t, !tc.wantReaped, + exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "bundle SpiceboxSession present") + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") + assert.Equal(t, joinedBefore.UTC(), admission.JoinedAt(&got).UTC(), "JoinedAt must not move: the session keeps its place in line") + + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + switch { + case tc.wantAdmittedNil: + assert.Nil(t, adm, "no Admitted condition") + case tc.wantMisfit: + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionMisfit, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") + assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting-timeout clock restarts at the release") + default: + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionTrue, adm.Status, "still admitted") + } + }) + } +} diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index e9f70b75..f145d8f2 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -1991,6 +1991,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu bundlesWereReady := meta.FindStatusCondition(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionBundlesReady) midSession := bundlesWereReady != nil && bundlesWereReady.Status == metav1.ConditionTrue stall, stalled := r.firstSchedulingStall(ctx, sess.Namespace, notReadyPods) + // Admission's safety net: an admitted set the scheduler still + // cannot place goes back to its line instead of holding the + // placed half (admission_gate.go). Ahead of the deadline, which + // a re-queued session must not be failed by. + if r.placementMisfit(&sess, stall, stalled && !midSession) { + sess.Status.BundleSessions = bundleSessionStatuses + return r.releaseMisfit(ctx, &sess, stall) + } bundleDeadline, held := r.capacityHoldDeadline(&sess, stall, stalled && !midSession, r.effectiveBundleReadyDeadline(ctx)) if bundleProvisioningTimedOut(capacityHoldAdjustedStart(&sess, earliestBundleStart, held, r.now()), r.now(), bundleDeadline) { sess.Status.BundleSessions = bundleSessionStatuses From 42e7d57c8f4042e9852ce0d6deca3a28e2211cb6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 13:25:19 -0500 Subject: [PATCH 10/33] Add burst admission scenario to bronzethread Bundles can now open several named sessions across more than one fake Channel, scope transcript steps to one session, hold a turn until other sessions reach a state, and run with whole-set admission against a fixed capacity. The driver watches every session for the whole run and checks that none holds part of a pod set, that admission never exceeds the capacity, and the bundle's ordering claims. admission-burst runs three reviewer sessions and one session of a second agent on room for two: the reviewer line is FIFO and the second agent is admitted ahead of the burst's tail. --- pkg/bronzethread/bundle.go | 19 + pkg/bronzethread/sessions.go | 160 +++++++ pkg/bronzethread/sessions_test.go | 127 +++++ pkg/bronzethread/userturn.go | 9 + test/e2e/admission.go | 101 ++++ .../testdata/admission-burst/bundle.json | 453 ++++++++++++++++++ .../admission-burst/manifests/00-agent.yaml | 144 ++++++ test/e2e/conversation.go | 21 +- test/e2e/harness.go | 15 +- test/e2e/threadrun/driver.go | 44 +- test/e2e/threadrun/sessions.go | 410 ++++++++++++++++ 11 files changed, 1493 insertions(+), 10 deletions(-) create mode 100644 pkg/bronzethread/sessions.go create mode 100644 pkg/bronzethread/sessions_test.go create mode 100644 test/e2e/admission.go create mode 100644 test/e2e/bronzethread/testdata/admission-burst/bundle.json create mode 100644 test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml create mode 100644 test/e2e/threadrun/sessions.go diff --git a/pkg/bronzethread/bundle.go b/pkg/bronzethread/bundle.go index 7771004a..a323d583 100644 --- a/pkg/bronzethread/bundle.go +++ b/pkg/bronzethread/bundle.go @@ -275,6 +275,11 @@ type Bundle struct { // raises a warning naming what stopped running. MetaToolReplies map[string]MetaToolReply `json:"metaToolReplies,omitempty"` + // Admission runs the bundle with whole-set session admission on, against + // a fixed capacity. Absent — every bundle but the admission scenarios — + // sessions are ungated, as before. See sessions.go. + Admission *Admission `json:"admission,omitempty"` + // LLM is the model's side, POSITIONAL: the nth Send gets the nth entry. LLM []LLMStep `json:"llm"` @@ -361,6 +366,10 @@ func (b Bundle) Validate() error { return err } + if err := b.validateSessions(); err != nil { + return err + } + return b.ValidateToolOutputs() } @@ -632,6 +641,12 @@ type LLMStep struct { // position. Steps for one class stay positional among themselves. AgentClass string `json:"agentClass,omitempty"` + // Thread scopes the step to the session a named userTurns thread opened, + // for the case AgentClass cannot tell apart: several sessions of ONE class + // at once. A request belongs to the thread whose opening text is its first + // user message. Steps for one thread stay positional among themselves. + Thread string `json:"thread,omitempty"` + // Expect is the divergence check. It is verified against the actual request // BEFORE the reply is returned, so a system that took a different path // fails loudly instead of being handed an answer to a question it did not @@ -883,6 +898,10 @@ type Assertions struct { // Channel asserts on the posts the bound output channel received. Channel *ChannelAssertions `json:"channel,omitempty"` + // Admission claims the order the line admitted and finished the run's + // sessions in. Requires Bundle.Admission. See sessions.go. + Admission *AdmissionAssertions `json:"admission,omitempty"` + // ToolsCalled names tools the run must have DISPATCHED, read back off the // replayed session's own transcript. // diff --git a/pkg/bronzethread/sessions.go b/pkg/bronzethread/sessions.go new file mode 100644 index 00000000..c9d9d1c8 --- /dev/null +++ b/pkg/bronzethread/sessions.go @@ -0,0 +1,160 @@ +package bronzethread + +import ( + "fmt" + "strings" +) + +// A bundle normally drives ONE session: every user turn lands on the same +// thread of the fixture's one conversational Channel, and the transcript is a +// single positional stream. A scenario about how several sessions share the +// cluster — the admission line — cannot be written that way: it needs several +// sessions, of more than one AgentClass, alive at once. +// +// Three optional fields open that up, and a bundle that sets none of them is +// exactly the single-session bundle it always was: +// +// - UserTurn.Thread (and Channel) start or continue a NAMED session. Each +// thread is its own channel key, so its own AgentSession. +// - LLMStep.Thread scopes a transcript step to one thread's session, the +// way AgentClass scopes it to a class. Several sessions of one class +// cannot be told apart by class, and an unscoped step is answered by +// whichever session asks next — which is a different transcript every +// run once sessions overlap. +// - UserTurn.WaitFor holds a turn until other sessions reach a state, so a +// scenario can say "while A is still working" without guessing at timing. + +// Barrier states a TurnBarrier can wait for. +const ( + // UntilAwaitingInput: the session is parked in await_user_message, still + // Running. The only state a follow-up turn can be delivered INTO rather + // than starting a fresh turn. + UntilAwaitingInput = "awaitingInput" + // UntilIdle: the session's turn ended (phase Idle). + UntilIdle = "idle" +) + +// TurnBarrier holds a user turn until the session on Thread reaches Until. +type TurnBarrier struct { + Thread string `json:"thread"` + Until string `json:"until"` +} + +// Admission turns on whole-set session admission for the run, against a +// fixed capacity, so a bundle can exercise the line itself. Absent, the run is +// ungated exactly as before — the harness then has no capacity to measure. +type Admission struct { + // CapacitySessions is how many sessions' whole pod sets the cluster can + // hold at once. The harness has no nodes, so this stands in for them. + CapacitySessions int `json:"capacitySessions"` +} + +// AdmissionAssertions are claims about the order the line admitted sessions +// in, read from what the run actually did. Threads are named as in userTurns. +// +// Ordering is by FIRST OBSERVATION: the driver watches every session for the +// whole run and records when each was first seen admitted and first seen +// finished. Two sessions admitted in one pass may be seen together, so only +// STRICT orderings are claimable — which is all a line promises. +type AdmissionAssertions struct { + // AdmittedBefore: each pair's First was admitted strictly before Then. + AdmittedBefore []ThreadOrder `json:"admittedBefore,omitempty"` + // CompletedBefore: each pair's First finished its work (reached Idle) + // strictly before Then did. + CompletedBefore []ThreadOrder `json:"completedBefore,omitempty"` + // PeakAdmitted is the EXACT largest number of sessions the run held + // admitted at once. Equal to the capacity proves the line actually + // queued; above it is the over-admission the line exists to prevent. + PeakAdmitted int `json:"peakAdmitted,omitempty"` +} + +// ThreadOrder is one strict ordering claim between two threads. +type ThreadOrder struct { + First string `json:"first"` + Then string `json:"then"` +} + +// Threads lists every named thread the bundle's user turns open, in the order +// they first appear. +func (b Bundle) Threads() []string { + var out []string + seen := map[string]bool{} + for _, ut := range b.UserTurns { + if ut.Thread != "" && !seen[ut.Thread] { + seen[ut.Thread] = true + out = append(out, ut.Thread) + } + } + return out +} + +// OpeningText is the text of the first user turn on thread: the message that +// opened that thread's session, and so the first user message of every model +// request the session makes. It is how a thread-scoped step recognizes its +// session's requests. +func (b Bundle) OpeningText(thread string) string { + for _, ut := range b.UserTurns { + if ut.Thread == thread { + return ut.Text + } + } + return "" +} + +// validateSessions refuses a multi-session bundle the driver could not route. +func (b Bundle) validateSessions() error { + threads := map[string]bool{} + for _, th := range b.Threads() { + threads[th] = true + } + if len(threads) > 0 && b.Trigger != nil { + return fmt.Errorf("named threads start sessions from userTurns; a triggered bundle has one session") + } + for i, ut := range b.UserTurns { + if len(threads) > 0 && ut.Thread == "" { + return fmt.Errorf("userTurns[%d] names no thread, but others do; once one turn names its session every turn must", i) + } + if ut.Channel != "" && ut.Thread == "" { + return fmt.Errorf("userTurns[%d] names a channel but no thread", i) + } + for j, w := range ut.WaitFor { + if !threads[w.Thread] { + return fmt.Errorf("userTurns[%d].waitFor[%d] names thread %q, which no user turn opens", i, j, w.Thread) + } + if w.Until != UntilAwaitingInput && w.Until != UntilIdle { + return fmt.Errorf("userTurns[%d].waitFor[%d].until is %q; want %q or %q", + i, j, w.Until, UntilAwaitingInput, UntilIdle) + } + } + } + // Each thread's opening text is how its steps find its requests, so no + // opening may contain another's: a request would then match both scopes. + for a := range threads { + for c := range threads { + if a != c && strings.Contains(b.OpeningText(a), b.OpeningText(c)) { + return fmt.Errorf("thread %q's opening text contains thread %q's, so their transcript steps cannot be told apart", a, c) + } + } + } + for i, step := range b.LLM { + if step.Thread != "" && !threads[step.Thread] { + return fmt.Errorf("llm[%d] is scoped to thread %q, which no user turn opens", i, step.Thread) + } + } + if b.Admission != nil && b.Admission.CapacitySessions < 1 { + return fmt.Errorf("admission.capacitySessions must be at least 1; a cluster with room for no session admits nothing") + } + if a := b.Assert.Admission; a != nil { + if b.Admission == nil { + return fmt.Errorf("assert.admission makes claims about a line, but the bundle turns admission on nowhere (admission)") + } + for _, list := range [][]ThreadOrder{a.AdmittedBefore, a.CompletedBefore} { + for _, o := range list { + if !threads[o.First] || !threads[o.Then] { + return fmt.Errorf("assert.admission orders %q before %q, but each must be a thread a user turn opens", o.First, o.Then) + } + } + } + } + return nil +} diff --git a/pkg/bronzethread/sessions_test.go b/pkg/bronzethread/sessions_test.go new file mode 100644 index 00000000..9881d3e6 --- /dev/null +++ b/pkg/bronzethread/sessions_test.go @@ -0,0 +1,127 @@ +package bronzethread_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + bt "github.com/authzed/openagentprimitives/pkg/bronzethread" +) + +// multiSessionBundle is the minimum runnable bundle with two named threads, a +// step scoped to one of them, admission on and one ordering claim. +func multiSessionBundle() bt.Bundle { + b := runnableBundle() + b.UserTurns = []bt.UserTurn{ + {Text: "review pull request 1", Thread: "a1"}, + {Text: "summarize the notes", Thread: "b", Channel: "notes-fake"}, + {Text: "wrap up", Thread: "a1", WaitFor: []bt.TurnBarrier{{Thread: "a1", Until: bt.UntilAwaitingInput}}}, + } + b.LLM = []bt.LLMStep{{Thread: "a1", Reply: []bt.ReplyPart{{Text: "none"}}}} + b.Admission = &bt.Admission{CapacitySessions: 1} + b.Assert.Admission = &bt.AdmissionAssertions{ + AdmittedBefore: []bt.ThreadOrder{{First: "a1", Then: "b"}}, + } + return b +} + +// TestBundle_ValidateSessions covers the preconditions a multi-session bundle +// adds: every turn routed, every scope and barrier naming a real thread, and +// admission claims only where admission is on. +func TestBundle_ValidateSessions(t *testing.T) { + cases := []struct { + name string + doctor func(b bt.Bundle) bt.Bundle + wantErr string // empty means the bundle must validate + }{ + { + name: "a multi-session admission bundle validates", + doctor: func(b bt.Bundle) bt.Bundle { return b }, + }, + { + name: "one turn names no thread while others do: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.UserTurns[1].Thread = ""; return b }, + wantErr: "every turn must", + }, + { + name: "a channel with no thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns = []bt.UserTurn{{Text: "hi", Channel: "notes-fake"}} + b.LLM[0].Thread = "" + b.Assert.Admission = nil + return b + }, + wantErr: "names a channel but no thread", + }, + { + name: "a barrier naming an unknown thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[2].WaitFor[0].Thread = "zz" + return b + }, + wantErr: "which no user turn opens", + }, + { + name: "a barrier with an unknown state: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[2].WaitFor[0].Until = "sleeping" + return b + }, + wantErr: "until is \"sleeping\"", + }, + { + name: "a step scoped to an unknown thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.LLM[0].Thread = "zz"; return b }, + wantErr: "llm[0] is scoped to thread \"zz\"", + }, + { + name: "one thread's opening text contains another's: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[1].Text = "please review pull request 1 too" + return b + }, + wantErr: "cannot be told apart", + }, + { + name: "admission with no capacity: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.Admission.CapacitySessions = 0; return b }, + wantErr: "capacitySessions must be at least 1", + }, + { + name: "admission claims with admission off: refused", + doctor: func(b bt.Bundle) bt.Bundle { b.Admission = nil; return b }, + wantErr: "turns admission on nowhere", + }, + { + name: "an admission claim naming an unknown thread: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.Assert.Admission.CompletedBefore = []bt.ThreadOrder{{First: "a1", Then: "zz"}} + return b + }, + wantErr: "each must be a thread a user turn opens", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.doctor(multiSessionBundle()).Validate() + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err, "this bundle must not be runnable") + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// Threads lists named threads once each, in first-appearance order, and +// OpeningText is the thread's FIRST turn — the message every request of that +// session starts from. +func TestBundle_ThreadsAndOpeningText(t *testing.T) { + b := multiSessionBundle() + assert.Equal(t, []string{"a1", "b"}, b.Threads()) + assert.Equal(t, "review pull request 1", b.OpeningText("a1"), "the first turn, not the follow-up") + assert.Empty(t, b.OpeningText("zz")) + assert.Empty(t, runnableBundle().Threads(), "a single-session bundle names no threads") +} diff --git a/pkg/bronzethread/userturn.go b/pkg/bronzethread/userturn.go index 4f1b8c08..d28a6510 100644 --- a/pkg/bronzethread/userturn.go +++ b/pkg/bronzethread/userturn.go @@ -45,6 +45,15 @@ type BundleAttachment struct { type UserTurn struct { Text string `json:"text"` Attachments []BundleAttachment `json:"attachments,omitempty"` + + // Thread names the session this turn lands on; Channel, the fake Channel + // it is sent through (empty: the fixture's only one). Both empty is the + // single-session bundle. See sessions.go. + Thread string `json:"thread,omitempty"` + Channel string `json:"channel,omitempty"` + + // WaitFor holds this turn until each named session reaches its state. + WaitFor []TurnBarrier `json:"waitFor,omitempty"` } // UnmarshalJSON accepts the bare-string and object forms. diff --git a/test/e2e/admission.go b/test/e2e/admission.go new file mode 100644 index 00000000..016505b3 --- /dev/null +++ b/test/e2e/admission.go @@ -0,0 +1,101 @@ +//go:build e2e + +package e2e + +import ( + "context" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/controllers/admission/admitter" +) + +// AdmissionOptions turns on whole-set session admission for one harness: the +// AgentSession reconciler gates every booting session's pods on Admitted=True +// (AdmissionEnabled), and the admitter runs against Capacity. +// +// Opt-in per scenario, like WithTokenAuthz. Nil — every scenario but an +// admission one — leaves sessions ungated, which is the reconciler's zero +// value and what every other test was written against. +type AdmissionOptions struct { + // Capacity is what the admitter places sets against. Required: envtest + // has no nodes, so the production KubeCapacitySource would report unknown + // capacity and admit every session at once — a line that never queues. + Capacity admission.CapacitySource +} + +// registerAdmitter adds the admitter to mgr when admission is on. Called for +// the first manager and again by RestartOperator, so a restarted operator +// resumes the line from AgentSession status as production's does. +func (h *Harness) registerAdmitter(t *testing.T, mgr manager.Manager) { + t.Helper() + if h.opts.Admission == nil { + return + } + if err := (&admitter.Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Capacity: h.opts.Admission.Capacity, + Enabled: true, + }).SetupWithManager(mgr); err != nil { + t.Fatalf("e2e: register admission controller: %v", err) + } +} + +// FixedSetCapacity is a cluster with room for exactly Sets whole session pod +// sets, whatever their size. Every Reserve of a non-empty demand takes one +// set's room, so with the harness's pods never binding to a node (there is +// no scheduler), each admitted session holds one slot until it is released. +// +// It stands in for nodes, not for the admission logic: the admitter still +// orders lines, applies fair share and reserves holders' unplaced demand +// through admission.Decide exactly as in production. +type FixedSetCapacity struct { + Sets int +} + +// Snapshot returns a fresh model with every set free; the admitter reserves +// current holders on it before deciding. +func (c FixedSetCapacity) Snapshot(context.Context) (admission.CapacityModel, error) { + return &setModel{free: c.Sets, sets: c.Sets}, nil +} + +// setModelSetCPUMilli and setModelSetMemBytes size one set for Totals, the +// denominator of a line's dominant share. Only the RATIO between lines' +// shares matters to Decide, so any positive size will do. +const ( + setModelSetCPUMilli = 4000 + setModelSetMemBytes = 8 << 30 +) + +type setModel struct { + free, sets int +} + +func (m *setModel) Known() bool { return true } + +func (m *setModel) Reserve(d admission.Demand) bool { + if len(d.Pods) == 0 { + return true + } + if m.free == 0 { + return false + } + m.free-- + return true +} + +// Free is a no-op: a model rebuilt every pass has no placed pods to return, +// and nothing in a scenario frees through the model mid-pass. +func (m *setModel) Free([]admission.PlacedPod) {} + +func (m *setModel) Clone() admission.CapacityModel { + c := *m + return &c +} + +func (m *setModel) Totals() (cpuMilli, memBytes int64) { + return int64(m.sets) * setModelSetCPUMilli, int64(m.sets) * setModelSetMemBytes +} diff --git a/test/e2e/bronzethread/testdata/admission-burst/bundle.json b/test/e2e/bronzethread/testdata/admission-burst/bundle.json new file mode 100644 index 00000000..be2f80ef --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-burst/bundle.json @@ -0,0 +1,453 @@ +{ + "name": "admission-burst", + "description": "A burst on a full cluster: three reviewer sessions arrive together, then a session of a second agent, on a cluster with room for exactly two sessions' whole pod sets. The line, not the scheduler, decides who runs.\n\nWhat is claimed, and how each claim is read from the run rather than from the transcript:\n- every session runs to completion and gives its room back (each reaches Idle after being admitted, then sleeps, which releases its admission);\n- the reviewer line is FIFO: a1 and a2 are admitted before a3, and a3 finishes after both;\n- fair share between lines: when a1 frees its room while a2 still holds one, the notes agent's session is admitted ahead of the burst's tail (b before a3);\n- never more than two sessions admitted at once, and exactly two at the peak (so the line actually queued);\n- no session ever holds part of a pod set: a waiting session holds none of its bundle SpiceboxSessions, a Running one holds all of them. The driver watches every session for the whole run to check this.\nEach session's sandbox call is asserted on its TOOL RESULT (lastToolResultContains / IsError), which proves the call reached that session's own bundle sandbox.\n\nWhy a1 and a2 park in await_user_message: a finished session keeps its admission until it sleeps, so WHICH holder frees its room first is an input to this scenario, not a race to leave to timing. a1 parks until every session has arrived, so its release cannot reach the line before b is waiting; a2 parks until b has finished, so it is still holding when a1's room frees, which is the moment fair share has to choose. The notes agent sleeps after 10s rather than 1s for the same reason: a3's room should come from a2 finishing, so a3 finishing last is a fact about the line, not a guess about speed.\n\nCapacity is two, not one, deliberately. With room for a single session no line ever HOLDS capacity at the moment a choice is made (the only holder is the session that just left), both lines tie at zero share, and the earlier arrival wins — so the second agent would be admitted last and fair share could not be observed at all.", + "agentDir": "testdata/admission-burst/manifests", + "agentClass": "burst-reviewer", + "admission": { + "capacitySessions": 2 + }, + "userTurns": [ + { + "text": "review pull request 101", + "thread": "a1", + "channel": "burst-reviewer-fake" + }, + { + "text": "review pull request 202", + "thread": "a2", + "channel": "burst-reviewer-fake" + }, + { + "text": "review pull request 303", + "thread": "a3", + "channel": "burst-reviewer-fake" + }, + { + "text": "check notes for pull request 404", + "thread": "b", + "channel": "burst-notes-fake" + }, + { + "text": "a1 go ahead and finish", + "thread": "a1", + "channel": "burst-reviewer-fake", + "waitFor": [ + { + "thread": "a1", + "until": "awaitingInput" + } + ] + }, + { + "text": "a2 go ahead and finish", + "thread": "a2", + "channel": "burst-reviewer-fake", + "waitFor": [ + { + "thread": "a2", + "until": "awaitingInput" + }, + { + "thread": "b", + "until": "idle" + } + ] + } + ], + "toolOutputs": { + "gitlike_gh": { + "stdout": "title: Update README.md\nstate: OPEN\nmergeable: MERGEABLE\n" + } + }, + "llm": [ + { + "thread": "a1", + "expect": { + "userTextContains": "review pull request 101" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 101" + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_a1": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_a1}}", + "_reason": "reading pull request 101", + "args": [ + "pr", + "view", + "101", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 101 is open and mergeable" + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "await_user_message", + "args": {} + } + } + ] + }, + { + "thread": "a2", + "expect": { + "userTextContains": "review pull request 202" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 202" + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_a2": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_a2}}", + "_reason": "reading pull request 202", + "args": [ + "pr", + "view", + "202", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 202 is open and mergeable" + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "await_user_message", + "args": {} + } + } + ] + }, + { + "thread": "a3", + "expect": { + "userTextContains": "review pull request 303" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 303" + } + } + } + ] + }, + { + "thread": "a3", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_a3": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_a3}}", + "_reason": "reading pull request 303", + "args": [ + "pr", + "view", + "303", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "a3", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 303 is open and mergeable" + } + } + } + ] + }, + { + "thread": "a3", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "looked at pull request 303" + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "userTextContains": "check notes for pull request 404" + }, + "reply": [ + { + "toolUse": { + "name": "new_operation", + "args": { + "description": "look at pull request 404" + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "lastToolResult": "new_operation", + "lastToolResultIsError": false, + "capture": { + "op_b": "\"operation_id\":\"([^\"]+)\"" + } + }, + "reply": [ + { + "toolUse": { + "name": "gitlike_gh", + "args": { + "operation_id": "{{op_b}}", + "_reason": "reading pull request 404", + "args": [ + "pr", + "view", + "404", + "--repo", + "demo-org/demo-repo" + ] + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "lastToolResult": "gitlike_gh", + "lastToolResultIsError": false, + "lastToolResultContains": "state: OPEN" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "pull request 404 is open and mergeable" + } + } + } + ] + }, + { + "thread": "b", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "looked at pull request 404" + } + } + } + ] + }, + { + "thread": "a1", + "expect": { + "userTextContains": "a1 go ahead and finish" + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "reviewed pull request 101" + } + } + } + ] + }, + { + "thread": "a2", + "expect": { + "userTextContains": "a2 go ahead and finish" + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "reviewed pull request 202" + } + } + } + ] + } + ], + "assert": { + "admission": { + "peakAdmitted": 2, + "admittedBefore": [ + { + "first": "a1", + "then": "a3" + }, + { + "first": "a2", + "then": "a3" + }, + { + "first": "a1", + "then": "b" + }, + { + "first": "a2", + "then": "b" + }, + { + "first": "b", + "then": "a3" + } + ], + "completedBefore": [ + { + "first": "a1", + "then": "a2" + }, + { + "first": "a1", + "then": "a3" + }, + { + "first": "a2", + "then": "a3" + }, + { + "first": "a1", + "then": "b" + } + ] + } + } +} diff --git a/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml b/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml new file mode 100644 index 00000000..6613a209 --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml @@ -0,0 +1,144 @@ +# Fixture for the admission-burst bundle: two agents sharing one cluster. +# +# burst-reviewer is the reviewbot shape — two sandbox bundles per session, so a +# session's pod set is more than one pod and "all or none" means something. +# burst-notes is a second agent with its own line. Both run with the plan gate +# off and tool calls permissive: what the bundle pins is WHEN each session gets +# to run, and a gate in front of the tool call would only add scenery. +# +# burst-reviewer's channels.sleepAfter is one second, on purpose. A finished +# (Idle) session keeps its admission until it sleeps, so the sleep reap is what +# hands its room to the next session in line; a long grace would only slow the +# scenario. +apiVersion: v1 +kind: Secret +metadata: {name: burst-placeholder, namespace: default} +type: Opaque +stringData: {api-key: "unused-by-the-test-provider"} +--- +apiVersion: v1 +kind: Secret +metadata: {name: burst-fake-creds, namespace: default} +type: Opaque +stringData: {placeholder: "unused-by-the-fake-channel"} +--- +apiVersion: v1 +kind: Secret +metadata: {name: burst-gh-token, namespace: default} +type: Opaque +stringData: {token: "ghp_unusedTheSandboxNeverExecutes"} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: SpiceboxToolspec +metadata: {name: ts-burst-gh, namespace: default} +spec: + name: ts-burst-gh + version: "1" + toolkit: {name: gh, revision: "2026-05-01"} + allowSubcommands: ["pr view"] +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: SpiceboxClass +metadata: {name: cls-burst-gh, namespace: default} +spec: + image: registry.example.com/gh-toolbelt:latest + resources: {cpu: "100m", memory: "64Mi", ephemeralStorage: "16Mi"} + tools: + - {name: gh, command: ["/usr/bin/gh"]} + toolspecs: + - {name: ts-burst-gh} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentIdentity +metadata: {name: burst-identity, namespace: default} +spec: + description: "Fixture identity for the admission burst bundle." + credentials: + # The gh toolkit declares a sensitive GITHUB_TOKEN, so a class is not Valid + # until a credential of this name backs it. Never read: nothing executes. + - name: github-token + type: static + static: {secretRef: {name: burst-gh-token, key: token}} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: burst-reviewer, namespace: default} +spec: + displayName: BurstReviewer + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test # production operator REFUSES this + name: scripted # cosmetic; ignored at runtime + apiKey: {name: burst-placeholder, key: api-key} + systemPrompt: + inline: | + You are a fixture reviewer for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You review pull requests with the gitlike_gh tool. + agentIdentity: burst-identity + authz: + toolCalls: {mode: permissive} + planGate: {mode: disabled} + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + channels: {sleepAfter: 1s} + toolBundles: + - {name: "gitlike", class: "cls-burst-gh", toolspecs: ["ts-burst-gh"]} + - {name: "scratch", class: "cls-burst-gh", toolspecs: ["ts-burst-gh"]} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: burst-notes, namespace: default} +spec: + displayName: BurstNotes + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test # production operator REFUSES this + name: scripted # cosmetic; ignored at runtime + apiKey: {name: burst-placeholder, key: api-key} + systemPrompt: + inline: | + You are a fixture note-taker for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You check pull request status with the gitlike_gh tool. + agentIdentity: burst-identity + authz: + toolCalls: {mode: permissive} + planGate: {mode: disabled} + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + # Longer than the reviewer's: a3's room should come from a2 finishing, not + # from this session sleeping. See the bundle description. + channels: {sleepAfter: 10s} + toolBundles: + - {name: "gitlike", class: "cls-burst-gh", toolspecs: ["ts-burst-gh"]} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: burst-reviewer-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: burst-reviewer + credentialsRef: {secretName: burst-fake-creds} + fake: {echo: false} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: burst-notes-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: burst-notes + credentialsRef: {secretName: burst-fake-creds} + fake: {echo: false} +--- +# The tuple that makes `gh pr view` on demo-org/demo-repo resolve to ALLOW, so +# every session's call succeeds for the session's own reasons. The harness also +# waits on RelationshipsApplied, which a schema-only bootstrap never gets. +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: SpiceDBBootstrap +metadata: {name: burst-repo, namespace: default} +spec: + relationships: + - resource: {type: github_repo_url, id: "aHR0cHM6Ly9naXRodWIuY29tL2RlbW8tb3JnL2RlbW8tcmVwbw"} + relation: owner + subject: {type: user, id: "user@example.com", canonicalize: true} diff --git a/test/e2e/conversation.go b/test/e2e/conversation.go index fb5d7284..07821a49 100644 --- a/test/e2e/conversation.go +++ b/test/e2e/conversation.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" "github.com/authzed/openagentprimitives/pkg/channels/channelkinds" fakekind "github.com/authzed/openagentprimitives/pkg/channels/channelkinds/fake" @@ -22,10 +24,15 @@ import ( type SendOption func(*sendCfg) type sendCfg struct { - user, thread string - attachments []channelkinds.InboundAttachment + user, thread, channel string + attachments []channelkinds.InboundAttachment } +// OnChannel sends through the named Channel instead of the fixture's only +// conversational one — for a fixture with one fake Channel per AgentClass, +// where singleChannel would refuse to pick. +func OnChannel(name string) SendOption { return func(c *sendCfg) { c.channel = name } } + // AsUser overrides the requester identity for this inbound — both the // channel-side ExternalID and the email that drives the SpiceDB // canonical-user computation. Pass "alice@example.com" etc; never echo a @@ -74,7 +81,15 @@ func (h *Harness) SendUserMessage(text string, opts ...SendOption) { o(&cfg) } - ch := h.singleChannel("SendUserMessage") + var ch *spiceboxv1alpha1.Channel + if cfg.channel != "" { + ch = &spiceboxv1alpha1.Channel{} + if err := h.K8s.Get(context.Background(), client.ObjectKey{Namespace: h.Namespace(), Name: cfg.channel}, ch); err != nil { + h.t.Fatalf("SendUserMessage: get Channel %q: %v", cfg.channel, err) + } + } else { + ch = h.singleChannel("SendUserMessage") + } // The fake listener registers its Driver lazily on Start (a goroutine), so // a Channel applied shortly before this send races the listener-starter. // Poll for the driver — mirroring ExpectAgentReply's poll — instead of a diff --git a/test/e2e/harness.go b/test/e2e/harness.go index 92d50b43..6c496680 100644 --- a/test/e2e/harness.go +++ b/test/e2e/harness.go @@ -288,6 +288,10 @@ type Options struct { // // nil — every scenario but a replay — leaves assembly untouched. ReplaceAssembledTool func(tool.Tool) (tool.Tool, error) + + // Admission, when non-nil, gates session pods on whole-set admission and + // runs the admitter against its fixed capacity. See AdmissionOptions. + Admission *AdmissionOptions } // delayedSchemaIO wraps a guardian SchemaIO and stalls WriteSchema, widening @@ -1325,9 +1329,12 @@ func (h *Harness) startManager(t *testing.T, env *testenv.Env, spdbCli *spicedb. // mandatory-dependency check. DeniedLister: spdbCli, SlotGrantCopier: spdbCli, + // Zero (ungated) unless the scenario opted into admission. + AdmissionEnabled: h.opts.Admission != nil, }).SetupWithManager(mgr); err != nil { t.Fatalf("e2e: register AgentSession controller: %v", err) } + h.registerAdmitter(t, mgr) // SubagentRequest — authorizes and performs delegation (roster/identity // checks, child creation, lineage tuples) AND propagates the child's @@ -1546,12 +1553,14 @@ func (h *Harness) RestartOperator(t *testing.T) { ForkChecker: h.SpiceDB, // StartChecker: the start gate is enforced in e2e exactly as in // production; StartRefusedNoticePublish is left nil (logged and dropped). - StartChecker: h.SpiceDB, - DeniedLister: h.SpiceDB, - SlotGrantCopier: h.SpiceDB, + StartChecker: h.SpiceDB, + DeniedLister: h.SpiceDB, + SlotGrantCopier: h.SpiceDB, + AdmissionEnabled: h.opts.Admission != nil, }).SetupWithManager(mgr); err != nil { t.Fatalf("e2e: RestartOperator: register AgentSession controller: %v", err) } + h.registerAdmitter(t, mgr) mgrCtx, mgrCancel := context.WithCancel(context.Background()) mgrDone := make(chan struct{}) diff --git a/test/e2e/threadrun/driver.go b/test/e2e/threadrun/driver.go index e4896d0f..39c73de8 100644 --- a/test/e2e/threadrun/driver.go +++ b/test/e2e/threadrun/driver.go @@ -183,6 +183,8 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { HoldToolsFromAssembly: toolCatalog.HeldFromAssembly(), FilterOfferedTools: toolCatalog.Filter(), ReplaceAssembledTool: metaCanner.Replacer(), + Admission: admissionOptionsFor(b), + AwaitIdleTTL: awaitIdleTTLFor(b), }) // Canned outputs for the tools whose disposition is ReplayOutput. Keyed by @@ -426,8 +428,21 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { // function's own doc for why this lives in the general turn loop rather // than a per-bundle hook, and for the no-op it is for every other bundle. workshopSessionBound := false + // The admission watch starts before the first turn: what it guards — no + // partial set, never over capacity — must hold from the first session on. + var watch *admissionWatch + if b.Admission != nil { + watch = startAdmissionWatch(t, h, b) + } + opened := map[string]bool{} for i, turn := range b.UserTurns { - h.SendUserMessage(turn.Text, e2e.InThread("bt-"+b.Name), e2e.WithAttachments(bundleAttachments(t, dir, i, turn)...)) + waitForBarriers(t, h, b, turn) + opts := append(turnSendOptions(b, turn), e2e.WithAttachments(bundleAttachments(t, dir, i, turn)...)) + h.SendUserMessage(turn.Text, opts...) + if b.Admission != nil && turn.Thread != "" && !opened[turn.Thread] { + awaitArrival(t, h, b, turn.Thread) + } + opened[turn.Thread] = true // Contains and NotContains are positional and run on the SAME reply, so a // turn asserting both awaits its reply once and checks presence + absence // together. A turn with neither is not awaited (some turns produce no @@ -445,6 +460,14 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { bindWorkshopSessionOnce(t, h, &workshopSessionBound) } + // An admission bundle's barrier: every session admitted, ran and gave its + // room back. Before the transcript check, which is only meaningful once + // the last queued session has had its turn. + if watch != nil { + waitAllReleased(t, h, b) + checkAdmission(t, watch, b) + } + checkTranscriptConsumed(t, b, tr) // The phase barrier runs FIRST among the state checks: for a bundle whose // run ends without an agent reply (a denied guest start, a forensic hold) @@ -466,7 +489,12 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { checkSpiceDB(t, h, b) checkTranscriptLength(t, h, b) checkToolsCalled(t, h, b) - checkGoldenTrace(t, h, dir) + // A golden trace is ONE session's log. A multi-session bundle has several + // and no defined way to fold them into one file, so it has no golden (yet) + // and asserts on its sessions through its own claims instead. + if len(b.Threads()) == 0 { + checkGoldenTrace(t, h, dir) + } checkMintedIDsConsumed(t, mintedIDs) checkProviderIDsConsumed(t, providerIDs) checkToolCatalogConsulted(t, toolCatalog) @@ -1120,6 +1148,7 @@ func registerTranscript(t *testing.T, h *e2e.Harness, b bt.Bundle) *transcript { parts := parseReplyParts(t, i, step.Reply) captures := compileCaptures(t, i, step.Expect.Capture) scope := scopes[step.AgentClass] + opening := b.OpeningText(step.Thread) // "" for an unscoped step h.LLM.On(func(req llm.Request) bool { // Scope FIRST: an unmatched scope must leave the step pending for // its own session, so a request from the other side falls through @@ -1128,6 +1157,9 @@ func registerTranscript(t *testing.T, h *e2e.Harness, b bt.Bundle) *transcript { if scope != "" && !strings.Contains(systemPromptOf(req), scope) { return false } + if opening != "" && !mentionsText(req, opening) { + return false + } checkExpect(t, i, step.Expect, req) applyCaptures(t, i, captures, req, vars) tr.markConsumed(i) @@ -1177,12 +1209,16 @@ func checkTranscriptConsumed(t *testing.T, b bt.Bundle, tr *transcript) { tr.mu.Lock() defer tr.mu.Unlock() for i, step := range b.LLM { - if step.AgentClass == "" { + scope := step.AgentClass + if step.Thread != "" { + scope = "thread " + step.Thread + } + if scope == "" { continue } assert.True(t, tr.consumed[i], "llm step %d was scoped to %q and never ran: that session did not take the turn the transcript says it did", - i, step.AgentClass) + i, scope) } } diff --git a/test/e2e/threadrun/sessions.go b/test/e2e/threadrun/sessions.go new file mode 100644 index 00000000..82ef7d80 --- /dev/null +++ b/test/e2e/threadrun/sessions.go @@ -0,0 +1,410 @@ +//go:build e2e + +package threadrun + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/agent/llm" + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + bt "github.com/authzed/openagentprimitives/pkg/bronzethread" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/test/e2e" +) + +// Multi-session bundles: named threads, thread-scoped steps, turn barriers, +// and the admission watch. See pkg/bronzethread/sessions.go for the format. +// A bundle that names no thread takes none of these paths. + +// sessionBarrierTimeout bounds every wait on another session's state. Long +// enough for a session to queue behind a whole other session's turn under +// suite contention; short enough that a line that never admits fails by name +// rather than by the suite timeout. +const sessionBarrierTimeout = 2 * time.Minute + +// threadKey is the channel key a turn is sent on. Each bundle gets its own +// thread (see Run), and each NAMED thread inside it its own key, so its own +// AgentSession. +func threadKey(b bt.Bundle, thread string) string { + if thread == "" { + return "bt-" + b.Name + } + return "bt-" + b.Name + "-" + thread +} + +// turnSendOptions routes one user turn: its thread, and its Channel when the +// fixture has more than one. +func turnSendOptions(b bt.Bundle, turn bt.UserTurn) []e2e.SendOption { + opts := []e2e.SendOption{e2e.InThread(threadKey(b, turn.Thread))} + if turn.Channel != "" { + opts = append(opts, e2e.OnChannel(turn.Channel)) + } + return opts +} + +// mentionsText reports whether any user text block of req contains text. A +// thread's opening message is in every request its session makes — and, by +// bt.Bundle.Validate, in no other session's. +func mentionsText(req llm.Request, text string) bool { + for _, m := range req.Messages { + if m.Role != "user" { + continue + } + for _, c := range m.Content { + if c.Type == "text" && strings.Contains(c.Text, text) { + return true + } + } + } + return false +} + +// threadSession returns the AgentSession a named thread opened, if it exists. +func threadSession(ctx context.Context, h *e2e.Harness, b bt.Bundle, thread string) (*spiceboxv1alpha1.AgentSession, error) { + var list spiceboxv1alpha1.AgentSessionList + if err := h.K8s.List(ctx, &list, client.InNamespace(h.Namespace())); err != nil { + return nil, err + } + key := threadKey(b, thread) + for i := range list.Items { + if in := list.Items[i].Spec.InputChannel; in != nil && in.Key == key { + return &list.Items[i], nil + } + } + return nil, nil +} + +// barrierMet reports whether sess has reached the barrier's state. +func barrierMet(sess *spiceboxv1alpha1.AgentSession, until string) bool { + if sess == nil { + return false + } + switch until { + case bt.UntilAwaitingInput: + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseRunning && sess.Status.AwaitingUserInputSince != nil + case bt.UntilIdle: + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseIdle + } + return false +} + +// waitForBarriers blocks until every session the turn waits on has reached +// its state. +func waitForBarriers(t *testing.T, h *e2e.Harness, b bt.Bundle, turn bt.UserTurn) { + t.Helper() + for _, w := range turn.WaitFor { + cutoff := time.Now().Add(sessionBarrierTimeout) + var last *spiceboxv1alpha1.AgentSession + for { + sess, err := threadSession(context.Background(), h, b, w.Thread) + if err == nil { + last = sess + } + if barrierMet(sess, w.Until) { + break + } + if time.Now().After(cutoff) { + phase := "" + if last != nil { + phase = last.Status.Phase + } + t.Fatalf("turn %q waits for thread %q to be %s; it was not within %s (phase %s)", + turn.Text, w.Thread, w.Until, sessionBarrierTimeout, phase) + } + time.Sleep(100 * time.Millisecond) + } + } +} + +// awaitArrival blocks until the session a turn just opened exists, then until +// the wall clock has moved past the second it was created in. +// +// The line orders by creationTimestamp, which the apiserver keeps to the +// second, and breaks a tie on name — which, for a session, is a hash of its +// channel key. Two sessions opened in one second would therefore line up in +// an order unrelated to which arrived first, and a bundle claiming arrival +// order would be claiming a coin toss. Only the bundles with admission on +// pay this. +func awaitArrival(t *testing.T, h *e2e.Harness, b bt.Bundle, thread string) { + t.Helper() + cutoff := time.Now().Add(sessionBarrierTimeout) + for { + sess, err := threadSession(context.Background(), h, b, thread) + if err == nil && sess != nil { + next := sess.CreationTimestamp.Time.Truncate(time.Second).Add(time.Second) + if wait := time.Until(next); wait > 0 { + time.Sleep(wait + 10*time.Millisecond) + } + return + } + if time.Now().After(cutoff) { + t.Fatalf("thread %q: no AgentSession appeared within %s; the turn never reached the pipeline", thread, sessionBarrierTimeout) + } + time.Sleep(50 * time.Millisecond) + } +} + +// admissionWatch observes every named thread's session for the whole run and +// records what the admission scenario asserts on: when each was first seen +// admitted and first seen finished, how many were admitted at once, and every +// moment a session held part of a pod set it should hold all or none of. +// +// It is a property check the run cannot opt out of: the reconciler creates a +// session's bundle SpiceboxSessions one by one, so the claim is not "a set +// appears atomically" but the two states that MUST be whole — a session that +// was never admitted holds none, and a Running session holds every one. +type admissionWatch struct { + b bt.Bundle + h *e2e.Harness + stop context.CancelFunc + done chan struct{} + + mu sync.Mutex + seq int + firstAdmitted map[string]int // thread -> observation number + firstIdle map[string]int + peak int + violations []string + lastErr error +} + +func startAdmissionWatch(t *testing.T, h *e2e.Harness, b bt.Bundle) *admissionWatch { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + w := &admissionWatch{ + b: b, h: h, stop: cancel, done: make(chan struct{}), + firstAdmitted: map[string]int{}, firstIdle: map[string]int{}, + } + go func() { + defer close(w.done) + tick := time.NewTicker(50 * time.Millisecond) + defer tick.Stop() + for { + w.observe(ctx) + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } + }() + t.Cleanup(w.halt) + return w +} + +func (w *admissionWatch) halt() { + w.stop() + <-w.done +} + +// observe takes one snapshot. A list error skips the snapshot (recorded, and +// reported only if nothing was ever observed): a missed snapshot can only +// lose an observation, never invent a violation. +// +// The lists are not one atomic read, so each claim is checked across a pair +// read in the order that makes it sound (h.K8s is the uncached client, so the +// reads are ordered at the apiserver): +// +// - "Running holds every bundle" reads sessions, THEN bundles: a session +// Running at the first read already had its whole set, and nothing +// reclaims a Running session's set. +// - "waiting holds no bundle" reads bundles, THEN sessions: a bundle seen at +// the first read was created after its session was admitted, so the +// session reads admitted at the second. Read the other way round, a +// session admitted and provisioned between the two reads would look like +// a waiting session holding its set. +func (w *admissionWatch) observe(ctx context.Context) { + ns := client.InNamespace(w.h.Namespace()) + var before, after spiceboxv1alpha1.AgentSessionList + var boxes spiceboxv1alpha1.SpiceboxSessionList + var classes spiceboxv1alpha1.AgentClassList + err := w.h.K8s.List(ctx, &classes, ns) + if err == nil { + err = w.h.K8s.List(ctx, &before, ns) + } + if err == nil { + err = w.h.K8s.List(ctx, &boxes, ns) + } + if err == nil { + err = w.h.K8s.List(ctx, &after, ns) + } + w.mu.Lock() + defer w.mu.Unlock() + if err != nil { + w.lastErr = err + return + } + w.seq++ + + bundlesOf := map[string]int{} + for i := range classes.Items { + bundlesOf[classes.Items[i].Name] = len(classes.Items[i].Spec.ToolBundles) + } + held := map[string]int{} + for i := range boxes.Items { + if boxes.Items[i].DeletionTimestamp == nil { + held[boxes.Items[i].Labels["agentprimitives.authzed.com/agentsession"]]++ + } + } + + prefix := threadKey(w.b, "") + "-" + threadOf := func(s *spiceboxv1alpha1.AgentSession) (string, bool) { + if s.Spec.InputChannel == nil || !strings.HasPrefix(s.Spec.InputChannel.Key, prefix) { + return "", false + } + return strings.TrimPrefix(s.Spec.InputChannel.Key, prefix), true + } + + for i := range before.Items { + s := &before.Items[i] + thread, ok := threadOf(s) + if !ok || s.Status.Phase != spiceboxv1alpha1.AgentSessionPhaseRunning { + continue + } + if n, want := held[s.Name], bundlesOf[s.Spec.Class]; n != want { + w.violations = append(w.violations, fmt.Sprintf( + "observation %d: thread %q (%s) is Running with %d of its %d bundle SpiceboxSessions", + w.seq, thread, s.Name, n, want)) + } + } + + admittedNow := 0 + for i := range after.Items { + s := &after.Items[i] + thread, ok := threadOf(s) + if !ok { + continue + } + phase := s.Status.Phase + admitted := admission.IsAdmitted(s) + terminal := phase == spiceboxv1alpha1.AgentSessionPhaseSucceeded || phase == spiceboxv1alpha1.AgentSessionPhaseFailed + if admitted && !terminal { + admittedNow++ + if w.firstAdmitted[thread] == 0 { + w.firstAdmitted[thread] = w.seq + } + } + if phase == spiceboxv1alpha1.AgentSessionPhaseIdle && w.firstAdmitted[thread] != 0 && w.firstIdle[thread] == 0 { + w.firstIdle[thread] = w.seq + } + booting := phase == "" || phase == spiceboxv1alpha1.AgentSessionPhasePending + if n := held[s.Name]; booting && !admitted && n > 0 { + w.violations = append(w.violations, fmt.Sprintf( + "observation %d: thread %q (%s) is waiting unadmitted yet holds %d of its %d bundle SpiceboxSessions", + w.seq, thread, s.Name, n, bundlesOf[s.Spec.Class])) + } + } + w.peak = max(w.peak, admittedNow) +} + +// waitAllReleased blocks until every named thread's session has finished and +// given its admission back through the sleep reap (status.sleptAt). That is +// the end of the line's story: every session admitted, ran, and freed its room. +func waitAllReleased(t *testing.T, h *e2e.Harness, b bt.Bundle) { + t.Helper() + cutoff := time.Now().Add(sessionBarrierTimeout) + for { + var pending []string + for _, th := range b.Threads() { + sess, err := threadSession(context.Background(), h, b, th) + if err != nil || sess == nil || sess.Status.SleptAt == nil { + pending = append(pending, th) + } + } + if len(pending) == 0 { + return + } + if time.Now().After(cutoff) { + t.Fatalf("threads %v never finished and released their admission within %s", pending, sessionBarrierTimeout) + } + time.Sleep(100 * time.Millisecond) + } +} + +// checkAdmission asserts the watch's record against the bundle's claims, and +// the properties every admission bundle holds without claiming them: no +// partial set, never more admitted than the capacity, and every thread +// admitted and finished. +func checkAdmission(t *testing.T, w *admissionWatch, b bt.Bundle) { + t.Helper() + w.halt() + w.mu.Lock() + defer w.mu.Unlock() + require.NotZero(t, w.seq, "the admission watch never observed the cluster (last error: %v)", w.lastErr) + t.Logf("admission record over %d observations: admitted %v, completed %v, peak %d admitted at once", + w.seq, sortedRecord(w.firstAdmitted), sortedRecord(w.firstIdle), w.peak) + + assert.Empty(t, w.violations, "a session held part of a pod set it must hold all or none of") + assert.LessOrEqual(t, w.peak, b.Admission.CapacitySessions, + "more sessions were admitted at once than the cluster has room for") + for _, th := range b.Threads() { + assert.NotZero(t, w.firstAdmitted[th], "thread %q was never admitted", th) + assert.NotZero(t, w.firstIdle[th], "thread %q never finished its work after admission", th) + } + + a := b.Assert.Admission + if a == nil { + return + } + if a.PeakAdmitted > 0 { + assert.Equal(t, a.PeakAdmitted, w.peak, "the most sessions admitted at once") + } + order := func(kind string, first map[string]int, claims []bt.ThreadOrder) { + for _, o := range claims { + f, s := first[o.First], first[o.Then] + assert.True(t, f != 0 && s != 0 && f < s, + "%s: thread %q must come strictly before %q (first observed at %d and %d; full record %v)", + kind, o.First, o.Then, f, s, sortedRecord(first)) + } + } + order("admitted", w.firstAdmitted, a.AdmittedBefore) + order("completed", w.firstIdle, a.CompletedBefore) +} + +// sortedRecord renders a first-observation map in observation order, so a +// failed ordering claim shows the order the run actually took. +func sortedRecord(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.SortFunc(keys, func(a, b string) int { return m[a] - m[b] }) + out := make([]string, 0, len(keys)) + for _, k := range keys { + out = append(out, fmt.Sprintf("%s@%d", k, m[k])) + } + return out +} + +// admissionOptionsFor turns a bundle's admission block into the harness +// option, or nil for every bundle that runs ungated. +func admissionOptionsFor(b bt.Bundle) *e2e.AdmissionOptions { + if b.Admission == nil { + return nil + } + return &e2e.AdmissionOptions{Capacity: e2e.FixedSetCapacity{Sets: b.Admission.CapacitySessions}} +} + +// awaitIdleTTLFor keeps a session parked in await_user_message for as long as +// a barrier may wait on it, for a bundle that waits on one. Every other bundle +// gets zero: await idle-exits at once, the path they were written against. +func awaitIdleTTLFor(b bt.Bundle) time.Duration { + for _, ut := range b.UserTurns { + for _, w := range ut.WaitFor { + if w.Until == bt.UntilAwaitingInput { + return 2 * sessionBarrierTimeout + } + } + } + return 0 +} From f62e3a1085aba70c607ff68ac1d8de6225d334dd Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 14:05:27 -0500 Subject: [PATCH 11/33] Keep unscoped transcript steps unscoped OpeningText for the unnamed thread returned the first user turn's text, which scoped every unscoped step to the parent session and stranded a delegated child's requests. --- pkg/bronzethread/sessions.go | 6 +++++- pkg/bronzethread/sessions_test.go | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/bronzethread/sessions.go b/pkg/bronzethread/sessions.go index c9d9d1c8..54bded5c 100644 --- a/pkg/bronzethread/sessions.go +++ b/pkg/bronzethread/sessions.go @@ -91,8 +91,12 @@ func (b Bundle) Threads() []string { // OpeningText is the text of the first user turn on thread: the message that // opened that thread's session, and so the first user message of every model // request the session makes. It is how a thread-scoped step recognizes its -// session's requests. +// session's requests. The unnamed thread has none: an unscoped step must match +// every session's requests, including a delegated child's. func (b Bundle) OpeningText(thread string) string { + if thread == "" { + return "" + } for _, ut := range b.UserTurns { if ut.Thread == thread { return ut.Text diff --git a/pkg/bronzethread/sessions_test.go b/pkg/bronzethread/sessions_test.go index 9881d3e6..6b623675 100644 --- a/pkg/bronzethread/sessions_test.go +++ b/pkg/bronzethread/sessions_test.go @@ -124,4 +124,6 @@ func TestBundle_ThreadsAndOpeningText(t *testing.T) { assert.Equal(t, "review pull request 1", b.OpeningText("a1"), "the first turn, not the follow-up") assert.Empty(t, b.OpeningText("zz")) assert.Empty(t, runnableBundle().Threads(), "a single-session bundle names no threads") + assert.Empty(t, runnableBundle().OpeningText(""), + "the unnamed thread has no opening text, so an unscoped step stays unscoped") } From d57412e035a289b80f3974873b902c3cf9b6ce0e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 14:24:34 -0500 Subject: [PATCH 12/33] Bound the misfit release and pin status.admission A placement-mismatch release now fails the session, as the waiting timeout does, once it has been in line longer than the capacity-hold timeout since it joined: a stall the admitter's model cannot see would otherwise re-admit and release forever, since each release restarts both the waiting clock and the bundle deadline. The release also clears the bundle stage's CapacityHold, whose give-up time described the pods it just reaped. status.admission is admitter-owned; the runner-forgery webhook now refuses a runner write to it. --- .../agentsession/admission_gate.go | 28 ++++++- .../agentsession/admission_gate_test.go | 76 +++++++++++++++---- .../webhooks/agentsession/webhook.go | 9 +++ .../webhooks/agentsession/webhook_test.go | 7 ++ 4 files changed, 104 insertions(+), 16 deletions(-) diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index fb6ead21..8e9e6ea2 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -148,12 +148,28 @@ func (r *Reconciler) placementMisfit(sess *spiceboxv1alpha1.AgentSession, stall // fairly; the model was wrong, not the session). admission.WaitingSince // restarts at this release, so the capacity-hold timeout gives it a fresh // window rather than the remainder of its first one. +// +// That fresh window is why the release is bounded by JoinedAt instead. A +// stall the admitter's model cannot see — a shortfall on the only nodes the +// pod tolerates, beside tainted nodes the model still counts — re-admits the +// session at once, stalls again, and releases again: WaitingSince and the +// bundle deadline both restart every cycle, so neither would ever fire. Once +// the session has been in line longer than the capacity-hold timeout since it +// joined, the release becomes the same failure the waiting timeout gives. +// +// The bundle stage's CapacityHold=True ("gives up at …") is released in the +// same write: its give-up time described pods that no longer exist, and left +// standing, every later stall would rewrite it into a fresh "queued" notice. func (r *Reconciler) releaseMisfit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, stall schedulingStall) (ctrl.Result, error) { logger := log.FromContext(ctx) - logger.Info("admitted session's pods could not be placed; releasing them and re-queueing it at its place in line", + timeout := r.effectiveCapacityHoldTimeout() + joined := admission.JoinedAt(sess) + expired := timeout > 0 && r.now().Sub(joined) > timeout + logger.Info("admitted session's pods could not be placed; releasing them", "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, "pod", stall.podName, "unschedulableFor", stall.pending.String(), - "schedulerMessage", stall.reason) + "schedulerMessage", stall.reason, "joinedAt", joined.UTC().Format(time.RFC3339), + "failingAfterCapacityHoldTimeout", expired) if _, err := r.reapSessionPods(ctx, sess); err != nil { return ctrl.Result{}, err // already logged inside reapSessionPods; retried next pass } @@ -161,9 +177,15 @@ func (r *Reconciler) releaseMisfit(ctx context.Context, sess *spiceboxv1alpha1.A Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, Reason: spiceboxv1alpha1.ReasonAdmissionMisfit, - Message: fmt.Sprintf("%s could not be placed for %s (%s); its pods were released and it waits at its place in line", stall.podName, stall.pending.Round(time.Second), strings.TrimSuffix(stall.reason, ".")), + Message: fmt.Sprintf("%s could not be placed for %s (%s); its pods were released", stall.podName, stall.pending.Round(time.Second), strings.TrimSuffix(stall.reason, ".")), LastTransitionTime: metav1.NewTime(r.now()), }) + if expired { + r.expireCapacityHold(sess, timeout) + return r.markBootFailed(ctx, sess, spiceboxv1alpha1.ReasonAgentSessionBundleFail, + capacityHoldExpiredMessage("sandbox bundle(s)", timeout, stall)) + } + r.releaseCapacityHold(sess) if err := r.applyStatus(ctx, sess); err != nil { logger.Info("misfit: recording the admission release failed; will retry", "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, "err", err.Error()) diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index a3441c8a..5d5d55c8 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -288,27 +288,37 @@ func ptrTo[T any](v T) *T { return &v } // An ADMITTED booting session whose bundle pod the scheduler still cannot // place after admissionMisfitGrace gives its pods back and returns to its line // (Admitted=False/PlacementMismatch), keeping its place: JoinedAt does not -// move. Short of the grace, with admission off, or on a stall another session -// finishing cannot cure, nothing is released. +// move, and the bundle stage's stale CapacityHold goes with the pods. Short of +// the grace, with admission off, or on a stall another session finishing +// cannot cure, nothing is released. Once the session has been in line longer +// than the capacity-hold timeout since it joined, the release fails it +// instead, so a stall the admitter's model cannot see cannot cycle forever. func TestAdmissionGate_Misfit(t *testing.T) { nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + // A shortfall on the nodes the pod tolerates, beside a tainted node the + // admitter's model still counts as room: capacityShortfall says yes, and + // the model re-admits at once after every release. + const mixedSchedMsg = "0/4 nodes are available: 3 Insufficient cpu, 1 node(s) had untolerated taint(s)." cases := []struct { - name string - enabled bool // Reconciler.AdmissionEnabled - admitted bool // the session carries Admitted=True - pending time.Duration // how long the bundle pod has been unschedulable - schedMsg string - woken bool // the session was cold-woken before this admission + name string + enabled bool // Reconciler.AdmissionEnabled + admitted bool // the session carries Admitted=True + pending time.Duration // how long the bundle pod has been unschedulable + createdAgo time.Duration // session creation (its JoinedAt); 0 = 20m + schedMsg string + woken bool // the session was cold-woken before this admission + heldBefore bool // the bundle stage already set CapacityHold=True/Waiting wantReaped bool wantMisfit bool // Admitted=False/PlacementMismatch at now + wantFailed bool // Failed/BundleFailed, CapacityHold=False/HoldExpired wantAdmittedNil bool // no Admitted condition at all }{ { - name: "admitted, pods unschedulable 2m: pods reaped, Admitted=False/PlacementMismatch, JoinedAt unchanged", - enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, + name: "admitted, pods unschedulable 2m: pods reaped, Admitted=False/PlacementMismatch, JoinedAt unchanged, CapacityHold released", + enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, heldBefore: true, wantReaped: true, wantMisfit: true, }, { @@ -316,6 +326,12 @@ func TestAdmissionGate_Misfit(t *testing.T) { enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, woken: true, wantReaped: true, wantMisfit: true, }, + { + name: "admitted, mixed taint+Insufficient stall, released and re-admitted repeatedly: fails once JoinedAt+timeout passes", + enabled: true, admitted: true, pending: 2 * time.Minute, createdAgo: 70 * time.Minute, + schedMsg: mixedSchedMsg, heldBefore: true, + wantReaped: true, wantMisfit: true, wantFailed: true, + }, { name: "admitted, pods unschedulable 1m (inside the misfit grace): pods kept, still admitted", enabled: true, admitted: true, pending: time.Minute, schedMsg: capacitySchedMsg, @@ -335,9 +351,13 @@ func TestAdmissionGate_Misfit(t *testing.T) { t.Run(tc.name, func(t *testing.T) { clock := func() time.Time { return nowT } started := nowT.Add(-tc.pending) + createdAgo := tc.createdAgo + if createdAgo == 0 { + createdAgo = 20 * time.Minute + } ac := classWithBundle("ac1", codeBundle) - sess := sessionCreatedAt("s1", "ac1", nowT.Add(-20*time.Minute)) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-createdAgo)) if tc.woken { wake := metav1.NewTime(nowT.Add(-10 * time.Minute)) sess.Status.LastWakeAt = &wake @@ -345,6 +365,14 @@ func TestAdmissionGate_Misfit(t *testing.T) { if tc.admitted { withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, started.Add(-time.Second)) } + if tc.heldBefore { + sess.Status.Conditions = append(sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionCapacityHold, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonCapacityHoldWaiting, + Message: "The cluster is out of room right now; this will start on its own once capacity frees up (gives up at 17:58 UTC).", + LastTransitionTime: metav1.NewTime(started.Add(30 * time.Second)), + }) + } joinedBefore := admission.JoinedAt(sess) bundle := bundleWithPod("s1", "code", "s1-code-pod", started) pod := pendingUnschedulablePod("s1-code-pod", "uid-1", started, tc.schedMsg) @@ -357,9 +385,29 @@ func TestAdmissionGate_Misfit(t *testing.T) { assert.Equal(t, !tc.wantReaped, exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), "bundle SpiceboxSession present") - assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") assert.Equal(t, joinedBefore.UTC(), admission.JoinedAt(&got).UTC(), "JoinedAt must not move: the session keeps its place in line") + hold := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionCapacityHold) + if tc.wantFailed { + require.Equal(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "a session in line past the timeout fails") + assert.Equal(t, spiceboxv1alpha1.ReasonAgentSessionBundleFail, got.Status.FailureReason) + if assert.NotNil(t, hold, "CapacityHold condition") { + assert.Equal(t, metav1.ConditionFalse, hold.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonCapacityHoldExpired, hold.Reason) + } + fc := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionFailed) + if assert.NotNil(t, fc, "Failed condition") { + assert.Contains(t, fc.Message, "waited 1h0m0s for cluster capacity") + assert.Contains(t, fc.Message, "Insufficient cpu") + } + } else { + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") + if tc.wantMisfit && tc.heldBefore && assert.NotNil(t, hold, "CapacityHold condition") { + assert.Equal(t, metav1.ConditionFalse, hold.Status, "the bundle stage's hold goes with the released pods") + assert.Equal(t, spiceboxv1alpha1.ReasonCapacityHoldReleased, hold.Reason) + } + } + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) switch { case tc.wantAdmittedNil: @@ -369,7 +417,9 @@ func TestAdmissionGate_Misfit(t *testing.T) { assert.Equal(t, metav1.ConditionFalse, adm.Status) assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionMisfit, adm.Reason) assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") - assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting-timeout clock restarts at the release") + if !tc.wantFailed { + assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting-timeout clock restarts at the release") + } default: require.NotNil(t, adm, "Admitted condition") assert.Equal(t, metav1.ConditionTrue, adm.Status, "still admitted") diff --git a/pkg/controllers/webhooks/agentsession/webhook.go b/pkg/controllers/webhooks/agentsession/webhook.go index efc3b075..9e9a9d5c 100644 --- a/pkg/controllers/webhooks/agentsession/webhook.go +++ b/pkg/controllers/webhooks/agentsession/webhook.go @@ -447,6 +447,15 @@ var pinnedStatusFields = []pinnedStatusField{ "fork gate and the recovery mechanism against a runner doing exactly this. channelsd also reads it as the " + "tie-breaker for which session owns a thread (newestActiveSession)", }, + { + name: "admission", + get: func(s *spiceboxv1alpha1.AgentSessionStatus) any { return s.Admission }, + why: "it is the admitter's record of this session's place in its AgentClass's admission line — position, line length " + + "and join time — and only the leader-only admitter writes it (pkg/controllers/admission/admitter). The admitter " + + "compares against the stored value to decide whether to write at all, so a forged value is not corrected until the " + + "real position next changes, and it is the source the 'Nth in line' notice to the waiting human is built from. " + + "A runner has no business in the queue it is waiting in", + }, { name: "effectiveSettings", get: func(s *spiceboxv1alpha1.AgentSessionStatus) any { return s.EffectiveSettings }, diff --git a/pkg/controllers/webhooks/agentsession/webhook_test.go b/pkg/controllers/webhooks/agentsession/webhook_test.go index 8e4ddb5d..a00507c1 100644 --- a/pkg/controllers/webhooks/agentsession/webhook_test.go +++ b/pkg/controllers/webhooks/agentsession/webhook_test.go @@ -625,6 +625,13 @@ func TestRunnerCannotForgeSessionIdentity(t *testing.T) { s.Status.AppliedInteractPermissionAt = &t }, }, + { + name: "runner rewrites status.admission to the head of its line: denied (the admitter alone owns the queue position a waiting human is shown)", + sub: "status", + mut: func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Admission = &spiceboxv1alpha1.AdmissionStatus{Position: 1, LineLength: 1} + }, + }, { // The operator turns spiceboxSessionName into BOTH a Role // resourceName and a memory-API token scope, on every reconcile, From 426fbecbaf010284db2e1cebc26905292c17bae9 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 15:16:32 -0500 Subject: [PATCH 13/33] Plan early reclaim of warm idle sessions Pure planning: oldest-idle warm sessions of the head's line, freed on a cloned model until the head fits. --- pkg/controllers/admission/reclaim.go | 49 +++++++++ pkg/controllers/admission/reclaim_test.go | 128 ++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 pkg/controllers/admission/reclaim.go create mode 100644 pkg/controllers/admission/reclaim_test.go diff --git a/pkg/controllers/admission/reclaim.go b/pkg/controllers/admission/reclaim.go new file mode 100644 index 00000000..8e6ab643 --- /dev/null +++ b/pkg/controllers/admission/reclaim.go @@ -0,0 +1,49 @@ +package admission + +import ( + "cmp" + "slices" + "time" + + "k8s.io/apimachinery/pkg/types" +) + +// WarmIdle is an Idle session whose pods still hold capacity and could be +// reclaimed early. +type WarmIdle struct { + Key types.NamespacedName + Line string + IdleSince time.Time + Pods []PlacedPod +} + +// PlanReclaim picks which warm Idle sessions of the head's own line to +// reclaim so the head fits. Only sessions idle at least minWarm are eligible, +// oldest first (ties by key), freed one at a time on a clone of model until +// the head's demand reserves; the freed prefix is returned. It returns nil +// when the model is unknown, when the head already fits, or when freeing every +// eligible session still does not fit. The caller's model is never mutated. +func PlanReclaim(head Candidate, warm []WarmIdle, model CapacityModel, now time.Time, minWarm time.Duration) []types.NamespacedName { + if !model.Known() || model.Clone().Reserve(head.Demand) { + return nil + } + var eligible []WarmIdle + for _, w := range warm { + if w.Line == head.Line && now.Sub(w.IdleSince) >= minWarm { + eligible = append(eligible, w) + } + } + slices.SortFunc(eligible, func(a, b WarmIdle) int { + return cmp.Or(a.IdleSince.Compare(b.IdleSince), cmp.Compare(a.Key.String(), b.Key.String())) + }) + scratch := model.Clone() + var plan []types.NamespacedName + for _, w := range eligible { + scratch.Free(w.Pods) + plan = append(plan, w.Key) + if scratch.Reserve(head.Demand) { + return plan + } + } + return nil +} diff --git a/pkg/controllers/admission/reclaim_test.go b/pkg/controllers/admission/reclaim_test.go new file mode 100644 index 00000000..12d44b69 --- /dev/null +++ b/pkg/controllers/admission/reclaim_test.go @@ -0,0 +1,128 @@ +package admission + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func warmOn(name, line string, idleFor time.Duration, now time.Time, cpuMilli int64) WarmIdle { + return WarmIdle{ + Key: types.NamespacedName{Namespace: "ns", Name: name}, + Line: line, + IdleSince: now.Add(-idleFor), + Pods: []PlacedPod{{Node: "a", PodDemand: pod(name, cpuMilli)}}, + } +} + +// fullNode is one 1-CPU node already holding two 500m pods, so it has no +// headroom until a warm session's pod is freed. +func fullNode() CapacityModel { + return NewNodeModel( + []corev1.Node{testNode("a", "1", "1Gi", "10")}, + []corev1.Pod{testPod("a", "500m"), testPod("a", "500m")}, + ) +} + +func TestPlanReclaim(t *testing.T) { + now := time.Date(2026, 11, 10, 12, 0, 0, 0, time.UTC) + head := func(cpu int64) Candidate { + return Candidate{Key: types.NamespacedName{Namespace: "ns", Name: "head"}, Line: "agent-a", Demand: Demand{Pods: []PodDemand{pod("h", cpu)}}} + } + key := func(n string) types.NamespacedName { return types.NamespacedName{Namespace: "ns", Name: n} } + + cases := []struct { + name string + head Candidate + warm []WarmIdle + model func() CapacityModel + minWarm time.Duration + want []types.NamespacedName + }{ + { + name: "head fits after freeing the oldest of two eligible: returns only the oldest", + head: head(500), + model: fullNode, + warm: []WarmIdle{ + warmOn("newer", "agent-a", 10*time.Minute, now, 500), + warmOn("oldest", "agent-a", 30*time.Minute, now, 500), + }, + minWarm: 5 * time.Minute, + want: []types.NamespacedName{key("oldest")}, + }, + { + name: "head needs both eligible sessions: returns both, oldest first", + head: head(1000), + model: fullNode, + warm: []WarmIdle{ + warmOn("newer", "agent-a", 10*time.Minute, now, 500), + warmOn("oldest", "agent-a", 30*time.Minute, now, 500), + }, + minWarm: 5 * time.Minute, + want: []types.NamespacedName{key("oldest"), key("newer")}, + }, + { + name: "idle-since tie: breaks on key order", + head: head(500), + model: fullNode, + warm: []WarmIdle{warmOn("b", "agent-a", 10*time.Minute, now, 500), warmOn("a", "agent-a", 10*time.Minute, now, 500)}, + minWarm: time.Minute, + want: []types.NamespacedName{key("a")}, + }, + { + name: "candidate idle 4m with minWarm 5m: not eligible, returns nil", + head: head(500), + model: fullNode, + warm: []WarmIdle{warmOn("young", "agent-a", 4*time.Minute, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "other line's warm session: never chosen", + head: head(500), + model: fullNode, + warm: []WarmIdle{warmOn("other", "agent-b", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "freeing every eligible session still does not fit: nil, nothing reclaimed", + head: head(1500), + model: fullNode, + warm: []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500), warmOn("w2", "agent-a", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "head already fits without freeing: nil", + head: head(500), + model: func() CapacityModel { return NewNodeModel([]corev1.Node{testNode("a", "1", "1Gi", "10")}, nil) }, + warm: []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + { + name: "unknown capacity: nil, unknown capacity already admits", + head: head(500), + model: func() CapacityModel { return NewNodeModel(nil, nil) }, + warm: []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500)}, + minWarm: 5 * time.Minute, + want: nil, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, PlanReclaim(tc.head, tc.warm, tc.model(), now, tc.minWarm)) + }) + } + + t.Run("plan does not mutate the caller's model", func(t *testing.T) { + m := fullNode() + w := []WarmIdle{warmOn("w1", "agent-a", time.Hour, now, 500)} + assert.Equal(t, []types.NamespacedName{key("w1")}, PlanReclaim(head(500), w, m, now, time.Minute)) + assert.False(t, m.Reserve(head(500).Demand), "caller's model must still be full after planning") + }) +} From 45ecac60f6b329758ceff5e71e08574df1822691 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 16:01:08 -0500 Subject: [PATCH 14/33] Reclaim warm idle sessions early when their agent's line is waiting After deciding, the admitter plans an early reclaim for each line whose head does not fit: warm Idle sessions of that line idle at least --admission-min-warm, oldest first, until the head fits. It asks for each through the existing sleep path with a reclaim-requested annotation. The AgentSession reconciler treats that annotation as "sleep now" (still behind the SessionHold guard), removes it once the session is slept, and clears it as stale when the session left Idle. --- internal/cmd/operator/main.go | 9 +- pkg/apis/v1alpha1/conditions.go | 9 ++ .../admission/admitter/controller.go | 123 ++++++++++++++++- .../admission/admitter/controller_test.go | 127 ++++++++++++++++++ pkg/controllers/agentsession/controller.go | 5 + pkg/controllers/agentsession/sleep.go | 69 ++++++++++ pkg/controllers/agentsession/sleep_test.go | 113 ++++++++++++++++ 7 files changed, 450 insertions(+), 5 deletions(-) diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index 37378c8b..d7a5a6be 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -764,9 +764,9 @@ type config struct { // sessionAdmission gates every booting AgentSession's pods on the // admitter's whole-set, fair-share decision; false admits at once. sessionAdmission bool - // admissionMinWarm is parsed and stored only; its consumer (how long an - // idle session keeps its admission before it may be reclaimed) is not - // wired yet. + // admissionMinWarm is how long an idle session keeps its admission (and + // warm pods) before the admitter may reclaim it early for a session of its + // AgentClass waiting in line. admissionMinWarm time.Duration oauthRefreshThreshold time.Duration workspaceStorageClass string @@ -896,7 +896,7 @@ func newCommand() *cobra.Command { fs.BoolVar(&cfg.sessionAdmission, "session-admission", true, "Admit each booting AgentSession's whole pod set against node capacity before any of its pods is created, in join order with fair share between AgentClasses; sessions that do not fit wait in line. Set to false to admit every session at once (pods then queue in the scheduler as before).") fs.DurationVar(&cfg.admissionMinWarm, "admission-min-warm", 5*time.Minute, - "Minimum time an idle AgentSession keeps its admission (and warm pods) before it may be reclaimed for a session waiting in line.") + "Minimum time an idle AgentSession keeps its admission (and warm pods) before the admitter may put it to sleep early, ahead of its sleepAfter grace, to make room for a session of the same AgentClass waiting in line. A held (SessionHold) session is never reclaimed. Only applies with --session-admission.") fs.DurationVar(&cfg.oauthRefreshThreshold, "oauth-refresh-threshold", 5*time.Minute, "Default refresh threshold for type=oauth AgentIdentity credentials. AgentIdentities with spec.refreshThreshold override this per-identity.") fs.StringVar(&cfg.workspaceStorageClass, "workspace-storage-class", "", @@ -2778,6 +2778,7 @@ func run(cfg *config) { Namespaces: slices.Sorted(maps.Keys(cacheNamespaces(cfg.watchNamespaces, operatorNamespace(cfg.podNamespace)))), Capacity: admissionCapacity, Enabled: cfg.sessionAdmission, + MinWarm: cfg.admissionMinWarm, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to register admission controller") os.Exit(1) diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index 7c512f22..c3738de7 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -1051,6 +1051,15 @@ const ( // coalesces rather than duplicating. AnnotationUIServeRequestedAt = "agentprimitives.authzed.com/ui-serve-requested-at" + // AnnotationReclaimRequested asks the operator to put a warm Idle session + // to sleep now, ahead of its sleepAfter grace, because a session of the + // same AgentClass is waiting in the admission line and freeing this one's + // pods lets it in. Written by the admitter (RFC3339 of the pass that asked; + // left alone if already present); the AgentSession reconciler's sleep path + // honors it — still behind the SessionHold guard — and removes it once the + // session is slept, or as stale once the session has left Idle. + AnnotationReclaimRequested = "agentprimitives.authzed.com/reclaim-requested" + // AnnotationSidecarFailureReported carries, as a JSON object, the // sidecar-name → failure-fingerprint map the runner has already reported to // the user — the durable half of its once-per-failure contract, since a diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 5fd10859..2142050e 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -15,6 +15,8 @@ import ( "context" "errors" "fmt" + "slices" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -33,9 +35,10 @@ import ( "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" "github.com/authzed/openagentprimitives/pkg/controllers/agentstatus" + "github.com/authzed/openagentprimitives/pkg/platform/schedfit" ) -// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions,verbs=get;list;watch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions,verbs=get;list;watch;patch // +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions/status,verbs=get;update;patch // +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentclasses,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch @@ -80,6 +83,10 @@ type Reconciler struct { // Enabled false admits every waiting session at once, with no capacity // check (--session-admission=false). Enabled bool + // MinWarm is how long an Idle session keeps its warm pods before a pass may + // ask for them back early for a session of its line that does not fit + // (--admission-min-warm). + MinWarm time.Duration // Now is the clock; nil means time.Now. Now func() time.Time // Demand prices a session's whole pod set. Nil means the default: get the @@ -190,6 +197,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R } pods, podsKnown := r.listPods(ctx) + var warm []admission.WarmIdle + warmByKey := map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{} for _, sess := range holding { d, err := r.demand(ctx, sess) if err != nil { @@ -206,6 +215,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R Demand: d, Unplaced: unplaced(sess.Namespace, d, pods, podsKnown), }) + if w, ok := warmIdle(sess, d, pods, podsKnown); ok { + warm = append(warm, w) + warmByKey[w.Key] = sess + } } in.Model = r.snapshot(ctx) @@ -227,9 +240,117 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R errs = append(errs, err) } } + if in.Model.Known() { + errs = append(errs, r.requestReclaims(ctx, in.Waiting, dec, warm, warmByKey, in.Model, now)...) + } return requeue, errors.Join(errs...) } +// isWarmIdle matches a holder whose pods could be given back early: Idle, +// channel-attached (the AgentSession reconciler's sleep path, which honors the +// reclaim request, runs only for those), not yet slept, with a known idle +// start for PlanReclaim's minWarm check. +func isWarmIdle(sess *spiceboxv1alpha1.AgentSession) bool { + return sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseIdle && + sess.Spec.InputChannel != nil && + sess.Status.SleptAt == nil && + sess.Status.LastIdleAt != nil +} + +// warmIdle builds the WarmIdle for a holder that is warm idle: its pod set's +// pods (matched by name, as unplaced does) that are bound to a node and still +// running, priced from the live pod spec — what the capacity snapshot counts +// against that node, so it is what a reclaim gives back. Not ok when the +// session is not warm idle, the pod list failed, or none of its pods holds +// capacity: there is nothing to free. +func warmIdle(sess *spiceboxv1alpha1.AgentSession, d admission.Demand, pods podIndex, known bool) (admission.WarmIdle, bool) { + if !known || !isWarmIdle(sess) { + return admission.WarmIdle{}, false + } + var placed []admission.PlacedPod + for _, p := range d.Pods { + pod, ok := pods[types.NamespacedName{Namespace: sess.Namespace, Name: p.Name}] + if !ok || pod.Spec.NodeName == "" || + pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + cpu, mem, eph := schedfit.EffectiveRequests(&pod.Spec) + placed = append(placed, admission.PlacedPod{ + Node: pod.Spec.NodeName, + PodDemand: admission.PodDemand{Name: pod.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph}, + }) + } + if len(placed) == 0 { + return admission.WarmIdle{}, false + } + return admission.WarmIdle{ + Key: client.ObjectKeyFromObject(sess), + Line: lineOf(sess), + IdleSince: sess.Status.LastIdleAt.Time, + Pods: placed, + }, true +} + +// requestReclaims asks for the warm Idle sessions PlanReclaim picks for each +// line's head (its still-waiting position 1) to be slept early. model is the +// one Decide reserved into, so this pass's admissions are respected. Lines go +// in name order; each plan's frees and its head's demand are applied to model +// before the next line plans, so two lines never count the same free space. +func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Candidate, dec admission.Decision, + warm []admission.WarmIdle, warmByKey map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, + model admission.CapacityModel, now time.Time, +) []error { + var heads []admission.Candidate + for _, c := range waiting { + if pos, ok := dec.Positions[c.Key]; ok && pos.Index == 1 { + heads = append(heads, c) + } + } + slices.SortFunc(heads, func(a, b admission.Candidate) int { return strings.Compare(a.Line, b.Line) }) + + pods := make(map[types.NamespacedName][]admission.PlacedPod, len(warm)) + for _, w := range warm { + pods[w.Key] = w.Pods + } + var errs []error + for _, head := range heads { + plan := admission.PlanReclaim(head, warm, model, now, r.MinWarm) + if len(plan) == 0 { + continue + } + for _, key := range plan { + model.Free(pods[key]) + if err := r.requestReclaim(ctx, warmByKey[key], head.Key, now); err != nil { + errs = append(errs, err) + } + } + // PlanReclaim proved the head fits once the plan's pods are freed. + _ = model.Reserve(head.Demand) + } + return errs +} + +// requestReclaim stamps AnnotationReclaimRequested on sess with a +// metadata-only merge patch, unless a request is already there: a later pass +// re-planning the same reclaim must leave the first request's time alone. +func (r *Reconciler) requestReclaim(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, head types.NamespacedName, now time.Time) error { + if _, ok := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]; ok { + return nil + } + logger := log.FromContext(ctx) + logger.Info("admission: requesting early reclaim of a warm idle session for its line's waiting head", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "head", head.String()) + patch := fmt.Appendf(nil, `{"metadata":{"annotations":{%q:%q}}}`, + spiceboxv1alpha1.AnnotationReclaimRequested, now.UTC().Format(time.RFC3339)) + target := &spiceboxv1alpha1.AgentSession{ObjectMeta: metav1.ObjectMeta{Name: sess.Name, Namespace: sess.Namespace}} + if err := r.Client.Patch(ctx, target, client.RawPatch(types.MergePatchType, patch)); err != nil { + logger.Info("admission: writing the reclaim request failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + return fmt.Errorf("request reclaim of %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + // listSessions lists every AgentSession the operator owns: cluster-wide with // no watch set, else one namespaced list per watched namespace. Any failed // list fails the pass (logged by the caller's error return) rather than diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index d0473782..b0f7d582 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -347,3 +347,130 @@ func TestAdmitterWatchNamespaces(t *testing.T) { }) } } + +// warmIdleSession is an admitted, channel-attached Idle session that went idle +// idleFor before the pass and has not slept: its pods still hold capacity. +func warmIdleSession(name string, idleFor time.Duration) *spiceboxv1alpha1.AgentSession { + s := waitingSession(name, 0) + s.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: name} + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + idleAt := metav1.NewTime(passTime.Add(-idleFor)) + s.Status.LastIdleAt = &idleAt + s.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + return s +} + +// boundRunnerPod is the session's oneCPUDemand pod, bound to node-a and +// requesting what oneCPUDemand prices. +func boundRunnerPod(sessName string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: sessName + "-runner", Namespace: "default"}, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{{ + Name: "runner", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Mi"), + }}, + }}, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } +} + +// cpuNode is node-a with cpus whole CPUs allocatable. +func cpuNode(cpus int64) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, + Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(cpus, resource.DecimalSI), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("110"), + }}, + } +} + +// TestAdmitterEarlyReclaim covers the reclaim request a pass makes for a line +// whose head does not fit: it annotates the warm Idle sessions of that line +// PlanReclaim picks, priced from the live pods the cluster snapshot also sees. +func TestAdmitterEarlyReclaim(t *testing.T) { + const minWarm = 5 * time.Minute + cases := []struct { + name string + objs []client.Object + wantAnnotated []string + wantClean []string + }{ + { + name: "line has a waiting head, one warm idle past minWarm: reclaim annotation set on it only", + objs: []client.Object{ + cpuNode(2), + warmIdleSession("rb-old", 10*time.Minute), boundRunnerPod("rb-old"), + warmIdleSession("rb-young", time.Minute), boundRunnerPod("rb-young"), + waitingSession("rb-wait", time.Minute), + }, + wantAnnotated: []string{"rb-old"}, + wantClean: []string{"rb-young", "rb-wait"}, + }, + { + name: "no waiting head: no annotation", + objs: []client.Object{ + cpuNode(1), + warmIdleSession("rb-old", 10*time.Minute), boundRunnerPod("rb-old"), + }, + wantClean: []string{"rb-old"}, + }, + { + name: "the head fits without a reclaim: admitted, no annotation", + objs: []client.Object{ + cpuNode(2), + warmIdleSession("rb-old", 10*time.Minute), boundRunnerPod("rb-old"), + waitingSession("rb-wait", time.Minute), + }, + wantClean: []string{"rb-old", "rb-wait"}, + }, + { + name: "only warm idle is younger than minWarm: no annotation", + objs: []client.Object{ + cpuNode(1), + warmIdleSession("rb-young", time.Minute), boundRunnerPod("rb-young"), + waitingSession("rb-wait", time.Minute), + }, + wantClean: []string{"rb-young", "rb-wait"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, c, _ := newReconciler(t, true, nil, tc.objs...) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + runPass(t, r) + + for _, name := range tc.wantAnnotated { + got := getSession(t, c, name) + assert.Equal(t, passTime.Format(time.RFC3339), got.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + "%s: reclaim requested at the pass's Now", name) + } + for _, name := range tc.wantClean { + assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "%s: no reclaim requested", name) + } + }) + } + + t.Run("a reclaim already requested: the annotation is not rewritten", func(t *testing.T) { + old := warmIdleSession("rb-old", 10*time.Minute) + earlier := baseTime.Format(time.RFC3339) + old.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: earlier} + r, c, _ := newReconciler(t, true, nil, cpuNode(1), old, boundRunnerPod("rb-old"), waitingSession("rb-wait", time.Minute)) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + runPass(t, r) + assert.Equal(t, earlier, getSession(t, c, "rb-old").Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + "the first request's time stands") + }) +} diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index f145d8f2..9833d1d5 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -2363,6 +2363,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // period sleep again instead of idleSleepDue reading it as already-slept. sess.Status.SleptAt = nil } + // The same catch-all for the admitter's reclaim request: one a session left + // Idle before honoring is stale, and must not sleep it on its next Idle. + if err := r.clearStaleReclaimRequest(ctx, &sess); err != nil { + return ctrl.Result{}, err + } // Forensic hold: an unreleased SessionHold naming this session forces // phase=Held and reaps the session's pods (containment). Must run before diff --git a/pkg/controllers/agentsession/sleep.go b/pkg/controllers/agentsession/sleep.go index 440af9e5..fcd8b5ea 100644 --- a/pkg/controllers/agentsession/sleep.go +++ b/pkg/controllers/agentsession/sleep.go @@ -12,6 +12,9 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" lifecyclecore "github.com/authzed/openagentprimitives/pkg/agent/session/lifecycle" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" @@ -56,6 +59,11 @@ func idleSleepDue(sleptAt, lastIdleAt *metav1.Time, sleepAfter time.Duration, no // and wakeable. Returns (slept, requeueAfter, err). Preconditions: caller // confirmed sess.Spec.InputChannel != nil, phase == Idle, and // !shouldWake(sess). +// +// The admitter's reclaim request (AnnotationReclaimRequested) makes the sleep +// due now, whatever SleepAfter says — including when idle sleep is disabled — +// but never past the SessionHold guard. The annotation is removed once the +// session is slept. func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, class *spiceboxv1alpha1.AgentClass) (bool, time.Duration, error) { // Evidence outranks GC. Archiving or reaping a held session would let the // forensic subject expire on a timer, which is exactly the failure mode that @@ -78,7 +86,20 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. if class != nil && class.Spec.Channels != nil && class.Spec.Channels.SleepAfter.Duration > 0 { sleepAfter = class.Spec.Channels.SleepAfter.Duration } + reclaim := reclaimRequested(sess) + if reclaim && sess.Status.SleptAt != nil { + // Already slept, so the request is fulfilled: a previous pass slept the + // session but failed to remove the annotation. Remove it now; sleeping + // again would reap nothing. + return false, 0, r.clearReclaimRequest(ctx, sess) + } due, remaining := idleSleepDue(sess.Status.SleptAt, sess.Status.LastIdleAt, sleepAfter, r.now()) + if reclaim && !due { + log.FromContext(ctx).Info("sleeping idle session early: its AgentClass has a session waiting for the capacity", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, + "reclaimRequested", sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]) + due, remaining = true, 0 + } if !due { return false, remaining, nil // remaining==0 when disabled/already-slept/no-clock } @@ -101,5 +122,53 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. if _, err := r.reapSessionPods(ctx, sess); err != nil { return false, 0, err // already logged inside; retry } + // The annotation is metadata and SleptAt is status: two subresources, so + // they cannot share the write above. It is removed only AFTER the status + // write and the reap succeed, so a failure anywhere earlier leaves the + // request in place for the retry; a failure here leaves a slept session + // still carrying it, which the already-slept branch above clears on the + // next pass without sleeping again. + if reclaim { + if err := r.clearReclaimRequest(ctx, sess); err != nil { + return true, 0, err + } + } return true, 0, nil } + +// reclaimRequested reports whether the admitter has asked for this session to +// be slept early (AnnotationReclaimRequested). +func reclaimRequested(sess *spiceboxv1alpha1.AgentSession) bool { + _, ok := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested] + return ok +} + +// clearStaleReclaimRequest removes a reclaim request from a session that is no +// longer Idle — it woke, or was held, before the sleep path acted on it. Left +// in place, it would sleep the session the moment it next went Idle, long +// after the admission line that asked for the capacity moved on. An Idle +// session's request is left for reconcileSleep. +func (r *Reconciler) clearStaleReclaimRequest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) error { + if sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseIdle || !reclaimRequested(sess) { + return nil + } + log.FromContext(ctx).Info("clearing a stale reclaim request: the session left Idle before it was slept", + "session", sess.Namespace+"/"+sess.Name, "class", sess.Spec.Class, "phase", sess.Status.Phase) + return r.clearReclaimRequest(ctx, sess) +} + +// clearReclaimRequest removes AnnotationReclaimRequested with a metadata-only +// merge patch, sent on a fresh object carrying just the key so the server's +// response cannot overwrite the caller's in-memory status; the caller's copy +// drops the annotation to match. +func (r *Reconciler) clearReclaimRequest(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) error { + patch := fmt.Appendf(nil, `{"metadata":{"annotations":{%q:null}}}`, spiceboxv1alpha1.AnnotationReclaimRequested) + target := &spiceboxv1alpha1.AgentSession{ObjectMeta: metav1.ObjectMeta{Name: sess.Name, Namespace: sess.Namespace}} + if err := r.Client.Patch(ctx, target, client.RawPatch(types.MergePatchType, patch)); err != nil { + log.FromContext(ctx).Info("removing the reclaim-requested annotation failed; retried on the next reconcile", + "session", sess.Namespace+"/"+sess.Name, "err", err.Error()) + return fmt.Errorf("clear reclaim-requested annotation on %s/%s: %w", sess.Namespace, sess.Name, err) + } + delete(sess.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested) + return nil +} diff --git a/pkg/controllers/agentsession/sleep_test.go b/pkg/controllers/agentsession/sleep_test.go index 056fe75b..1efce78a 100644 --- a/pkg/controllers/agentsession/sleep_test.go +++ b/pkg/controllers/agentsession/sleep_test.go @@ -7,13 +7,18 @@ package agentsession import ( + "context" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/admission" + "github.com/authzed/openagentprimitives/pkg/memory" ) // TestSandboxProvisioningDesired pins the lazy-provisioning gate: only an @@ -67,3 +72,111 @@ func TestIdleSleepDue(t *testing.T) { }) } } + +// withReclaimRequest seeds the fixture session as Admitted=True and carrying +// the admitter's reclaim-requested annotation, then reloads f.sess from the +// fake client so reconcileSleep sees what a reconcile would read. +func (f *sleepFixture) withReclaimRequest(t *testing.T) { + t.Helper() + ctx := memory.WithSystemApproval(context.Background(), "test") + f.sess.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(f.now.Add(-time.Hour)), + }} + require.NoError(t, f.c.Status().Update(ctx, f.sess), "seed Admitted=True") + live := f.getSession(t) + live.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: f.now.Format(time.RFC3339)} + require.NoError(t, f.c.Update(ctx, live), "seed reclaim annotation") + f.sess = f.getSession(t) +} + +func TestReconcileSleepReclaimRequested(t *testing.T) { + cases := []struct { + name string + sleepAfter time.Duration + }{ + {name: "reclaim annotation on an Idle session within SleepAfter: slept now, Admitted=False/Reclaimed, annotation cleared", sleepAfter: 10 * time.Minute}, + {name: "reclaim annotation with idle sleep disabled: slept now, Admitted=False/Reclaimed, annotation cleared", sleepAfter: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, tc.sleepAfter) // idle 2m: not due on its own + f.withReclaimRequest(t) + + slept, requeueAfter, err := f.r.reconcileSleep(memory.WithSystemApproval(context.Background(), "test"), f.sess, f.class) + require.NoError(t, err) + assert.True(t, slept, "a reclaim request makes the sleep due now") + assert.Zero(t, requeueAfter) + + got := f.getSession(t) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhaseIdle, got.Status.Phase, "stays Idle (wakeable)") + assert.NotNil(t, got.Status.SleptAt, "SleptAt stamped") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if assert.NotNil(t, adm, "Admitted condition") { + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + } + assert.NotContains(t, got.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "annotation cleared") + assert.True(t, f.bundlesGone(t), "pods reaped") + }) + } + + t.Run("reclaim annotation on a held session (SessionHold): not slept", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + require.NoError(t, f.c.Create(ctx, &spiceboxv1alpha1.SessionHold{ + ObjectMeta: metav1.ObjectMeta{Name: "h1", Namespace: "default"}, + Spec: spiceboxv1alpha1.SessionHoldSpec{SessionRef: spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "s1"}}, + Status: spiceboxv1alpha1.SessionHoldStatus{Phase: spiceboxv1alpha1.SessionHoldPhaseActive}, + }), "create SessionHold") + + slept, _, err := f.r.reconcileSleep(ctx, f.sess, f.class) + require.NoError(t, err) + assert.False(t, slept, "the hold guard outranks a reclaim request") + + got := f.getSession(t) + assert.Nil(t, got.Status.SleptAt, "SleptAt not stamped") + assert.True(t, admission.IsAdmitted(got), "still admitted") + assert.False(t, f.bundlesGone(t), "pods kept") + }) + + t.Run("reclaim annotation on an already-slept session: annotation cleared, nothing reaped again", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + sleptAt := metav1.NewTime(f.now.Add(-time.Minute)) + f.sess.Status.SleptAt = &sleptAt + require.NoError(t, f.c.Status().Update(ctx, f.sess), "seed already-slept status") + f.sess = f.getSession(t) + + slept, _, err := f.r.reconcileSleep(ctx, f.sess, f.class) + require.NoError(t, err) + assert.False(t, slept, "already slept: no second sleep") + assert.NotContains(t, f.getSession(t).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "fulfilled request cleared") + assert.False(t, f.bundlesGone(t), "not reaped a second time") + }) + + t.Run("stale reclaim annotation on a session that woke (Running): annotation cleared, not slept", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + f.sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + require.NoError(t, f.c.Status().Update(ctx, f.sess), "seed Running") + f.sess = f.getSession(t) + + require.NoError(t, f.r.clearStaleReclaimRequest(ctx, f.sess)) + got := f.getSession(t) + assert.NotContains(t, got.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "stale request cleared") + assert.NotContains(t, f.sess.Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "in-memory copy agrees") + assert.Nil(t, got.Status.SleptAt, "not slept") + assert.False(t, f.bundlesGone(t), "pods kept") + }) + + t.Run("reclaim annotation on an Idle session: clearStaleReclaimRequest leaves it for the sleep path", func(t *testing.T) { + f := newSleepFixture(t, -2*time.Minute, 10*time.Minute) + f.withReclaimRequest(t) + require.NoError(t, f.r.clearStaleReclaimRequest(memory.WithSystemApproval(context.Background(), "test"), f.sess)) + assert.Contains(t, f.getSession(t).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "an Idle session's request is not stale") + }) +} From b33286c8d09dd4f645bee1aaaa177482f626b7d3 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 16:41:34 -0500 Subject: [PATCH 15/33] Credit in-flight reclaims before planning more A reclaimed session's pods stay in the capacity snapshot until they are gone, and each status write and pod event in between wakes the admitter, which then saw the head still not fit and reclaimed the next-oldest warm session too. Before a line plans, free the still-bound pods of its in-flight reclaims (requested but not yet slept, or slept and Reclaimed with pods still terminating) and keep them out of the candidates. Log, rather than discard, a head that does not fit after its planned reclaims are freed. --- .../admission/admitter/controller.go | 163 +++++++++++++----- .../admission/admitter/controller_test.go | 81 ++++++++- 2 files changed, 204 insertions(+), 40 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 2142050e..5702ccfd 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -197,8 +197,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R } pods, podsKnown := r.listPods(ctx) - var warm []admission.WarmIdle - warmByKey := map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{} + pool := reclaimPool{ + sessions: map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{}, + inFlight: map[string][]admission.PlacedPod{}, + } for _, sess := range holding { d, err := r.demand(ctx, sess) if err != nil { @@ -215,11 +217,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R Demand: d, Unplaced: unplaced(sess.Namespace, d, pods, podsKnown), }) - if w, ok := warmIdle(sess, d, pods, podsKnown); ok { - warm = append(warm, w) - warmByKey[w.Key] = sess + if podsKnown && isWarmIdle(sess) { + pool.addWarm(sess, boundPods(sess.Namespace, d, pods)) } } + if podsKnown { + r.addDrainingReclaims(ctx, &pool, sessions, pods) + } in.Model = r.snapshot(ctx) if !in.Model.Known() { @@ -241,7 +245,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R } } if in.Model.Known() { - errs = append(errs, r.requestReclaims(ctx, in.Waiting, dec, warm, warmByKey, in.Model, now)...) + errs = append(errs, r.requestReclaims(ctx, in.Waiting, dec, &pool, in.Model, now)...) } return requeue, errors.Join(errs...) } @@ -257,19 +261,15 @@ func isWarmIdle(sess *spiceboxv1alpha1.AgentSession) bool { sess.Status.LastIdleAt != nil } -// warmIdle builds the WarmIdle for a holder that is warm idle: its pod set's -// pods (matched by name, as unplaced does) that are bound to a node and still -// running, priced from the live pod spec — what the capacity snapshot counts -// against that node, so it is what a reclaim gives back. Not ok when the -// session is not warm idle, the pod list failed, or none of its pods holds -// capacity: there is nothing to free. -func warmIdle(sess *spiceboxv1alpha1.AgentSession, d admission.Demand, pods podIndex, known bool) (admission.WarmIdle, bool) { - if !known || !isWarmIdle(sess) { - return admission.WarmIdle{}, false - } +// boundPods is the part of a session's pod set the capacity snapshot counts +// against a node: its demand's pods (matched by name, as unplaced does) that +// are bound and not finished, priced from the live pod spec. A pod already +// marked for deletion is included — the snapshot still counts it until it is +// gone — so it is exactly what a reclaim gives back. +func boundPods(namespace string, d admission.Demand, pods podIndex) []admission.PlacedPod { var placed []admission.PlacedPod for _, p := range d.Pods { - pod, ok := pods[types.NamespacedName{Namespace: sess.Namespace, Name: p.Name}] + pod, ok := pods[types.NamespacedName{Namespace: namespace, Name: p.Name}] if !ok || pod.Spec.NodeName == "" || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { continue @@ -280,25 +280,103 @@ func warmIdle(sess *spiceboxv1alpha1.AgentSession, d admission.Demand, pods podI PodDemand: admission.PodDemand{Name: pod.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph}, }) } + return placed +} + +// reclaimPool is what a pass may plan reclaims from, per line. +// +// inFlight is the capacity reclaims requested on an earlier pass are already +// giving back: the bound pods of a session carrying the reclaim request but +// not yet slept, and of a slept session released for a reclaim whose pods are +// still being torn down. The snapshot still counts those pods, so without +// crediting them every pass between the request and the last pod's deletion — +// and the admitter is woken by each of those status writes and pod events — +// would see the head still not fit and reclaim the next-oldest warm session +// too, cascading through the line for one head. +type reclaimPool struct { + warm []admission.WarmIdle + sessions map[types.NamespacedName]*spiceboxv1alpha1.AgentSession + inFlight map[string][]admission.PlacedPod +} + +// addWarm files a warm Idle holder's bound pods: as in flight when a reclaim +// is already requested (so it is not chosen again), else as a candidate. A +// session with no bound pods has nothing to give back and is skipped. +func (p *reclaimPool) addWarm(sess *spiceboxv1alpha1.AgentSession, placed []admission.PlacedPod) { if len(placed) == 0 { - return admission.WarmIdle{}, false - } - return admission.WarmIdle{ - Key: client.ObjectKeyFromObject(sess), - Line: lineOf(sess), - IdleSince: sess.Status.LastIdleAt.Time, - Pods: placed, - }, true + return + } + line := lineOf(sess) + if _, requested := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]; requested { + p.inFlight[line] = append(p.inFlight[line], placed...) + return + } + key := client.ObjectKeyFromObject(sess) + p.warm = append(p.warm, admission.WarmIdle{Key: key, Line: line, IdleSince: sess.Status.LastIdleAt.Time, Pods: placed}) + p.sessions[key] = sess +} + +// isDrainingReclaim matches a session the sleep path has already released for +// a reclaim (slept, Admitted=False/Reclaimed). It is neither a holder nor +// waiting, but its pods may still be terminating. +func isDrainingReclaim(sess *spiceboxv1alpha1.AgentSession) bool { + if sess.Status.SleptAt == nil { + return false + } + cond := meta.FindStatusCondition(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + return cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == spiceboxv1alpha1.ReasonAdmissionReclaimed +} + +// addDrainingReclaims credits the still-listed bound pods of every draining +// reclaim to its line. A slept session stays Admitted=False/Reclaimed for as +// long as it sleeps, so pricing is skipped unless one of its pods (runner pod +// by session label, sandbox pod by its bundle SpiceboxSession's label) is +// still listed — the common case is a long-gone pod set. +func (r *Reconciler) addDrainingReclaims(ctx context.Context, pool *reclaimPool, sessions []spiceboxv1alpha1.AgentSession, pods podIndex) { + labelled := map[types.NamespacedName]bool{} + for _, pod := range pods { + for _, l := range []string{sessionPodLabel, sandboxPodLabel} { + if v := pod.Labels[l]; v != "" { + labelled[types.NamespacedName{Namespace: pod.Namespace, Name: v}] = true + } + } + } + hasPods := func(sess *spiceboxv1alpha1.AgentSession) bool { + if labelled[types.NamespacedName{Namespace: sess.Namespace, Name: sess.Name}] { + return true + } + for _, b := range sess.Status.BundleSessions { + if labelled[types.NamespacedName{Namespace: sess.Namespace, Name: b.SpiceboxSessionName}] { + return true + } + } + return false + } + for i := range sessions { + sess := &sessions[i] + if !isDrainingReclaim(sess) || !hasPods(sess) { + continue + } + d, err := r.demand(ctx, sess) + if err != nil { + log.FromContext(ctx).Info("admission: cannot price a reclaimed session's terminating pods; not crediting them to its line", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) + continue + } + line := lineOf(sess) + pool.inFlight[line] = append(pool.inFlight[line], boundPods(sess.Namespace, d, pods)...) + } } // requestReclaims asks for the warm Idle sessions PlanReclaim picks for each // line's head (its still-waiting position 1) to be slept early. model is the -// one Decide reserved into, so this pass's admissions are respected. Lines go -// in name order; each plan's frees and its head's demand are applied to model -// before the next line plans, so two lines never count the same free space. +// one Decide reserved into, so this pass's admissions are respected. Before a +// line plans, its in-flight reclaims are freed on model, so only the shortfall +// they do not already cover is reclaimed. Lines go in name order; each line's +// frees and its head's demand are applied to model before the next line plans, +// so two lines never count the same free space. func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Candidate, dec admission.Decision, - warm []admission.WarmIdle, warmByKey map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, - model admission.CapacityModel, now time.Time, + pool *reclaimPool, model admission.CapacityModel, now time.Time, ) []error { var heads []admission.Candidate for _, c := range waiting { @@ -308,24 +386,31 @@ func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Ca } slices.SortFunc(heads, func(a, b admission.Candidate) int { return strings.Compare(a.Line, b.Line) }) - pods := make(map[types.NamespacedName][]admission.PlacedPod, len(warm)) - for _, w := range warm { + pods := make(map[types.NamespacedName][]admission.PlacedPod, len(pool.warm)) + for _, w := range pool.warm { pods[w.Key] = w.Pods } var errs []error for _, head := range heads { - plan := admission.PlanReclaim(head, warm, model, now, r.MinWarm) - if len(plan) == 0 { - continue - } + inFlight := pool.inFlight[head.Line] + model.Free(inFlight) + plan := admission.PlanReclaim(head, pool.warm, model, now, r.MinWarm) for _, key := range plan { model.Free(pods[key]) - if err := r.requestReclaim(ctx, warmByKey[key], head.Key, now); err != nil { + if err := r.requestReclaim(ctx, pool.sessions[key], head.Key, now); err != nil { errs = append(errs, err) } } - // PlanReclaim proved the head fits once the plan's pods are freed. - _ = model.Reserve(head.Demand) + if len(plan) == 0 && len(inFlight) == 0 { + continue + } + // Hold the freed space for this head so a later line cannot plan on it. + // With a plan, PlanReclaim proved the head fits; a failure means that + // invariant broke. With only in-flight frees it may simply not fit yet. + if !model.Reserve(head.Demand) && len(plan) > 0 { + log.FromContext(ctx).Info("admission: head does not fit after its planned reclaims were freed; the plan's fit check and the model disagree", + "class", head.Line, "head", head.Key.String(), "reclaims", len(plan)) + } } return errs } diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index b0f7d582..8aa5b1ed 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -367,7 +367,10 @@ func warmIdleSession(name string, idleFor time.Duration) *spiceboxv1alpha1.Agent // requesting what oneCPUDemand prices. func boundRunnerPod(sessName string) *corev1.Pod { return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: sessName + "-runner", Namespace: "default"}, + ObjectMeta: metav1.ObjectMeta{ + Name: sessName + "-runner", Namespace: "default", + Labels: map[string]string{sessionPodLabel: sessName}, + }, Spec: corev1.PodSpec{ NodeName: "node-a", Containers: []corev1.Container{{ @@ -474,3 +477,79 @@ func TestAdmitterEarlyReclaim(t *testing.T) { "the first request's time stands") }) } + +// TestAdmitterReclaimInFlightIsCredited pins that one head gets one reclaim: +// between the request and the reclaimed pods actually leaving the node, every +// pass (each woken by the sleep's status write or a pod event) still sees the +// head not fit in the snapshot. Crediting the in-flight reclaim's bound pods +// keeps those passes from reclaiming the next-oldest warm session too. +func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { + const minWarm = 5 * time.Minute + cases := []struct { + name string + advance func(t *testing.T, c client.Client) // what happened to rb-1 between passes + }{ + { + name: "reclaim requested but not yet slept: second pass annotates nothing new", + advance: func(*testing.T, client.Client) {}, + }, + { + name: "slept and Admitted=False/Reclaimed, pod still bound: second pass annotates nothing new", + advance: func(t *testing.T, c client.Client) { + markReclaimed(t, c, "rb-1") + }, + }, + { + name: "slept and Admitted=False/Reclaimed, pod terminating (DeletionTimestamp): second pass annotates nothing new", + advance: func(t *testing.T, c client.Client) { + markReclaimed(t, c, "rb-1") + var pod corev1.Pod + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "rb-1-runner"}, &pod)) + require.NoError(t, c.Delete(context.Background(), &pod), "delete held by a finalizer: pod lingers terminating") + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(&pod), &pod)) + require.NotNil(t, pod.DeletionTimestamp, "pod is terminating") + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pod1 := boundRunnerPod("rb-1") + pod1.Finalizers = []string{"test/hold"} + r, c, _ := newReconciler(t, true, nil, + cpuNode(3), + warmIdleSession("rb-1", 30*time.Minute), pod1, + warmIdleSession("rb-2", 20*time.Minute), boundRunnerPod("rb-2"), + warmIdleSession("rb-3", 10*time.Minute), boundRunnerPod("rb-3"), + waitingSession("rb-wait", time.Minute), + ) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + + runPass(t, r) + require.Contains(t, getSession(t, c, "rb-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") + require.NotContains(t, getSession(t, c, "rb-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims only one") + + tc.advance(t, c) + runPass(t, r) + for _, name := range []string{"rb-2", "rb-3"} { + assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "%s: the in-flight reclaim already covers the head", name) + } + }) + } +} + +// markReclaimed does what the sleep path's status write does: SleptAt and +// Admitted=False/Reclaimed. The reclaim annotation stays (it is removed only +// after the reap) and the pods are left for the test to dispose of. +func markReclaimed(t *testing.T, c client.Client, name string) { + t.Helper() + s := getSession(t, c, name) + slept := metav1.NewTime(passTime) + s.Status.SleptAt = &slept + meta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionReclaimed, LastTransitionTime: metav1.NewTime(passTime), + }) + require.NoError(t, c.Status().Update(context.Background(), s), "mark %s reclaimed", name) +} From 09bad9a7b30e5c2bc6d7b3d526aac866ec2f3572 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 17:25:47 -0500 Subject: [PATCH 16/33] Cap concurrent sessions per agent; let delegated children through AgentClass gains spec.maxConcurrentSessions. The admitter reads it for every line with a waiting session and hands it to Decide as that line's cap; delegated children are neither counted nor blocked. A class whose cap cannot be read holds its non-child sessions back for the pass. The admission-child-at-cap bundle delegates into a capped line that is already full and asserts the delegate tool result is the child's answer. --- ...ntprimitives.authzed.com_agentclasses.yaml | 8 + pkg/apis/v1alpha1/agentclass_types.go | 7 + pkg/apis/v1alpha1/zz_generated.deepcopy.go | 5 + .../admission/admitter/controller.go | 46 ++++ .../admission/admitter/controller_test.go | 92 ++++++++ pkg/platform/manifests/install.yaml | 8 + pkg/web/workshopprojectsrv/whitelist_test.go | 3 + .../workshopprojectsrv/workshopprojectsrv.go | 2 + .../admission-child-at-cap/bundle.json | 214 ++++++++++++++++++ .../manifests/00-agent.yaml | 85 +++++++ 10 files changed, 470 insertions(+) create mode 100644 test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json create mode 100644 test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml diff --git a/config/crds/agentprimitives.authzed.com_agentclasses.yaml b/config/crds/agentprimitives.authzed.com_agentclasses.yaml index d5f1ab3b..2c2f4b46 100644 --- a/config/crds/agentprimitives.authzed.com_agentclasses.yaml +++ b/config/crds/agentprimitives.authzed.com_agentclasses.yaml @@ -1542,6 +1542,14 @@ spec: required: - prompt type: object + maxConcurrentSessions: + description: |- + MaxConcurrentSessions: at most this many of this agent's sessions hold + capacity at once. Delegated child sessions are not counted and are + never blocked by it. Unset means no limit. + format: int32 + minimum: 1 + type: integer mcpServers: description: |- MCPServers references MCPServer CRs in the same namespace whose diff --git a/pkg/apis/v1alpha1/agentclass_types.go b/pkg/apis/v1alpha1/agentclass_types.go index 61cf1a0d..e2b424ce 100644 --- a/pkg/apis/v1alpha1/agentclass_types.go +++ b/pkg/apis/v1alpha1/agentclass_types.go @@ -175,6 +175,13 @@ type AgentClassSpec struct { // +optional Budget *BudgetConfig `json:"budget,omitempty"` + // MaxConcurrentSessions: at most this many of this agent's sessions hold + // capacity at once. Delegated child sessions are not counted and are + // never blocked by it. Unset means no limit. + // +optional + // +kubebuilder:validation:Minimum=1 + MaxConcurrentSessions *int32 `json:"maxConcurrentSessions,omitempty"` + // BoundEntities is the PRE-RENAME spelling of spec.authz.slots. // // Still deserialized on purpose. This is an AUTHZ field: dropping it from diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index d71f22eb..bd1aa892 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -183,6 +183,11 @@ func (in *AgentClassSpec) DeepCopyInto(out *AgentClassSpec) { *out = new(BudgetConfig) **out = **in } + if in.MaxConcurrentSessions != nil { + in, out := &in.MaxConcurrentSessions, &out.MaxConcurrentSessions + *out = new(int32) + **out = **in + } if in.BoundEntities != nil { in, out := &in.BoundEntities, &out.BoundEntities *out = make([]AuthzSlot, len(*in)) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 5702ccfd..fc71159d 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -20,6 +20,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -195,6 +196,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R Demand: d, }) } + in.Caps, in.Waiting = r.capLines(ctx, in.Waiting, byKey) pods, podsKnown := r.listPods(ctx) pool := reclaimPool{ @@ -250,6 +252,50 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R return requeue, errors.Join(errs...) } +// capLines reads spec.maxConcurrentSessions from the AgentClass of every line +// with a waiting candidate, keyed by line as Decide expects. A class that does +// not exist has no cap (pricing has already refused its sessions when it +// prices from the class). A class that cannot be read fails closed: that +// line's non-child candidates sit this pass out, since admitting them could +// exceed a cap nobody could check; its children stay, as a cap never blocks +// them. +func (r *Reconciler) capLines(ctx context.Context, waiting []admission.Candidate, + sessions map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, +) (map[string]int32, []admission.Candidate) { + caps := map[string]int32{} + unreadable := map[string]bool{} + checked := map[string]bool{} + for _, c := range waiting { + if checked[c.Line] { + continue + } + checked[c.Line] = true + sess := sessions[c.Key] + var ac spiceboxv1alpha1.AgentClass + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: sess.Namespace, Name: sess.Spec.Class}, &ac); err != nil { + if !apierrors.IsNotFound(err) { + log.FromContext(ctx).Info("admission: cannot read the class's session cap; its non-delegated sessions keep waiting this pass", + "class", c.Line, "err", err.Error()) + unreadable[c.Line] = true + } + continue + } + if limit := ac.Spec.MaxConcurrentSessions; limit != nil { + caps[c.Line] = *limit + } + } + if len(unreadable) == 0 { + return caps, waiting + } + kept := waiting[:0:0] + for _, c := range waiting { + if c.Child || !unreadable[c.Line] { + kept = append(kept, c) + } + } + return caps, kept +} + // isWarmIdle matches a holder whose pods could be given back early: Idle, // channel-attached (the AgentSession reconciler's sleep path, which honors the // reclaim request, runs only for those), not yet slept, with a known idle diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 8aa5b1ed..140dc183 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -553,3 +553,95 @@ func markReclaimed(t *testing.T, c client.Client, name string) { }) require.NoError(t, c.Status().Update(context.Background(), s), "mark %s reclaimed", name) } + +// cappedClass is testClass with spec.maxConcurrentSessions set. +func cappedClass(limit int32) *spiceboxv1alpha1.AgentClass { + return &spiceboxv1alpha1.AgentClass{ + ObjectMeta: metav1.ObjectMeta{Name: testClass, Namespace: "default"}, + Spec: spiceboxv1alpha1.AgentClassSpec{MaxConcurrentSessions: &limit}, + } +} + +// admittedRunning is a non-child session already holding its admission. +func admittedRunning(name string) *spiceboxv1alpha1.AgentSession { + s := waitingSession(name, 0) + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + s.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), + }} + return s +} + +// childOf marks s as a session delegated by parent. +func childOf(s *spiceboxv1alpha1.AgentSession, parent string) *spiceboxv1alpha1.AgentSession { + s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: parent} + return s +} + +func TestAdmitterConcurrencyCap(t *testing.T) { + t.Run("class cap 1 with one admitted parent: its waiting child is admitted; a waiting non-child is not", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(1), + admittedRunning("rb-parent"), + waitingSession("rb-other", time.Minute), + childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent"), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "rb-child")) + assertPosition(t, getSession(t, c, "rb-other"), 1, 1) + }) + + t.Run("class cap 2 with one admitted parent: the oldest waiting non-child takes the second slot, the next waits", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(2), + admittedRunning("rb-parent"), + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "rb-a")) + assertPosition(t, getSession(t, c, "rb-b"), 1, 1) + }) + + t.Run("no AgentClass to read a cap from: the line is uncapped, every waiting session that fits is admitted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + admittedRunning("rb-parent"), + waitingSession("rb-a", time.Minute), + waitingSession("rb-b", 2*time.Minute), + ) + runPass(t, r) + assertAdmitted(t, getSession(t, c, "rb-a")) + assertAdmitted(t, getSession(t, c, "rb-b")) + }) + + t.Run("AgentClass read fails: the line's non-children keep waiting unpositioned, its child is still admitted", func(t *testing.T) { + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects( + cappedClass(1), + admittedRunning("rb-parent"), + waitingSession("rb-other", time.Minute), + childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent"), + ). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*spiceboxv1alpha1.AgentClass); ok { + return fmt.Errorf("apiserver unavailable") + } + return cl.Get(ctx, key, obj, opts...) + }, + }). + Build() + r := &Reconciler{ + Client: c, Capacity: roomFor(5), Enabled: true, + Now: func() time.Time { return passTime }, Demand: oneCPUDemand, + } + runPass(t, r) + assertAdmitted(t, getSession(t, c, "rb-child")) + other := getSession(t, c, "rb-other") + assert.False(t, admission.IsAdmitted(other), "rb-other not admitted past a cap it could not read") + assert.Nil(t, other.Status.Admission, "rb-other not positioned") + }) +} diff --git a/pkg/platform/manifests/install.yaml b/pkg/platform/manifests/install.yaml index 3265860c..72a348a0 100644 --- a/pkg/platform/manifests/install.yaml +++ b/pkg/platform/manifests/install.yaml @@ -1580,6 +1580,14 @@ spec: required: - prompt type: object + maxConcurrentSessions: + description: |- + MaxConcurrentSessions: at most this many of this agent's sessions hold + capacity at once. Delegated child sessions are not counted and are + never blocked by it. Unset means no limit. + format: int32 + minimum: 1 + type: integer mcpServers: description: |- MCPServers references MCPServer CRs in the same namespace whose diff --git a/pkg/web/workshopprojectsrv/whitelist_test.go b/pkg/web/workshopprojectsrv/whitelist_test.go index bda91b82..7fa31fcd 100644 --- a/pkg/web/workshopprojectsrv/whitelist_test.go +++ b/pkg/web/workshopprojectsrv/whitelist_test.go @@ -54,6 +54,9 @@ var notProjectedFields = map[string]bool{ "WorkspaceSource": true, "ToolSessionLog": true, "Budget": true, + // MaxConcurrentSessions caps the source agent's own sessions; a stand-in + // is a different class with no line worth limiting. + "MaxConcurrentSessions": true, "BoundEntities": true, "Channels": true, "Authz": true, diff --git a/pkg/web/workshopprojectsrv/workshopprojectsrv.go b/pkg/web/workshopprojectsrv/workshopprojectsrv.go index ac878ce8..f3f6f773 100644 --- a/pkg/web/workshopprojectsrv/workshopprojectsrv.go +++ b/pkg/web/workshopprojectsrv/workshopprojectsrv.go @@ -544,6 +544,8 @@ func reachable(agents []workshopthreadsrv.AgentInThread, ns, class string) bool // Model absent means the tier default applies, Harness absent means // ap-native. // - Budget — the tier default applies. +// - MaxConcurrentSessions — the source agent's session cap; a stand-in is +// a different class, with no line of the source's to limit. // - Subagents, SubagentModes, BoundEntities (deprecated) — DROPPED. A // source class's own roster would almost certainly name classes this // workshop does not have, which the AgentClass reconciler marks diff --git a/test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json b/test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json new file mode 100644 index 00000000..e466195f --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-child-at-cap/bundle.json @@ -0,0 +1,214 @@ +{ + "name": "admission-child-at-cap", + "description": "A lead delegates to a coder whose class is already at its concurrency cap, and the delegation still completes: the delegate tool result the lead is handed is the child's own answer, not a timeout.\n\ncap-coder sets maxConcurrentSessions: 1. The holder thread is a direct coder session that takes the line's one slot and parks in await_user_message, so it is still holding when the lead delegates. The queued thread is a second direct coder session that arrives while the holder holds, and waits behind the cap. The lead's delegated child then joins the SAME coder line, behind queued in arrival order, at cap. A delegated child is never blocked by its class's cap (and is not counted toward it), and children go first in their line, so the child is admitted at once and its answer reaches the lead.\n\nWhat is claimed, and how each claim is read from the run:\n- the delegation completed: the lead's delegate step asserts the TOOL RESULT is not an error and contains the child's return_result text (lastToolResultContains). A child held behind the cap would leave the parent's delegate call polling until it gave up, and the step would never see that text;\n- the cap was in force while the child ran: queued, which arrived before the lead, is admitted strictly after the lead and after the holder. Without the cap queued would be admitted as soon as it arrived.\n\nThe holder finishes only once the lead is Idle (waitFor), so the coder line is at cap for the lead's whole delegation, and queued's slot comes from the holder sleeping afterwards. Capacity is four session pod sets, more than the run ever holds, so the cap and not cluster room is what queues queued.", + "agentDir": "testdata/admission-child-at-cap/manifests", + "agentClass": "cap-lead", + "defaultUser": "user@example.com", + "admission": { + "capacitySessions": 4 + }, + "userTurns": [ + { + "text": "hold the standing coder slot", + "thread": "holder", + "channel": "cap-coder-fake" + }, + { + "text": "tidy the import blocks when you can", + "thread": "queued", + "channel": "cap-coder-fake", + "waitFor": [ + { + "thread": "holder", + "until": "awaitingInput" + } + ] + }, + { + "text": "have the coder fix the flaky retry test", + "thread": "lead", + "channel": "cap-lead-fake", + "waitFor": [ + { + "thread": "holder", + "until": "awaitingInput" + } + ] + }, + { + "text": "holder may finish now", + "thread": "holder", + "channel": "cap-coder-fake", + "waitFor": [ + { + "thread": "lead", + "until": "idle" + } + ] + } + ], + "llm": [ + { + "thread": "holder", + "expect": { + "userTextContains": "hold the standing coder slot" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "holding the coder slot" + } + } + } + ] + }, + { + "thread": "holder", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "await_user_message", + "args": {} + } + } + ] + }, + { + "thread": "holder", + "expect": { + "userTextContains": "holder may finish now" + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "held the coder slot" + } + } + } + ] + }, + { + "thread": "queued", + "expect": { + "userTextContains": "tidy the import blocks when you can" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "import blocks tidied" + } + } + } + ] + }, + { + "thread": "queued", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "tidied the import blocks" + } + } + } + ] + }, + { + "thread": "lead", + "expect": { + "userTextContains": "have the coder fix the flaky retry test" + }, + "reply": [ + { + "toolUse": { + "name": "delegate", + "args": { + "agent": "cap-coder", + "task": "repair the payments retry test so it stops flaking" + } + } + } + ] + }, + { + "agentClass": "cap-coder", + "expect": { + "userTextContains": "repair the payments retry test so it stops flaking" + }, + "reply": [ + { + "toolUse": { + "name": "return_result", + "args": { + "result": "Payments retry test repaired: the backoff now resets between attempts." + } + } + } + ] + }, + { + "thread": "lead", + "expect": { + "lastToolResult": "delegate", + "lastToolResultIsError": false, + "lastToolResultContains": "the backoff now resets between attempts" + }, + "reply": [ + { + "toolUse": { + "name": "respond_to_user", + "args": { + "text": "The coder repaired the retry test." + } + } + } + ] + }, + { + "thread": "lead", + "expect": { + "lastToolResult": "respond_to_user", + "lastToolResultIsError": false + }, + "reply": [ + { + "toolUse": { + "name": "agent_work_complete", + "args": { + "summary": "had the coder repair the retry test" + } + } + } + ] + } + ], + "assert": { + "admission": { + "admittedBefore": [ + { + "first": "holder", + "then": "queued" + }, + { + "first": "lead", + "then": "queued" + } + ] + } + } +} diff --git a/test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml b/test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml new file mode 100644 index 00000000..a69497bc --- /dev/null +++ b/test/e2e/bronzethread/testdata/admission-child-at-cap/manifests/00-agent.yaml @@ -0,0 +1,85 @@ +# Fixture for the admission-child-at-cap bundle: a lead that delegates to a +# coder whose class is capped at one concurrent session. +# +# The cap is on cap-coder, the CHILD's class, because that is the line the +# delegated child joins: a child session runs under the class it was +# delegated to. A cap on cap-lead would never see the child at all. A class +# cannot delegate to itself (a roster cycle is refused), so parent and child +# are necessarily of different classes, and the coder line is put at its cap +# by a direct (non-delegated) coder session, the "holder" thread. +# +# Both classes are toolless beyond their meta-tool grants and are +# channel-attached with a one-second sleepAfter: a finished session keeps its +# admission until it sleeps, and the sleep is what hands the coder line's one +# slot to the queued session at the end. +apiVersion: v1 +kind: Secret +metadata: {name: capchild-placeholder, namespace: default} +type: Opaque +stringData: {api-key: "unused-by-the-test-provider"} +--- +apiVersion: v1 +kind: Secret +metadata: {name: capchild-fake-creds, namespace: default} +type: Opaque +stringData: {placeholder: "unused-by-the-fake-channel"} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: cap-lead, namespace: default} +spec: + displayName: CapLead + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test # production operator REFUSES this + name: scripted # cosmetic; ignored at runtime + apiKey: {name: capchild-placeholder, key: api-key} + capabilities: + subagents: {} + subagents: ["cap-coder"] + systemPrompt: + inline: | + You are a fixture lead for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You hand bounded coding tasks to the coder with the delegate tool. + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + channels: {sleepAfter: 1s} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: AgentClass +metadata: {name: cap-coder, namespace: default} +spec: + displayName: CapCoder + description: "E2E fixture agent. Do not deploy to a real cluster." + model: + provider: test + name: scripted + apiKey: {name: capchild-placeholder, key: api-key} + systemPrompt: + inline: | + You are a fixture coder for the agentprimitives e2e harness. + DO NOT deploy to a real cluster — your model provider is `test`. + You complete coding tasks, whether a person or a lead asked. + budget: {maxTurns: 50, maxTokens: 100000, maxDuration: "1h"} + channels: {sleepAfter: 1s} + maxConcurrentSessions: 1 +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: cap-lead-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: cap-lead + credentialsRef: {secretName: capchild-fake-creds} + fake: {echo: false} +--- +apiVersion: agentprimitives.authzed.com/v1alpha1 +kind: Channel +metadata: {name: cap-coder-fake, namespace: default} +spec: + kind: fake + role: both + agentClass: cap-coder + credentialsRef: {secretName: capchild-fake-creds} + fake: {echo: false} From 28f0453a1cabf932f574c57adc10b9829b69aa2c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:06:08 -0500 Subject: [PATCH 17/33] Tell queued sessions where they are in line The capacity-hold notice now appends the session's place in its class's admission line when status.admission records one. --- pkg/channels/sessionnotice/sessionnotice.go | 17 ++++++++-- .../sessionnotice/sessionnotice_test.go | 31 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/pkg/channels/sessionnotice/sessionnotice.go b/pkg/channels/sessionnotice/sessionnotice.go index 09e27a53..6dc10fcf 100644 --- a/pkg/channels/sessionnotice/sessionnotice.go +++ b/pkg/channels/sessionnotice/sessionnotice.go @@ -15,6 +15,9 @@ package sessionnotice import ( + "fmt" + + "github.com/dustin/go-humanize" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -209,10 +212,18 @@ func CapacityHold(sess *spiceboxv1alpha1.AgentSession) (string, bool) { if c == nil || c.Status != metav1.ConditionTrue { return "", false } - if c.Message == "" { - return SchedulingLead, true + text := c.Message + if text == "" { + text = SchedulingLead + } + // The admitter records where the session stands in its class's line. The + // notice is posted once per hold episode, so this is the position at the + // time it posts. The class name stands in for a display name: the session + // status carries none, and this package stays a pure derivation (no lookup). + if a := sess.Status.Admission; a != nil && a.Position > 0 { + text += fmt.Sprintf(" You're %s in line for %s.", humanize.Ordinal(int(a.Position)), sess.Spec.Class) } - return c.Message, true + return text, true } // CapacityHoldExpired reports whether the session failed because its capacity diff --git a/pkg/channels/sessionnotice/sessionnotice_test.go b/pkg/channels/sessionnotice/sessionnotice_test.go index 3c98bb19..162ce3ae 100644 --- a/pkg/channels/sessionnotice/sessionnotice_test.go +++ b/pkg/channels/sessionnotice/sessionnotice_test.go @@ -175,3 +175,34 @@ func TestDerive_HealthySessionIsSilent(t *testing.T) { assert.Empty(t, sessionnotice.Derive(sess(spiceboxv1alpha1.AgentSessionPhaseRunning))) assert.Empty(t, sessionnotice.Derive(nil), "a nil session is not a panic") } + +func TestCapacityHoldPosition(t *testing.T) { + const holdMsg = "The cluster is out of room right now; this will start on its own once capacity frees up (gives up at 17:23 UTC)." + hold := cond(spiceboxv1alpha1.AgentSessionConditionCapacityHold, metav1.ConditionTrue, + spiceboxv1alpha1.ReasonCapacityHoldWaiting, holdMsg) + + cases := []struct { + name string + admission *spiceboxv1alpha1.AdmissionStatus + want string + }{ + {name: "position 3 of 5: hold text plus ordinal sentence naming the class", + admission: &spiceboxv1alpha1.AdmissionStatus{Position: 3, LineLength: 5}, + want: holdMsg + " You're 3rd in line for reviewbot."}, + {name: "no admission status: hold text unchanged", + want: holdMsg}, + {name: "position 0: hold text unchanged", + admission: &spiceboxv1alpha1.AdmissionStatus{Position: 0, LineLength: 5}, + want: holdMsg}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := sess(spiceboxv1alpha1.AgentSessionPhasePending, hold) + s.Spec.Class = "reviewbot" + s.Status.Admission = tc.admission + got, held := sessionnotice.CapacityHold(s) + require.True(t, held) + assert.Equal(t, tc.want, got) + }) + } +} From a1d1131dd7996f325db17a607d39b75e872b798c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:27:44 -0500 Subject: [PATCH 18/33] Queue a session in line when it reaches the admission gate A session's wait was measured from creation, so one that cleared a start approval, identity choice or credential gate more than the capacity-hold timeout after creation failed on its first gated pass. The admitter also admitted sessions that had a Pending phase and resolved settings but had not reached the gate (parked, fatal settings, pin-blocked), reserving capacity for them. The gate now stamps Admitted=False/Queued at arrival; the waiting timeout and give-up notice run from it, and the admitter considers only queued sessions. WriteOwned persists a deliberately re-stamped transition time on a same-status reason change, so a released session arriving again is queued from now. --- pkg/apis/v1alpha1/conditions.go | 6 ++ .../admission/admitter/controller.go | 12 ++- .../admission/admitter/controller_test.go | 27 +++++- .../admitter/restart_integration_test.go | 15 ++- pkg/controllers/admission/line.go | 14 ++- pkg/controllers/admission/line_test.go | 32 +++++++ .../agentsession/admission_gate.go | 42 ++++++++- .../agentsession/admission_gate_test.go | 94 ++++++++++++++++--- pkg/controllers/agentstatus/statuswrite.go | 18 ++++ .../agentstatus/statuswrite_test.go | 60 ++++++++++++ 10 files changed, 294 insertions(+), 26 deletions(-) diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index c3738de7..ac87bd25 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -523,6 +523,12 @@ const ( // AgentSessionConditionAdmitted reasons. const ( ReasonAdmitted = "Admitted" + // ReasonAdmissionQueued marks a session that has reached the admission + // gate and is waiting in line. Its transition time is the moment it + // arrived, which the waiting timeout is measured from; the admitter + // considers only sessions carrying it, so a session parked at an earlier + // gate never reserves anything. + ReasonAdmissionQueued = "Queued" // ReasonAdmissionReclaimed marks a session released so a session ahead of // it in line could be admitted. ReasonAdmissionReclaimed = "Reclaimed" diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index fc71159d..b071fa4a 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -121,12 +121,16 @@ func isTerminal(sess *spiceboxv1alpha1.AgentSession) bool { sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhaseFailed } -// isWaiting matches the AgentSession reconciler's gate: a booting session -// ("" or Pending) that is not admitted. A session parked at a pre-start gate -// has another phase and is not in line yet. +// isWaiting matches a session the AgentSession reconciler's gate has put in +// line: booting ("" or Pending) and marked Admitted=False/Queued. The marker +// is required, not just the phase: a session's settings are persisted, and +// its phase set to Pending, before the start-approval, identity and +// credential gates park it and before a fatal-settings or image-pin block +// stops it — a session in any of those states never reached the line and must +// never be admitted, or it would reserve capacity it cannot use. func isWaiting(sess *spiceboxv1alpha1.AgentSession) bool { return (sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending) && - !admission.IsAdmitted(sess) + admission.IsQueued(sess) } func (r *Reconciler) demand(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 140dc183..6225aa75 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -61,15 +61,23 @@ func testScheme(t *testing.T) *runtime.Scheme { } // waitingSession is a booting, never-admitted session created at -// baseTime+offset. +// baseTime+offset that reached the admission gate at once +// (Admitted=False/Queued at creation). func waitingSession(name string, offset time.Duration) *spiceboxv1alpha1.AgentSession { return &spiceboxv1alpha1.AgentSession{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "default", CreationTimestamp: metav1.NewTime(baseTime.Add(offset)), }, - Spec: spiceboxv1alpha1.AgentSessionSpec{Class: testClass}, - Status: spiceboxv1alpha1.AgentSessionStatus{Phase: spiceboxv1alpha1.AgentSessionPhasePending}, + Spec: spiceboxv1alpha1.AgentSessionSpec{Class: testClass}, + Status: spiceboxv1alpha1.AgentSessionStatus{ + Phase: spiceboxv1alpha1.AgentSessionPhasePending, + Conditions: []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, + LastTransitionTime: metav1.NewTime(baseTime.Add(offset)), + }}, + }, } } @@ -163,6 +171,19 @@ func TestAdmitterPass(t *testing.T) { } }) + t.Run("Pending without the Queued marker (parked at an earlier gate, or fatal settings): not admitted, no position", func(t *testing.T) { + parked := waitingSession("rb-parked", 0) + parked.Status.Conditions = nil + r, c, _ := newReconciler(t, true, roomFor(5), parked, waitingSession("rb-a", time.Minute)) + runPass(t, r) + + got := getSession(t, c, "rb-parked") + assert.Nil(t, meta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + "a session that never reached the gate is never admitted") + assert.Nil(t, got.Status.Admission, "and holds no line position") + assertAdmitted(t, getSession(t, c, "rb-a")) + }) + t.Run("a position write is skipped when unchanged: second pass sends no status patch", func(t *testing.T) { r, c, patches := newReconciler(t, true, roomFor(1), waitingSession("rb-a", time.Minute), diff --git a/pkg/controllers/admission/admitter/restart_integration_test.go b/pkg/controllers/admission/admitter/restart_integration_test.go index b8bac3b3..409ab6cc 100644 --- a/pkg/controllers/admission/admitter/restart_integration_test.go +++ b/pkg/controllers/admission/admitter/restart_integration_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -140,13 +141,23 @@ func TestAdmitterRestartRebuildsLine(t *testing.T) { stop := startAdmitter(t, env) for i := 1; i <= 5; i++ { - require.NoError(t, c.Create(ctx, &spiceboxv1alpha1.AgentSession{ + sess := &spiceboxv1alpha1.AgentSession{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("s%d", i), Namespace: restartNamespace}, Spec: spiceboxv1alpha1.AgentSessionSpec{ Class: testClass, Prompt: spiceboxv1alpha1.PromptSource{Inline: "review this"}, }, - }), "create s%d", i) + } + require.NoError(t, c.Create(ctx, sess), "create s%d", i) + // envtest runs no AgentSession reconciler: put the session in line the + // way its admission gate does, Admitted=False/Queued at arrival. + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + meta.SetStatusCondition(&sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, Message: "waiting in line", + LastTransitionTime: sess.CreationTimestamp, + }) + require.NoError(t, c.Status().Update(ctx, sess), "queue s%d", i) } waitForAdmitted(t, c, 2) diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go index 57a19f61..4b03951c 100644 --- a/pkg/controllers/admission/line.go +++ b/pkg/controllers/admission/line.go @@ -65,7 +65,10 @@ func JoinedAt(sess *v1alpha1.AgentSession) time.Time { } // WaitingSince is when the session's current wait began: JoinedAt, or the -// later moment it was released from admission. +// later moment its Admitted condition last went False. The gate stamps +// Admitted=False/Queued at the moment a session reaches it, so for a session +// in line this is its gate arrival — not its creation, however long it was +// parked at a start-approval, identity or credential gate first. func WaitingSince(sess *v1alpha1.AgentSession) time.Time { joined := JoinedAt(sess) if c := admittedCondition(sess); c != nil && c.Status == metav1.ConditionFalse && @@ -75,6 +78,15 @@ func WaitingSince(sess *v1alpha1.AgentSession) time.Time { return joined } +// IsQueued reports whether the session has reached the admission gate and is +// waiting in line (Admitted=False/Queued). Only such a session is a candidate +// for admission: one parked at an earlier gate, or blocked before it, has not +// reached the line. +func IsQueued(sess *v1alpha1.AgentSession) bool { + c := admittedCondition(sess) + return c != nil && c.Status == metav1.ConditionFalse && c.Reason == v1alpha1.ReasonAdmissionQueued +} + // IsAdmitted reports whether the session's Admitted condition is True. func IsAdmitted(sess *v1alpha1.AgentSession) bool { return meta.IsStatusConditionTrue(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) diff --git a/pkg/controllers/admission/line_test.go b/pkg/controllers/admission/line_test.go index 3839a610..d154a18e 100644 --- a/pkg/controllers/admission/line_test.go +++ b/pkg/controllers/admission/line_test.go @@ -92,6 +92,11 @@ func TestWaitingSince(t *testing.T) { withAdmitted(sessionAt(0), metav1.ConditionTrue, 5*time.Minute), at(0), }, + { + "queued at the gate two hours after creation (a start approval in between): the gate-arrival time", + withAdmitted(sessionAt(0), metav1.ConditionFalse, 2*time.Hour), + at(2 * time.Hour), + }, { "cold wake after release: the later wake time", withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, time.Minute), 2*time.Minute), @@ -111,6 +116,33 @@ func TestIsAdmitted(t *testing.T) { assert.False(t, IsAdmitted(withAdmitted(sessionAt(0), metav1.ConditionFalse, 0)), "Admitted=False") } +func TestIsQueued(t *testing.T) { + queued := func(status metav1.ConditionStatus, reason string) *v1alpha1.AgentSession { + s := sessionAt(0) + s.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.AgentSessionConditionAdmitted, Status: status, Reason: reason, + LastTransitionTime: *mtime(time.Minute), + }} + return s + } + cases := []struct { + name string + sess *v1alpha1.AgentSession + want bool + }{ + {"no Admitted condition: not queued", sessionAt(0), false}, + {"Admitted=False/Queued: queued", queued(metav1.ConditionFalse, v1alpha1.ReasonAdmissionQueued), true}, + {"Admitted=False/Reclaimed: not queued (released, has not reached the gate again)", queued(metav1.ConditionFalse, v1alpha1.ReasonAdmissionReclaimed), false}, + {"Admitted=False/PlacementMismatch: not queued", queued(metav1.ConditionFalse, v1alpha1.ReasonAdmissionMisfit), false}, + {"Admitted=True: not queued", queued(metav1.ConditionTrue, v1alpha1.ReasonAdmitted), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, IsQueued(tc.sess)) + }) + } +} + func TestDemandTotal(t *testing.T) { d := Demand{Pods: []PodDemand{ {Name: "runner", CPUMilli: 500, MemBytes: 1 << 20, EphemeralBytes: 9}, diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index 8e9e6ea2..ae1c66ff 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -47,17 +47,23 @@ func (r *Reconciler) awaitingAdmission(sess *spiceboxv1alpha1.AgentSession) bool } // waitForAdmission is the whole reconcile tail of a booting session that is -// not admitted: give up once it has waited past the capacity-hold timeout, -// otherwise surface the wait and requeue. It creates nothing. +// not admitted: mark its arrival in line, give up once it has waited past the +// capacity-hold timeout, otherwise surface the wait and requeue. It creates +// nothing. // -// The wait is measured from admission.WaitingSince — the later of joining the -// line and the last release — so a session released back to the line (a -// placement misfit) gets a full window, not the remainder of its first one. +// The first gated pass of a wait stamps Admitted=False/Queued at now (see +// markQueued), and the wait is measured from admission.WaitingSince — that +// arrival time. A session parked for hours at a start-approval, identity or +// credential gate therefore gets a full window from the moment it actually +// joins the line, and so does one released back to it (a placement misfit or +// a reclaim followed by a cold wake), rather than the remainder of an earlier +// one. // // A negative --capacity-hold-timeout (hold disabled) means no give-up time: // there is no pod whose ordinary readiness deadline could apply instead, so // the session waits for admission with no CapacityHold notice. func (r *Reconciler) waitForAdmission(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (ctrl.Result, error) { + r.markQueued(sess) timeout := r.effectiveCapacityHoldTimeout() since := admission.WaitingSince(sess) until := since.Add(timeout) @@ -80,6 +86,32 @@ func (r *Reconciler) waitForAdmission(ctx context.Context, sess *spiceboxv1alpha return ctrl.Result{RequeueAfter: requeue}, r.applyStatus(ctx, sess) } +// markQueued records, on sess, that it has reached the admission gate: +// Admitted=False/Queued with now as the transition time, unless it already +// carries that marker (a later pass of the same wait keeps the original +// arrival). The admitter considers only queued sessions, so this marker is +// what puts a session in line; a session parked or blocked before the gate +// never carries it and never reserves anything. The caller persists it. +// +// The transition time is set explicitly rather than left to +// meta.SetStatusCondition, which keeps the old time on a False→False change: +// a session released earlier (Reclaimed, PlacementMismatch) arrives in line +// again now, not at its release. +func (r *Reconciler) markQueued(sess *spiceboxv1alpha1.AgentSession) { + if admission.IsQueued(sess) { + return + } + now := metav1.NewTime(r.now()) + conditions.Set(sess, &sess.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, + Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, + Message: "waiting in line for cluster capacity", + LastTransitionTime: now, + }) + conditions.Find(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted).LastTransitionTime = now +} + // admissionWaitExpiredMessage is the Failed condition message when a session // waited unadmitted past the capacity-hold timeout. No pod was ever created, // so unlike capacityHoldExpiredMessage there is no scheduler detail to quote. diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index 5d5d55c8..9cec7160 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -63,17 +63,26 @@ func TestAdmissionGate(t *testing.T) { wantHoldStatus metav1.ConditionStatus // "" = CapacityHold absent wantHoldReason string wantMsg []string // substrings of the CapacityHold (held) or Failed message + // wantQueuedAt, when set, is the expected Admitted=False/Queued + // transition time: the moment the session reached the gate. + wantQueuedAt time.Time }{ { - name: "Pending, not admitted: no SpiceboxSession; CapacityHold=True/WaitingForAdmission after grace", + name: "Pending, queued 2m ago: no SpiceboxSession; CapacityHold=True/WaitingForAdmission after grace", enabled: true, bundles: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-2*time.Minute)) + }, wantHoldStatus: metav1.ConditionTrue, wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, wantMsg: []string{"out of room", "gives up at 17:58 UTC"}, }, { - name: "Pending, not admitted, no bundles: no runner pod", + name: "Pending, queued 2m ago, no bundles: no runner pod", enabled: true, createdAt: -2 * time.Minute, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-2*time.Minute)) + }, wantHoldStatus: metav1.ConditionTrue, wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, }, @@ -81,6 +90,11 @@ func TestAdmissionGate(t *testing.T) { name: "Pending, not admitted, inside the grace: no pods and no hold notice yet", enabled: true, bundles: true, createdAt: -10 * time.Second, }, + { + name: "first reaches the gate 3h after creation: queued from now, not failed, no hold notice yet", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + wantQueuedAt: nowT, + }, { name: "Pending, admitted: bundle SpiceboxSession created as today, no hold", enabled: true, bundles: true, createdAt: -2 * time.Minute, @@ -123,22 +137,23 @@ func TestAdmissionGate(t *testing.T) { wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldReleased, }, { - name: "waiting past the timeout from join time: Failed, CapacityHold=False/HoldExpired", - enabled: true, bundles: true, createdAt: -61 * time.Minute, + name: "queued past the timeout: Failed, CapacityHold=False/HoldExpired", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-61*time.Minute)) + }, wantFailed: true, wantHoldStatus: metav1.ConditionFalse, wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldExpired, wantMsg: []string{"waited 1h0m0s", "cluster capacity"}, }, { - name: "released long after creation: the wait is measured from the release, not failed", + name: "released long after creation: re-queued at the gate, the wait restarts there, not failed", enabled: true, bundles: true, createdAt: -3 * time.Hour, prep: func(s *spiceboxv1alpha1.AgentSession) { withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionMisfit, nowT.Add(-5*time.Minute)) }, - wantHoldStatus: metav1.ConditionTrue, - wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldWaitingAdmission, - wantMsg: []string{"gives up at 17:55 UTC"}, + wantQueuedAt: nowT, }, { name: "AdmissionEnabled=false: bundle SpiceboxSession created without Admitted", @@ -188,6 +203,14 @@ func TestAdmissionGate(t *testing.T) { assert.Equal(t, tc.wantHoldReason, hold.Reason) } + if !tc.wantQueuedAt.IsZero() { + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if assert.NotNil(t, adm, "Admitted condition") { + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason) + assert.Equal(t, tc.wantQueuedAt, adm.LastTransitionTime.Time.UTC(), "queued at gate arrival") + } + } if !tc.wantFailed { assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "session must not fail") if hold != nil && hold.Status == metav1.ConditionTrue { @@ -208,6 +231,51 @@ func TestAdmissionGate(t *testing.T) { } } +// A session that clears its start approval two hours after creation reaches +// the admission gate then, not at creation: it is queued from that moment +// (Admitted=False/Queued at now), is not failed for having "waited" through +// the approval, creates nothing, and provisions normally once admitted. +func TestAdmissionGate_QueuedOnArrivalAfterStartApproval(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + clock := func() time.Time { return nowT } + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + ac := classWithBundle("ac1", codeBundle) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-2*time.Hour)) + // Approved: channelsd removed the park marker; the phase still says parked. + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingStartApproval + parked := metav1.NewTime(nowT.Add(-2 * time.Hour)) + sess.Status.StartApprovalParkedAt = &parked + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = true + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + require.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase, "an approval wait is not a wait in line") + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, got.Status.Phase) + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, adm, "Admitted condition written at the gate") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason) + assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "queued at gate arrival") + assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting timeout runs from gate arrival") + assert.False(t, exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "no pods before admission") + + // The admitter admits it. + apimeta.SetStatusCondition(&got.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(nowT), + }) + require.NoError(t, c.Status().Update(context.Background(), &got), "admit") + runReconciles(t, r, "s1", 3) + + got = getSession(t, c, "s1") + assert.NotEqual(t, spiceboxv1alpha1.AgentSessionPhaseFailed, got.Status.Phase) + assert.True(t, exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "admitted: the bundle SpiceboxSession is created") +} + // A terminal reap releases admission in the status write that accompanies it, // and never invents an Admitted condition on a session that never had one. func TestAdmissionGate_TerminalReapReleases(t *testing.T) { @@ -317,7 +385,7 @@ func TestAdmissionGate_Misfit(t *testing.T) { wantAdmittedNil bool // no Admitted condition at all }{ { - name: "admitted, pods unschedulable 2m: pods reaped, Admitted=False/PlacementMismatch, JoinedAt unchanged, CapacityHold released", + name: "admitted, pods unschedulable 2m: pods reaped, released and re-queued, JoinedAt unchanged, CapacityHold released", enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, heldBefore: true, wantReaped: true, wantMisfit: true, }, @@ -415,9 +483,13 @@ func TestAdmissionGate_Misfit(t *testing.T) { case tc.wantMisfit: require.NotNil(t, adm, "Admitted condition") assert.Equal(t, metav1.ConditionFalse, adm.Status) - assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionMisfit, adm.Reason) assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") - if !tc.wantFailed { + if tc.wantFailed { + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionMisfit, adm.Reason) + } else { + // Released (PlacementMismatch), then back at the gate on the + // next pass: queued again, its wait restarting there. + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason, "the released session is back in line") assert.Equal(t, nowT, admission.WaitingSince(&got).UTC(), "the waiting-timeout clock restarts at the release") } default: diff --git a/pkg/controllers/agentstatus/statuswrite.go b/pkg/controllers/agentstatus/statuswrite.go index cbcf30ef..3f886e3d 100644 --- a/pkg/controllers/agentstatus/statuswrite.go +++ b/pkg/controllers/agentstatus/statuswrite.go @@ -78,6 +78,7 @@ func WriteOwned(ctx context.Context, c client.Client, sess, original *spiceboxv1 prior := fresh.DeepCopy() for i := range set { meta.SetStatusCondition(&fresh.Status.Conditions, set[i]) + carryRestampedTransition(fresh.Status.Conditions, original, set[i]) } for _, t := range removed { meta.RemoveStatusCondition(&fresh.Status.Conditions, t) @@ -91,6 +92,23 @@ func WriteOwned(ctx context.Context, c client.Client, sess, original *spiceboxv1 }) } +// carryRestampedTransition persists a transition time the writer moved on +// purpose. meta.SetStatusCondition keeps the stored LastTransitionTime +// whenever the status is unchanged, and a writer that builds its conditions +// through it never moves the time on such a change — so a time later than the +// one in original, on an unchanged status, is a deliberate re-stamp (a +// session arriving in line again: Admitted False/Reclaimed → False/Queued at +// now) and is copied onto the stored condition. +func carryRestampedTransition(stored []metav1.Condition, original *spiceboxv1alpha1.AgentSession, cur metav1.Condition) { + old := meta.FindStatusCondition(original.Status.Conditions, cur.Type) + if old == nil || old.Status != cur.Status || !cur.LastTransitionTime.After(old.LastTransitionTime.Time) { + return + } + if c := meta.FindStatusCondition(stored, cur.Type); c != nil && c.Status == cur.Status { + c.LastTransitionTime = cur.LastTransitionTime + } +} + // diffOwnedConditions returns the owner's conditions that were added or changed // (set) and those removed, between original and sess. func diffOwnedConditions(original, sess *spiceboxv1alpha1.AgentSession, owner spiceboxv1alpha1.StatusFieldOwner) (set []metav1.Condition, removed []string) { diff --git a/pkg/controllers/agentstatus/statuswrite_test.go b/pkg/controllers/agentstatus/statuswrite_test.go index 4b1dae38..51853f22 100644 --- a/pkg/controllers/agentstatus/statuswrite_test.go +++ b/pkg/controllers/agentstatus/statuswrite_test.go @@ -3,6 +3,7 @@ package agentstatus import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -131,3 +132,62 @@ func TestWriteOwned_ConditionWrite_DoesNotResendOperatorOwnedFields(t *testing.T assert.True(t, got.Status.EffectiveSettings.ReportSessionCost, "operator-owned effectiveSettings.reportSessionCost must be preserved by a condition write") } + +// A writer that deliberately re-stamps a condition's transition time on a +// reason change with no status change (Admitted False/Reclaimed → +// False/Queued: the session arrives in line now, not at its release) must +// have that time persisted. meta.SetStatusCondition alone keeps the stored +// time whenever the status is unchanged; a writer that did not move the time +// keeps that behaviour. +func TestWriteOwned_CarriesDeliberateTransitionTimeOnSameStatus(t *testing.T) { + released := metav1.NewTime(time.Now().Add(-48 * time.Hour).Truncate(time.Second)) + queued := metav1.NewTime(time.Now().Truncate(time.Second)) + cases := []struct { + name string + wantLTT metav1.Time + mutate func(c *metav1.Condition) + }{ + { + name: "reason changed and transition time moved later: the new time is persisted", + wantLTT: queued, + mutate: func(c *metav1.Condition) { + c.Reason = spiceboxv1alpha1.ReasonAdmissionQueued + c.LastTransitionTime = queued + }, + }, + { + name: "reason changed, transition time untouched: the stored time is kept", + wantLTT: released, + mutate: func(c *metav1.Condition) { + c.Reason = spiceboxv1alpha1.ReasonAdmissionQueued + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + original := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: "sess", Namespace: "default"}, + Status: spiceboxv1alpha1.AgentSessionStatus{Conditions: []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionReclaimed, LastTransitionTime: released, + }}}, + } + c := fake.NewClientBuilder().WithScheme(writeOwnedScheme(t)). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithObjects(original.DeepCopy()).Build() + var base spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(original), &base)) + sess := base.DeepCopy() + tc.mutate(&sess.Status.Conditions[0]) + + require.NoError(t, WriteOwned(context.Background(), c, sess, &base, spiceboxv1alpha1.OwnerOperator)) + + var got spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(original), &got)) + cond := meta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, cond) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, cond.Reason) + assert.True(t, tc.wantLTT.Equal(&cond.LastTransitionTime), "want %v got %v", tc.wantLTT, cond.LastTransitionTime) + }) + } +} From 38fc4714c679b6712b12c986f41a80f8c87d11c1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:28:29 -0500 Subject: [PATCH 19/33] Order the line by last wake, not by admission transitions JoinedAt fell back to creation once a cold-woken session had been admitted (or when its wake and the admission transition shared a second), so a misfit release measured its wait from creation and failed any channel session older than the capacity-hold timeout. JoinedAt is now the last wake if set, else creation. --- pkg/controllers/admission/line.go | 20 +++++++++---------- pkg/controllers/admission/line_test.go | 9 +++++++-- .../agentsession/admission_gate.go | 9 +++++---- .../agentsession/admission_gate_test.go | 3 ++- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go index 4b03951c..16e8eb74 100644 --- a/pkg/controllers/admission/line.go +++ b/pkg/controllers/admission/line.go @@ -49,19 +49,17 @@ func admittedCondition(sess *v1alpha1.AgentSession) *metav1.Condition { return meta.FindStatusCondition(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) } -// JoinedAt is when the session took its place in line. A wake that came after -// the last admission transition re-queues the session, so it counts; a wake -// the session was later released after (e.g. a placement misfit) does not, and -// the session keeps its original place at creationTimestamp. +// JoinedAt is when the session took its place in line, which orders it there: +// its last wake if it has one, else its creation. A cold wake re-joins the +// back of the line, so it counts; a warm wake never left the line, and a +// release (a placement misfit) keeps the place the session already had. The +// waiting timeout is not measured from here but from gate arrival +// (WaitingSince). func JoinedAt(sess *v1alpha1.AgentSession) time.Time { - wake := sess.Status.LastWakeAt - if wake == nil { - return sess.CreationTimestamp.Time + if wake := sess.Status.LastWakeAt; wake != nil { + return wake.Time } - if c := admittedCondition(sess); c != nil && !wake.Time.After(c.LastTransitionTime.Time) { - return sess.CreationTimestamp.Time - } - return wake.Time + return sess.CreationTimestamp.Time } // WaitingSince is when the session's current wait began: JoinedAt, or the diff --git a/pkg/controllers/admission/line_test.go b/pkg/controllers/admission/line_test.go index d154a18e..41c22d8a 100644 --- a/pkg/controllers/admission/line_test.go +++ b/pkg/controllers/admission/line_test.go @@ -58,9 +58,14 @@ func TestJoinedAt(t *testing.T) { at(2 * time.Minute), }, { - "released by misfit after its last wake (lastWakeAt t1, Admitted=False at t2>t1): creationTimestamp", + "cold wake, admitted, then released by a misfit (lastWakeAt t1, Admitted=False at t2>t1): lastWakeAt", withWake(withAdmitted(sessionAt(0), metav1.ConditionFalse, 2*time.Minute), time.Minute), - at(0), + at(time.Minute), + }, + { + "wake in the same second as the admission transition: lastWakeAt", + withWake(withAdmitted(sessionAt(0), metav1.ConditionTrue, time.Minute), time.Minute), + at(time.Minute), }, { "woke with no Admitted condition yet: lastWakeAt", diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index ae1c66ff..2e0ee4b8 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -175,11 +175,12 @@ func (r *Reconciler) placementMisfit(sess *spiceboxv1alpha1.AgentSession, stall // status write leaves the session admitted, and the next pass decides afresh // from whatever still stands. // -// The session keeps its place: admission.JoinedAt reads only creation and -// wake times, so it re-joins at the front of its line (it was admitted +// The session keeps its place: admission.JoinedAt is its last wake (a cold +// wake re-joined the line then) or its creation, never an admission +// transition, so it re-joins at the front of its line (it was admitted // fairly; the model was wrong, not the session). admission.WaitingSince -// restarts at this release, so the capacity-hold timeout gives it a fresh -// window rather than the remainder of its first one. +// restarts when it is queued again at the gate, so the capacity-hold timeout +// gives it a fresh window rather than the remainder of its first one. // // That fresh window is why the release is bounded by JoinedAt instead. A // stall the admitter's model cannot see — a shortfall on the only nodes the diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index 9cec7160..b5d6fbeb 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -390,8 +390,9 @@ func TestAdmissionGate_Misfit(t *testing.T) { wantReaped: true, wantMisfit: true, }, { - name: "admitted after a cold wake, pods unschedulable 2m: released, JoinedAt unchanged", + name: "admitted after a cold wake 10m ago, session created 3h ago, pods unschedulable 2m: released and re-queued, not failed, JoinedAt is the wake", enabled: true, admitted: true, pending: 2 * time.Minute, schedMsg: capacitySchedMsg, woken: true, + createdAgo: 3 * time.Hour, wantReaped: true, wantMisfit: true, }, { From b53444b21dd5d493f6e17c3f5814e039ccc7ac7c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:33:55 -0500 Subject: [PATCH 20/33] Adopt sessions whose pods predate admission Admission defaults on, so at upgrade every session already booting, running or warm idle had no Admitted condition. A booting one was gated with its pods still up, and the admitter priced its whole set on top of headroom that already subtracted those pods; a running one was never counted toward its class's cap or share. A non-terminal session with no Admitted condition whose status records pods, and that is not slept or held, is pre-admission: the gate lets it through without queueing it, and the admitter adopts it (Admitted=True/Adopted) once it sees one of its pods, so it counts as a holder. A pre-admission session that sleeps is released like an admitted one, so its next wake queues. --- pkg/apis/v1alpha1/conditions.go | 4 + .../admission/admitter/controller.go | 102 +++++++++++---- .../admission/admitter/controller_test.go | 77 ++++++++++++ .../admitter/restart_integration_test.go | 119 +++++++++++++++++- pkg/controllers/admission/line.go | 30 +++++ pkg/controllers/admission/line_test.go | 51 ++++++++ .../agentsession/admission_gate.go | 21 +++- .../agentsession/admission_gate_test.go | 55 ++++++++ pkg/controllers/agentsession/sleep.go | 7 +- 9 files changed, 430 insertions(+), 36 deletions(-) diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index ac87bd25..d5807018 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -529,6 +529,10 @@ const ( // considers only sessions carrying it, so a session parked at an earlier // gate never reserves anything. ReasonAdmissionQueued = "Queued" + // ReasonAdmissionAdopted marks a session whose pods were created before + // admission was switched on: the admitter adopts it as admitted, so it is + // counted as a holder rather than gated with its pods already up. + ReasonAdmissionAdopted = "Adopted" // ReasonAdmissionReclaimed marks a session released so a session ahead of // it in line could be admitted. ReasonAdmissionReclaimed = "Reclaimed" diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index b071fa4a..82656862 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -153,7 +153,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R if err != nil { return ctrl.Result{}, err } - var waiting, holding []*spiceboxv1alpha1.AgentSession + var waiting, holding, preAdmission []*spiceboxv1alpha1.AgentSession for i := range sessions { sess := &sessions[i] switch { @@ -162,11 +162,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R // pods it has placed already reduce node headroom in the snapshot. case admission.IsAdmitted(sess): holding = append(holding, sess) + case admission.PreAdmission(sess): + preAdmission = append(preAdmission, sess) case isWaiting(sess): waiting = append(waiting, sess) } } - if len(waiting) == 0 { + if len(waiting) == 0 && (len(preAdmission) == 0 || !r.Enabled) { return requeue, nil } now := r.now() @@ -174,13 +176,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R if !r.Enabled { var errs []error for _, sess := range waiting { - if err := r.admit(ctx, sess, now, "Session admission is disabled; admitted at once."); err != nil { + if err := r.admit(ctx, sess, now, spiceboxv1alpha1.ReasonAdmitted, "Session admission is disabled; admitted at once."); err != nil { errs = append(errs, err) } } return requeue, errors.Join(errs...) } + var errs []error + pods, podsKnown := r.listPods(ctx) + adopted, adoptErrs := r.adoptPreAdmission(ctx, preAdmission, pods, podsKnown, now) + errs = append(errs, adoptErrs...) + holding = append(holding, adopted...) + if len(waiting) == 0 { + return requeue, errors.Join(errs...) + } + in := admission.Input{} byKey := make(map[types.NamespacedName]*spiceboxv1alpha1.AgentSession, len(waiting)) for _, sess := range waiting { @@ -202,7 +213,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R } in.Caps, in.Waiting = r.capLines(ctx, in.Waiting, byKey) - pods, podsKnown := r.listPods(ctx) pool := reclaimPool{ sessions: map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{}, inFlight: map[string][]admission.PlacedPod{}, @@ -239,9 +249,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R dec := admission.Decide(in) - var errs []error for _, key := range dec.Admit { - if err := r.admit(ctx, byKey[key], now, "The session's whole pod set fits the cluster."); err != nil { + if err := r.admit(ctx, byKey[key], now, spiceboxv1alpha1.ReasonAdmitted, "The session's whole pod set fits the cluster."); err != nil { errs = append(errs, err) } } @@ -377,13 +386,14 @@ func isDrainingReclaim(sess *spiceboxv1alpha1.AgentSession) bool { return cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == spiceboxv1alpha1.ReasonAdmissionReclaimed } -// addDrainingReclaims credits the still-listed bound pods of every draining -// reclaim to its line. A slept session stays Admitted=False/Reclaimed for as -// long as it sleeps, so pricing is skipped unless one of its pods (runner pod -// by session label, sandbox pod by its bundle SpiceboxSession's label) is -// still listed — the common case is a long-gone pod set. -func (r *Reconciler) addDrainingReclaims(ctx context.Context, pool *reclaimPool, sessions []spiceboxv1alpha1.AgentSession, pods podIndex) { - labelled := map[types.NamespacedName]bool{} +// sessionsWithPods is every session (by namespace/name) that has at least one +// listed pod: its runner, detector or sidecar pods by the session label, its +// sandbox pods by their bundle SpiceboxSession's label — which a pod adopted +// from a SandboxWarmPool also carries, under a pool-minted name. +type sessionsWithPods map[types.NamespacedName]bool + +func indexSessionsWithPods(pods podIndex) sessionsWithPods { + labelled := sessionsWithPods{} for _, pod := range pods { for _, l := range []string{sessionPodLabel, sandboxPodLabel} { if v := pod.Labels[l]; v != "" { @@ -391,20 +401,63 @@ func (r *Reconciler) addDrainingReclaims(ctx context.Context, pool *reclaimPool, } } } - hasPods := func(sess *spiceboxv1alpha1.AgentSession) bool { - if labelled[types.NamespacedName{Namespace: sess.Namespace, Name: sess.Name}] { + return labelled +} + +// has reports whether sess has a listed pod. +func (l sessionsWithPods) has(sess *spiceboxv1alpha1.AgentSession) bool { + if l[types.NamespacedName{Namespace: sess.Namespace, Name: sess.Name}] { + return true + } + for _, b := range sess.Status.BundleSessions { + if l[types.NamespacedName{Namespace: sess.Namespace, Name: b.SpiceboxSessionName}] { return true } - for _, b := range sess.Status.BundleSessions { - if labelled[types.NamespacedName{Namespace: sess.Namespace, Name: b.SpiceboxSessionName}] { - return true - } + } + return false +} + +// adoptPreAdmission writes Admitted=True/Adopted on every pre-admission +// session (admission.PreAdmission) that still has a listed pod, and returns +// them so this pass counts them as holders. One with no listed pod holds +// nothing and is left alone; so is every one when the pod list failed, since +// adopting a session with no pods would reserve capacity for nothing. +func (r *Reconciler) adoptPreAdmission(ctx context.Context, sessions []*spiceboxv1alpha1.AgentSession, + pods podIndex, podsKnown bool, now time.Time, +) ([]*spiceboxv1alpha1.AgentSession, []error) { + if len(sessions) == 0 || !podsKnown { + return nil, nil + } + withPods := indexSessionsWithPods(pods) + var adopted []*spiceboxv1alpha1.AgentSession + var errs []error + for _, sess := range sessions { + if !withPods.has(sess) { + continue } - return false + log.FromContext(ctx).Info("admission: adopting a session whose pods predate admission", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "phase", sess.Status.Phase) + if err := r.admit(ctx, sess, now, spiceboxv1alpha1.ReasonAdmissionAdopted, + "The session's pods were created before admission was enabled; adopted as admitted."); err != nil { + errs = append(errs, err) + } + // Counted as a holder either way: its pods are up whether or not the + // write landed, and the next pass retries the write. + adopted = append(adopted, sess) } + return adopted, errs +} + +// addDrainingReclaims credits the still-listed bound pods of every draining +// reclaim to its line. A slept session stays Admitted=False/Reclaimed for as +// long as it sleeps, so pricing is skipped unless one of its pods (runner pod +// by session label, sandbox pod by its bundle SpiceboxSession's label) is +// still listed — the common case is a long-gone pod set. +func (r *Reconciler) addDrainingReclaims(ctx context.Context, pool *reclaimPool, sessions []spiceboxv1alpha1.AgentSession, pods podIndex) { + withPods := indexSessionsWithPods(pods) for i := range sessions { sess := &sessions[i] - if !isDrainingReclaim(sess) || !hasPods(sess) { + if !isDrainingReclaim(sess) || !withPods.has(sess) { continue } d, err := r.demand(ctx, sess) @@ -564,13 +617,14 @@ func unplaced(namespace string, d admission.Demand, pods podIndex, known bool) a return out } -// admit writes Admitted=True at now and clears the session's line position. -func (r *Reconciler) admit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, now time.Time, message string) error { +// admit writes Admitted=True with reason at now and clears the session's line +// position. +func (r *Reconciler) admit(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, now time.Time, reason, message string) error { original := sess.DeepCopy() meta.SetStatusCondition(&sess.Status.Conditions, metav1.Condition{ Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, - Reason: spiceboxv1alpha1.ReasonAdmitted, + Reason: reason, Message: message, ObservedGeneration: sess.Generation, LastTransitionTime: metav1.NewTime(now), diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 6225aa75..b532c9b6 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -666,3 +666,80 @@ func TestAdmitterConcurrencyCap(t *testing.T) { assert.Nil(t, other.Status.Admission, "rb-other not positioned") }) } + +// preAdmissionSession is a session that started its pods before admission was +// switched on: in phase, with its runner pod recorded on status, and no +// Admitted condition at all. +func preAdmissionSession(name string, phase string) *spiceboxv1alpha1.AgentSession { + s := waitingSession(name, 0) + s.Status.Phase = phase + s.Status.Conditions = nil + s.Status.RunnerPodName = name + "-runner" + return s +} + +// TestAdmitterAdoptsPreAdmissionSessions pins the rollout: a session whose +// pods predate admission is adopted (Admitted=True/Adopted) and counted as a +// holder — toward its line's cap, and its unplaced pods reserved — rather +// than gated with its pods up or left uncounted. +func TestAdmitterAdoptsPreAdmissionSessions(t *testing.T) { + assertAdopted := func(t *testing.T, s *spiceboxv1alpha1.AgentSession) { + t.Helper() + cond := meta.FindStatusCondition(s.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if assert.NotNil(t, cond, "%s: Admitted condition", s.Name) { + assert.Equal(t, metav1.ConditionTrue, cond.Status, "%s: adopted as admitted", s.Name) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionAdopted, cond.Reason, "%s: reason", s.Name) + } + } + + t.Run("Running pre-admission session with a bound pod, class cap 1: adopted, and the cap holds the waiting session back", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(1), + preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("rb-old"), + waitingSession("rb-a", time.Minute), + ) + runPass(t, r) + assertAdopted(t, getSession(t, c, "rb-old")) + assertPosition(t, getSession(t, c, "rb-a"), 1, 1) + }) + + t.Run("Pending pre-admission session with an unbound pod, room for one: adopted, its unplaced pod reserved, the waiting session waits", func(t *testing.T) { + pod := boundRunnerPod("rb-old") + pod.Spec.NodeName = "" + pod.Status.Phase = corev1.PodPending + r, c, _ := newReconciler(t, true, roomFor(1), + preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhasePending), pod, + waitingSession("rb-a", time.Minute), + ) + runPass(t, r) + assertAdopted(t, getSession(t, c, "rb-old")) + assertPosition(t, getSession(t, c, "rb-a"), 1, 1) + }) + + t.Run("pre-admission session with nothing waiting: still adopted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseIdle), boundRunnerPod("rb-old"), + ) + runPass(t, r) + assertAdopted(t, getSession(t, c, "rb-old")) + }) + + t.Run("pre-admission status but no pod listed: not adopted (nothing to hold)", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), + preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseRunning), + waitingSession("rb-a", time.Minute), + ) + runPass(t, r) + assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "rb-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + "no pods: not a holder") + assertAdmitted(t, getSession(t, c, "rb-a")) + }) + + t.Run("Enabled=false: nothing adopted", func(t *testing.T) { + r, c, _ := newReconciler(t, false, roomFor(1), + preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("rb-old"), + ) + runPass(t, r) + assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "rb-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted)) + }) +} diff --git a/pkg/controllers/admission/admitter/restart_integration_test.go b/pkg/controllers/admission/admitter/restart_integration_test.go index 409ab6cc..8bcd5bc7 100644 --- a/pkg/controllers/admission/admitter/restart_integration_test.go +++ b/pkg/controllers/admission/admitter/restart_integration_test.go @@ -31,9 +31,22 @@ func TestMain(m *testing.M) { os.Exit(testenv.RunPackage(m)) } const restartNamespace = "admission-restart" +// demandIn prices sessions in namespace ns with oneCPUDemand and every other +// session as needing nothing. The admitter lists cluster-wide, and envtest is +// shared: another test's sessions must not take this test's room. +func demandIn(ns string) func(context.Context, *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + return func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Namespace != ns { + return admission.Demand{}, nil + } + return oneCPUDemand(ctx, sess) + } +} + // startAdmitter runs one admitter in its own manager, as a fresh operator // process would, and returns a stop func that blocks until the manager exits. -func startAdmitter(t *testing.T, env *testenv.Env) (stop func()) { +// Only sessions in ns are priced (demandIn). +func startAdmitter(t *testing.T, env *testenv.Env, ns string) (stop func()) { t.Helper() mgr, err := ctrl.NewManager(env.Cfg, ctrl.Options{ Scheme: env.Scheme, @@ -49,7 +62,7 @@ func startAdmitter(t *testing.T, env *testenv.Env) (stop func()) { Enabled: true, // envtest runs no AgentSession reconciler, so settings are never // resolved; every session needs the same one-CPU pod set. - Demand: oneCPUDemand, + Demand: demandIn(ns), }).SetupWithManager(mgr), "admitter SetupWithManager") ctx, cancel := context.WithCancel(context.Background()) @@ -139,7 +152,7 @@ func TestAdmitterRestartRebuildsLine(t *testing.T) { require.NoError(t, err, "create namespace") } - stop := startAdmitter(t, env) + stop := startAdmitter(t, env, restartNamespace) for i := 1; i <= 5; i++ { sess := &spiceboxv1alpha1.AgentSession{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("s%d", i), Namespace: restartNamespace}, @@ -164,7 +177,7 @@ func TestAdmitterRestartRebuildsLine(t *testing.T) { assertStaysAdmitted(t, c, []string{"s1", "s2"}, []string{"s1", "s2"}) stop() - startAdmitter(t, env) + startAdmitter(t, env, restartNamespace) // Release a holder the way a finished session does: it goes terminal. var s1 spiceboxv1alpha1.AgentSession require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: restartNamespace, Name: "s1"}, &s1), "get s1") @@ -180,3 +193,101 @@ func TestAdmitterRestartRebuildsLine(t *testing.T) { assert.Equal(t, int32(1), s4.Status.Admission.Position, "s4 heads the line") assert.Equal(t, int32(2), s4.Status.Admission.LineLength, "two still waiting") } + +const adoptNamespace = "admission-adopt" + +// TestAdmitterAdoptsPreAdmissionSessionsEnvtest is the rollout property +// against a real apiserver: sessions whose pods were created before admission +// was switched on — a Running one with a bound runner pod, a Pending one +// mid-boot with an unbound sandbox pod under a warm-pool name — carry no +// Admitted condition. The admitter adopts both (Admitted=True/Adopted) and +// counts them as holders: the Pending one's unplaced pod takes one of the two +// CPUs, so of two queued sessions exactly one is admitted. +func TestAdmitterAdoptsPreAdmissionSessionsEnvtest(t *testing.T) { + env := testenv.Shared(t) + c := env.Client + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: adoptNamespace}}); !apierrors.IsAlreadyExists(err) { + require.NoError(t, err, "create namespace") + } + + create := func(name string, mutate func(s *spiceboxv1alpha1.AgentSession)) { + t.Helper() + sess := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: adoptNamespace}, + Spec: spiceboxv1alpha1.AgentSessionSpec{ + Class: testClass, + Prompt: spiceboxv1alpha1.PromptSource{Inline: "review this"}, + }, + } + require.NoError(t, c.Create(ctx, sess), "create %s", name) + mutate(sess) + require.NoError(t, c.Status().Update(ctx, sess), "status %s", name) + } + pod := func(name string, labels map[string]string, node string) { + t.Helper() + require.NoError(t, c.Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: adoptNamespace, Labels: labels}, + Spec: corev1.PodSpec{ + NodeName: node, + Containers: []corev1.Container{{Name: "c", Image: "busybox"}}, + }, + }), "create pod %s", name) + } + + create("old-running", func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning + s.Status.RunnerPodName = "old-running-runner" + }) + pod("old-running-runner", map[string]string{sessionPodLabel: "old-running"}, "node-a") + create("old-booting", func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + s.Status.BundleSessions = []spiceboxv1alpha1.ResolvedBundle{{Name: "code", SpiceboxSessionName: "old-booting-code"}} + }) + pod("warmpool-x7k2p", map[string]string{sandboxPodLabel: "old-booting-code"}, "") + for _, name := range []string{"new-a", "new-b"} { + create(name, func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + meta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, + Reason: spiceboxv1alpha1.ReasonAdmissionQueued, Message: "waiting in line", + LastTransitionTime: s.CreationTimestamp, + }) + }) + } + + startAdmitter(t, env, adoptNamespace) + + reasons := func() map[string]string { + var list spiceboxv1alpha1.AgentSessionList + require.NoError(t, c.List(ctx, &list, client.InNamespace(adoptNamespace)), "list sessions") + out := map[string]string{} + for i := range list.Items { + if cond := meta.FindStatusCondition(list.Items[i].Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted); cond != nil && cond.Status == metav1.ConditionTrue { + out[list.Items[i].Name] = cond.Reason + } + } + return out + } + deadline := time.Now().Add(30 * time.Second) + var got map[string]string + for time.Now().Before(deadline) { + got = reasons() + if len(got) >= 3 { + break + } + time.Sleep(100 * time.Millisecond) + } + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionAdopted, got["old-running"], "running pre-admission session adopted") + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionAdopted, got["old-booting"], "booting pre-admission session (pool-named pod) adopted") + assert.Equal(t, spiceboxv1alpha1.ReasonAdmitted, got["new-a"], "the oldest queued session takes the free CPU") + + // Over a window, the adopted holders keep new-b out. + window := time.Now().Add(2 * time.Second) + for time.Now().Before(window) { + if !assert.NotContains(t, reasons(), "new-b", "new-b stays in line: the adopted holders count") { + return + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go index 16e8eb74..a16a7e1d 100644 --- a/pkg/controllers/admission/line.go +++ b/pkg/controllers/admission/line.go @@ -85,6 +85,36 @@ func IsQueued(sess *v1alpha1.AgentSession) bool { return c != nil && c.Status == metav1.ConditionFalse && c.Reason == v1alpha1.ReasonAdmissionQueued } +// PreAdmission reports whether the session started its pods before admission +// was switched on: it has never had an Admitted condition, is not terminal, +// and its status records pods — a runner pod or a bundle SpiceboxSession. The +// reconciler's gate lets such a session through (it already holds what it +// would have been admitted for, and gating it would freeze it with its pods +// up), and the admitter adopts it (Admitted=True/Adopted) so it is counted as +// a holder. +// +// A session whose pods were reaped while it stays non-terminal — slept, or +// Held — is excluded: its status still names the pods it had, but it holds +// nothing, and its next wake must queue like any cold wake. +func PreAdmission(sess *v1alpha1.AgentSession) bool { + switch sess.Status.Phase { + case v1alpha1.AgentSessionPhaseSucceeded, v1alpha1.AgentSessionPhaseFailed, v1alpha1.AgentSessionPhaseHeld: + return false + } + if sess.Status.SleptAt != nil || admittedCondition(sess) != nil { + return false + } + if sess.Status.RunnerPodName != "" { + return true + } + for _, b := range sess.Status.BundleSessions { + if b.SpiceboxSessionName != "" { + return true + } + } + return false +} + // IsAdmitted reports whether the session's Admitted condition is True. func IsAdmitted(sess *v1alpha1.AgentSession) bool { return meta.IsStatusConditionTrue(sess.Status.Conditions, v1alpha1.AgentSessionConditionAdmitted) diff --git a/pkg/controllers/admission/line_test.go b/pkg/controllers/admission/line_test.go index 41c22d8a..72775cf2 100644 --- a/pkg/controllers/admission/line_test.go +++ b/pkg/controllers/admission/line_test.go @@ -148,6 +148,57 @@ func TestIsQueued(t *testing.T) { } } +func TestPreAdmission(t *testing.T) { + withPods := func(mut func(s *v1alpha1.AgentSession)) *v1alpha1.AgentSession { + s := sessionAt(0) + s.Status.Phase = v1alpha1.AgentSessionPhasePending + mut(s) + return s + } + runner := func(s *v1alpha1.AgentSession) { s.Status.RunnerPodName = "s-runner" } + bundle := func(s *v1alpha1.AgentSession) { + s.Status.BundleSessions = []v1alpha1.ResolvedBundle{{Name: "code", SpiceboxSessionName: "s-code"}} + } + cases := []struct { + name string + sess *v1alpha1.AgentSession + want bool + }{ + {"Pending, bundle sessions recorded, no Admitted condition: pre-admission", withPods(bundle), true}, + {"Running, runner pod recorded, no Admitted condition: pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseRunning + }), true}, + {"warm Idle, runner pod recorded, no Admitted condition: pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseIdle + }), true}, + {"Pending, nothing recorded: a new session, not pre-admission", withPods(func(*v1alpha1.AgentSession) {}), false}, + {"has an Admitted condition: admission already knows it", withPods(func(s *v1alpha1.AgentSession) { + bundle(s) + s.Status.Conditions = []metav1.Condition{{Type: v1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionFalse, Reason: v1alpha1.ReasonAdmissionQueued}} + }), false}, + {"slept (pods reaped): not pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseIdle + s.Status.SleptAt = mtime(time.Minute) + }), false}, + {"Held (pods reaped): not pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseHeld + }), false}, + {"terminal: not pre-admission", withPods(func(s *v1alpha1.AgentSession) { + runner(s) + s.Status.Phase = v1alpha1.AgentSessionPhaseSucceeded + }), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, PreAdmission(tc.sess)) + }) + } +} + func TestDemandTotal(t *testing.T) { d := Demand{Pods: []PodDemand{ {Name: "runner", CPUMilli: 500, MemBytes: 1 << 20, EphemeralBytes: 9}, diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index 2e0ee4b8..1abdb9a2 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -39,8 +39,13 @@ import ( // // A warm wake — Idle back to Pending while still Admitted=True — is not // gated: the session never gave up its place, so it provisions at once. +// +// Nor is a session that booted before admission was switched on +// (admission.PreAdmission): it already has pods, so gating it would leave +// them up while it waited for a second copy of its set. It is let through, +// with no Queued marker, and the admitter adopts it as a holder. func (r *Reconciler) awaitingAdmission(sess *spiceboxv1alpha1.AgentSession) bool { - if !r.AdmissionEnabled || admission.IsAdmitted(sess) { + if !r.AdmissionEnabled || admission.IsAdmitted(sess) || admission.PreAdmission(sess) { return false } return sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending @@ -121,11 +126,17 @@ func admissionWaitExpiredMessage(waited time.Duration) string { // releaseAdmissionOnReclaim records, on sess, that the session's pods are being // reclaimed: Admitted True→False/Reclaimed. The caller persists it in the same -// status write that records the reclaim. A session that is not Admitted=True -// is left untouched — admission off, or never admitted — so no condition -// appears on sessions that never had one. Reports whether it changed sess. +// status write that records the reclaim, and calls it before marking the +// session slept or Held. A session that is neither Admitted=True nor (with +// admission on) pre-admission is left untouched — admission off, or never +// admitted — so no condition appears on sessions that never had one. +// +// A pre-admission session not yet adopted gets the release too: once its +// pods are gone, a wake that clears SleptAt would otherwise make it look +// pre-admission again and let it past the gate with nothing reserved. +// Reports whether it changed sess. func (r *Reconciler) releaseAdmissionOnReclaim(sess *spiceboxv1alpha1.AgentSession) bool { - if !admission.IsAdmitted(sess) { + if !admission.IsAdmitted(sess) && (!r.AdmissionEnabled || !admission.PreAdmission(sess)) { return false } // r.now() for the transition time: admission.JoinedAt and WaitingSince diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index b5d6fbeb..84cb43f4 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -66,6 +66,8 @@ func TestAdmissionGate(t *testing.T) { // wantQueuedAt, when set, is the expected Admitted=False/Queued // transition time: the moment the session reached the gate. wantQueuedAt time.Time + // wantNoAdmittedCnd: the gate wrote no Admitted condition at all. + wantNoAdmittedCnd bool }{ { name: "Pending, queued 2m ago: no SpiceboxSession; CapacityHold=True/WaitingForAdmission after grace", @@ -155,6 +157,18 @@ func TestAdmissionGate(t *testing.T) { }, wantQueuedAt: nowT, }, + { + name: "pre-admission: booted before admission (bundle recorded, no Admitted condition), long ago: proceeds, not queued, not failed", + enabled: true, bundles: true, createdAt: -3 * time.Hour, + prep: func(s *spiceboxv1alpha1.AgentSession) { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhasePending + s.Status.BundleSessions = []spiceboxv1alpha1.ResolvedBundle{{ + Name: codeBundle.Name, SpiceboxSessionName: agentsession.BundleSessionName(s, codeBundle), + }} + }, + wantPods: true, + wantNoAdmittedCnd: true, + }, { name: "AdmissionEnabled=false: bundle SpiceboxSession created without Admitted", bundles: true, createdAt: -2 * time.Minute, @@ -203,6 +217,10 @@ func TestAdmissionGate(t *testing.T) { assert.Equal(t, tc.wantHoldReason, hold.Reason) } + if tc.wantNoAdmittedCnd { + assert.Nil(t, apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + "no Admitted condition: the gate let it through without queueing it") + } if !tc.wantQueuedAt.IsZero() { adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) if assert.NotNil(t, adm, "Admitted condition") { @@ -351,6 +369,43 @@ func TestAdmissionGate_SleepReleasesInOneWrite(t *testing.T) { assert.Equal(t, nowT, adm.LastTransitionTime.Time.UTC(), "transition stamped from the reconciler clock") } +// A pre-admission session (pods started before admission, no Admitted +// condition) that sleeps gives its place up like an admitted one: +// Admitted=False/Reclaimed in the sleep's write. Without it the session would +// look pre-admission again once a wake clears SleptAt, and bypass the gate +// with no pods. +func TestAdmissionGate_PreAdmissionSleepReleases(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + clock := func() time.Time { return nowT } + + idle := metav1.NewTime(nowT.Add(-time.Hour)) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-2*time.Hour)) + sess.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + sess.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + sess.Status.LastIdleAt = &idle + sess.Status.RunnerPodName = "s1-runner" + ac := classWithBundle("ac1") + + r, c := fakeReconciler(t, clock, ac, sess) + r.AdmissionEnabled = true + r.DefaultSessionSleepAfter = 10 * time.Minute + + live := getSession(t, c, "s1") + require.True(t, admission.PreAdmission(&live), "fixture is pre-admission") + slept, _, err := r.ReconcileSleepForTest(ctx, &live, ac) + require.NoError(t, err) + require.True(t, slept, "past-grace Idle session sleeps") + + got := getSession(t, c, "s1") + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionReclaimed, adm.Reason) + got.Status.SleptAt = nil + assert.False(t, admission.PreAdmission(&got), "after a wake clears SleptAt it is a normal session that queues") +} + func ptrTo[T any](v T) *T { return &v } // An ADMITTED booting session whose bundle pod the scheduler still cannot diff --git a/pkg/controllers/agentsession/sleep.go b/pkg/controllers/agentsession/sleep.go index fcd8b5ea..0121d36c 100644 --- a/pkg/controllers/agentsession/sleep.go +++ b/pkg/controllers/agentsession/sleep.go @@ -109,13 +109,14 @@ func (r *Reconciler) reconcileSleep(ctx context.Context, sess *spiceboxv1alpha1. return false, 0, fmt.Errorf("apply Sleep event: %w", err) } patched := sess.DeepCopy() - now := metav1.NewTime(r.now()) - patched.Status.SleptAt = &now // The reap below gives the session's place back to its line. Recorded in // THIS write, beside SleptAt: a wake clears SleptAt, so a release written // separately could be lost in between, leaving the session admitted with - // no pods and its capacity counted twice. + // no pods and its capacity counted twice. Decided before SleptAt is set, + // which would hide a pre-admission session from the release. r.releaseAdmissionOnReclaim(patched) + now := metav1.NewTime(r.now()) + patched.Status.SleptAt = &now if err := agentstatus.WriteOwned(ctx, r.Client, patched, sess, spiceboxv1alpha1.OwnerOperator); err != nil { return false, 0, fmt.Errorf("stamp SleptAt: %w", err) } From 3f39e34a9a3e564f15205092e2003351db86769d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:37:04 -0500 Subject: [PATCH 21/33] Stop the misfit release looping on an optimistic capacity model Three cases let the model admit sets the scheduler could not place, so the misfit release reaped, re-admitted and re-noticed the same session every few minutes: tainted pools counted as room, a namespace-scoped operator not seeing other namespaces' pods, and unknown capacity admitting everything. The node model now records each node's NoSchedule/NoExecute taints and places a pod only where its tolerations (carried on PodDemand from the built spec) tolerate them. A namespace-scoped admitter treats capacity as unknown. Sessions admitted under unknown capacity get the reason AdmittedUnknownCapacity, which the misfit release skips, leaving them to the pod-level hold. Unknown capacity is logged on transition only. --- pkg/apis/v1alpha1/conditions.go | 6 ++ .../admission/admitter/controller.go | 59 +++++++++++++++---- .../admission/admitter/controller_test.go | 31 +++++++++- pkg/controllers/admission/capacity.go | 48 ++++++++++++++- pkg/controllers/admission/capacity_test.go | 53 +++++++++++++++++ pkg/controllers/admission/line.go | 4 ++ .../agentsession/admission_gate.go | 9 +++ .../agentsession/admission_gate_test.go | 29 ++++++--- pkg/controllers/agentsession/demand.go | 7 +-- 9 files changed, 216 insertions(+), 30 deletions(-) diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index d5807018..71651939 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -533,6 +533,12 @@ const ( // admission was switched on: the admitter adopts it as admitted, so it is // counted as a holder rather than gated with its pods already up. ReasonAdmissionAdopted = "Adopted" + // ReasonAdmittedUnknownCapacity marks a session admitted while the + // admitter had no reliable view of cluster capacity (no node data, a + // failed snapshot, or a namespace-scoped operator that cannot see other + // namespaces' pods). Nothing proved its set fits, so a stall is not + // treated as a misfit: its pods keep the ordinary pod-level hold. + ReasonAdmittedUnknownCapacity = "AdmittedUnknownCapacity" // ReasonAdmissionReclaimed marks a session released so a session ahead of // it in line could be admitted. ReasonAdmissionReclaimed = "Reclaimed" diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 82656862..0ccfc844 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -17,6 +17,7 @@ import ( "fmt" "slices" "strings" + "sync/atomic" "time" corev1 "k8s.io/api/core/v1" @@ -36,7 +37,6 @@ import ( "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" "github.com/authzed/openagentprimitives/pkg/controllers/agentstatus" - "github.com/authzed/openagentprimitives/pkg/platform/schedfit" ) // +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions,verbs=get;list;watch;patch @@ -88,6 +88,10 @@ type Reconciler struct { // ask for them back early for a session of its line that does not fit // (--admission-min-warm). MinWarm time.Duration + // capacityUnknown is whether the previous pass found capacity unknown, so + // the reason is logged once on a known→unknown transition rather than on + // every pass (the admitter is woken by every session and pod event). + capacityUnknown atomic.Bool // Now is the clock; nil means time.Now. Now func() time.Time // Demand prices a session's whole pod set. Nil means the default: get the @@ -241,16 +245,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R r.addDrainingReclaims(ctx, &pool, sessions, pods) } - in.Model = r.snapshot(ctx) - if !in.Model.Known() { - logger.Info("admission: cluster capacity unknown; admitting in line order", "waiting", len(in.Waiting)) - } + in.Model = r.snapshot(ctx, len(in.Waiting)) in.TotalCPUMilli, in.TotalMemBytes = in.Model.Totals() dec := admission.Decide(in) + // Under an unknown model nothing proved the set fits, so the reconciler + // must not read a later stall as the model being wrong: the reason tells + // its misfit release to leave the session to the pod-level hold. + reason, message := spiceboxv1alpha1.ReasonAdmitted, "The session's whole pod set fits the cluster." + if !in.Model.Known() { + reason, message = spiceboxv1alpha1.ReasonAdmittedUnknownCapacity, + "Cluster capacity is unknown to the operator; admitted in line order without a fit check." + } for _, key := range dec.Admit { - if err := r.admit(ctx, byKey[key], now, spiceboxv1alpha1.ReasonAdmitted, "The session's whole pod set fits the cluster."); err != nil { + if err := r.admit(ctx, byKey[key], now, reason, message); err != nil { errs = append(errs, err) } } @@ -333,10 +342,9 @@ func boundPods(namespace string, d admission.Demand, pods podIndex) []admission. pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { continue } - cpu, mem, eph := schedfit.EffectiveRequests(&pod.Spec) placed = append(placed, admission.PlacedPod{ Node: pod.Spec.NodeName, - PodDemand: admission.PodDemand{Name: pod.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph}, + PodDemand: admission.PodDemandFor(pod.Name, &pod.Spec), }) } return placed @@ -563,15 +571,40 @@ func (r *Reconciler) listSessions(ctx context.Context) ([]spiceboxv1alpha1.Agent } // snapshot returns the capacity model, or an unknown one when there is no -// source or the snapshot fails (logged): missing data must not block sessions. -func (r *Reconciler) snapshot(ctx context.Context) admission.CapacityModel { - if r.Capacity == nil { +// source, when the snapshot fails, or when the operator is namespace-scoped: +// missing data must not block sessions. The reason is logged once per +// known→unknown transition (see capacityUnknown). +// +// A namespace-scoped operator (a watch set) sees only its own namespaces' +// pods, so node headroom computed from them overstates what is free by every +// pod in every other namespace. An optimistic model admits sets the scheduler +// cannot place, and the misfit release then reaps and re-admits them in a +// loop; an unknown one admits in line order and leaves stalls to the +// pod-level hold. +func (r *Reconciler) snapshot(ctx context.Context, waiting int) admission.CapacityModel { + unknown := func(why string, kv ...any) admission.CapacityModel { + if !r.capacityUnknown.Swap(true) { + log.FromContext(ctx).Info("admission: cluster capacity unknown; admitting in line order without a fit check", + append([]any{"reason", why, "waiting", waiting}, kv...)...) + } return admission.NewNodeModel(nil, nil) } + if len(r.Namespaces) > 0 { + return unknown("the operator is namespace-scoped (--watch-namespaces) and cannot see other namespaces' pods", + "namespaces", r.Namespaces) + } + if r.Capacity == nil { + return unknown("no capacity source is configured") + } m, err := r.Capacity.Snapshot(ctx) if err != nil { - log.FromContext(ctx).Info("admission: capacity snapshot failed; treating capacity as unknown", "err", err.Error()) - return admission.NewNodeModel(nil, nil) + return unknown("the capacity snapshot failed", "err", err.Error()) + } + if !m.Known() { + return unknown("no node reports allocatable resources") + } + if r.capacityUnknown.Swap(false) { + log.FromContext(ctx).Info("admission: cluster capacity known again; admitting by fit") } return m } diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index b532c9b6..d63fab29 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -255,17 +255,44 @@ func TestAdmitterPass(t *testing.T) { assertAdmitted(t, getSession(t, c, "rb-b")) }) - t.Run("capacity snapshot fails: capacity unknown, every waiting session admitted in line order", func(t *testing.T) { + t.Run("capacity snapshot fails: capacity unknown, every waiting session admitted as AdmittedUnknownCapacity", func(t *testing.T) { r, c, _ := newReconciler(t, true, failingSource{}, waitingSession("rb-a", time.Minute), waitingSession("rb-b", 2*time.Minute), ) runPass(t, r) + assertAdmittedUnknown(t, getSession(t, c, "rb-a")) + assertAdmittedUnknown(t, getSession(t, c, "rb-b")) + }) + + t.Run("namespace-scoped operator (watch set): capacity treated as unknown even with no room, admitted as AdmittedUnknownCapacity", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(0), + waitingSession("rb-a", time.Minute), + ) + r.Namespaces = []string{"default"} + runPass(t, r) + assertAdmittedUnknown(t, getSession(t, c, "rb-a")) + }) + + t.Run("known capacity: admitted with reason Admitted", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(1), waitingSession("rb-a", time.Minute)) + runPass(t, r) assertAdmitted(t, getSession(t, c, "rb-a")) - assertAdmitted(t, getSession(t, c, "rb-b")) }) } +// assertAdmittedUnknown asserts s was admitted under an unknown capacity +// model, which the reconciler's misfit release must leave alone. +func assertAdmittedUnknown(t *testing.T, s *spiceboxv1alpha1.AgentSession) { + t.Helper() + cond := meta.FindStatusCondition(s.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if !assert.NotNil(t, cond, "%s: Admitted condition", s.Name) { + return + } + assert.Equal(t, metav1.ConditionTrue, cond.Status, "%s: Admitted status", s.Name) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmittedUnknownCapacity, cond.Reason, "%s: Admitted reason", s.Name) +} + type failingSource struct{} func (failingSource) Snapshot(context.Context) (admission.CapacityModel, error) { diff --git a/pkg/controllers/admission/capacity.go b/pkg/controllers/admission/capacity.go index 0050626f..cc0dbe10 100644 --- a/pkg/controllers/admission/capacity.go +++ b/pkg/controllers/admission/capacity.go @@ -7,6 +7,7 @@ import ( "slices" corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/authzed/openagentprimitives/pkg/platform/schedfit" @@ -61,9 +62,21 @@ func (s KubeCapacitySource) Snapshot(ctx context.Context) (CapacityModel, error) return NewNodeModel(nodes.Items, pods.Items), nil } +// PodDemandFor prices one pod from its spec the way the capacity snapshot +// counts it (schedfit.EffectiveRequests), carrying its tolerations so a node +// model places it only where the scheduler could. +func PodDemandFor(name string, spec *corev1.PodSpec) PodDemand { + cpu, mem, eph := schedfit.EffectiveRequests(spec) + return PodDemand{Name: name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph, Tolerations: spec.Tolerations} +} + type nodeSlot struct { name string cpuMilli, memBytes, ephemeralBytes int64 + // taints are the node's NoSchedule and NoExecute taints: a pod is placed + // here only if it tolerates every one. PreferNoSchedule is a preference + // the scheduler may override, so it is not recorded. + taints []corev1.Taint // pods is the remaining pod slots; unlimitedPods is set when the node // reports no `pods` allocatable (fake or unusual nodes), so it is not // read as zero. @@ -73,7 +86,36 @@ type nodeSlot struct { func (n *nodeSlot) fits(p PodDemand) bool { return p.CPUMilli <= n.cpuMilli && p.MemBytes <= n.memBytes && - p.EphemeralBytes <= n.ephemeralBytes && (n.unlimitedPods || n.pods >= 1) + p.EphemeralBytes <= n.ephemeralBytes && (n.unlimitedPods || n.pods >= 1) && + n.tolerates(p.Tolerations) +} + +// tolerates reports whether tols tolerate every one of the node's recorded +// taints, matched by the upstream Toleration.ToleratesTaint. Comparison +// operators (Lt/Gt) are matched as disabled, the scheduler's default: a pod +// relying on them is kept off the node rather than admitted onto it. +func (n *nodeSlot) tolerates(tols []corev1.Toleration) bool { + logger := klog.Background() + for i := range n.taints { + if !slices.ContainsFunc(tols, func(t corev1.Toleration) bool { + return t.ToleratesTaint(logger, &n.taints[i], false) + }) { + return false + } + } + return true +} + +// blockingTaints is the subset of a node's taints that keep an intolerant pod +// off it. +func blockingTaints(node corev1.Node) []corev1.Taint { + var out []corev1.Taint + for _, t := range node.Spec.Taints { + if t.Effect == corev1.TaintEffectNoSchedule || t.Effect == corev1.TaintEffectNoExecute { + out = append(out, t) + } + } + return out } func (n *nodeSlot) take(p PodDemand) { @@ -92,7 +134,8 @@ type nodeModel struct { } // NewNodeModel builds a model from node headroom. Unschedulable nodes are -// skipped. With no nodes, or none reporting allocatable, the model is unknown. +// skipped, and a node's NoSchedule/NoExecute taints are recorded so Reserve +// places only pods that tolerate them. With no nodes, or none reporting allocatable, the model is unknown. func NewNodeModel(nodes []corev1.Node, pods []corev1.Pod) CapacityModel { m := &nodeModel{} if !schedfit.CeilingFromNodes(nodes).Known { @@ -115,6 +158,7 @@ func NewNodeModel(nodes []corev1.Node, pods []corev1.Pod) CapacityModel { ephemeralBytes: h.EphemeralBytes, pods: h.Pods, unlimitedPods: !hasPods, + taints: blockingTaints(nodes[i]), }) } return m diff --git a/pkg/controllers/admission/capacity_test.go b/pkg/controllers/admission/capacity_test.go index 439fa72f..73cc7335 100644 --- a/pkg/controllers/admission/capacity_test.go +++ b/pkg/controllers/admission/capacity_test.go @@ -211,3 +211,56 @@ func TestNodeModelTotals(t *testing.T) { }) } } + +// A node's NoSchedule/NoExecute taints keep a pod off it unless the pod +// tolerates them, as the scheduler would; PreferNoSchedule does not. Without +// this the model counts a tainted pool as room, admits a set the scheduler +// then cannot place, and the misfit safety net reaps and re-admits it in a +// loop. +func TestNodeModelReserveRespectsTaints(t *testing.T) { + gpuTaint := corev1.Taint{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule} + tainted := func(effect corev1.TaintEffect) corev1.Node { + n := testNode("gpu", "8", "8Gi", "10") + taint := gpuTaint + taint.Effect = effect + n.Spec.Taints = []corev1.Taint{taint} + return n + } + tolerating := pod("p", 1000) + tolerating.Tolerations = []corev1.Toleration{{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}} + cases := []struct { + name string + nodes []corev1.Node + pod PodDemand + want bool + }{ + {"only a NoSchedule-tainted node, pod does not tolerate: does not fit", []corev1.Node{tainted(corev1.TaintEffectNoSchedule)}, pod("p", 1000), false}, + {"only a NoExecute-tainted node, pod does not tolerate: does not fit", []corev1.Node{tainted(corev1.TaintEffectNoExecute)}, pod("p", 1000), false}, + {"only a NoSchedule-tainted node, pod tolerates it: fits", []corev1.Node{tainted(corev1.TaintEffectNoSchedule)}, tolerating, true}, + {"only a PreferNoSchedule-tainted node: fits (a preference, not a bar)", []corev1.Node{tainted(corev1.TaintEffectPreferNoSchedule)}, pod("p", 1000), true}, + {"a tainted big node beside an untainted small one, pod too big for the small: does not fit", + []corev1.Node{tainted(corev1.TaintEffectNoSchedule), testNode("small", "500m", "1Gi", "10")}, pod("p", 1000), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := NewNodeModel(tc.nodes, nil) + require.True(t, m.Known()) + assert.Equal(t, tc.want, m.Reserve(Demand{Pods: []PodDemand{tc.pod}})) + }) + } +} + +func TestPodDemandFor(t *testing.T) { + tol := corev1.Toleration{Key: "pool", Operator: corev1.TolerationOpEqual, Value: "agents", Effect: corev1.TaintEffectNoSchedule} + spec := corev1.PodSpec{ + Tolerations: []corev1.Toleration{tol}, + Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), corev1.ResourceMemory: resource.MustParse("64Mi"), + }}}}, + } + got := PodDemandFor("runner", &spec) + assert.Equal(t, "runner", got.Name) + assert.Equal(t, int64(250), got.CPUMilli) + assert.Equal(t, int64(64<<20), got.MemBytes) + assert.Equal(t, []corev1.Toleration{tol}, got.Tolerations, "the pod's tolerations ride along for the node model") +} diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go index a16a7e1d..fe8b68f4 100644 --- a/pkg/controllers/admission/line.go +++ b/pkg/controllers/admission/line.go @@ -8,6 +8,7 @@ import ( "slices" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -19,6 +20,9 @@ import ( type PodDemand struct { Name string CPUMilli, MemBytes, EphemeralBytes int64 + // Tolerations are the pod's own, from its built spec: a node model places + // the pod only on a node whose NoSchedule/NoExecute taints it tolerates. + Tolerations []corev1.Toleration } // Demand is the whole pod set a session needs admitted together. diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index 1abdb9a2..65575f11 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -171,10 +171,19 @@ const admissionMisfitGrace = 2 * time.Minute // forever with its waiting-timeout clock restarting at every release. Left // alone, it keeps the ordinary readiness deadline and its specific failure // message. +// +// A session admitted under an unknown capacity model +// (ReasonAdmittedUnknownCapacity) is skipped as well: no fit check admitted +// it, so a stall is not the model being wrong, and releasing it would only +// re-admit it at once into the same stall. Its pods keep the pod-level +// capacity hold and that hold's deadline. func (r *Reconciler) placementMisfit(sess *spiceboxv1alpha1.AgentSession, stall schedulingStall, stalled bool) bool { if !r.AdmissionEnabled || !stalled || !admission.IsAdmitted(sess) { return false } + if c := conditions.Find(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted); c.Reason == spiceboxv1alpha1.ReasonAdmittedUnknownCapacity { + return false + } booting := sess.Status.Phase == "" || sess.Status.Phase == spiceboxv1alpha1.AgentSessionPhasePending return booting && capacityShortfall(stall.reason) && stall.pending >= admissionMisfitGrace } diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index 84cb43f4..f35cd9cc 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -425,14 +425,16 @@ func TestAdmissionGate_Misfit(t *testing.T) { const mixedSchedMsg = "0/4 nodes are available: 3 Insufficient cpu, 1 node(s) had untolerated taint(s)." cases := []struct { - name string - enabled bool // Reconciler.AdmissionEnabled - admitted bool // the session carries Admitted=True - pending time.Duration // how long the bundle pod has been unschedulable - createdAgo time.Duration // session creation (its JoinedAt); 0 = 20m - schedMsg string - woken bool // the session was cold-woken before this admission - heldBefore bool // the bundle stage already set CapacityHold=True/Waiting + name string + enabled bool // Reconciler.AdmissionEnabled + admitted bool // the session carries Admitted=True + // admittedReason is the Admitted=True reason; "" = Admitted. + admittedReason string + pending time.Duration // how long the bundle pod has been unschedulable + createdAgo time.Duration // session creation (its JoinedAt); 0 = 20m + schedMsg string + woken bool // the session was cold-woken before this admission + heldBefore bool // the bundle stage already set CapacityHold=True/Waiting wantReaped bool wantMisfit bool // Admitted=False/PlacementMismatch at now @@ -456,6 +458,11 @@ func TestAdmissionGate_Misfit(t *testing.T) { schedMsg: mixedSchedMsg, heldBefore: true, wantReaped: true, wantMisfit: true, wantFailed: true, }, + { + name: "admitted under unknown capacity, pods unschedulable 2m: no misfit release, pods kept under the pod-level hold, still admitted", + enabled: true, admitted: true, admittedReason: spiceboxv1alpha1.ReasonAdmittedUnknownCapacity, + pending: 2 * time.Minute, schedMsg: capacitySchedMsg, + }, { name: "admitted, pods unschedulable 1m (inside the misfit grace): pods kept, still admitted", enabled: true, admitted: true, pending: time.Minute, schedMsg: capacitySchedMsg, @@ -487,7 +494,11 @@ func TestAdmissionGate_Misfit(t *testing.T) { sess.Status.LastWakeAt = &wake } if tc.admitted { - withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, started.Add(-time.Second)) + reason := tc.admittedReason + if reason == "" { + reason = spiceboxv1alpha1.ReasonAdmitted + } + withAdmittedCondition(sess, metav1.ConditionTrue, reason, started.Add(-time.Second)) } if tc.heldBefore { sess.Status.Conditions = append(sess.Status.Conditions, metav1.Condition{ diff --git a/pkg/controllers/agentsession/demand.go b/pkg/controllers/agentsession/demand.go index b1a8089c..8b94d181 100644 --- a/pkg/controllers/agentsession/demand.go +++ b/pkg/controllers/agentsession/demand.go @@ -13,7 +13,6 @@ import ( "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" "github.com/authzed/openagentprimitives/pkg/platform/podspec" - "github.com/authzed/openagentprimitives/pkg/platform/schedfit" "github.com/authzed/openagentprimitives/pkg/tools/toolchain/resolve" "github.com/authzed/openagentprimitives/pkg/web/secretoutsrv" ) @@ -30,7 +29,8 @@ const demandPlaceholderImage = "demand-estimate" // creates nothing. // // Each pod is rendered by the same builder the reconciler uses to create it, -// then priced by schedfit.EffectiveRequests: +// then priced by admission.PodDemandFor (schedfit.EffectiveRequests, plus the +// pod's tolerations for the node model's taint check): // // - the runner (BuildRunnerPod, as PodRunnerFactory calls it), with the // in-pod sidecars it carries; @@ -120,8 +120,7 @@ func DemandFor(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.Agen d := admission.Demand{Pods: make([]admission.PodDemand, 0, len(pods))} for _, p := range pods { - cpu, mem, eph := schedfit.EffectiveRequests(&p.Spec) - d.Pods = append(d.Pods, admission.PodDemand{Name: p.Name, CPUMilli: cpu, MemBytes: mem, EphemeralBytes: eph}) + d.Pods = append(d.Pods, admission.PodDemandFor(p.Name, &p.Spec)) } return d, nil } From 6669f35f010938a41fc869d84dcb612202176a3e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:38:47 -0500 Subject: [PATCH 22/33] Reclaim a warm idle session when its line is blocked by the cap Warm idle holders count toward an AgentClass's session cap, but early reclaim only planned for heads that did not fit, so a line held at its cap by idle sessions waited out their full sleep grace. Decide now reports which lines' heads wait on the cap. For such a head the admitter asks the line's oldest non-child warm idle session past the minimum warm time to sleep early, one per head, and not while an earlier request for the line is still in flight. --- .../admission/admitter/controller.go | 70 +++++++++++------ .../admission/admitter/controller_test.go | 75 +++++++++++++++++++ pkg/controllers/admission/decide.go | 24 +++++- pkg/controllers/admission/decide_test.go | 53 +++++++++++++ pkg/controllers/admission/reclaim.go | 29 +++++++ pkg/controllers/admission/reclaim_test.go | 52 +++++++++++++ 6 files changed, 276 insertions(+), 27 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 0ccfc844..a70d993f 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -218,8 +218,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R in.Caps, in.Waiting = r.capLines(ctx, in.Waiting, byKey) pool := reclaimPool{ - sessions: map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{}, - inFlight: map[string][]admission.PlacedPod{}, + sessions: map[types.NamespacedName]*spiceboxv1alpha1.AgentSession{}, + inFlight: map[string][]admission.PlacedPod{}, + requested: map[string]int{}, } for _, sess := range holding { d, err := r.demand(ctx, sess) @@ -268,9 +269,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R errs = append(errs, err) } } - if in.Model.Known() { - errs = append(errs, r.requestReclaims(ctx, in.Waiting, dec, &pool, in.Model, now)...) - } + // Capacity reclaims plan only on a known model (PlanReclaim returns nil + // otherwise); cap reclaims need no capacity data, so they run either way. + errs = append(errs, r.requestReclaims(ctx, in.Waiting, dec, &pool, in.Model, now)...) return requeue, errors.Join(errs...) } @@ -360,26 +361,35 @@ func boundPods(namespace string, d admission.Demand, pods podIndex) []admission. // and the admitter is woken by each of those status writes and pod events — // would see the head still not fit and reclaim the next-oldest warm session // too, cascading through the line for one head. +// +// requested counts, per line, the non-child holders already carrying a +// reclaim request and not yet slept: each is already giving a cap slot back, +// so a cap-blocked head waits for it rather than asking for another. type reclaimPool struct { - warm []admission.WarmIdle - sessions map[types.NamespacedName]*spiceboxv1alpha1.AgentSession - inFlight map[string][]admission.PlacedPod + warm []admission.WarmIdle + sessions map[types.NamespacedName]*spiceboxv1alpha1.AgentSession + inFlight map[string][]admission.PlacedPod + requested map[string]int } // addWarm files a warm Idle holder's bound pods: as in flight when a reclaim // is already requested (so it is not chosen again), else as a candidate. A // session with no bound pods has nothing to give back and is skipped. func (p *reclaimPool) addWarm(sess *spiceboxv1alpha1.AgentSession, placed []admission.PlacedPod) { - if len(placed) == 0 { - return - } line := lineOf(sess) + child := sess.Spec.Parent != nil if _, requested := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]; requested { p.inFlight[line] = append(p.inFlight[line], placed...) + if !child { + p.requested[line]++ + } + return + } + if len(placed) == 0 { return } key := client.ObjectKeyFromObject(sess) - p.warm = append(p.warm, admission.WarmIdle{Key: key, Line: line, IdleSince: sess.Status.LastIdleAt.Time, Pods: placed}) + p.warm = append(p.warm, admission.WarmIdle{Key: key, Line: line, IdleSince: sess.Status.LastIdleAt.Time, Pods: placed, Child: child}) p.sessions[key] = sess } @@ -479,13 +489,18 @@ func (r *Reconciler) addDrainingReclaims(ctx context.Context, pool *reclaimPool, } } -// requestReclaims asks for the warm Idle sessions PlanReclaim picks for each -// line's head (its still-waiting position 1) to be slept early. model is the -// one Decide reserved into, so this pass's admissions are respected. Before a -// line plans, its in-flight reclaims are freed on model, so only the shortfall -// they do not already cover is reclaimed. Lines go in name order; each line's -// frees and its head's demand are applied to model before the next line plans, -// so two lines never count the same free space. +// requestReclaims asks for the warm Idle sessions picked for each line's head +// (its still-waiting position 1) to be slept early. model is the one Decide +// reserved into, so this pass's admissions are respected. Lines go in name +// order; each line's frees and its head's demand are applied to model before +// the next line plans, so two lines never count the same free space. +// +// A head whose line is at its cap (dec.CapBlocked) needs a slot, not space: +// PlanCapReclaim picks one warm non-child holder of the line, unless one +// already requested is still in flight (it is already giving a slot back). +// Any other head gets PlanReclaim's capacity plan; before it plans, the +// line's in-flight reclaims are freed on model, so only the shortfall they do +// not already cover is reclaimed. func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Candidate, dec admission.Decision, pool *reclaimPool, model admission.CapacityModel, now time.Time, ) []error { @@ -505,7 +520,14 @@ func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Ca for _, head := range heads { inFlight := pool.inFlight[head.Line] model.Free(inFlight) - plan := admission.PlanReclaim(head, pool.warm, model, now, r.MinWarm) + var plan []types.NamespacedName + if dec.CapBlocked[head.Line] { + if key, ok := admission.PlanCapReclaim(head, pool.warm, now, r.MinWarm); ok && pool.requested[head.Line] == 0 { + plan = []types.NamespacedName{key} + } + } else { + plan = admission.PlanReclaim(head, pool.warm, model, now, r.MinWarm) + } for _, key := range plan { model.Free(pods[key]) if err := r.requestReclaim(ctx, pool.sessions[key], head.Key, now); err != nil { @@ -516,9 +538,11 @@ func (r *Reconciler) requestReclaims(ctx context.Context, waiting []admission.Ca continue } // Hold the freed space for this head so a later line cannot plan on it. - // With a plan, PlanReclaim proved the head fits; a failure means that - // invariant broke. With only in-flight frees it may simply not fit yet. - if !model.Reserve(head.Demand) && len(plan) > 0 { + // With a capacity plan, PlanReclaim proved the head fits; a failure + // means that invariant broke. With only in-flight frees, or a cap + // reclaim (which frees a slot, not necessarily enough space), it may + // simply not fit yet. + if !model.Reserve(head.Demand) && len(plan) > 0 && !dec.CapBlocked[head.Line] { log.FromContext(ctx).Info("admission: head does not fit after its planned reclaims were freed; the plan's fit check and the model disagree", "class", head.Line, "head", head.Key.String(), "reclaims", len(plan)) } diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index d63fab29..a23d7df3 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -770,3 +770,78 @@ func TestAdmitterAdoptsPreAdmissionSessions(t *testing.T) { assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "rb-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted)) }) } + +// TestAdmitterCapReclaim pins early reclaim for a line blocked by its cap +// rather than by capacity: warm idle holders count toward the cap, so the +// oldest one past minWarm is asked back, one per head, and not while an +// earlier request for the line is still in flight. +func TestAdmitterCapReclaim(t *testing.T) { + const minWarm = 5 * time.Minute + requested := func(s *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.AgentSession { + s.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: baseTime.Format(time.RFC3339)} + return s + } + cases := []struct { + name string + objs []client.Object + wantAnnotated []string + wantClean []string + }{ + { + name: "cap 2 held by two warm idle sessions, plenty of room: the oldest is reclaimed, only it", + objs: []client.Object{ + cpuNode(8), cappedClass(2), + warmIdleSession("rb-old", 30*time.Minute), boundRunnerPod("rb-old"), + warmIdleSession("rb-mid", 20*time.Minute), boundRunnerPod("rb-mid"), + waitingSession("rb-wait", time.Minute), + }, + wantAnnotated: []string{"rb-old"}, + wantClean: []string{"rb-mid", "rb-wait"}, + }, + { + name: "a cap reclaim already in flight for the line: no second one", + objs: []client.Object{ + cpuNode(8), cappedClass(2), + requested(warmIdleSession("rb-old", 30*time.Minute)), boundRunnerPod("rb-old"), + warmIdleSession("rb-mid", 20*time.Minute), boundRunnerPod("rb-mid"), + waitingSession("rb-wait", time.Minute), + }, + wantClean: []string{"rb-mid", "rb-wait"}, + }, + { + name: "the cap is held by a running session, the warm one is too young: nothing reclaimed", + objs: []client.Object{ + cpuNode(8), cappedClass(2), + admittedRunning("rb-run"), + warmIdleSession("rb-young", time.Minute), boundRunnerPod("rb-young"), + waitingSession("rb-wait", time.Minute), + }, + wantClean: []string{"rb-young", "rb-wait"}, + }, + { + name: "under the cap and the head fits: admitted, nothing reclaimed", + objs: []client.Object{ + cpuNode(8), cappedClass(3), + warmIdleSession("rb-old", 30*time.Minute), boundRunnerPod("rb-old"), + waitingSession("rb-wait", time.Minute), + }, + wantClean: []string{"rb-old"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, c, _ := newReconciler(t, true, nil, tc.objs...) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = minWarm + runPass(t, r) + for _, name := range tc.wantAnnotated { + assert.Equal(t, passTime.Format(time.RFC3339), getSession(t, c, name).Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + "%s: reclaim requested at the pass's Now", name) + } + for _, name := range tc.wantClean { + assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "%s: no reclaim requested", name) + } + }) + } +} diff --git a/pkg/controllers/admission/decide.go b/pkg/controllers/admission/decide.go index e32bd2a7..1c8e6fee 100644 --- a/pkg/controllers/admission/decide.go +++ b/pkg/controllers/admission/decide.go @@ -36,11 +36,17 @@ type Position struct { Index, LineLength int32 } -// Decision is the outcome of one pass: who to admit now, and where everyone -// still waiting stands in their line. +// Decision is the outcome of one pass: who to admit now, where everyone +// still waiting stands in their line, and which lines' heads wait on their +// cap rather than on room. type Decision struct { Admit []types.NamespacedName Positions map[types.NamespacedName]Position + // CapBlocked holds every line whose head is still waiting, is not a + // delegated child, and whose non-child holders (counting this pass's + // admissions) have reached the line's cap. Freeing space does nothing for + // such a line; only a holder giving its slot back does. + CapBlocked map[string]bool } // lineState is one line's remaining queue and what it already holds. @@ -57,7 +63,13 @@ func (l *lineState) eligible() bool { if len(l.queue) == 0 { return false } - return l.queue[0].Child || l.capacity <= 0 || l.nonChild < l.capacity + return !l.capBlocked() +} + +// capBlocked reports whether the line's head waits on the cap: a non-child +// head, a capped line, its non-child holders at the cap. +func (l *lineState) capBlocked() bool { + return len(l.queue) > 0 && !l.queue[0].Child && l.capacity > 0 && l.nonChild >= l.capacity } // Decide admits waiting sessions, whole pod set at a time, repeatedly picking @@ -152,10 +164,14 @@ func Decide(in Input) Decision { } positions := map[types.NamespacedName]Position{} + capBlocked := map[string]bool{} for _, l := range lines { for i, c := range l.queue { positions[c.Key] = Position{Index: int32(i + 1), LineLength: int32(len(l.queue))} } + if l.capBlocked() { + capBlocked[l.name] = true + } } - return Decision{Admit: admit, Positions: positions} + return Decision{Admit: admit, Positions: positions, CapBlocked: capBlocked} } diff --git a/pkg/controllers/admission/decide_test.go b/pkg/controllers/admission/decide_test.go index b32e674f..44db23fc 100644 --- a/pkg/controllers/admission/decide_test.go +++ b/pkg/controllers/admission/decide_test.go @@ -190,3 +190,56 @@ func TestDecidePositions(t *testing.T) { key("b2"): {Index: 1, LineLength: 1}, }, got.Positions, "positions are 1-based per line, cover only still-waiting candidates") } + +// CapBlocked names the lines whose head is still waiting because the line is +// at its cap, not because it does not fit — the lines a cap reclaim can help. +func TestDecideCapBlocked(t *testing.T) { + cases := []struct { + name string + in Input + want map[string]bool + }{ + { + name: "cap 1 with one holder, room to spare: the line is cap-blocked", + in: Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100)}, + Holders: []Holder{holder("h", "l", false, 100)}, + Caps: map[string]int32{"l": 1}, + Model: &slotModel{known: true, slots: 5}, + }, + want: map[string]bool{"l": true}, + }, + { + name: "uncapped line whose head does not fit: not cap-blocked", + in: Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100, 100)}, + Model: &slotModel{known: true, slots: 1}, + }, + want: map[string]bool{}, + }, + { + name: "under its cap, head does not fit: not cap-blocked", + in: Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 100, 100)}, + Holders: []Holder{holder("h", "l", false, 100)}, + Caps: map[string]int32{"l": 2}, + Model: &slotModel{known: true, slots: 1}, + }, + want: map[string]bool{}, + }, + { + name: "cap reached by this pass's own admission: cap-blocked", + in: Input{ + Waiting: []Candidate{cand("a", "l", 1, false, 100), cand("b", "l", 2, false, 100)}, + Caps: map[string]int32{"l": 1}, + Model: &slotModel{known: true, slots: 5}, + }, + want: map[string]bool{"l": true}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, Decide(tc.in).CapBlocked) + }) + } +} diff --git a/pkg/controllers/admission/reclaim.go b/pkg/controllers/admission/reclaim.go index 8e6ab643..62d9507f 100644 --- a/pkg/controllers/admission/reclaim.go +++ b/pkg/controllers/admission/reclaim.go @@ -15,6 +15,9 @@ type WarmIdle struct { Line string IdleSince time.Time Pods []PlacedPod + // Child is set for a delegated child session, which never counts toward + // its line's cap and so frees no cap slot. + Child bool } // PlanReclaim picks which warm Idle sessions of the head's own line to @@ -47,3 +50,29 @@ func PlanReclaim(head Candidate, warm []WarmIdle, model CapacityModel, now time. } return nil } + +// PlanCapReclaim picks the warm Idle session to reclaim for a head whose line +// is at its cap (Decision.CapBlocked): the head needs a slot, not space, and +// one reclaimed non-child holder gives it one. Of head's line's non-child +// warm sessions idle at least minWarm, the oldest (ties by key) is returned. +// ok is false for a child head (a cap never blocks it) or when none is +// eligible. +func PlanCapReclaim(head Candidate, warm []WarmIdle, now time.Time, minWarm time.Duration) (types.NamespacedName, bool) { + if head.Child { + return types.NamespacedName{}, false + } + var pick *WarmIdle + for i := range warm { + w := &warm[i] + if w.Line != head.Line || w.Child || now.Sub(w.IdleSince) < minWarm { + continue + } + if pick == nil || cmp.Or(w.IdleSince.Compare(pick.IdleSince), cmp.Compare(w.Key.String(), pick.Key.String())) < 0 { + pick = w + } + } + if pick == nil { + return types.NamespacedName{}, false + } + return pick.Key, true +} diff --git a/pkg/controllers/admission/reclaim_test.go b/pkg/controllers/admission/reclaim_test.go index 12d44b69..c9238c85 100644 --- a/pkg/controllers/admission/reclaim_test.go +++ b/pkg/controllers/admission/reclaim_test.go @@ -126,3 +126,55 @@ func TestPlanReclaim(t *testing.T) { assert.False(t, m.Reserve(head(500).Demand), "caller's model must still be full after planning") }) } + +// A line at its cap needs a SLOT, not space: PlanCapReclaim picks the single +// oldest non-child warm idle session of the head's line past minWarm. +func TestPlanCapReclaim(t *testing.T) { + now := time.Date(2026, 11, 10, 12, 0, 0, 0, time.UTC) + head := Candidate{Key: types.NamespacedName{Namespace: "ns", Name: "head"}, Line: "agent-a"} + childWarm := func(w WarmIdle) WarmIdle { w.Child = true; return w } + cases := []struct { + name string + head Candidate + warm []WarmIdle + want string + wantOK bool + }{ + { + name: "two eligible warm sessions: the oldest only", + head: head, + warm: []WarmIdle{ + warmOn("young", "agent-a", 10*time.Minute, now, 100), + warmOn("old", "agent-a", 30*time.Minute, now, 100), + }, + want: "old", wantOK: true, + }, + { + name: "oldest is younger than minWarm, another line's is older: nothing", + head: head, + warm: []WarmIdle{ + warmOn("young", "agent-a", time.Minute, now, 100), + warmOn("other", "agent-b", time.Hour, now, 100), + }, + }, + { + name: "only a delegated child is warm (children do not count toward the cap): nothing", + head: head, + warm: []WarmIdle{childWarm(warmOn("kid", "agent-a", time.Hour, now, 100))}, + }, + { + name: "child head (never cap-blocked): nothing", + head: Candidate{Key: head.Key, Line: "agent-a", Child: true}, + warm: []WarmIdle{warmOn("old", "agent-a", time.Hour, now, 100)}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := PlanCapReclaim(tc.head, tc.warm, now, 5*time.Minute) + assert.Equal(t, tc.wantOK, ok) + if tc.wantOK { + assert.Equal(t, tc.want, got.Name) + } + }) + } +} From ad681f905c9328b72a5b3ee6b84650eac06a4063 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:40:25 -0500 Subject: [PATCH 23/33] Release admission when a forensic hold reaps a session's pods A held session's pods were reaped but it stayed Admitted=True, so it kept a full reservation (all of it unplaced) and a cap slot for as long as the hold stood. The Held projection's status write now records Admitted=False/Reclaimed; on release the session rejoins the line through the gate. --- pkg/controllers/agentsession/hold.go | 10 +++ pkg/controllers/agentsession/hold_test.go | 82 +++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/pkg/controllers/agentsession/hold.go b/pkg/controllers/agentsession/hold.go index ba25113b..1e4422ca 100644 --- a/pkg/controllers/agentsession/hold.go +++ b/pkg/controllers/agentsession/hold.go @@ -220,6 +220,16 @@ func (r *Reconciler) reconcileHold(ctx context.Context, sess *spiceboxv1alpha1.A } } + // The reap below gives the session's admission back + // (Admitted=False/Reclaimed), recorded in the status write that persists + // the Held projection — a held session reserves nothing for its line, and + // on release it rejoins the back of the line through the gate like any + // cold wake. Decided before the projection: a pre-admission session is + // recognised only while its phase is not yet Held. A session admission + // never knew of is left without a condition. Idempotent on later passes: + // a released session is no longer Admitted=True. + r.releaseAdmissionOnReclaim(sess) + // Project the Held phase through the lifecycle machine, exactly as the // other early-return paths do, so phase stays a projection of the log // rather than a string this file invents. diff --git a/pkg/controllers/agentsession/hold_test.go b/pkg/controllers/agentsession/hold_test.go index 857fcc7a..e6bf7a14 100644 --- a/pkg/controllers/agentsession/hold_test.go +++ b/pkg/controllers/agentsession/hold_test.go @@ -916,3 +916,85 @@ func TestReconcile_activeHoldOnRunningSessionParksAndReaps(t *testing.T) { err = c.Get(ctx, client.ObjectKey{Namespace: "default", Name: RunnerPodName(sess)}, &gotRunner) assert.True(t, apierrors.IsNotFound(err), "the runner pod must still be reaped, got err=%v", err) } + +// TestReconcileHold_releasesAdmission pins that a forensic hold gives the +// session's admission back with its pods: Admitted=False/Reclaimed, persisted +// with the Held projection, so a held session holds no phantom reservation. +// A session admission never knew of is left without a condition; one that +// was admitted rejoins the line through the gate once its hold is released. +func TestReconcileHold_releasesAdmission(t *testing.T) { + newHold := func(phase string) *spiceboxv1alpha1.SessionHold { + return &spiceboxv1alpha1.SessionHold{ + ObjectMeta: metav1.ObjectMeta{Name: "h1", Namespace: "demo"}, + Spec: spiceboxv1alpha1.SessionHoldSpec{ + SessionRef: spiceboxv1alpha1.NamespacedRef{Namespace: "demo", Name: "s1"}, + Reason: "denial streak", + }, + Status: spiceboxv1alpha1.SessionHoldStatus{Phase: phase}, + } + } + cases := []struct { + name string + admitted bool + wantReason string // "" = no Admitted condition + }{ + {name: "admitted Running session held: Admitted=False/Reclaimed persisted", admitted: true, wantReason: spiceboxv1alpha1.ReasonAdmissionReclaimed}, + {name: "session with no Admitted condition and no pods recorded held: no condition written"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + sess := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: "s1", Namespace: "demo"}, + Status: spiceboxv1alpha1.AgentSessionStatus{Phase: spiceboxv1alpha1.AgentSessionPhaseRunning}, + } + if tc.admitted { + sess.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Hour)), + }} + } + c, r := newParkTestReconciler(t, sess, newHold(spiceboxv1alpha1.SessionHoldPhaseActive)) + r.AdmissionEnabled = true + + _, proceed, err := r.reconcileHold(ctx, sess) + require.NoError(t, err) + require.False(t, proceed) + + var got spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(sess), &got)) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhaseHeld, got.Status.Phase) + adm := conditionsFind(got.Status.Conditions) + if tc.wantReason == "" { + assert.Nil(t, adm, "no Admitted condition invented") + return + } + require.NotNil(t, adm, "Admitted condition") + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, tc.wantReason, adm.Reason) + + // Released: back to Pending, and through the gate like any arrival. + require.NoError(t, c.Delete(ctx, newHold(spiceboxv1alpha1.SessionHoldPhaseActive))) + require.NoError(t, c.Create(ctx, func() *spiceboxv1alpha1.SessionHold { + h := newHold(spiceboxv1alpha1.SessionHoldPhaseReleased) + h.Status.ReleasedBy = "user:approver-1" + return h + }())) + r.LifecycleMemory = memory.NewLocal(inmem.NewBackend()) + _, proceed, err = r.reconcileHold(ctx, &got) + require.NoError(t, err) + require.True(t, proceed) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, got.Status.Phase) + assert.True(t, r.awaitingAdmission(&got), "a released hold rejoins the line at the gate") + }) + } +} + +func conditionsFind(conds []metav1.Condition) *metav1.Condition { + for i := range conds { + if conds[i].Type == spiceboxv1alpha1.AgentSessionConditionAdmitted { + return &conds[i] + } + } + return nil +} From 1640274e7bb59816fc31abff7bca200ace87e564 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:42:29 -0500 Subject: [PATCH 24/33] Match a session's sandbox pods by label, not by name A sandbox pod adopted from a SandboxWarmPool keeps its pool-minted name, so matching demand entries to pods by name never found it: the holder's whole bundle demand stayed unplaced and was reserved on top of a snapshot that already counted the pod, for the session's lifetime, and an early reclaim could not credit it. A bundle's demand entry now carries its SpiceboxSession, and the admitter finds its pod by that SpiceboxSession's label (which adoption injects); runner, detector and sidecar entries still match by their deterministic names. --- .../admission/admitter/controller.go | 86 +++++++++++++++---- .../admission/admitter/controller_test.go | 45 ++++++++-- pkg/controllers/admission/line.go | 5 ++ pkg/controllers/agentsession/demand.go | 8 +- pkg/controllers/agentsession/demand_test.go | 6 +- 5 files changed, 123 insertions(+), 27 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index a70d993f..3ad8f5ef 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -331,16 +331,15 @@ func isWarmIdle(sess *spiceboxv1alpha1.AgentSession) bool { } // boundPods is the part of a session's pod set the capacity snapshot counts -// against a node: its demand's pods (matched by name, as unplaced does) that -// are bound and not finished, priced from the live pod spec. A pod already -// marked for deletion is included — the snapshot still counts it until it is -// gone — so it is exactly what a reclaim gives back. +// against a node: its demand's pods (matched by podIndex.lookup, as unplaced +// does) that are bound and not finished, priced from the live pod spec. A pod +// already marked for deletion is included — the snapshot still counts it +// until it is gone — so it is exactly what a reclaim gives back. func boundPods(namespace string, d admission.Demand, pods podIndex) []admission.PlacedPod { var placed []admission.PlacedPod for _, p := range d.Pods { - pod, ok := pods[types.NamespacedName{Namespace: namespace, Name: p.Name}] - if !ok || pod.Spec.NodeName == "" || - pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + pod := pods.lookup(namespace, p) + if pod == nil || !isBound(pod) { continue } placed = append(placed, admission.PlacedPod{ @@ -412,7 +411,7 @@ type sessionsWithPods map[types.NamespacedName]bool func indexSessionsWithPods(pods podIndex) sessionsWithPods { labelled := sessionsWithPods{} - for _, pod := range pods { + for _, pod := range pods.byName { for _, l := range []string{sessionPodLabel, sandboxPodLabel} { if v := pod.Labels[l]; v != "" { labelled[types.NamespacedName{Namespace: pod.Namespace, Name: v}] = true @@ -633,8 +632,61 @@ func (r *Reconciler) snapshot(ctx context.Context, waiting int) admission.Capaci return m } -// podIndex is the cluster's pods by namespace/name. -type podIndex map[types.NamespacedName]*corev1.Pod +// podIndex is the cluster's pods, looked up for a session's demand entries by +// role: a sandbox pod by its bundle SpiceboxSession's label (sandboxPodLabel), +// every other pod by name. +// +// The split is because a pod adopted from a SandboxWarmPool keeps the +// pool-minted name it was created under; tryAdopt injects the +// SpiceboxSession label onto it, and that label is the only stable link. The +// runner, detector and separate-pod sidecar pods are always created by this +// operator under the deterministic name their PodDemand carries. +type podIndex struct { + byName map[types.NamespacedName]*corev1.Pod + bySandbox map[types.NamespacedName][]*corev1.Pod +} + +func newPodIndex(pods []corev1.Pod) podIndex { + idx := podIndex{ + byName: make(map[types.NamespacedName]*corev1.Pod, len(pods)), + bySandbox: map[types.NamespacedName][]*corev1.Pod{}, + } + for i := range pods { + p := &pods[i] + idx.byName[client.ObjectKeyFromObject(p)] = p + if v := p.Labels[sandboxPodLabel]; v != "" { + k := types.NamespacedName{Namespace: p.Namespace, Name: v} + idx.bySandbox[k] = append(idx.bySandbox[k], p) + } + } + return idx +} + +// lookup finds the pod a demand entry in namespace describes, or nil. Of +// several pods labelled for one sandbox (a replacement beside a terminating +// predecessor), a bound, unfinished one is preferred: that is the pod the +// snapshot is counting against a node. +func (idx podIndex) lookup(namespace string, p admission.PodDemand) *corev1.Pod { + if p.SandboxSession == "" { + return idx.byName[types.NamespacedName{Namespace: namespace, Name: p.Name}] + } + cands := idx.bySandbox[types.NamespacedName{Namespace: namespace, Name: p.SandboxSession}] + for _, pod := range cands { + if isBound(pod) { + return pod + } + } + if len(cands) > 0 { + return cands[0] + } + return nil +} + +// isBound reports whether the snapshot counts pod against a node: bound and +// not finished. +func isBound(pod *corev1.Pod) bool { + return pod.Spec.NodeName != "" && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed +} // listPods indexes every pod for the unplaced-demand check. A list error is // logged and reported as not known, so every holder pod counts as unplaced. @@ -642,18 +694,14 @@ func (r *Reconciler) listPods(ctx context.Context) (podIndex, bool) { var pods corev1.PodList if err := r.Client.List(ctx, &pods); err != nil { log.FromContext(ctx).Info("admission: list pods failed; counting every admitted session's pods as unplaced", "err", err.Error()) - return nil, false - } - idx := make(podIndex, len(pods.Items)) - for i := range pods.Items { - idx[client.ObjectKeyFromObject(&pods.Items[i])] = &pods.Items[i] + return podIndex{}, false } - return idx, true + return newPodIndex(pods.Items), true } // unplaced is the part of a holder's demand the capacity snapshot does not -// already account for, matched to pods by name in the session's namespace -// (DemandFor names each PodDemand after the pod its builder creates): +// already account for, each entry matched to its pod in the session's +// namespace by podIndex.lookup: // // - a pod bound to a node (spec.nodeName set) is in the snapshot: placed; // - a pod that exists but is not yet bound, or does not exist yet, is not: @@ -665,7 +713,7 @@ func unplaced(namespace string, d admission.Demand, pods podIndex, known bool) a var out admission.Demand for _, p := range d.Pods { if known { - if pod, ok := pods[types.NamespacedName{Namespace: namespace, Name: p.Name}]; ok && pod.Spec.NodeName != "" { + if pod := pods.lookup(namespace, p); pod != nil && pod.Spec.NodeName != "" { continue } } diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index a23d7df3..4a6343c7 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -305,19 +305,26 @@ func (failingSource) Snapshot(context.Context) (admission.CapacityModel, error) func TestUnplacedDemand(t *testing.T) { d := admission.Demand{Pods: []admission.PodDemand{ {Name: "bound", CPUMilli: 1}, {Name: "pending", CPUMilli: 2}, {Name: "missing", CPUMilli: 4}, + // A sandbox pod adopted from a SandboxWarmPool: bound under a + // pool-minted name, found by its bundle SpiceboxSession's label. + {Name: "s1-code-pod", SandboxSession: "s1-code", CPUMilli: 8}, }} - pods := podIndex{ - {Namespace: "default", Name: "bound"}: &corev1.Pod{Spec: corev1.PodSpec{NodeName: "node-a"}}, - {Namespace: "default", Name: "pending"}: &corev1.Pod{}, - } + pods := newPodIndex([]corev1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "bound", Namespace: "default"}, Spec: corev1.PodSpec{NodeName: "node-a"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "pending", Namespace: "default"}}, + { + ObjectMeta: metav1.ObjectMeta{Name: "warmpool-x7k2p", Namespace: "default", Labels: map[string]string{sandboxPodLabel: "s1-code"}}, + Spec: corev1.PodSpec{NodeName: "node-a"}, + }, + }) cases := []struct { name string pods podIndex known bool want []string }{ - {name: "pods listed: pending and missing pods are unplaced", pods: pods, known: true, want: []string{"pending", "missing"}}, - {name: "pod list failed: every pod counted unplaced", known: false, want: []string{"bound", "pending", "missing"}}, + {name: "pods listed: pending and missing pods are unplaced, the pool-named sandbox pod is placed", pods: pods, known: true, want: []string{"pending", "missing"}}, + {name: "pod list failed: every pod counted unplaced", known: false, want: []string{"bound", "pending", "missing", "s1-code-pod"}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -845,3 +852,29 @@ func TestAdmitterCapReclaim(t *testing.T) { }) } } + +// A warm idle holder whose sandbox pod was adopted from a SandboxWarmPool has +// that pod under a pool-minted name. Matched by name, it was never counted as +// bound: its demand stayed "unplaced" (reserved on top of the snapshot, which +// already counts it — a double count for life), and an early reclaim could +// not credit what it gives back. Matched by its bundle SpiceboxSession's +// label, it is the bundle entry's pod. +func TestBoundPodsMatchesPoolNamedSandboxPod(t *testing.T) { + d := admission.Demand{Pods: []admission.PodDemand{ + {Name: "rb-runner", CPUMilli: 1000}, + {Name: "rb-code-pod", SandboxSession: "rb-code", CPUMilli: 500}, + }} + runner := boundRunnerPod("rb") + sandbox := boundRunnerPod("ignored") + sandbox.Name = "warmpool-x7k2p" + sandbox.Labels = map[string]string{sandboxPodLabel: "rb-code"} + pods := newPodIndex([]corev1.Pod{*runner, *sandbox}) + + got := boundPods("default", d, pods) + var names []string + for _, p := range got { + names = append(names, p.Name) + } + assert.ElementsMatch(t, []string{"rb-runner", "warmpool-x7k2p"}, names, "both the runner and the pool-named sandbox pod are bound") + assert.Empty(t, unplaced("default", d, pods, true).Pods, "nothing left unplaced") +} diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go index fe8b68f4..fc87eb71 100644 --- a/pkg/controllers/admission/line.go +++ b/pkg/controllers/admission/line.go @@ -23,6 +23,11 @@ type PodDemand struct { // Tolerations are the pod's own, from its built spec: a node model places // the pod only on a node whose NoSchedule/NoExecute taints it tolerates. Tolerations []corev1.Toleration + // SandboxSession is set on a tool bundle's sandbox pod: the name of the + // bundle SpiceboxSession it runs for. Such a pod is found by that + // SpiceboxSession's label rather than by Name, because one adopted from a + // SandboxWarmPool carries a pool-minted name. + SandboxSession string } // Demand is the whole pod set a session needs admitted together. diff --git a/pkg/controllers/agentsession/demand.go b/pkg/controllers/agentsession/demand.go index 8b94d181..27ee0e33 100644 --- a/pkg/controllers/agentsession/demand.go +++ b/pkg/controllers/agentsession/demand.go @@ -90,12 +90,16 @@ func DemandFor(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.Agen } pods = append(pods, runner) + // A bundle's sandbox pod is keyed by its SpiceboxSession, not its name: a + // pod adopted from a SandboxWarmPool has a pool-minted one. + sandboxSessions := make(map[string]string, len(ac.Spec.ToolBundles)) for _, b := range ac.Spec.ToolBundles { pod, err := demandBundlePod(ctx, c, sess, ac, b) if err != nil { return admission.Demand{}, err } pods = append(pods, pod) + sandboxSessions[pod.Name] = BundleSessionName(sess, b) } for _, det := range dets { @@ -120,7 +124,9 @@ func DemandFor(ctx context.Context, c client.Reader, sess *spiceboxv1alpha1.Agen d := admission.Demand{Pods: make([]admission.PodDemand, 0, len(pods))} for _, p := range pods { - d.Pods = append(d.Pods, admission.PodDemandFor(p.Name, &p.Spec)) + pd := admission.PodDemandFor(p.Name, &p.Spec) + pd.SandboxSession = sandboxSessions[p.Name] + d.Pods = append(d.Pods, pd) } return d, nil } diff --git a/pkg/controllers/agentsession/demand_test.go b/pkg/controllers/agentsession/demand_test.go index 733ffb92..902772be 100644 --- a/pkg/controllers/agentsession/demand_test.go +++ b/pkg/controllers/agentsession/demand_test.go @@ -129,9 +129,13 @@ func TestDemandForMatchesBuiltPods(t *testing.T) { {one, 1000, 1024}, } { bundle := spiceboxv1alpha1.ToolBundle{Name: "b-" + tc.class.Name, Class: tc.class.Name} - pod, berr := podspec.Build(BuildBundleSession(sess, bundle, "", "", nil, nil, nil), tc.class.Spec) + bs := BuildBundleSession(sess, bundle, "", "", nil, nil, nil) + pod, berr := podspec.Build(bs, tc.class.Spec) require.NoError(t, berr, "build bundle pod for %s", tc.class.Name) want := podDemandOf(pod) + // A sandbox pod is matched to its demand by its bundle SpiceboxSession's + // label, not its name: a pod adopted from a warm pool has a pool name. + want.SandboxSession = bs.Name assert.Equal(t, want, byName[pod.Name], "bundle %s demand equals the built pod's effective requests", bundle.Name) assert.Equal(t, tc.cpuMilli, byName[pod.Name].CPUMilli, From a623bac4b5d896e1110b83e51949e421eb3da7d1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 18:45:06 -0500 Subject: [PATCH 25/33] Tidy admission minors from the branch review Reserve a holder's unplaced pods one at a time, so one pod the model cannot place no longer drops the reservation for the rest. Clear a stale line position from sessions that are no longer waiting. Wrap a content-guard detector's config error and give a nil detector spec its own message. gofmt the AgentSession webhook. --- .../admission/admitter/controller.go | 28 ++++++++- .../admission/admitter/controller_test.go | 18 ++++++ pkg/controllers/admission/decide.go | 12 +++- pkg/controllers/admission/decide_test.go | 18 ++++++ pkg/controllers/agentsession/demand_test.go | 58 +++++++++++++++++++ pkg/controllers/agentsession/sidecarpod.go | 7 ++- .../webhooks/agentsession/webhook.go | 8 +-- 7 files changed, 137 insertions(+), 12 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 3ad8f5ef..abce9779 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -158,8 +158,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R return ctrl.Result{}, err } var waiting, holding, preAdmission []*spiceboxv1alpha1.AgentSession + var errs []error for i := range sessions { sess := &sessions[i] + if !isWaiting(sess) { + if err := r.clearPosition(ctx, sess); err != nil { + errs = append(errs, err) + } + } switch { case isTerminal(sess): // Never a holder, even while still Admitted=True with pods: any @@ -173,12 +179,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R } } if len(waiting) == 0 && (len(preAdmission) == 0 || !r.Enabled) { - return requeue, nil + return requeue, errors.Join(errs...) } now := r.now() if !r.Enabled { - var errs []error for _, sess := range waiting { if err := r.admit(ctx, sess, now, spiceboxv1alpha1.ReasonAdmitted, "Session admission is disabled; admitted at once."); err != nil { errs = append(errs, err) @@ -187,7 +192,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R return requeue, errors.Join(errs...) } - var errs []error pods, podsKnown := r.listPods(ctx) adopted, adoptErrs := r.adoptPreAdmission(ctx, preAdmission, pods, podsKnown, now) errs = append(errs, adoptErrs...) @@ -743,6 +747,24 @@ func (r *Reconciler) admit(ctx context.Context, sess *spiceboxv1alpha1.AgentSess return nil } +// clearPosition removes a line position left on a session that is no longer +// waiting — it went terminal, was parked, or was admitted by a write that did +// not clear it — so its thread never shows a place in line it does not have. +// A no-op, with no write, when there is nothing to clear. +func (r *Reconciler) clearPosition(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) error { + if sess.Status.Admission == nil { + return nil + } + original := sess.DeepCopy() + sess.Status.Admission = nil + if err := agentstatus.WriteOwned(ctx, r.Client, sess, original, spiceboxv1alpha1.OwnerOperator); err != nil { + log.FromContext(ctx).Info("admission: clearing a stale line position failed; the next pass retries", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "phase", sess.Status.Phase, "err", err.Error()) + return fmt.Errorf("clear admission position for %s/%s: %w", sess.Namespace, sess.Name, err) + } + return nil +} + // writePosition records a waiting session's place in line, only when its // position, line length or join time changed. func (r *Reconciler) writePosition(ctx context.Context, sess *spiceboxv1alpha1.AgentSession, pos admission.Position) error { diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 4a6343c7..174af126 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -878,3 +878,21 @@ func TestBoundPodsMatchesPoolNamedSandboxPod(t *testing.T) { assert.ElementsMatch(t, []string{"rb-runner", "warmpool-x7k2p"}, names, "both the runner and the pool-named sandbox pod are bound") assert.Empty(t, unplaced("default", d, pods, true).Pods, "nothing left unplaced") } + +// A session that is no longer waiting must not keep showing a line position +// ("3rd in line") it no longer has: the pass clears a stale status.admission. +func TestAdmitterClearsStalePosition(t *testing.T) { + stale := func(s *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.AgentSession { + s.Status.Admission = &spiceboxv1alpha1.AdmissionStatus{Position: 3, LineLength: 5, JoinedAt: s.CreationTimestamp} + return s + } + failed := stale(waitingSession("rb-failed", 0)) + failed.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseFailed + parked := stale(waitingSession("rb-parked", 0)) + parked.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingCredentials + + r, c, _ := newReconciler(t, true, roomFor(1), failed, parked) + runPass(t, r) + assert.Nil(t, getSession(t, c, "rb-failed").Status.Admission, "a Failed session holds no line position") + assert.Nil(t, getSession(t, c, "rb-parked").Status.Admission, "a parked session holds no line position") +} diff --git a/pkg/controllers/admission/decide.go b/pkg/controllers/admission/decide.go index 1c8e6fee..b910d071 100644 --- a/pkg/controllers/admission/decide.go +++ b/pkg/controllers/admission/decide.go @@ -84,9 +84,15 @@ func (l *lineState) capBlocked() bool { // in.Holders. func Decide(in Input) Decision { for _, h := range in.Holders { - // A failure means the pods already landed on nodes the snapshot - // accounts for; the holder is admitted either way. - _ = in.Model.Reserve(h.Unplaced) + // Each unplaced pod is reserved on its own, best effort: they are not + // yet on any node the snapshot counts, but they will be, so the room + // they take must not go to a waiting head. A pod the model cannot + // place (the model disagrees with the scheduler, or the pod waits on a + // node to free) is simply not reserved; the holder stays admitted + // either way, and its other pods are still reserved. + for _, p := range h.Unplaced.Pods { + in.Model.Reserve(Demand{Pods: []PodDemand{p}}) + } } lines := map[string]*lineState{} diff --git a/pkg/controllers/admission/decide_test.go b/pkg/controllers/admission/decide_test.go index 44db23fc..f2bc7ecb 100644 --- a/pkg/controllers/admission/decide_test.go +++ b/pkg/controllers/admission/decide_test.go @@ -243,3 +243,21 @@ func TestDecideCapBlocked(t *testing.T) { }) } } + +// A holder's unplaced pods are reserved one at a time, best effort: one that +// cannot be placed (the model is wrong about it, or it is waiting on a node +// to free) must not stop the others from being reserved, or a waiting head +// would be admitted into space the holder's other pods are about to take. +func TestDecideReservesHolderUnplacedPodsIndividually(t *testing.T) { + in := Input{ + Waiting: []Candidate{cand("w", "l", 1, false, 600)}, + Holders: []Holder{{ + Key: key("h"), Line: "l", + Demand: Demand{Pods: []PodDemand{pod("h-small", 600), pod("h-huge", 4000)}}, + // h-huge fits no node; h-small will land on n1. + Unplaced: Demand{Pods: []PodDemand{pod("h-small", 600), pod("h-huge", 4000)}}, + }}, + Model: NewNodeModel([]corev1.Node{testNode("n1", "1", "8Gi", "10")}, nil), + } + assert.Empty(t, Decide(in).Admit, "the holder's placeable pod is reserved even though its other pod is not") +} diff --git a/pkg/controllers/agentsession/demand_test.go b/pkg/controllers/agentsession/demand_test.go index 902772be..b8fdbe03 100644 --- a/pkg/controllers/agentsession/demand_test.go +++ b/pkg/controllers/agentsession/demand_test.go @@ -2,6 +2,9 @@ package agentsession import ( "context" + "encoding/json" + "errors" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -16,7 +19,9 @@ import ( _ "github.com/authzed/openagentprimitives/pkg/agent/harness/apnative" // the default harness the runner pod is built with spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/authz/contentguard" _ "github.com/authzed/openagentprimitives/pkg/authz/contentguard/kinds/promptinjection" // a DetectorProvider inspector + contentguardregistry "github.com/authzed/openagentprimitives/pkg/authz/contentguard/registry" "github.com/authzed/openagentprimitives/pkg/controllers/admission" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession/cosidecar" "github.com/authzed/openagentprimitives/pkg/platform/podspec" @@ -225,3 +230,56 @@ func TestDemandForErrors(t *testing.T) { }) } } + +// nilDetectorInspector is a DetectorProvider that resolves no detector. +type nilDetectorInspector struct{} + +func (nilDetectorInspector) ID() string { return "test-nil-detector" } +func (nilDetectorInspector) Configure(json.RawMessage) (contentguard.Instance, error) { + return nil, errors.New("not used") +} +func (nilDetectorInspector) Detector(json.RawMessage) (*contentguard.DetectorSpec, error) { + return nil, nil +} + +var registerNilDetector sync.Once + +// promptInjectionID is the promptinjection kind's registry ID. +const promptInjectionID = "prompt-injection" + +// A detector-requiring inspector that cannot produce its detector fails +// closed either way, but says which way: a config error is wrapped (so a +// caller can inspect it) and a nil spec gets its own message rather than +// "detector X: ". +func TestResolveContentGuardDetectorsErrors(t *testing.T) { + registerNilDetector.Do(func() { contentguardregistry.Register(nilDetectorInspector{}) }) + cases := []struct { + name string + inspector spiceboxv1alpha1.ContentInspectorConfig + wantMsg string + wantWrap bool + }{ + { + name: "config the inspector rejects: error wraps the inspector's", + inspector: spiceboxv1alpha1.ContentInspectorConfig{ID: promptInjectionID, Config: apiextv1.JSON{Raw: []byte(`{not json`)}}, + wantMsg: `content-guard detector "` + promptInjectionID + `"`, + wantWrap: true, + }, + { + name: "inspector resolves no detector spec: its own message, no ", + inspector: spiceboxv1alpha1.ContentInspectorConfig{ID: "test-nil-detector", Config: apiextv1.JSON{Raw: []byte(`{}`)}}, + wantMsg: `content-guard detector "test-nil-detector": the inspector resolved no detector spec`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := resolveContentGuardDetectors(&spiceboxv1alpha1.EffectiveSettings{ + ContentInspectors: []spiceboxv1alpha1.ContentInspectorConfig{tc.inspector}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantMsg) + assert.NotContains(t, err.Error(), "") + assert.Equal(t, tc.wantWrap, errors.Unwrap(err) != nil, "wrapped with %%w") + }) + } +} diff --git a/pkg/controllers/agentsession/sidecarpod.go b/pkg/controllers/agentsession/sidecarpod.go index 6c90216f..60aaf0c9 100644 --- a/pkg/controllers/agentsession/sidecarpod.go +++ b/pkg/controllers/agentsession/sidecarpod.go @@ -88,8 +88,11 @@ func resolveContentGuardDetectors(eff *spiceboxv1alpha1.EffectiveSettings) ([]sp continue } ds, err := dp.Detector(ci.Config.Raw) - if err != nil || ds == nil { - return nil, fmt.Errorf("content-guard detector %q: %v", ci.ID, err) + if err != nil { + return nil, fmt.Errorf("content-guard detector %q: %w", ci.ID, err) + } + if ds == nil { + return nil, fmt.Errorf("content-guard detector %q: the inspector resolved no detector spec for its config", ci.ID) } // Use the image exactly as declared in the inspector config: pinned // by digest it launches the exact bytes, by tag it launches by tag. diff --git a/pkg/controllers/webhooks/agentsession/webhook.go b/pkg/controllers/webhooks/agentsession/webhook.go index 9e9a9d5c..b9d64a95 100644 --- a/pkg/controllers/webhooks/agentsession/webhook.go +++ b/pkg/controllers/webhooks/agentsession/webhook.go @@ -603,10 +603,10 @@ var runnerWritableStatusFields = map[string]string{ "session's own retry of an already-idempotent write, denying its own starter no more than the runner could " + "already withhold by declining to run at all — self-scoped, and any other session sharing that class/subject " + "still attempts the same touch", - "slotPins": "a display-only mirror of slot pin state written by StatusPatcher (pkg/agent/runner/status.go). " + - "Every enforcement decision reads SpiceDB directly (the slot_pin tuple and the MUST_MATCH preconditions on " + - "grant writes), and the approval card derives move state from a live SpiceDB ReadPin rather than this field, " + - "so a forged value only misdescribes the forging session's own `oap session show` output and buys no authority anywhere", + "slotPins": "a display-only mirror of slot pin state written by StatusPatcher (pkg/agent/runner/status.go). " + + "Every enforcement decision reads SpiceDB directly (the slot_pin tuple and the MUST_MATCH preconditions on " + + "grant writes), and the approval card derives move state from a live SpiceDB ReadPin rather than this field, " + + "so a forged value only misdescribes the forging session's own `oap session show` output and buys no authority anywhere", } // Webhook validates AgentSession update admission requests. From 2841733e7608c09d3217d91ceb37d35e465b20f5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 20:37:55 -0500 Subject: [PATCH 26/33] Queue sessions slept or held before admission when they resume --- pkg/controllers/admission/line.go | 5 +- .../agentsession/admission_gate.go | 26 ++++- .../agentsession/admission_gate_test.go | 103 ++++++++++++++++++ pkg/controllers/agentsession/controller.go | 7 ++ pkg/controllers/agentsession/hold.go | 30 ++++- pkg/controllers/agentsession/hold_test.go | 11 +- 6 files changed, 170 insertions(+), 12 deletions(-) diff --git a/pkg/controllers/admission/line.go b/pkg/controllers/admission/line.go index fc87eb71..0e31fe9f 100644 --- a/pkg/controllers/admission/line.go +++ b/pkg/controllers/admission/line.go @@ -104,7 +104,10 @@ func IsQueued(sess *v1alpha1.AgentSession) bool { // // A session whose pods were reaped while it stays non-terminal — slept, or // Held — is excluded: its status still names the pods it had, but it holds -// nothing, and its next wake must queue like any cold wake. +// nothing, and its next wake must queue like any cold wake. Its reap normally +// records Admitted=False/Reclaimed; one reaped before admission was switched +// on has the condition recorded by its wake, archive wake or hold release +// instead, since those clear the exclusion while the pod names stay. func PreAdmission(sess *v1alpha1.AgentSession) bool { switch sess.Status.Phase { case v1alpha1.AgentSessionPhaseSucceeded, v1alpha1.AgentSessionPhaseFailed, v1alpha1.AgentSessionPhaseHeld: diff --git a/pkg/controllers/agentsession/admission_gate.go b/pkg/controllers/agentsession/admission_gate.go index 65575f11..5fbc87da 100644 --- a/pkg/controllers/agentsession/admission_gate.go +++ b/pkg/controllers/agentsession/admission_gate.go @@ -139,6 +139,31 @@ func (r *Reconciler) releaseAdmissionOnReclaim(sess *spiceboxv1alpha1.AgentSessi if !admission.IsAdmitted(sess) && (!r.AdmissionEnabled || !admission.PreAdmission(sess)) { return false } + r.setAdmissionReclaimed(sess) + return true +} + +// releaseUnrecordedAdmission records Admitted=False/Reclaimed on a session +// that is resuming from a reap admission never saw — slept, archived or Held +// before admission was switched on, or while it was off — and so carries no +// Admitted condition. Its status still names the pods it had (RunnerPodName +// and BundleSessions are never cleared), so once the resume puts it back in a +// booting phase it would look pre-admission and pass the gate with nothing +// reserved. With the condition it queues like any cold wake. The caller +// persists it, in the same write as the resume. +// +// Inert with admission off, and on a session that already has an Admitted +// condition (its own reap released it). Reports whether it changed sess. +func (r *Reconciler) releaseUnrecordedAdmission(sess *spiceboxv1alpha1.AgentSession) bool { + if !r.AdmissionEnabled || conditions.Find(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) != nil { + return false + } + r.setAdmissionReclaimed(sess) + return true +} + +// setAdmissionReclaimed writes Admitted=False/Reclaimed onto sess. +func (r *Reconciler) setAdmissionReclaimed(sess *spiceboxv1alpha1.AgentSession) { // r.now() for the transition time: admission.JoinedAt and WaitingSince // compare it against status.lastWakeAt, and the admitter orders the line // by them. @@ -149,7 +174,6 @@ func (r *Reconciler) releaseAdmissionOnReclaim(sess *spiceboxv1alpha1.AgentSessi Message: "the session's pods were reclaimed; a later wake rejoins the line", LastTransitionTime: metav1.NewTime(r.now()), }) - return true } // admissionMisfitGrace is how long an admitted session's bundle pod may stay diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index f35cd9cc..be60c72c 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -406,6 +406,109 @@ func TestAdmissionGate_PreAdmissionSleepReleases(t *testing.T) { assert.False(t, admission.PreAdmission(&got), "after a wake clears SleptAt it is a normal session that queues") } +// A session slept or held before admission was switched on (an upgrade, or a +// run with admission off) has no Admitted condition, and its status still +// names the pods it had before they were reaped. Resumed with admission on — +// a cold wake, an archive wake, or its hold released — it must queue at the gate like any +// cold wake and create none of its pods, not pass as pre-admission on those +// stale names. With admission off it boots at once, as it always has. +func TestAdmissionGate_ResumeWithoutAdmittedConditionQueues(t *testing.T) { + nowT := time.Now().UTC().Truncate(time.Second) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + + // slept: an Idle channel session whose pods the sleep reaped, with a wake + // requested after it slept. + slept := func(s *spiceboxv1alpha1.AgentSession) []client.Object { + idle := metav1.NewTime(nowT.Add(-2 * time.Hour)) + sleptAt := metav1.NewTime(nowT.Add(-time.Hour)) + s.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseIdle + s.Status.LastIdleAt = &idle + s.Status.SleptAt = &sleptAt + s.Annotations = map[string]string{ + spiceboxv1alpha1.AnnotationWakeRequestedAt: nowT.Add(-time.Second).Format(time.RFC3339Nano), + } + return nil + } + // archived: swept to Succeeded with its pods reaped, then woken. + archived := func(s *spiceboxv1alpha1.AgentSession) []client.Object { + finished := metav1.NewTime(nowT.Add(-time.Hour)) + s.Spec.InputChannel = &spiceboxv1alpha1.ChannelBinding{Name: "chan-1", Kind: "fake", Key: "thread-1"} + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded + s.Status.FinishedAt = &finished + s.Status.Conditions = []metav1.Condition{{ + Type: spiceboxv1alpha1.AgentSessionConditionIdle, Status: metav1.ConditionTrue, + Reason: spiceboxv1alpha1.ReasonAgentSessionArchived, LastTransitionTime: finished, + }} + s.Annotations = map[string]string{ + spiceboxv1alpha1.AnnotationWakeRequestedAt: nowT.Add(-time.Second).Format(time.RFC3339Nano), + } + return nil + } + // held: parked at Held with its pods reaped, and the hold since released. + held := func(s *spiceboxv1alpha1.AgentSession) []client.Object { + s.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseHeld + return []client.Object{&spiceboxv1alpha1.SessionHold{ + ObjectMeta: metav1.ObjectMeta{Name: "h1", Namespace: "default"}, + Spec: spiceboxv1alpha1.SessionHoldSpec{ + SessionRef: spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "s1"}, + Reason: "denial streak", + }, + Status: spiceboxv1alpha1.SessionHoldStatus{ + Phase: spiceboxv1alpha1.SessionHoldPhaseReleased, ReleasedBy: "user:approver-1", + }, + }} + } + + cases := []struct { + name string + enabled bool + prep func(s *spiceboxv1alpha1.AgentSession) []client.Object + }{ + {name: "slept before admission, woken with admission on: Admitted=False/Queued, no pods created", enabled: true, prep: slept}, + {name: "held before admission, released with admission on: Admitted=False/Queued, no pods created", enabled: true, prep: held}, + {name: "archived before admission, woken with admission on: Admitted=False/Queued, no pods created", enabled: true, prep: archived}, + {name: "slept, woken with admission off: boots at once, no Admitted condition", prep: slept}, + {name: "held, released with admission off: boots at once, no Admitted condition", prep: held}, + {name: "archived, woken with admission off: boots at once, no Admitted condition", prep: archived}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + ac := classWithBundle("ac1", codeBundle) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-3*time.Hour)) + extra := tc.prep(sess) + // The pods it had before they were reaped, still named on status. + sess.Status.RunnerPodName = agentsession.RunnerPodName(sess) + sess.Status.BundleSessions = []spiceboxv1alpha1.ResolvedBundle{{ + Name: codeBundle.Name, SpiceboxSessionName: agentsession.BundleSessionName(sess, codeBundle), + }} + require.False(t, admission.PreAdmission(sess), "fixture holds no pods while parked") + + r, c := fakeReconciler(t, clock, append([]client.Object{ac, sess}, extra...)...) + r.AdmissionEnabled = tc.enabled + runReconciles(t, r, "s1", 3) + + got := getSession(t, c, "s1") + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, got.Status.Phase, "resumed to Pending") + bundlePresent := exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}) + adm := apimeta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + if !tc.enabled { + assert.True(t, bundlePresent, "admission off: the bundle SpiceboxSession is created at once") + assert.Nil(t, adm, "admission off: no Admitted condition written") + return + } + assert.False(t, bundlePresent, "no bundle SpiceboxSession before admission") + assert.False(t, exists(t, c, agentsession.RunnerPodName(&got), &corev1.Pod{}), "no runner pod before admission") + if assert.NotNil(t, adm, "Admitted condition") { + assert.Equal(t, metav1.ConditionFalse, adm.Status) + assert.Equal(t, spiceboxv1alpha1.ReasonAdmissionQueued, adm.Reason) + } + assert.True(t, admission.IsQueued(&got), "in line for the admitter") + }) + } +} + func ptrTo[T any](v T) *T { return &v } // An ADMITTED booting session whose bundle pod the scheduler still cannot diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index 9833d1d5..9b0f25cf 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -3096,6 +3096,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // is the source of truth for real container crashes, and the // maxRunnerRestarts kill-switch must fire only on those. sess.Status.LastIdleAt = nil + // A slept or archived session's pods are gone. One reaped before + // admission knew of it has no Admitted condition, and with SleptAt + // cleared its stale pod names would read as pre-admission: record the + // release here, in the wake's own status write, so it queues instead. + if sess.Status.SleptAt != nil || wasArchivedBySweep { + r.releaseUnrecordedAdmission(&sess) + } // Clear SleptAt explicitly, not just via the LastIdleAt catch-all: this // path nils LastIdleAt in the same pass that applies WakeRequested, and // that catch-all ran earlier in this call while phase was still Idle, so diff --git a/pkg/controllers/agentsession/hold.go b/pkg/controllers/agentsession/hold.go index 1e4422ca..b1b83b72 100644 --- a/pkg/controllers/agentsession/hold.go +++ b/pkg/controllers/agentsession/hold.go @@ -269,12 +269,15 @@ func (r *Reconciler) reconcileHold(ctx context.Context, sess *spiceboxv1alpha1.A // session off PhaseHeld instead of re-reading a log whose last phase-changing // event is still Held. // -// proceed is always true here — either a Released event was just appended and -// the rest of Reconcile should run so normal provisioning resumes, or there -// is no released hold to attribute the release to (a session manually forced -// into PhaseHeld with no SessionHold ever naming it), in which case there is -// nothing this method can do and the session is left exactly as it already -// was, same as if no hold had ever been observed. +// proceed is true here unless the released session must wait for admission — +// either a Released event was just appended and the rest of Reconcile should +// run so normal provisioning resumes, or there is no released hold to +// attribute the release to (a session manually forced into PhaseHeld with no +// SessionHold ever naming it), in which case there is nothing this method can +// do and the session is left exactly as it already was, same as if no hold +// had ever been observed. With admission on, a released session that is not +// admitted persists its release and requeues (proceed=false) so it reaches +// the admission gate before any pod is created. func (r *Reconciler) reconcileHoldRelease(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (ctrl.Result, bool, error) { released, err := r.releasedHoldFor(ctx, sess) if err != nil { @@ -293,6 +296,21 @@ func (r *Reconciler) reconcileHoldRelease(ctx context.Context, sess *spiceboxv1a }); err != nil { return ctrl.Result{}, false, fmt.Errorf("apply Released for %s/%s: %w", sess.Namespace, sess.Name, err) } + // A session held before admission knew of it was reaped with no Admitted + // condition, and back at Pending its stale pod names would read as + // pre-admission. Record the release so it queues like any cold wake. + r.releaseUnrecordedAdmission(sess) + + // This runs below the admission gate in Reconcile, so a released session + // that must wait in line cannot continue this pass: it would fall through + // to pod creation unadmitted. Persist the Pending projection (and the + // release above) and requeue; the next pass meets the gate first. + if r.awaitingAdmission(sess) { + if err := r.applyStatus(ctx, sess); err != nil { + return ctrl.Result{}, false, fmt.Errorf("record hold release for %s/%s: %w", sess.Namespace, sess.Name, err) + } + return ctrl.Result{RequeueAfter: time.Second}, false, nil + } return ctrl.Result{}, true, nil } diff --git a/pkg/controllers/agentsession/hold_test.go b/pkg/controllers/agentsession/hold_test.go index e6bf7a14..294a6df4 100644 --- a/pkg/controllers/agentsession/hold_test.go +++ b/pkg/controllers/agentsession/hold_test.go @@ -981,11 +981,14 @@ func TestReconcileHold_releasesAdmission(t *testing.T) { return h }())) r.LifecycleMemory = memory.NewLocal(inmem.NewBackend()) - _, proceed, err = r.reconcileHold(ctx, &got) + res, proceed, err := r.reconcileHold(ctx, &got) require.NoError(t, err) - require.True(t, proceed) - assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, got.Status.Phase) - assert.True(t, r.awaitingAdmission(&got), "a released hold rejoins the line at the gate") + assert.False(t, proceed, "an unadmitted release stops this pass, which runs below the admission gate") + assert.Positive(t, res.RequeueAfter, "requeued so the next pass meets the gate") + var persisted spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(sess), &persisted)) + assert.Equal(t, spiceboxv1alpha1.AgentSessionPhasePending, persisted.Status.Phase, "Pending projection persisted") + assert.True(t, r.awaitingAdmission(&persisted), "a released hold rejoins the line at the gate") }) } } From 5cb8027bfffcf9adc1b05aca724ee78dc399a537 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 21:18:03 -0500 Subject: [PATCH 27/33] Use neutral fixture names in admission tests --- internal/cmd/operator/main.go | 2 +- pkg/channels/sessionnotice/sessionnotice_test.go | 4 ++-- pkg/controllers/admission/admitter/controller_test.go | 4 ++-- .../testdata/admission-burst/manifests/00-agent.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index d7a5a6be..0833c698 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -2768,7 +2768,7 @@ func run(cfg *config) { // create pods for. Leader-only (the controller-runtime default), so one // operator admits against the free space at a time. Capacity is declared // as the interface and assigned a concrete value, never a typed-nil - // pointer (CLAUDE.md, nil interfaces). + // pointer (see the repo conventions on nil interfaces). var admissionCapacity sessionadmission.CapacitySource = sessionadmission.KubeCapacitySource{Reader: mgr.GetClient()} if err := (&admitter.Reconciler{ Client: mgr.GetClient(), diff --git a/pkg/channels/sessionnotice/sessionnotice_test.go b/pkg/channels/sessionnotice/sessionnotice_test.go index 162ce3ae..754d3139 100644 --- a/pkg/channels/sessionnotice/sessionnotice_test.go +++ b/pkg/channels/sessionnotice/sessionnotice_test.go @@ -188,7 +188,7 @@ func TestCapacityHoldPosition(t *testing.T) { }{ {name: "position 3 of 5: hold text plus ordinal sentence naming the class", admission: &spiceboxv1alpha1.AdmissionStatus{Position: 3, LineLength: 5}, - want: holdMsg + " You're 3rd in line for reviewbot."}, + want: holdMsg + " You're 3rd in line for demo-agent."}, {name: "no admission status: hold text unchanged", want: holdMsg}, {name: "position 0: hold text unchanged", @@ -198,7 +198,7 @@ func TestCapacityHoldPosition(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := sess(spiceboxv1alpha1.AgentSessionPhasePending, hold) - s.Spec.Class = "reviewbot" + s.Spec.Class = "demo-agent" s.Status.Admission = tc.admission got, held := sessionnotice.CapacityHold(s) require.True(t, held) diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 174af126..2d387b7d 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -25,7 +25,7 @@ import ( "github.com/authzed/openagentprimitives/pkg/controllers/admission" ) -const testClass = "reviewbot" +const testClass = "demo-agent" var ( baseTime = time.Date(2026, 11, 10, 9, 0, 0, 0, time.UTC) @@ -145,7 +145,7 @@ func assertPosition(t *testing.T, s *spiceboxv1alpha1.AgentSession, index, lengt } func TestAdmitterPass(t *testing.T) { - t.Run("three waiting reviewbot sessions, room for one: oldest gets Admitted=True, others get Position 1 and 2 of 2", func(t *testing.T) { + t.Run("three waiting demo-agent sessions, room for one: oldest gets Admitted=True, others get Position 1 and 2 of 2", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(1), waitingSession("rb-c", 3*time.Minute), waitingSession("rb-a", time.Minute), diff --git a/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml b/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml index 6613a209..034c6168 100644 --- a/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml +++ b/test/e2e/bronzethread/testdata/admission-burst/manifests/00-agent.yaml @@ -1,6 +1,6 @@ # Fixture for the admission-burst bundle: two agents sharing one cluster. # -# burst-reviewer is the reviewbot shape — two sandbox bundles per session, so a +# burst-reviewer is a code-review agent shape — two sandbox bundles per session, so a # session's pod set is more than one pod and "all or none" means something. # burst-notes is a second agent with its own line. Both run with the plan gate # off and tool calls permissive: what the bundle pins is WHEN each session gets From dfa9fc3ec7859d1bf3d92d4f7c2f7498a75e72b1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 21:50:58 -0500 Subject: [PATCH 28/33] Exempt only operator-recorded delegated children from the admission cap --- .../admission/admitter/controller.go | 86 +++++++++- .../admission/admitter/controller_test.go | 158 +++++++++++++++++- 2 files changed, 233 insertions(+), 11 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index abce9779..f9017e96 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -42,6 +42,7 @@ import ( // +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions,verbs=get;list;watch;patch // +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentsessions/status,verbs=get;update;patch // +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=agentclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups=agentprimitives.authzed.com,resources=subagentrequests,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch @@ -192,6 +193,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R return requeue, errors.Join(errs...) } + delegated, err := r.listDelegations(ctx) + if err != nil { + errs = append(errs, err) + } pods, podsKnown := r.listPods(ctx) adopted, adoptErrs := r.adoptPreAdmission(ctx, preAdmission, pods, podsKnown, now) errs = append(errs, adoptErrs...) @@ -215,7 +220,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R Key: key, Line: lineOf(sess), JoinedAt: admission.JoinedAt(sess), - Child: sess.Spec.Parent != nil, + Child: delegated.isChild(ctx, sess), Demand: d, }) } @@ -235,15 +240,16 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R logger.Info("admission: cannot price an admitted session's pod set; counting it with no unplaced demand", "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) } + child := delegated.isChild(ctx, sess) in.Holders = append(in.Holders, admission.Holder{ Key: client.ObjectKeyFromObject(sess), Line: lineOf(sess), - Child: sess.Spec.Parent != nil, + Child: child, Demand: d, Unplaced: unplaced(sess.Namespace, d, pods, podsKnown), }) if podsKnown && isWarmIdle(sess) { - pool.addWarm(sess, boundPods(sess.Namespace, d, pods)) + pool.addWarm(sess, child, boundPods(sess.Namespace, d, pods)) } } if podsKnown { @@ -377,10 +383,10 @@ type reclaimPool struct { // addWarm files a warm Idle holder's bound pods: as in flight when a reclaim // is already requested (so it is not chosen again), else as a candidate. A -// session with no bound pods has nothing to give back and is skipped. -func (p *reclaimPool) addWarm(sess *spiceboxv1alpha1.AgentSession, placed []admission.PlacedPod) { +// session with no bound pods has nothing to give back and is skipped. child is +// the holder's proven delegation (delegations.isChild). +func (p *reclaimPool) addWarm(sess *spiceboxv1alpha1.AgentSession, child bool, placed []admission.PlacedPod) { line := lineOf(sess) - child := sess.Spec.Parent != nil if _, requested := sess.Annotations[spiceboxv1alpha1.AnnotationReclaimRequested]; requested { p.inFlight[line] = append(p.inFlight[line], placed...) if !child { @@ -574,6 +580,67 @@ func (r *Reconciler) requestReclaim(ctx context.Context, sess *spiceboxv1alpha1. return nil } +// delegation is what a SubagentRequest's status records about the child it +// created: the request (its UID, which the child's controller owner reference +// names) and the parent it delegated for. +type delegation struct { + request types.UID + parent spiceboxv1alpha1.NamespacedRef +} + +// delegations indexes, by child session, every delegation the SubagentRequest +// controller has recorded. +// +// A session's own spec.parent is not proof that it is a delegated child: +// anyone who may create an AgentSession can set it, and a child skips its +// class's cap and goes first in its line. A request's status.childRef is +// written only by the operator (no namespace principal is granted +// subagentrequests/status), and only once it has created that child itself, +// so the exemption is read from there. +type delegations map[types.NamespacedName]delegation + +// listDelegations indexes every SubagentRequest that names its child. A list +// error returns an empty index, so the pass treats every session as +// non-delegated rather than trusting spec.parent; the caller reports the +// error so the pass is retried. +func (r *Reconciler) listDelegations(ctx context.Context) (delegations, error) { + var list spiceboxv1alpha1.SubagentRequestList + if err := r.Client.List(ctx, &list); err != nil { + log.FromContext(ctx).Info("admission: list SubagentRequests failed; no session is treated as a delegated child this pass", + "err", err.Error()) + return delegations{}, fmt.Errorf("admission pass: list SubagentRequests: %w", err) + } + out := make(delegations, len(list.Items)) + for i := range list.Items { + sr := &list.Items[i] + if ref := sr.Status.ChildRef; ref != nil { + out[types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}] = delegation{request: sr.UID, parent: sr.Spec.Parent} + } + } + return out, nil +} + +// isChild reports whether sess is a delegated child: it names a parent, and a +// SubagentRequest that owns it records creating it for that same parent. A +// session claiming a parent with no such record is treated as non-delegated +// and logged. That includes a genuine child in the moment between its +// creation and its request's status write; it waits as an ordinary session +// until the write lands, which wakes the pass (SetupWithManager). +func (d delegations) isChild(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) bool { + claimed := sess.Spec.Parent + if claimed == nil { + return false + } + if got, ok := d[client.ObjectKeyFromObject(sess)]; ok && got.parent == *claimed { + if owner := metav1.GetControllerOf(sess); owner != nil && owner.UID == got.request { + return true + } + } + log.FromContext(ctx).Info("admission: session names a parent but no SubagentRequest records delegating it; treating it as a non-delegated session", + "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "claimedParent", claimed.Namespace+"/"+claimed.Name) + return false +} + // listSessions lists every AgentSession the operator owns: cluster-wide with // no watch set, else one namespaced list per watched namespace. Any failed // list fails the pass (logged by the caller's error return) rather than @@ -789,8 +856,10 @@ func (r *Reconciler) writePosition(ctx context.Context, sess *spiceboxv1alpha1.A return nil } -// SetupWithManager maps every AgentSession event, and every event on a pod -// carrying a session label, to the single Key. +// SetupWithManager maps every AgentSession event, every SubagentRequest event +// (its status.childRef write is what proves a waiting session a delegated +// child), and every event on a pod carrying a session label, to the single +// Key. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { toKey := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { return []reconcile.Request{{NamespacedName: Key}} @@ -804,6 +873,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). Named("admission"). Watches(&spiceboxv1alpha1.AgentSession{}, toKey). + Watches(&spiceboxv1alpha1.SubagentRequest{}, toKey). Watches(&corev1.Pod{}, toKey, builder.WithPredicates(sessionPod)). Complete(r) } diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 2d387b7d..25892882 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -594,6 +594,37 @@ func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { } } +// A draining reclaim whose pod set cannot be priced is not credited to its +// line: its terminating pods still fill the snapshot, so the head still does +// not fit and the next pass reclaims the next-oldest warm session. +func TestAdmitterDrainingReclaimUnpriceable(t *testing.T) { + r, c, _ := newReconciler(t, true, nil, + cpuNode(3), + warmIdleSession("rb-1", 30*time.Minute), boundRunnerPod("rb-1"), + warmIdleSession("rb-2", 20*time.Minute), boundRunnerPod("rb-2"), + warmIdleSession("rb-3", 10*time.Minute), boundRunnerPod("rb-3"), + waitingSession("rb-wait", time.Minute), + ) + r.Capacity = admission.KubeCapacitySource{Reader: c} + r.MinWarm = 5 * time.Minute + + runPass(t, r) + require.Contains(t, getSession(t, c, "rb-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") + + markReclaimed(t, c, "rb-1") + r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Name == "rb-1" { + return admission.Demand{}, fmt.Errorf("AgentClass gone") + } + return oneCPUDemand(ctx, sess) + } + runPass(t, r) + assert.Contains(t, getSession(t, c, "rb-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "rb-1's pods are uncredited, so the next-oldest is reclaimed") + assert.NotContains(t, getSession(t, c, "rb-3").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "one more reclaim covers the head") +} + // markReclaimed does what the sleep path's status write does: SleptAt and // Admitted=False/Reclaimed. The reclaim annotation stays (it is removed only // after the reap) and the pods are left for the test to dispose of. @@ -628,19 +659,40 @@ func admittedRunning(name string) *spiceboxv1alpha1.AgentSession { return s } -// childOf marks s as a session delegated by parent. +// childOf makes s a session delegated by parent the way the SubagentRequest +// controller builds one: spec.parent set and controlled by its request. +// requestFor is that request, with the child recorded on its status; a test +// leaves it out to make s a session that only claims a parent. func childOf(s *spiceboxv1alpha1.AgentSession, parent string) *spiceboxv1alpha1.AgentSession { s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: parent} + s.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(requestFor(s), + spiceboxv1alpha1.SchemeGroupVersion.WithKind("SubagentRequest"))} return s } +// requestFor is the SubagentRequest that created child (see childOf). +func requestFor(child *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.SubagentRequest { + sr := &spiceboxv1alpha1.SubagentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: child.Name + "-req", Namespace: child.Namespace, UID: types.UID(child.Name + "-req-uid")}, + Spec: spiceboxv1alpha1.SubagentRequestSpec{Class: testClass, Task: "do the delegated work"}, + Status: spiceboxv1alpha1.SubagentRequestStatus{ + ChildRef: &spiceboxv1alpha1.NamespacedRef{Namespace: child.Namespace, Name: child.Name}, + }, + } + if child.Spec.Parent != nil { + sr.Spec.Parent = *child.Spec.Parent + } + return sr +} + func TestAdmitterConcurrencyCap(t *testing.T) { t.Run("class cap 1 with one admitted parent: its waiting child is admitted; a waiting non-child is not", func(t *testing.T) { + child := childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent") r, c, _ := newReconciler(t, true, roomFor(5), cappedClass(1), admittedRunning("rb-parent"), waitingSession("rb-other", time.Minute), - childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent"), + child, requestFor(child), ) runPass(t, r) assertAdmitted(t, getSession(t, c, "rb-child")) @@ -670,14 +722,32 @@ func TestAdmitterConcurrencyCap(t *testing.T) { assertAdmitted(t, getSession(t, c, "rb-b")) }) + t.Run("an unpriceable holder still takes its cap slot: the waiting session is cap-blocked despite room", func(t *testing.T) { + r, c, _ := newReconciler(t, true, roomFor(5), + cappedClass(1), + admittedRunning("rb-holder"), + waitingSession("rb-wait", time.Minute), + ) + r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { + if sess.Name == "rb-holder" { + return admission.Demand{}, fmt.Errorf("settings not resolved") + } + return oneCPUDemand(ctx, sess) + } + runPass(t, r) + assert.True(t, admission.IsAdmitted(getSession(t, c, "rb-holder")), "the unpriceable holder stays admitted") + assertPosition(t, getSession(t, c, "rb-wait"), 1, 1) + }) + t.Run("AgentClass read fails: the line's non-children keep waiting unpositioned, its child is still admitted", func(t *testing.T) { + child := childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent") c := fake.NewClientBuilder(). WithScheme(testScheme(t)). WithObjects( cappedClass(1), admittedRunning("rb-parent"), waitingSession("rb-other", time.Minute), - childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent"), + child, requestFor(child), ). WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). WithInterceptorFuncs(interceptor.Funcs{ @@ -701,6 +771,88 @@ func TestAdmitterConcurrencyCap(t *testing.T) { }) } +// TestAdmitterChildNeedsProvenance pins that spec.parent alone does not make +// a session a delegated child: anyone who may create an AgentSession can set +// it. Without a SubagentRequest that created it for that parent, the session +// is capped and waits in arrival order like any other. +func TestAdmitterChildNeedsProvenance(t *testing.T) { + cases := []struct { + name string + // objs is the claimed child's own objects: the session, and any request. + objs func(claimed *spiceboxv1alpha1.AgentSession) []client.Object + // listFails fails every SubagentRequest list. + listFails bool + }{ + { + name: "spec.parent with no SubagentRequest: capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "rb-parent"} + return []client.Object{s} + }, + }, + { + name: "the request records a different parent: capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + sr := requestFor(childOf(s, "rb-parent")) + s.Spec.Parent.Name = "rb-elsewhere" + return []client.Object{s, sr} + }, + }, + { + name: "the request records this child but does not own it: capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + sr := requestFor(childOf(s, "rb-parent")) + s.OwnerReferences = nil + return []client.Object{s, sr} + }, + }, + { + name: "SubagentRequest list fails: even a genuine child is capped, behind the earlier session", + objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { + s = childOf(s, "rb-parent") + return []client.Object{s, requestFor(s)} + }, + listFails: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + objs := append([]client.Object{ + cappedClass(1), + admittedRunning("rb-parent"), + waitingSession("rb-other", time.Minute), + }, tc.objs(waitingSession("rb-claimed", 2*time.Minute))...) + var funcs interceptor.Funcs + if tc.listFails { + funcs.List = func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*spiceboxv1alpha1.SubagentRequestList); ok { + return fmt.Errorf("apiserver unavailable") + } + return cl.List(ctx, list, opts...) + } + } + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). + WithInterceptorFuncs(funcs). + Build() + r := &Reconciler{ + Client: c, Capacity: roomFor(5), Enabled: true, + Now: func() time.Time { return passTime }, Demand: oneCPUDemand, + } + _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: Key}) + if tc.listFails { + assert.ErrorContains(t, err, "list SubagentRequests", "the failed list is reported so the pass retries") + } else { + require.NoError(t, err, "Reconcile") + } + assertPosition(t, getSession(t, c, "rb-other"), 1, 2) + assertPosition(t, getSession(t, c, "rb-claimed"), 2, 2) + }) + } +} + // preAdmissionSession is a session that started its pods before admission was // switched on: in phase, with its runner pod recorded on status, and no // Admitted condition at all. From 6db6471cd3b08a1998d9b0a3628e92d9fa5ec7ce Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 21:54:11 -0500 Subject: [PATCH 29/33] Cover the admission branches the review found untested and document the starvation tradeoff --- internal/cmd/operator/main.go | 8 ++ .../admission/admitter/controller.go | 7 +- pkg/controllers/admission/decide.go | 9 ++ pkg/controllers/admission/decide_test.go | 20 ++++ .../agentsession/admission_gate_test.go | 113 ++++++++++++++++++ .../bundle_epoch_reconcile_test.go | 9 ++ 6 files changed, 163 insertions(+), 3 deletions(-) diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index 0833c698..0f33313e 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -2769,6 +2769,14 @@ func run(cfg *config) { // operator admits against the free space at a time. Capacity is declared // as the interface and assigned a concrete value, never a typed-nil // pointer (see the repo conventions on nil interfaces). + // + // The capacity snapshot reads nodes and pods from the cached client while + // sessions are listed through the APIReader. Only the session list must + // see the admitter's own last writes (a lagging cache would show an + // admitted session still waiting and admit a second into its space); + // node and pod state the admitter never writes, so a cache a moment + // behind costs at most one pass's accuracy, and an uncached list of every + // pod on each pass would be a full apiserver read per wake. var admissionCapacity sessionadmission.CapacitySource = sessionadmission.KubeCapacitySource{Reader: mgr.GetClient()} if err := (&admitter.Reconciler{ Client: mgr.GetClient(), diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index f9017e96..3d20128b 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -234,9 +234,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R for _, sess := range holding { d, err := r.demand(ctx, sess) if err != nil { - // Still a holder: it counts toward its line's share and cap. Its - // placed pods are in the snapshot; only its unplaced ones go - // unreserved, which is the accepted cost of an unpriceable holder. + // Still a holder: it counts toward its line's cap. With no + // demand it adds nothing to the line's share, and its unplaced + // pods go unreserved (its placed ones are in the snapshot), which + // is the accepted cost of an unpriceable holder. logger.Info("admission: cannot price an admitted session's pod set; counting it with no unplaced demand", "session", sess.Namespace+"/"+sess.Name, "class", lineOf(sess), "err", err.Error()) } diff --git a/pkg/controllers/admission/decide.go b/pkg/controllers/admission/decide.go index b910d071..79daf54a 100644 --- a/pkg/controllers/admission/decide.go +++ b/pkg/controllers/admission/decide.go @@ -79,6 +79,15 @@ func (l *lineState) capBlocked() bool { // ordered ahead of others in their line and are never blocked by, nor counted // toward, the cap. // +// There is no reservation across lines. A head that does not fit holds no +// space for itself, so other lines' smaller heads keep being admitted into +// whatever frees, and a large head can be starved for as long as they keep +// arriving: work-conserving admission is chosen over a cross-line reservation +// that would idle free capacity while the large head waits for enough to +// free at once. The wait is bounded, not unbounded: a starved head fails +// once it has waited past --capacity-hold-timeout, as any unadmitted session +// does. +// // Decide mutates in.Model: Unplaced holder demand and every admitted // candidate's demand are reserved on it. It does not modify in.Waiting or // in.Holders. diff --git a/pkg/controllers/admission/decide_test.go b/pkg/controllers/admission/decide_test.go index f2bc7ecb..09a95bcb 100644 --- a/pkg/controllers/admission/decide_test.go +++ b/pkg/controllers/admission/decide_test.go @@ -106,6 +106,26 @@ func TestDecide(t *testing.T) { }, admit: keys("b"), }, + { + name: "equal shares across lines: admitted in head JoinedAt order, then by line name", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("c", "C", 2, false, 100), cand("a", "A", 2, false, 100), cand("b", "B", 1, false, 100)}, + Model: &slotModel{known: true, slots: 3}, + } + }, + admit: keys("b", "a", "c"), + }, + { + name: "one free slot: a later child is admitted ahead of an earlier non-child in its line", + in: func() Input { + return Input{ + Waiting: []Candidate{cand("early", "l", 1, false, 100), cand("kid", "l", 2, true, 100)}, + Model: &slotModel{known: true, slots: 1}, + } + }, + admit: keys("kid"), + }, { name: "cap 2 with 2 holders: non-child head waits, child head admitted", in: func() Input { diff --git a/pkg/controllers/agentsession/admission_gate_test.go b/pkg/controllers/agentsession/admission_gate_test.go index be60c72c..cc5bafc7 100644 --- a/pkg/controllers/agentsession/admission_gate_test.go +++ b/pkg/controllers/agentsession/admission_gate_test.go @@ -9,6 +9,7 @@ package agentsession_test import ( "context" + "errors" "testing" "time" @@ -19,7 +20,9 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" "github.com/authzed/openagentprimitives/pkg/controllers/admission" @@ -57,6 +60,8 @@ func TestAdmissionGate(t *testing.T) { createdAt time.Duration // session creation, relative to now // prep stamps admission state onto the session before it is loaded. prep func(sess *spiceboxv1alpha1.AgentSession) + // holdTimeout is Reconciler.CapacityHoldTimeout; 0 = the default. + holdTimeout time.Duration wantPods bool // the bundle SpiceboxSession (bundles) or runner pod (no bundles) exists wantFailed bool @@ -149,6 +154,14 @@ func TestAdmissionGate(t *testing.T) { wantHoldReason: spiceboxv1alpha1.ReasonCapacityHoldExpired, wantMsg: []string{"waited 1h0m0s", "cluster capacity"}, }, + { + name: "hold disabled (negative timeout), queued 3h ago: waits on, not failed, no CapacityHold give-up time posted", + enabled: true, bundles: true, createdAt: -3 * time.Hour, holdTimeout: -1, + prep: func(s *spiceboxv1alpha1.AgentSession) { + withAdmittedCondition(s, metav1.ConditionFalse, spiceboxv1alpha1.ReasonAdmissionQueued, nowT.Add(-3*time.Hour)) + }, + wantQueuedAt: nowT.Add(-3 * time.Hour), + }, { name: "released long after creation: re-queued at the gate, the wait restarts there, not failed", enabled: true, bundles: true, createdAt: -3 * time.Hour, @@ -196,6 +209,7 @@ func TestAdmissionGate(t *testing.T) { r, c := fakeReconciler(t, clock, ac, sess) r.AdmissionEnabled = tc.enabled + r.CapacityHoldTimeout = tc.holdTimeout runReconciles(t, r, "s1", 3) got := getSession(t, c, "s1") @@ -669,3 +683,102 @@ func TestAdmissionGate_Misfit(t *testing.T) { }) } } + +// releaseMisfit gives admission back only once the session's pods are gone: +// released while they still stood, the admitter would count their capacity +// twice. A failed reap writes no release; a failed release write leaves the +// session admitted (its pods already reaped), and the error is returned so +// the next pass decides afresh. +func TestAdmissionGate_MisfitReleaseFailures(t *testing.T) { + nowT := time.Date(2026, 11, 10, 17, 0, 0, 0, time.UTC) + codeBundle := spiceboxv1alpha1.ToolBundle{Name: "code", Class: "toolbelt"} + errBoom := errors.New("apiserver unavailable") + carriesRelease := func(obj client.Object) bool { + sess, ok := obj.(*spiceboxv1alpha1.AgentSession) + if !ok { + return false + } + c := apimeta.FindStatusCondition(sess.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted) + return c != nil && c.Reason == spiceboxv1alpha1.ReasonAdmissionMisfit + } + + cases := []struct { + name string + // funcs is built per case so it can record onto releaseWrites. + funcs func(releaseWrites *int) interceptor.Funcs + wantErr string + wantReaped bool + }{ + { + name: "reap fails: error returned, pods kept, still admitted, no release written", + funcs: func(releaseWrites *int) interceptor.Funcs { + return interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*spiceboxv1alpha1.SpiceboxSession); ok { + return errBoom + } + return cl.Delete(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + if carriesRelease(obj) { + *releaseWrites++ + } + return cl.SubResource(sub).Patch(ctx, obj, p, opts...) + }, + } + }, + wantErr: "reap bundle SpiceboxSession", + }, + { + name: "release write fails after the reap: error returned, pods reaped, still admitted", + funcs: func(releaseWrites *int) interceptor.Funcs { + return interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + if carriesRelease(obj) { + *releaseWrites++ + return errBoom + } + return cl.SubResource(sub).Patch(ctx, obj, p, opts...) + }, + } + }, + wantErr: "record placement-mismatch release", + wantReaped: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clock := func() time.Time { return nowT } + started := nowT.Add(-2 * time.Minute) + sess := sessionCreatedAt("s1", "ac1", nowT.Add(-20*time.Minute)) + withAdmittedCondition(sess, metav1.ConditionTrue, spiceboxv1alpha1.ReasonAdmitted, started.Add(-time.Second)) + bundle := bundleWithPod("s1", "code", "s1-code-pod", started) + pod := pendingUnschedulablePod("s1-code-pod", "uid-1", started, capacitySchedMsg) + + var releaseWrites int + r, c := fakeReconcilerWithFuncs(t, clock, tc.funcs(&releaseWrites), classWithBundle("ac1", codeBundle), sess, bundle, pod) + r.AdmissionEnabled = true + + ctx := memory.WithSystemApproval(context.Background(), "test") + var err error + for range 3 { + if _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "s1"}}); err != nil { + break + } + } + require.Error(t, err, "the failed step is returned so the reconcile retries") + assert.ErrorContains(t, err, tc.wantErr) + + got := getSession(t, c, "s1") + assert.Equal(t, !tc.wantReaped, + exists(t, c, agentsession.BundleSessionName(&got, codeBundle), &spiceboxv1alpha1.SpiceboxSession{}), + "bundle SpiceboxSession present") + assert.True(t, admission.IsAdmitted(&got), "no release persisted: still Admitted=True") + if !tc.wantReaped { + assert.Zero(t, releaseWrites, "no release is written before the reap succeeds") + } else { + assert.Equal(t, 1, releaseWrites, "the release is written once the reap succeeded") + } + }) + } +} diff --git a/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go b/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go index b459b517..447854e5 100644 --- a/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go +++ b/pkg/controllers/agentsession/bundle_epoch_reconcile_test.go @@ -36,6 +36,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" "github.com/authzed/openagentprimitives/pkg/controllers/agentsession" @@ -54,6 +55,13 @@ import ( // reconcile clock pinned to clock(). RunnerFactory is wired (unused before the // bundle/detector gates the tests exercise). func fakeReconciler(t *testing.T, clock func() time.Time, objs ...client.Object) (*agentsession.Reconciler, client.Client) { + t.Helper() + return fakeReconcilerWithFuncs(t, clock, interceptor.Funcs{}, objs...) +} + +// fakeReconcilerWithFuncs is fakeReconciler with funcs intercepting the fake +// client's calls, for a test that fails one write. +func fakeReconcilerWithFuncs(t *testing.T, clock func() time.Time, funcs interceptor.Funcs, objs ...client.Object) (*agentsession.Reconciler, client.Client) { t.Helper() scheme := testfixtures.NewScheme(t, rbacv1.AddToScheme, networkingv1.AddToScheme) c := fake.NewClientBuilder(). @@ -67,6 +75,7 @@ func fakeReconciler(t *testing.T, clock func() time.Time, objs ...client.Object) // the deduced converter handles it. A fake-client limitation only — the // same applies work against a real apiserver (netpol_envtest_test.go). WithTypeConverters(managedfields.NewDeducedTypeConverter()). + WithInterceptorFuncs(funcs). Build() r := &agentsession.Reconciler{ Client: c, From 5eae7170a072eb9234ba560212db5eef30fbbd0a Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 11 Nov 2026 00:22:29 -0500 Subject: [PATCH 30/33] Defer an admission pass when delegation records cannot be read --- .../admission/admitter/controller.go | 6 +- .../admission/admitter/controller_test.go | 336 +++++++++--------- 2 files changed, 176 insertions(+), 166 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index 3d20128b..a94534b3 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -195,7 +195,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.R delegated, err := r.listDelegations(ctx) if err != nil { - errs = append(errs, err) + // Defer the whole pass rather than decide on a view known to be wrong: + // with no delegation index every genuine child would read as a + // non-child, losing its cap exemption and, if warm idle, becoming a + // cap-reclaim candidate during an API blip. + return requeue, errors.Join(append(errs, err)...) } pods, podsKnown := r.listPods(ctx) adopted, adoptErrs := r.adoptPreAdmission(ctx, preAdmission, pods, podsKnown, now) diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 25892882..807526fc 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -147,48 +147,48 @@ func assertPosition(t *testing.T, s *spiceboxv1alpha1.AgentSession, index, lengt func TestAdmitterPass(t *testing.T) { t.Run("three waiting demo-agent sessions, room for one: oldest gets Admitted=True, others get Position 1 and 2 of 2", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(1), - waitingSession("rb-c", 3*time.Minute), - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), + waitingSession("sess-c", 3*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), ) res := runPass(t, r) assert.Equal(t, 15*time.Second, res.RequeueAfter, "periodic requeue") - assertAdmitted(t, getSession(t, c, "rb-a")) - assertPosition(t, getSession(t, c, "rb-b"), 1, 2) - assertPosition(t, getSession(t, c, "rb-c"), 2, 2) + assertAdmitted(t, getSession(t, c, "sess-a")) + assertPosition(t, getSession(t, c, "sess-b"), 1, 2) + assertPosition(t, getSession(t, c, "sess-c"), 2, 2) }) t.Run("Enabled=false: every waiting session admitted, with no room at all", func(t *testing.T) { r, c, _ := newReconciler(t, false, roomFor(0), - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), - waitingSession("rb-c", 3*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + waitingSession("sess-c", 3*time.Minute), ) runPass(t, r) - for _, name := range []string{"rb-a", "rb-b", "rb-c"} { + for _, name := range []string{"sess-a", "sess-b", "sess-c"} { assertAdmitted(t, getSession(t, c, name)) } }) t.Run("Pending without the Queued marker (parked at an earlier gate, or fatal settings): not admitted, no position", func(t *testing.T) { - parked := waitingSession("rb-parked", 0) + parked := waitingSession("sess-parked", 0) parked.Status.Conditions = nil - r, c, _ := newReconciler(t, true, roomFor(5), parked, waitingSession("rb-a", time.Minute)) + r, c, _ := newReconciler(t, true, roomFor(5), parked, waitingSession("sess-a", time.Minute)) runPass(t, r) - got := getSession(t, c, "rb-parked") + got := getSession(t, c, "sess-parked") assert.Nil(t, meta.FindStatusCondition(got.Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), "a session that never reached the gate is never admitted") assert.Nil(t, got.Status.Admission, "and holds no line position") - assertAdmitted(t, getSession(t, c, "rb-a")) + assertAdmitted(t, getSession(t, c, "sess-a")) }) t.Run("a position write is skipped when unchanged: second pass sends no status patch", func(t *testing.T) { r, c, patches := newReconciler(t, true, roomFor(1), - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), - waitingSession("rb-c", 3*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), + waitingSession("sess-c", 3*time.Minute), ) runPass(t, r) require.Positive(t, patches.Load(), "first pass writes admission and positions") @@ -196,88 +196,88 @@ func TestAdmitterPass(t *testing.T) { runPass(t, r) assert.Zero(t, patches.Load(), "nothing changed, so nothing is written") - assertPosition(t, getSession(t, c, "rb-b"), 1, 2) + assertPosition(t, getSession(t, c, "sess-b"), 1, 2) }) t.Run("a holder still occupying the room: nobody is admitted, the line starts at 1", func(t *testing.T) { - holder := waitingSession("rb-holder", 0) + holder := waitingSession("sess-holder", 0) holder.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseRunning holder.Status.Conditions = []metav1.Condition{{ Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), }} r, c, _ := newReconciler(t, true, roomFor(1), holder, - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), ) runPass(t, r) - assertPosition(t, getSession(t, c, "rb-a"), 1, 2) - assertPosition(t, getSession(t, c, "rb-b"), 2, 2) + assertPosition(t, getSession(t, c, "sess-a"), 1, 2) + assertPosition(t, getSession(t, c, "sess-b"), 2, 2) }) t.Run("a terminal session still Admitted=True: not a holder, so the head is admitted", func(t *testing.T) { - done := waitingSession("rb-done", 0) + done := waitingSession("sess-done", 0) done.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseSucceeded done.Status.Conditions = []metav1.Condition{{ Type: spiceboxv1alpha1.AgentSessionConditionAdmitted, Status: metav1.ConditionTrue, Reason: spiceboxv1alpha1.ReasonAdmitted, LastTransitionTime: metav1.NewTime(baseTime), }} - r, c, _ := newReconciler(t, true, roomFor(1), done, waitingSession("rb-a", time.Minute)) + r, c, _ := newReconciler(t, true, roomFor(1), done, waitingSession("sess-a", time.Minute)) runPass(t, r) - assertAdmitted(t, getSession(t, c, "rb-a")) + assertAdmitted(t, getSession(t, c, "sess-a")) }) t.Run("a session parked at a pre-start gate: not waiting, never admitted or positioned", func(t *testing.T) { - parked := waitingSession("rb-parked", 0) + parked := waitingSession("sess-parked", 0) parked.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingDecision r, c, _ := newReconciler(t, true, roomFor(5), parked) runPass(t, r) - got := getSession(t, c, "rb-parked") + got := getSession(t, c, "sess-parked") assert.False(t, admission.IsAdmitted(got), "parked session not admitted") assert.Nil(t, got.Status.Admission, "parked session has no line position") }) t.Run("demand fails for the oldest: it keeps waiting unpositioned, the next is admitted", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(1), - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), ) r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { - if sess.Name == "rb-a" { + if sess.Name == "sess-a" { return admission.Demand{}, fmt.Errorf("settings not resolved") } return oneCPUDemand(ctx, sess) } runPass(t, r) - a := getSession(t, c, "rb-a") - assert.False(t, admission.IsAdmitted(a), "rb-a not admitted") - assert.Nil(t, a.Status.Admission, "rb-a not positioned") - assertAdmitted(t, getSession(t, c, "rb-b")) + a := getSession(t, c, "sess-a") + assert.False(t, admission.IsAdmitted(a), "sess-a not admitted") + assert.Nil(t, a.Status.Admission, "sess-a not positioned") + assertAdmitted(t, getSession(t, c, "sess-b")) }) t.Run("capacity snapshot fails: capacity unknown, every waiting session admitted as AdmittedUnknownCapacity", func(t *testing.T) { r, c, _ := newReconciler(t, true, failingSource{}, - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), ) runPass(t, r) - assertAdmittedUnknown(t, getSession(t, c, "rb-a")) - assertAdmittedUnknown(t, getSession(t, c, "rb-b")) + assertAdmittedUnknown(t, getSession(t, c, "sess-a")) + assertAdmittedUnknown(t, getSession(t, c, "sess-b")) }) t.Run("namespace-scoped operator (watch set): capacity treated as unknown even with no room, admitted as AdmittedUnknownCapacity", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(0), - waitingSession("rb-a", time.Minute), + waitingSession("sess-a", time.Minute), ) r.Namespaces = []string{"default"} runPass(t, r) - assertAdmittedUnknown(t, getSession(t, c, "rb-a")) + assertAdmittedUnknown(t, getSession(t, c, "sess-a")) }) t.Run("known capacity: admitted with reason Admitted", func(t *testing.T) { - r, c, _ := newReconciler(t, true, roomFor(1), waitingSession("rb-a", time.Minute)) + r, c, _ := newReconciler(t, true, roomFor(1), waitingSession("sess-a", time.Minute)) runPass(t, r) - assertAdmitted(t, getSession(t, c, "rb-a")) + assertAdmitted(t, getSession(t, c, "sess-a")) }) } @@ -368,9 +368,9 @@ func TestAdmitterWatchNamespaces(t *testing.T) { c := fake.NewClientBuilder(). WithScheme(testScheme(t)). WithObjects( - inNamespace(waitingSession("rb-a", time.Minute), "team-a"), - inNamespace(waitingSession("rb-b", 2*time.Minute), "team-b"), - inNamespace(waitingSession("rb-c", 3*time.Minute), "other"), + inNamespace(waitingSession("sess-a", time.Minute), "team-a"), + inNamespace(waitingSession("sess-b", 2*time.Minute), "team-b"), + inNamespace(waitingSession("sess-c", 3*time.Minute), "other"), ). WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). WithInterceptorFuncs(interceptor.Funcs{ @@ -467,38 +467,38 @@ func TestAdmitterEarlyReclaim(t *testing.T) { name: "line has a waiting head, one warm idle past minWarm: reclaim annotation set on it only", objs: []client.Object{ cpuNode(2), - warmIdleSession("rb-old", 10*time.Minute), boundRunnerPod("rb-old"), - warmIdleSession("rb-young", time.Minute), boundRunnerPod("rb-young"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-old", 10*time.Minute), boundRunnerPod("sess-old"), + warmIdleSession("sess-young", time.Minute), boundRunnerPod("sess-young"), + waitingSession("sess-wait", time.Minute), }, - wantAnnotated: []string{"rb-old"}, - wantClean: []string{"rb-young", "rb-wait"}, + wantAnnotated: []string{"sess-old"}, + wantClean: []string{"sess-young", "sess-wait"}, }, { name: "no waiting head: no annotation", objs: []client.Object{ cpuNode(1), - warmIdleSession("rb-old", 10*time.Minute), boundRunnerPod("rb-old"), + warmIdleSession("sess-old", 10*time.Minute), boundRunnerPod("sess-old"), }, - wantClean: []string{"rb-old"}, + wantClean: []string{"sess-old"}, }, { name: "the head fits without a reclaim: admitted, no annotation", objs: []client.Object{ cpuNode(2), - warmIdleSession("rb-old", 10*time.Minute), boundRunnerPod("rb-old"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-old", 10*time.Minute), boundRunnerPod("sess-old"), + waitingSession("sess-wait", time.Minute), }, - wantClean: []string{"rb-old", "rb-wait"}, + wantClean: []string{"sess-old", "sess-wait"}, }, { name: "only warm idle is younger than minWarm: no annotation", objs: []client.Object{ cpuNode(1), - warmIdleSession("rb-young", time.Minute), boundRunnerPod("rb-young"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-young", time.Minute), boundRunnerPod("sess-young"), + waitingSession("sess-wait", time.Minute), }, - wantClean: []string{"rb-young", "rb-wait"}, + wantClean: []string{"sess-young", "sess-wait"}, }, } for _, tc := range cases { @@ -521,14 +521,14 @@ func TestAdmitterEarlyReclaim(t *testing.T) { } t.Run("a reclaim already requested: the annotation is not rewritten", func(t *testing.T) { - old := warmIdleSession("rb-old", 10*time.Minute) + old := warmIdleSession("sess-old", 10*time.Minute) earlier := baseTime.Format(time.RFC3339) old.Annotations = map[string]string{spiceboxv1alpha1.AnnotationReclaimRequested: earlier} - r, c, _ := newReconciler(t, true, nil, cpuNode(1), old, boundRunnerPod("rb-old"), waitingSession("rb-wait", time.Minute)) + r, c, _ := newReconciler(t, true, nil, cpuNode(1), old, boundRunnerPod("sess-old"), waitingSession("sess-wait", time.Minute)) r.Capacity = admission.KubeCapacitySource{Reader: c} r.MinWarm = minWarm runPass(t, r) - assert.Equal(t, earlier, getSession(t, c, "rb-old").Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], + assert.Equal(t, earlier, getSession(t, c, "sess-old").Annotations[spiceboxv1alpha1.AnnotationReclaimRequested], "the first request's time stands") }) } @@ -551,15 +551,15 @@ func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { { name: "slept and Admitted=False/Reclaimed, pod still bound: second pass annotates nothing new", advance: func(t *testing.T, c client.Client) { - markReclaimed(t, c, "rb-1") + markReclaimed(t, c, "sess-1") }, }, { name: "slept and Admitted=False/Reclaimed, pod terminating (DeletionTimestamp): second pass annotates nothing new", advance: func(t *testing.T, c client.Client) { - markReclaimed(t, c, "rb-1") + markReclaimed(t, c, "sess-1") var pod corev1.Pod - require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "rb-1-runner"}, &pod)) + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "sess-1-runner"}, &pod)) require.NoError(t, c.Delete(context.Background(), &pod), "delete held by a finalizer: pod lingers terminating") require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(&pod), &pod)) require.NotNil(t, pod.DeletionTimestamp, "pod is terminating") @@ -568,25 +568,25 @@ func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - pod1 := boundRunnerPod("rb-1") + pod1 := boundRunnerPod("sess-1") pod1.Finalizers = []string{"test/hold"} r, c, _ := newReconciler(t, true, nil, cpuNode(3), - warmIdleSession("rb-1", 30*time.Minute), pod1, - warmIdleSession("rb-2", 20*time.Minute), boundRunnerPod("rb-2"), - warmIdleSession("rb-3", 10*time.Minute), boundRunnerPod("rb-3"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-1", 30*time.Minute), pod1, + warmIdleSession("sess-2", 20*time.Minute), boundRunnerPod("sess-2"), + warmIdleSession("sess-3", 10*time.Minute), boundRunnerPod("sess-3"), + waitingSession("sess-wait", time.Minute), ) r.Capacity = admission.KubeCapacitySource{Reader: c} r.MinWarm = minWarm runPass(t, r) - require.Contains(t, getSession(t, c, "rb-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") - require.NotContains(t, getSession(t, c, "rb-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims only one") + require.Contains(t, getSession(t, c, "sess-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") + require.NotContains(t, getSession(t, c, "sess-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims only one") tc.advance(t, c) runPass(t, r) - for _, name := range []string{"rb-2", "rb-3"} { + for _, name := range []string{"sess-2", "sess-3"} { assert.NotContains(t, getSession(t, c, name).Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "%s: the in-flight reclaim already covers the head", name) } @@ -600,28 +600,28 @@ func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { func TestAdmitterDrainingReclaimUnpriceable(t *testing.T) { r, c, _ := newReconciler(t, true, nil, cpuNode(3), - warmIdleSession("rb-1", 30*time.Minute), boundRunnerPod("rb-1"), - warmIdleSession("rb-2", 20*time.Minute), boundRunnerPod("rb-2"), - warmIdleSession("rb-3", 10*time.Minute), boundRunnerPod("rb-3"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-1", 30*time.Minute), boundRunnerPod("sess-1"), + warmIdleSession("sess-2", 20*time.Minute), boundRunnerPod("sess-2"), + warmIdleSession("sess-3", 10*time.Minute), boundRunnerPod("sess-3"), + waitingSession("sess-wait", time.Minute), ) r.Capacity = admission.KubeCapacitySource{Reader: c} r.MinWarm = 5 * time.Minute runPass(t, r) - require.Contains(t, getSession(t, c, "rb-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") + require.Contains(t, getSession(t, c, "sess-1").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "pass 1 reclaims the oldest") - markReclaimed(t, c, "rb-1") + markReclaimed(t, c, "sess-1") r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { - if sess.Name == "rb-1" { + if sess.Name == "sess-1" { return admission.Demand{}, fmt.Errorf("AgentClass gone") } return oneCPUDemand(ctx, sess) } runPass(t, r) - assert.Contains(t, getSession(t, c, "rb-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, - "rb-1's pods are uncredited, so the next-oldest is reclaimed") - assert.NotContains(t, getSession(t, c, "rb-3").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + assert.Contains(t, getSession(t, c, "sess-2").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, + "sess-1's pods are uncredited, so the next-oldest is reclaimed") + assert.NotContains(t, getSession(t, c, "sess-3").Annotations, spiceboxv1alpha1.AnnotationReclaimRequested, "one more reclaim covers the head") } @@ -687,66 +687,66 @@ func requestFor(child *spiceboxv1alpha1.AgentSession) *spiceboxv1alpha1.Subagent func TestAdmitterConcurrencyCap(t *testing.T) { t.Run("class cap 1 with one admitted parent: its waiting child is admitted; a waiting non-child is not", func(t *testing.T) { - child := childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent") + child := childOf(waitingSession("sess-child", 2*time.Minute), "sess-parent") r, c, _ := newReconciler(t, true, roomFor(5), cappedClass(1), - admittedRunning("rb-parent"), - waitingSession("rb-other", time.Minute), + admittedRunning("sess-parent"), + waitingSession("sess-other", time.Minute), child, requestFor(child), ) runPass(t, r) - assertAdmitted(t, getSession(t, c, "rb-child")) - assertPosition(t, getSession(t, c, "rb-other"), 1, 1) + assertAdmitted(t, getSession(t, c, "sess-child")) + assertPosition(t, getSession(t, c, "sess-other"), 1, 1) }) t.Run("class cap 2 with one admitted parent: the oldest waiting non-child takes the second slot, the next waits", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(5), cappedClass(2), - admittedRunning("rb-parent"), - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), + admittedRunning("sess-parent"), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), ) runPass(t, r) - assertAdmitted(t, getSession(t, c, "rb-a")) - assertPosition(t, getSession(t, c, "rb-b"), 1, 1) + assertAdmitted(t, getSession(t, c, "sess-a")) + assertPosition(t, getSession(t, c, "sess-b"), 1, 1) }) t.Run("no AgentClass to read a cap from: the line is uncapped, every waiting session that fits is admitted", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(5), - admittedRunning("rb-parent"), - waitingSession("rb-a", time.Minute), - waitingSession("rb-b", 2*time.Minute), + admittedRunning("sess-parent"), + waitingSession("sess-a", time.Minute), + waitingSession("sess-b", 2*time.Minute), ) runPass(t, r) - assertAdmitted(t, getSession(t, c, "rb-a")) - assertAdmitted(t, getSession(t, c, "rb-b")) + assertAdmitted(t, getSession(t, c, "sess-a")) + assertAdmitted(t, getSession(t, c, "sess-b")) }) t.Run("an unpriceable holder still takes its cap slot: the waiting session is cap-blocked despite room", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(5), cappedClass(1), - admittedRunning("rb-holder"), - waitingSession("rb-wait", time.Minute), + admittedRunning("sess-holder"), + waitingSession("sess-wait", time.Minute), ) r.Demand = func(ctx context.Context, sess *spiceboxv1alpha1.AgentSession) (admission.Demand, error) { - if sess.Name == "rb-holder" { + if sess.Name == "sess-holder" { return admission.Demand{}, fmt.Errorf("settings not resolved") } return oneCPUDemand(ctx, sess) } runPass(t, r) - assert.True(t, admission.IsAdmitted(getSession(t, c, "rb-holder")), "the unpriceable holder stays admitted") - assertPosition(t, getSession(t, c, "rb-wait"), 1, 1) + assert.True(t, admission.IsAdmitted(getSession(t, c, "sess-holder")), "the unpriceable holder stays admitted") + assertPosition(t, getSession(t, c, "sess-wait"), 1, 1) }) t.Run("AgentClass read fails: the line's non-children keep waiting unpositioned, its child is still admitted", func(t *testing.T) { - child := childOf(waitingSession("rb-child", 2*time.Minute), "rb-parent") + child := childOf(waitingSession("sess-child", 2*time.Minute), "sess-parent") c := fake.NewClientBuilder(). WithScheme(testScheme(t)). WithObjects( cappedClass(1), - admittedRunning("rb-parent"), - waitingSession("rb-other", time.Minute), + admittedRunning("sess-parent"), + waitingSession("sess-other", time.Minute), child, requestFor(child), ). WithStatusSubresource(&spiceboxv1alpha1.AgentSession{}). @@ -764,17 +764,18 @@ func TestAdmitterConcurrencyCap(t *testing.T) { Now: func() time.Time { return passTime }, Demand: oneCPUDemand, } runPass(t, r) - assertAdmitted(t, getSession(t, c, "rb-child")) - other := getSession(t, c, "rb-other") - assert.False(t, admission.IsAdmitted(other), "rb-other not admitted past a cap it could not read") - assert.Nil(t, other.Status.Admission, "rb-other not positioned") + assertAdmitted(t, getSession(t, c, "sess-child")) + other := getSession(t, c, "sess-other") + assert.False(t, admission.IsAdmitted(other), "sess-other not admitted past a cap it could not read") + assert.Nil(t, other.Status.Admission, "sess-other not positioned") }) } // TestAdmitterChildNeedsProvenance pins that spec.parent alone does not make // a session a delegated child: anyone who may create an AgentSession can set // it. Without a SubagentRequest that created it for that parent, the session -// is capped and waits in arrival order like any other. +// is capped and waits in arrival order like any other. A failed request list +// defers the pass instead of deciding without the index. func TestAdmitterChildNeedsProvenance(t *testing.T) { cases := []struct { name string @@ -786,30 +787,30 @@ func TestAdmitterChildNeedsProvenance(t *testing.T) { { name: "spec.parent with no SubagentRequest: capped, behind the earlier session", objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { - s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "rb-parent"} + s.Spec.Parent = &spiceboxv1alpha1.NamespacedRef{Namespace: "default", Name: "sess-parent"} return []client.Object{s} }, }, { name: "the request records a different parent: capped, behind the earlier session", objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { - sr := requestFor(childOf(s, "rb-parent")) - s.Spec.Parent.Name = "rb-elsewhere" + sr := requestFor(childOf(s, "sess-parent")) + s.Spec.Parent.Name = "sess-elsewhere" return []client.Object{s, sr} }, }, { name: "the request records this child but does not own it: capped, behind the earlier session", objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { - sr := requestFor(childOf(s, "rb-parent")) + sr := requestFor(childOf(s, "sess-parent")) s.OwnerReferences = nil return []client.Object{s, sr} }, }, { - name: "SubagentRequest list fails: even a genuine child is capped, behind the earlier session", + name: "SubagentRequest list fails: the pass is deferred, nobody is admitted or positioned", objs: func(s *spiceboxv1alpha1.AgentSession) []client.Object { - s = childOf(s, "rb-parent") + s = childOf(s, "sess-parent") return []client.Object{s, requestFor(s)} }, listFails: true, @@ -819,9 +820,9 @@ func TestAdmitterChildNeedsProvenance(t *testing.T) { t.Run(tc.name, func(t *testing.T) { objs := append([]client.Object{ cappedClass(1), - admittedRunning("rb-parent"), - waitingSession("rb-other", time.Minute), - }, tc.objs(waitingSession("rb-claimed", 2*time.Minute))...) + admittedRunning("sess-parent"), + waitingSession("sess-other", time.Minute), + }, tc.objs(waitingSession("sess-claimed", 2*time.Minute))...) var funcs interceptor.Funcs if tc.listFails { funcs.List = func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { @@ -844,11 +845,16 @@ func TestAdmitterChildNeedsProvenance(t *testing.T) { _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: Key}) if tc.listFails { assert.ErrorContains(t, err, "list SubagentRequests", "the failed list is reported so the pass retries") - } else { - require.NoError(t, err, "Reconcile") + for _, name := range []string{"sess-other", "sess-claimed"} { + s := getSession(t, c, name) + assert.False(t, admission.IsAdmitted(s), "%s not admitted on a deferred pass", name) + assert.Nil(t, s.Status.Admission, "%s not positioned on a deferred pass", name) + } + return } - assertPosition(t, getSession(t, c, "rb-other"), 1, 2) - assertPosition(t, getSession(t, c, "rb-claimed"), 2, 2) + require.NoError(t, err, "Reconcile") + assertPosition(t, getSession(t, c, "sess-other"), 1, 2) + assertPosition(t, getSession(t, c, "sess-claimed"), 2, 2) }) } } @@ -881,52 +887,52 @@ func TestAdmitterAdoptsPreAdmissionSessions(t *testing.T) { t.Run("Running pre-admission session with a bound pod, class cap 1: adopted, and the cap holds the waiting session back", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(5), cappedClass(1), - preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("rb-old"), - waitingSession("rb-a", time.Minute), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("sess-old"), + waitingSession("sess-a", time.Minute), ) runPass(t, r) - assertAdopted(t, getSession(t, c, "rb-old")) - assertPosition(t, getSession(t, c, "rb-a"), 1, 1) + assertAdopted(t, getSession(t, c, "sess-old")) + assertPosition(t, getSession(t, c, "sess-a"), 1, 1) }) t.Run("Pending pre-admission session with an unbound pod, room for one: adopted, its unplaced pod reserved, the waiting session waits", func(t *testing.T) { - pod := boundRunnerPod("rb-old") + pod := boundRunnerPod("sess-old") pod.Spec.NodeName = "" pod.Status.Phase = corev1.PodPending r, c, _ := newReconciler(t, true, roomFor(1), - preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhasePending), pod, - waitingSession("rb-a", time.Minute), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhasePending), pod, + waitingSession("sess-a", time.Minute), ) runPass(t, r) - assertAdopted(t, getSession(t, c, "rb-old")) - assertPosition(t, getSession(t, c, "rb-a"), 1, 1) + assertAdopted(t, getSession(t, c, "sess-old")) + assertPosition(t, getSession(t, c, "sess-a"), 1, 1) }) t.Run("pre-admission session with nothing waiting: still adopted", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(1), - preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseIdle), boundRunnerPod("rb-old"), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseIdle), boundRunnerPod("sess-old"), ) runPass(t, r) - assertAdopted(t, getSession(t, c, "rb-old")) + assertAdopted(t, getSession(t, c, "sess-old")) }) t.Run("pre-admission status but no pod listed: not adopted (nothing to hold)", func(t *testing.T) { r, c, _ := newReconciler(t, true, roomFor(1), - preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseRunning), - waitingSession("rb-a", time.Minute), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseRunning), + waitingSession("sess-a", time.Minute), ) runPass(t, r) - assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "rb-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), + assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "sess-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted), "no pods: not a holder") - assertAdmitted(t, getSession(t, c, "rb-a")) + assertAdmitted(t, getSession(t, c, "sess-a")) }) t.Run("Enabled=false: nothing adopted", func(t *testing.T) { r, c, _ := newReconciler(t, false, roomFor(1), - preAdmissionSession("rb-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("rb-old"), + preAdmissionSession("sess-old", spiceboxv1alpha1.AgentSessionPhaseRunning), boundRunnerPod("sess-old"), ) runPass(t, r) - assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "rb-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted)) + assert.Nil(t, meta.FindStatusCondition(getSession(t, c, "sess-old").Status.Conditions, spiceboxv1alpha1.AgentSessionConditionAdmitted)) }) } @@ -950,41 +956,41 @@ func TestAdmitterCapReclaim(t *testing.T) { name: "cap 2 held by two warm idle sessions, plenty of room: the oldest is reclaimed, only it", objs: []client.Object{ cpuNode(8), cappedClass(2), - warmIdleSession("rb-old", 30*time.Minute), boundRunnerPod("rb-old"), - warmIdleSession("rb-mid", 20*time.Minute), boundRunnerPod("rb-mid"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-old", 30*time.Minute), boundRunnerPod("sess-old"), + warmIdleSession("sess-mid", 20*time.Minute), boundRunnerPod("sess-mid"), + waitingSession("sess-wait", time.Minute), }, - wantAnnotated: []string{"rb-old"}, - wantClean: []string{"rb-mid", "rb-wait"}, + wantAnnotated: []string{"sess-old"}, + wantClean: []string{"sess-mid", "sess-wait"}, }, { name: "a cap reclaim already in flight for the line: no second one", objs: []client.Object{ cpuNode(8), cappedClass(2), - requested(warmIdleSession("rb-old", 30*time.Minute)), boundRunnerPod("rb-old"), - warmIdleSession("rb-mid", 20*time.Minute), boundRunnerPod("rb-mid"), - waitingSession("rb-wait", time.Minute), + requested(warmIdleSession("sess-old", 30*time.Minute)), boundRunnerPod("sess-old"), + warmIdleSession("sess-mid", 20*time.Minute), boundRunnerPod("sess-mid"), + waitingSession("sess-wait", time.Minute), }, - wantClean: []string{"rb-mid", "rb-wait"}, + wantClean: []string{"sess-mid", "sess-wait"}, }, { name: "the cap is held by a running session, the warm one is too young: nothing reclaimed", objs: []client.Object{ cpuNode(8), cappedClass(2), - admittedRunning("rb-run"), - warmIdleSession("rb-young", time.Minute), boundRunnerPod("rb-young"), - waitingSession("rb-wait", time.Minute), + admittedRunning("sess-run"), + warmIdleSession("sess-young", time.Minute), boundRunnerPod("sess-young"), + waitingSession("sess-wait", time.Minute), }, - wantClean: []string{"rb-young", "rb-wait"}, + wantClean: []string{"sess-young", "sess-wait"}, }, { name: "under the cap and the head fits: admitted, nothing reclaimed", objs: []client.Object{ cpuNode(8), cappedClass(3), - warmIdleSession("rb-old", 30*time.Minute), boundRunnerPod("rb-old"), - waitingSession("rb-wait", time.Minute), + warmIdleSession("sess-old", 30*time.Minute), boundRunnerPod("sess-old"), + waitingSession("sess-wait", time.Minute), }, - wantClean: []string{"rb-old"}, + wantClean: []string{"sess-old"}, }, } for _, tc := range cases { @@ -1013,13 +1019,13 @@ func TestAdmitterCapReclaim(t *testing.T) { // label, it is the bundle entry's pod. func TestBoundPodsMatchesPoolNamedSandboxPod(t *testing.T) { d := admission.Demand{Pods: []admission.PodDemand{ - {Name: "rb-runner", CPUMilli: 1000}, - {Name: "rb-code-pod", SandboxSession: "rb-code", CPUMilli: 500}, + {Name: "sess-runner", CPUMilli: 1000}, + {Name: "sess-code-pod", SandboxSession: "sess-code", CPUMilli: 500}, }} runner := boundRunnerPod("rb") sandbox := boundRunnerPod("ignored") sandbox.Name = "warmpool-x7k2p" - sandbox.Labels = map[string]string{sandboxPodLabel: "rb-code"} + sandbox.Labels = map[string]string{sandboxPodLabel: "sess-code"} pods := newPodIndex([]corev1.Pod{*runner, *sandbox}) got := boundPods("default", d, pods) @@ -1027,7 +1033,7 @@ func TestBoundPodsMatchesPoolNamedSandboxPod(t *testing.T) { for _, p := range got { names = append(names, p.Name) } - assert.ElementsMatch(t, []string{"rb-runner", "warmpool-x7k2p"}, names, "both the runner and the pool-named sandbox pod are bound") + assert.ElementsMatch(t, []string{"sess-runner", "warmpool-x7k2p"}, names, "both the runner and the pool-named sandbox pod are bound") assert.Empty(t, unplaced("default", d, pods, true).Pods, "nothing left unplaced") } @@ -1038,13 +1044,13 @@ func TestAdmitterClearsStalePosition(t *testing.T) { s.Status.Admission = &spiceboxv1alpha1.AdmissionStatus{Position: 3, LineLength: 5, JoinedAt: s.CreationTimestamp} return s } - failed := stale(waitingSession("rb-failed", 0)) + failed := stale(waitingSession("sess-failed", 0)) failed.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseFailed - parked := stale(waitingSession("rb-parked", 0)) + parked := stale(waitingSession("sess-parked", 0)) parked.Status.Phase = spiceboxv1alpha1.AgentSessionPhaseAwaitingCredentials r, c, _ := newReconciler(t, true, roomFor(1), failed, parked) runPass(t, r) - assert.Nil(t, getSession(t, c, "rb-failed").Status.Admission, "a Failed session holds no line position") - assert.Nil(t, getSession(t, c, "rb-parked").Status.Admission, "a parked session holds no line position") + assert.Nil(t, getSession(t, c, "sess-failed").Status.Admission, "a Failed session holds no line position") + assert.Nil(t, getSession(t, c, "sess-parked").Status.Admission, "a parked session holds no line position") } From a8d51744a64644b5923912eb85d4098df741a17b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 11 Nov 2026 00:23:10 -0500 Subject: [PATCH 31/33] Finish renaming admitter test fixtures --- pkg/controllers/admission/admitter/controller_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller_test.go b/pkg/controllers/admission/admitter/controller_test.go index 807526fc..e63d571b 100644 --- a/pkg/controllers/admission/admitter/controller_test.go +++ b/pkg/controllers/admission/admitter/controller_test.go @@ -355,11 +355,11 @@ func TestAdmitterWatchNamespaces(t *testing.T) { { name: "watch set: one namespaced list per watched namespace, unwatched sessions untouched", namespaces: []string{"team-a", "team-b"}, - wantLists: []string{"team-a", "team-b"}, wantAdmitted: []string{"team-a/rb-a", "team-b/rb-b"}, + wantLists: []string{"team-a", "team-b"}, wantAdmitted: []string{"team-a/sess-a", "team-b/sess-b"}, }, { name: "no watch set: one cluster-wide list, every namespace admitted", - wantLists: []string{""}, wantAdmitted: []string{"other/rb-c", "team-a/rb-a", "team-b/rb-b"}, + wantLists: []string{""}, wantAdmitted: []string{"other/sess-c", "team-a/sess-a", "team-b/sess-b"}, }, } for _, tc := range cases { @@ -542,7 +542,7 @@ func TestAdmitterReclaimInFlightIsCredited(t *testing.T) { const minWarm = 5 * time.Minute cases := []struct { name string - advance func(t *testing.T, c client.Client) // what happened to rb-1 between passes + advance func(t *testing.T, c client.Client) // what happened to sess-1 between passes }{ { name: "reclaim requested but not yet slept: second pass annotates nothing new", @@ -1022,7 +1022,7 @@ func TestBoundPodsMatchesPoolNamedSandboxPod(t *testing.T) { {Name: "sess-runner", CPUMilli: 1000}, {Name: "sess-code-pod", SandboxSession: "sess-code", CPUMilli: 500}, }} - runner := boundRunnerPod("rb") + runner := boundRunnerPod("sess") sandbox := boundRunnerPod("ignored") sandbox.Name = "warmpool-x7k2p" sandbox.Labels = map[string]string{sandboxPodLabel: "sess-code"} From 26f16fa87a4d100c79d07f964afee3f859531ae5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 11 Nov 2026 01:02:59 -0500 Subject: [PATCH 32/33] Charge the e2e capacity model one set per session, not per reservation Admission reserves an admitted session's unplaced pods one at a time, and the fixed-set model charged a whole set for each, so a single admitted reviewer filled the burst scenario's two-set cluster. Whenever the first session was admitted alone, the second waited for it to leave. Charge each set to its session's runner pod instead. --- test/e2e/admission.go | 29 ++++++++++++++++------ test/e2e/admission_test.go | 51 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 test/e2e/admission_test.go diff --git a/test/e2e/admission.go b/test/e2e/admission.go index 016505b3..c068a3ab 100644 --- a/test/e2e/admission.go +++ b/test/e2e/admission.go @@ -4,6 +4,7 @@ package e2e import ( "context" + "strings" "testing" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -45,9 +46,10 @@ func (h *Harness) registerAdmitter(t *testing.T, mgr manager.Manager) { } // FixedSetCapacity is a cluster with room for exactly Sets whole session pod -// sets, whatever their size. Every Reserve of a non-empty demand takes one -// set's room, so with the harness's pods never binding to a node (there is -// no scheduler), each admitted session holds one slot until it is released. +// sets, whatever their size. A set's room is charged to its runner pod, the +// one pod every session's set has exactly one of, so with the harness's pods +// never binding to a node (there is no scheduler), each admitted session holds +// one slot until it is released. // // It stands in for nodes, not for the admission logic: the admitter still // orders lines, applies fair share and reserves holders' unplaced demand @@ -76,17 +78,30 @@ type setModel struct { func (m *setModel) Known() bool { return true } +// Reserve charges one set per runner pod in d. Charging per call instead +// would be wrong: Decide reserves a holder's unplaced pods one at a time, so a +// three-pod holder would take three sets. A pod that is not a runner (sandbox, +// detector, sidecar) rides on its session's set and is free. func (m *setModel) Reserve(d admission.Demand) bool { - if len(d.Pods) == 0 { - return true + sets := 0 + for _, p := range d.Pods { + if isRunnerPod(p) { + sets++ + } } - if m.free == 0 { + if sets > m.free { return false } - m.free-- + m.free -= sets return true } +// isRunnerPod matches the runner pod of a session's demand, named by +// agentsession.RunnerPodName. +func isRunnerPod(p admission.PodDemand) bool { + return p.SandboxSession == "" && strings.HasSuffix(p.Name, "-runner") +} + // Free is a no-op: a model rebuilt every pass has no placed pods to return, // and nothing in a scenario frees through the model mid-pass. func (m *setModel) Free([]admission.PlacedPod) {} diff --git a/test/e2e/admission_test.go b/test/e2e/admission_test.go new file mode 100644 index 00000000..252aef59 --- /dev/null +++ b/test/e2e/admission_test.go @@ -0,0 +1,51 @@ +//go:build e2e + +package e2e + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/controllers/admission" +) + +// setDemand is one session's pod set as DemandFor prices it: the runner pod +// plus one sandbox pod per tool bundle. +func setDemand(session string, bundles ...string) admission.Demand { + d := admission.Demand{Pods: []admission.PodDemand{{Name: session + "-runner", CPUMilli: 100, MemBytes: 1 << 20}}} + for _, b := range bundles { + d.Pods = append(d.Pods, admission.PodDemand{ + Name: session + "-" + b + "-pod", CPUMilli: 100, MemBytes: 1 << 20, SandboxSession: session + "-" + b, + }) + } + return d +} + +// TestFixedSetCapacityCountsWholeSets pins that the harness model charges one +// set per SESSION, however Decide splits that session's pods across Reserve +// calls. Decide reserves a holder's unplaced pods one at a time; a model that +// charged a set per call let one three-pod holder fill a two-set cluster, so +// the second arrival waited for the first to leave. +func TestFixedSetCapacityCountsWholeSets(t *testing.T) { + holder := setDemand("first", "gitlike", "scratch") + model, err := FixedSetCapacity{Sets: 2}.Snapshot(context.Background()) + require.NoError(t, err) + + key := func(n string) types.NamespacedName { return types.NamespacedName{Namespace: "default", Name: n} } + dec := admission.Decide(admission.Input{ + Holders: []admission.Holder{{Key: key("first"), Line: "default/reviewer", Demand: holder, Unplaced: holder}}, + Waiting: []admission.Candidate{ + {Key: key("second"), Line: "default/reviewer", Demand: setDemand("second", "gitlike", "scratch")}, + {Key: key("third"), Line: "default/reviewer", Demand: setDemand("third", "gitlike", "scratch")}, + }, + Model: model, + }) + + assert.Equal(t, []types.NamespacedName{key("second")}, dec.Admit, + "a two-set cluster with one holder has room for exactly one more session") + assert.Equal(t, admission.Position{Index: 1, LineLength: 1}, dec.Positions[key("third")]) +} From 005e40ffa8ddade00a3656eddb3ea6a52eedaa3d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 11 Nov 2026 01:13:33 -0500 Subject: [PATCH 33/33] Describe the deferred pass in the delegation list's comment and log --- pkg/controllers/admission/admitter/controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/admission/admitter/controller.go b/pkg/controllers/admission/admitter/controller.go index a94534b3..a2614a63 100644 --- a/pkg/controllers/admission/admitter/controller.go +++ b/pkg/controllers/admission/admitter/controller.go @@ -605,13 +605,13 @@ type delegation struct { type delegations map[types.NamespacedName]delegation // listDelegations indexes every SubagentRequest that names its child. A list -// error returns an empty index, so the pass treats every session as -// non-delegated rather than trusting spec.parent; the caller reports the -// error so the pass is retried. +// error is returned, and the caller defers the whole pass on it: nothing is +// decided, admitted or reclaimed until a later pass can read the delegation +// records, since without them no session could be proven a delegated child. func (r *Reconciler) listDelegations(ctx context.Context) (delegations, error) { var list spiceboxv1alpha1.SubagentRequestList if err := r.Client.List(ctx, &list); err != nil { - log.FromContext(ctx).Info("admission: list SubagentRequests failed; no session is treated as a delegated child this pass", + log.FromContext(ctx).Info("admission: list SubagentRequests failed; deferring this admission pass", "err", err.Error()) return delegations{}, fmt.Errorf("admission pass: list SubagentRequests: %w", err) }