From b3f9a9ea5dc9084205444ec769d8e86c33e155d2 Mon Sep 17 00:00:00 2001 From: Sohan Maheshwar <1119120+sohanmaheshwar@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:43:55 +0100 Subject: [PATCH] channelsd: don't resume a deleted session's transcript under a reused name A channelsd session's name is derived from (channel, key), and its memory scope is keyed by that name, not the CR's UID. The transcript outlives the CR on purpose (append-only, signed audit evidence). So when a session was deleted, by hand or by the archive sweep followed by session GC, the next message on the same key minted a session into its predecessor's transcript. The runner read that as a resume: turn 0 already existed, so the new message (spec.prompt) was never placed. A predecessor that ended on the agent's reply parked without a word; one that ended on a tool result ran the old conversation again. Either way the message was lost, and the only trace was the runner's "cold-start: skipped" log line; Slack had already shown "starting". Seen live as a Slack DM that got no reply after the user's DM session was deleted. The old conversation also came back without the rules that govern continuing one. While a session's CR exists, lifecycle.ContinuationDisposition refuses an inbound to a policy-halted or held session, and a finished session's follow-up starts a new session that inherits only under the takeover rules. Those rules read the CR. Once it is gone, nothing applied them. This fix chooses a fresh start, which the design leaves open. While the CR exists, continuity is intended: an archived session resumes, and a DM is one ongoing session per user. But everything tied to a session incarnation (lifecycle.ForIncarnation, sweepBeforeMint) treats a reused name as a new session, and the continuation rules cannot run without the CR. So a message after deletion starts a new session. The cost: a DM user whose session was garbage-collected gets a session with no memory of the old conversation. Before this, they got no reply. The fresh-session path now picks its name with freshSessionName. It walks makeSessionName's base, then base-g2, base-g3, ... (at most 512), and takes the first generation whose memory scope holds no turns. - The walk is deterministic, so concurrent replicas still collapse onto one session through AlreadyExists. - Each step reads turns before the CR. A replica that sees another replica's just-written turns therefore also sees its CR and joins it. - A session that is being deleted counts as a predecessor. - Follow-ups still route by the channel labels, which the successor carries. - A memory read error fails closed (InternalError, no session minted), like sweepBeforeMint, and runs after the thread-ownership guard. - The longest name stays within the 63-byte Job-name budget ("ws-apply-"). Nothing in memory is deleted or rewritten. Audit chains and `oap audit verify` are unchanged. The bronze bundle recreated-session-starts-fresh covers it end to end: user turn, session deleted, user turn on the same key. The second session's first request opens on the new message, and the first session's tool output appears nowhere in its context. Supporting bundle format additions: - userTurns[].deleteSessionFirst - expect.requestNotContains, which steelthread's expectAssertsNothing now counts as an assertion - Harness.DeleteSession Webhook triggers are covered: they enter through the same Deliver path. Not covered: - A message that arrives while a session is being deleted (its finalizer is running) still finds it: correlateSessions matches by label and does not check DeletionTimestamp, so the message is delivered into a session that is about to vanish. - Sessions created outside channelsd can still start on a predecessor's transcript. That includes kubectl/oap, scheduled goal sessions (named from the occurrence digest, goals.SessionName), and fork, inherit and takeover children, whose names are deterministic. A runner-side fix should tie turn-0 placement to the CR's UID; lifecycle.ForIncarnation (pkg/memory/kinds/lifecycle/order.go:45-77) already separates incarnations in a name-keyed scope. The controller should also stop adopting a predecessor's per-session Secret (check IsControlledBy): today a fast delete and apply hands the successor the old memory token and audit seed until owner-reference GC runs. --- pkg/bronzethread/bundle.go | 10 + pkg/bronzethread/bundle_test.go | 17 ++ pkg/bronzethread/userturn.go | 10 + pkg/channels/channelsd/pipeline/pipeline.go | 113 ++++++++- .../channelsd/pipeline/reused_name_test.go | 216 ++++++++++++++++++ pkg/steelthread/selfcheck.go | 1 + pkg/steelthread/selfcheck_test.go | 13 ++ .../bundle.json | 67 ++++++ test/e2e/session_delete.go | 42 ++++ test/e2e/threadrun/driver.go | 22 ++ test/e2e/threadrun/helpers.go | 20 ++ test/e2e/threadrun/helpers_request_test.go | 34 +++ 12 files changed, 562 insertions(+), 3 deletions(-) create mode 100644 pkg/channels/channelsd/pipeline/reused_name_test.go create mode 100644 test/e2e/bronzethread/testdata/recreated-session-starts-fresh/bundle.json create mode 100644 test/e2e/session_delete.go create mode 100644 test/e2e/threadrun/helpers_request_test.go diff --git a/pkg/bronzethread/bundle.go b/pkg/bronzethread/bundle.go index 7771004a..6e3e31d6 100644 --- a/pkg/bronzethread/bundle.go +++ b/pkg/bronzethread/bundle.go @@ -326,6 +326,9 @@ func (b Bundle) Validate() error { if len(b.UserTurns) == 0 && b.Trigger == nil { return fmt.Errorf("a bundle needs either userTurns or a trigger; nothing would start the run") } + if len(b.UserTurns) > 0 && b.UserTurns[0].DeleteSessionFirst { + return fmt.Errorf("userTurns[0]: deleteSessionFirst needs a session to delete, and none exists before the first turn") + } if b.Trigger != nil { if b.Trigger.Channel == "" { return fmt.Errorf("trigger needs a channel") @@ -669,6 +672,13 @@ type Expect struct { // denied nothing. LastToolResultNotContains string `json:"lastToolResultNotContains,omitempty"` + // RequestNotContains asserts something appears in NO message of the + // request — no text, no tool input, no tool result, however far back. + // LastToolResultNotContains says one result was clean; this says a value + // is not in the model's context at all, which is the claim to make when the + // question is whether another session's conversation leaked into this one. + RequestNotContains string `json:"requestNotContains,omitempty"` + // LastToolResultIsError is a pointer so a bundle can assert "must NOT be an // error" (false) distinctly from not caring (unset). LastToolResultIsError *bool `json:"lastToolResultIsError,omitempty"` diff --git a/pkg/bronzethread/bundle_test.go b/pkg/bronzethread/bundle_test.go index d02d4085..6db8a386 100644 --- a/pkg/bronzethread/bundle_test.go +++ b/pkg/bronzethread/bundle_test.go @@ -139,6 +139,23 @@ func TestBundle_Validate(t *testing.T) { }, wantErr: "trigger needs a channelKey", }, + { + name: "deleteSessionFirst on a later turn validates", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns = append(b.UserTurns, bt.UserTurn{Text: "again", DeleteSessionFirst: true}) + return b + }, + }, + { + // Before the first turn there is no session to delete; the driver + // would fail on a lookup minutes into a fixture boot instead. + name: "deleteSessionFirst on the first turn: refused", + doctor: func(b bt.Bundle) bt.Bundle { + b.UserTurns[0].DeleteSessionFirst = true + return b + }, + wantErr: "userTurns[0]: deleteSessionFirst", + }, { // Folded in rather than left as a separate call, so both consumers // get it from the one method. diff --git a/pkg/bronzethread/userturn.go b/pkg/bronzethread/userturn.go index 4f1b8c08..f1b5ffab 100644 --- a/pkg/bronzethread/userturn.go +++ b/pkg/bronzethread/userturn.go @@ -45,6 +45,16 @@ type BundleAttachment struct { type UserTurn struct { Text string `json:"text"` Attachments []BundleAttachment `json:"attachments,omitempty"` + + // DeleteSessionFirst deletes the conversation's session, and waits until + // it is gone, before this turn is sent. The turn then reaches a channel key + // whose session no longer exists while that session's transcript still + // does: turns are append-only audit evidence that outlives the CR, in a + // scope keyed by the session's NAME. That is the state a person returning + // to a deleted (or archived-then-collected) conversation is in, and no + // other step can produce it. Refused on the first turn, where there is + // nothing to delete. + DeleteSessionFirst bool `json:"deleteSessionFirst,omitempty"` } // UnmarshalJSON accepts the bare-string and object forms. diff --git a/pkg/channels/channelsd/pipeline/pipeline.go b/pkg/channels/channelsd/pipeline/pipeline.go index f131cda6..3e56495e 100644 --- a/pkg/channels/channelsd/pipeline/pipeline.go +++ b/pkg/channels/channelsd/pipeline/pipeline.go @@ -1398,8 +1398,6 @@ func (p *Pipeline) Deliver(ctx context.Context, ev channelkinds.InboundEvent) (c // Fresh-session path (no terminal predecessor: archived == nil below). caps := bindingCaps(p.Capabilities(ev.Channel.Spec.Kind)) - sessName := makeSessionName(ev.Channel, ev.ChannelKey) - prefix := channelevents.SubjectPrefix(ev.Channel.Namespace, sessName) // No InheritFrom on this path: a terminal predecessor is either forked // (NewInheriting) or refused above, so nothing reaching here has one. @@ -1436,6 +1434,21 @@ func (p *Pipeline) Deliver(ctx context.Context, ev channelkinds.InboundEvent) (c } } + // The name to mint under — never makeSessionName directly: its answer can + // name a scope a DELETED predecessor's transcript still occupies (see + // freshSessionName). AFTER the thread-ownership guard, so a message another + // agent's session owns keeps its silent refusal even while memory is + // unreadable. Fail CLOSED, like sweepBeforeMint below: the scope's contents + // could not be established, and minting anyway is exactly how a + // predecessor's transcript becomes this session's working context. + sessName, err := p.freshSessionName(ctx, ev.Channel, ev.ChannelKey) + if err != nil { + log.FromContext(ctx).Info("fresh session: could not establish an unoccupied name; refusing to mint", + "channel", channelRefForLog(ev.Channel), "channelKey", ev.ChannelKey, "err", err.Error()) + return channelkinds.InboundDecision{Outcome: channelkinds.OutcomeInternalError}, err + } + prefix := channelevents.SubjectPrefix(ev.Channel.Namespace, sessName) + // Session-START gate (start_gate.go): on an org-membership-attributing // channel, a non-member needs agentclass#start_session — or a platform // admin's approval — before a session runs for them at all. AFTER the @@ -2721,8 +2734,84 @@ func (p *Pipeline) sweepBeforeMint(ctx context.Context, ns, name string) error { return nil } +// maxSessionNameGenerations bounds freshSessionName's walk. Each step past the +// first is a session for this exact (channel, key) that was deleted after it +// had spoken: the archive sweep and session GC produce at most one a month for +// a conversation that goes quiet, so this is decades of them, and reaching it +// means something is wrong rather than busy — say so instead of looping. It +// also caps the name suffix at three digits, which the 63-byte name budget +// depends on (see sessionNameGeneration). +const maxSessionNameGenerations = 512 + +// freshSessionName picks the name a fresh session for (ch, key) is minted +// under: the first generation (see sessionNameGeneration) whose memory scope +// holds no turns. +// +// Why it cannot simply be makeSessionName. A memory scope is keyed by +// namespace/name, not by the CR's UID, and its transcript outlives the CR on +// purpose: the turns are append-only, signed audit evidence that session +// deletion and GC cannot remove. makeSessionName is deterministic, so once the +// session it names has been deleted — by hand, or by the archive sweep followed +// by session GC — the next message on the same key would mint a session INTO +// its predecessor's transcript. The runner then reads that as a resume: turn 0 +// already exists, so the new message (spec.prompt, placed only when no turn 0 +// exists) is never placed. What follows depends on how the old conversation +// ended — on the agent's reply, it parks without a word; on a tool result +// (agent_work_complete's, say), it runs the OLD conversation again. Either way +// the person's message is lost, and the predecessor's tool output — never +// re-authorized for whoever is asking now — is the new session's working +// context. sweepBeforeMint already clears what a reused +// name leaves in SpiceDB; this is the memory half, and since the memory cannot +// be cleared, the name moves instead. +// +// Deterministic across replicas, so the AlreadyExists race the base name relies +// on still collapses two replicas onto one session: every replica reads the +// same retained scopes and walks the same sequence. Follow-up messages never +// recompute the name — correlateSessions finds the session by its channel +// labels, which a successor carries exactly as the first generation did. +// +// The order of the two reads per step is what keeps the race safe. A scope +// holding turns is either a dead predecessor's or a LIVE session's — another +// replica that minted this same inbound a moment ago and has already written +// into it. Turns are read FIRST and the CR second, so a replica that sees +// another's turns is guaranteed to see that replica's CR too (it was created +// before anything was written under its name) and joins it rather than walking +// past it to mint a duplicate. p.K8s is the uncached client. +// +// Any read error is returned, never guessed past: the caller refuses to mint. +func (p *Pipeline) freshSessionName(ctx context.Context, ch *spiceboxv1alpha1.Channel, key string) (string, error) { + base := makeSessionName(ch, key) + for g := 1; g <= maxSessionNameGenerations; g++ { + name := sessionNameGeneration(base, g) + turns, err := p.Memory.ReadAll(ctx, ch.Namespace, name) + if err != nil { + return "", fmt.Errorf("read memory scope %s/%s before minting a session under it: %w", ch.Namespace, name, err) + } + if len(turns) == 0 { + return name, nil + } + var existing spiceboxv1alpha1.AgentSession + switch err := p.K8s.Get(ctx, types.NamespacedName{Namespace: ch.Namespace, Name: name}, &existing); { + case err == nil && existing.DeletionTimestamp == nil: + // Live: a concurrent mint of this conversation. The Create below + // meets AlreadyExists and joins it, exactly as for the base name. + return name, nil + case err != nil && !apierrors.IsNotFound(err): + return "", fmt.Errorf("check whether %s/%s exists: %w", ch.Namespace, name, err) + } + // Gone, or being deleted (its finalizer is running): a predecessor + // either way, and joining one about to vanish would strand the inbound. + log.FromContext(ctx).Info("fresh session: memory scope holds a deleted session's transcript; minting under the next name instead", + "occupied", ch.Namespace+"/"+name, "turns", len(turns), + "channel", channelRefForLog(ch), "channelKey", key) + } + return "", fmt.Errorf("every one of %d candidate names for %s key %q holds a deleted session's transcript", + maxSessionNameGenerations, channelRefForLog(ch), key) +} + // makeSessionName produces a deterministic-ish name based on the channel -// and key. Truncated + suffixed with a hash so k8s names stay valid. +// and key. Truncated + suffixed with a hash so k8s names stay valid. A fresh +// mint goes through freshSessionName, which starts here. func makeSessionName(ch *spiceboxv1alpha1.Channel, key string) string { h := sha256.Sum256([]byte(ch.Name + "/" + key)) suffix := hex.EncodeToString(h[:])[:8] @@ -2733,6 +2822,24 @@ func makeSessionName(ch *spiceboxv1alpha1.Channel, key string) string { return base + "-" + suffix } +// sessionNameGeneration names the g-th session minted for one (channel, key): +// base itself (makeSessionName's answer) for the first, base-g after it. A +// counter rather than a hash so the sequence is the same on every replica and +// readable in `kubectl get agentsessions`. +// +// The limit that binds is 63, not 253: a session's workspace Jobs are named +// "ws-apply-" (agentsession rbac.go), and a Job name must fit a 63-byte +// label value. base is at most 49 bytes (40 of channel name, "-", 8 hex), and +// maxSessionNameGenerations keeps the suffix to at most "-g512", 5 bytes — so +// the longest name is 54 and its Job name exactly 63. A larger bound, or a +// longer suffix, must shorten base to match. +func sessionNameGeneration(base string, g int) string { + if g <= 1 { + return base + } + return base + "-g" + strconv.Itoa(g) +} + // renderMention returns the channel kind's clickable mention syntax for // externalID (Slack: "<@U123>"), falling back to the bare externalID when // the kind isn't registered or its RenderMention returns empty. Shared by diff --git a/pkg/channels/channelsd/pipeline/reused_name_test.go b/pkg/channels/channelsd/pipeline/reused_name_test.go new file mode 100644 index 00000000..a55cf71a --- /dev/null +++ b/pkg/channels/channelsd/pipeline/reused_name_test.go @@ -0,0 +1,216 @@ +package pipeline + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/channels/channelkey" + "github.com/authzed/openagentprimitives/pkg/channels/channelkinds" +) + +// A session name is derived from (channel, key), and a session's memory scope +// is keyed by that name — not by the CR's UID — and outlives the CR by design +// (the transcript is append-only audit evidence). So when a session is deleted +// and the same person messages the same channel key again, the deterministic +// name lands the fresh session on its predecessor's transcript: the runner +// replays it as a resume and never places the new message, with the +// predecessor's tool output as working context. +// +// These cases pin the fix: a fresh session is never minted over a scope that +// already holds turns. +func TestFreshMint_ReusedNameWithRetainedTranscript(t *testing.T) { + const key = "dm:U1" + ctx := context.Background() + predecessorTurns := []MemTurn{ + {Index: 0, Role: "user", Content: []MemContent{{Type: "text", Text: "what is in the shared drive?"}}}, + {Index: 1, Role: "assistant", Content: []MemContent{{Type: "text", Text: "the payroll sheet"}}}, + } + + cases := []struct { + name string + // seededGenerations names, by generation, the scopes that already hold + // a dead predecessor's turns: 1 is makeSessionName's own name, 2 its + // first successor, and so on. + seededGenerations []int + // liveAt, when non-zero, pre-creates a session CR (carrying no channel + // labels, so Deliver's correlation misses it — exactly the state a + // concurrent replica's just-minted session is in) at that generation. + liveAt int + // terminating marks the liveAt session as already being deleted. + terminating bool + readErr error + wantErr bool + wantOut channelkinds.Outcome + // wantGeneration is the generation the inbound's session ends up under; + // 0 means no session may be created at all. + wantGeneration int + }{ + { + name: "empty scope: the deterministic name is used unchanged", + wantOut: channelkinds.OutcomeRouted, + wantGeneration: 1, + }, + { + name: "scope holds a deleted predecessor's transcript: minted under the next generation", + seededGenerations: []int{1}, + wantOut: channelkinds.OutcomeRouted, + wantGeneration: 2, + }, + { + name: "two predecessors deleted in turn: walks past both to the first empty scope", + seededGenerations: []int{1, 2}, + wantOut: channelkinds.OutcomeRouted, + wantGeneration: 3, + }, + { + name: "a live session holds the occupied scope (replica race): joins it, mints no second session", + seededGenerations: []int{1}, + liveAt: 1, + wantOut: channelkinds.OutcomeRouted, + wantGeneration: 1, + }, + { + // Its finalizer is running: it is the predecessor, not a concurrent + // mint, and joining it would route the inbound into a session that + // is about to vanish. + name: "the session holding the occupied scope is being deleted: minted under the next generation", + seededGenerations: []int{1}, + liveAt: 1, + terminating: true, + wantOut: channelkinds.OutcomeRouted, + wantGeneration: 2, + }, + { + name: "memory unreadable: fail closed, InternalError and no session minted", + readErr: errors.New("memory backend unavailable"), + wantErr: true, + wantOut: channelkinds.OutcomeInternalError, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ch := newChannel("c1") + objs := []client.Object{ch} + if tc.liveAt != 0 { + existing := &spiceboxv1alpha1.AgentSession{ + ObjectMeta: metav1.ObjectMeta{Name: generationName(ch, key, tc.liveAt), Namespace: ch.Namespace}, + Spec: spiceboxv1alpha1.AgentSessionSpec{Class: "ac1"}, + } + if tc.terminating { + now := metav1.Now() + existing.DeletionTimestamp = &now + existing.Finalizers = []string{spiceboxv1alpha1.FinalizerAgentSession} + } + objs = append(objs, existing) + } + p, _, mem, _, cli := newPipeline(t, objs...) + for _, g := range tc.seededGenerations { + mem.seeded[ch.Namespace+"/"+generationName(ch, key, g)] = predecessorTurns + } + mem.readErr = tc.readErr + + dec, err := p.Deliver(ctx, channelkinds.InboundEvent{ + Channel: ch, + ExternalIDs: channelkinds.ExternalIdentity{Kind: "fake", ExternalID: "U1", Email: "a@example.com"}, + ChannelKey: key, + MessageText: "hello again", + }) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantOut, dec.Outcome) + + var sessions spiceboxv1alpha1.AgentSessionList + require.NoError(t, cli.List(ctx, &sessions)) + live := slices.DeleteFunc(sessions.Items, func(s spiceboxv1alpha1.AgentSession) bool { + return s.DeletionTimestamp != nil + }) + if tc.wantGeneration == 0 { + assert.Empty(t, live, "no session may be minted over a scope whose contents could not be read") + return + } + require.Len(t, live, 1, "exactly one live session for one inbound") + got := live[0] + assert.Equal(t, generationName(ch, key, tc.wantGeneration), got.Name) + if tc.liveAt == tc.wantGeneration { + return // the pre-existing session is not this inbound's to shape + } + assert.Equal(t, "hello again", got.Spec.Prompt.Inline, + "the inbound must be the new session's opening prompt") + assert.Equal(t, channelkey.LabelValue(key), got.Labels[spiceboxv1alpha1.LabelChannelKey], + "the correlation labels are what route the next message in this conversation here") + require.NotNil(t, got.Spec.InputChannel) + assert.Equal(t, key, got.Spec.InputChannel.Key) + assert.Contains(t, got.Spec.InputChannel.NATSSubjectPrefix, got.Name, + "the subject prefix must follow the name actually minted, not the occupied one") + }) + } +} + +// The follow-up half: a session minted under a successor name must still be +// the one the NEXT message in the same conversation reaches. Routing is by the +// correlation labels, never by recomputing the name, so this pins that the +// successor carries them and nothing on the routed path re-derives the name. +func TestFreshMint_SuccessorNameReceivesTheFollowUp(t *testing.T) { + const key = "dm:U1" + ctx := context.Background() + ch := newChannel("c1") + p, _, mem, _, cli := newPipeline(t, ch) + mem.seeded[ch.Namespace+"/"+makeSessionName(ch, key)] = []MemTurn{ + {Index: 0, Role: "user", Content: []MemContent{{Type: "text", Text: "old question"}}}, + {Index: 1, Role: "assistant", Content: []MemContent{{Type: "text", Text: "old answer"}}}, + } + ev := func(text string) channelkinds.InboundEvent { + return channelkinds.InboundEvent{ + Channel: ch, + ExternalIDs: channelkinds.ExternalIdentity{Kind: "fake", ExternalID: "U1", Email: "a@example.com"}, + ChannelKey: key, + MessageText: text, + } + } + + dec, err := p.Deliver(ctx, ev("hello again")) + require.NoError(t, err, "first Deliver mints the successor") + require.Equal(t, channelkinds.OutcomeRouted, dec.Outcome) + successor := generationName(ch, key, 2) + + dec, err = p.Deliver(ctx, ev("and a follow-up")) + require.NoError(t, err, "second Deliver routes into the successor") + assert.Equal(t, channelkinds.OutcomeRouted, dec.Outcome) + assert.Equal(t, successor, dec.Session.Name, "the follow-up must reach the successor session") + + var sessions spiceboxv1alpha1.AgentSessionList + require.NoError(t, cli.List(ctx, &sessions)) + assert.Len(t, sessions.Items, 1, "a follow-up must not mint another session") + require.Len(t, mem.appends, 1, "the follow-up is the only turn channelsd writes") + assert.Equal(t, successor, mem.appends[0].name, "into the successor's scope, never the predecessor's") + require.NotEmpty(t, mem.appends[0].turn.Content) + assert.Equal(t, "and a follow-up", mem.appends[0].turn.Content[0].Text) +} + +// The longest name the walk can mint must still leave room for the objects +// named after a session. The tightest is the workspace Job, "ws-apply-", +// whose name must fit a 63-byte label value. +func TestSessionNameGeneration_LongestNameFitsAJobName(t *testing.T) { + ch := newChannel(strings.Repeat("c", 80)) // truncated to 40 by makeSessionName + name := sessionNameGeneration(makeSessionName(ch, "dm:U1"), maxSessionNameGenerations) + assert.LessOrEqual(t, len("ws-apply-"+name), 63, "longest successor name %q", name) +} + +// generationName is the name the g-th session for (ch, key) is minted under: +// makeSessionName's own for the first, then the successors the mint path +// walks to when a predecessor's transcript still occupies the scope. +func generationName(ch *spiceboxv1alpha1.Channel, key string, g int) string { + return sessionNameGeneration(makeSessionName(ch, key), g) +} diff --git a/pkg/steelthread/selfcheck.go b/pkg/steelthread/selfcheck.go index 6a45ebed..0e2ef03b 100644 --- a/pkg/steelthread/selfcheck.go +++ b/pkg/steelthread/selfcheck.go @@ -1659,6 +1659,7 @@ func expectAssertsNothing(e bt.Expect) bool { e.LastToolResult == "" && e.LastToolResultContains == "" && e.LastToolResultNotContains == "" && + e.RequestNotContains == "" && e.LastToolResultIsError == nil && len(e.ToolResultCounts) == 0 && e.ToolOffered == "" && diff --git a/pkg/steelthread/selfcheck_test.go b/pkg/steelthread/selfcheck_test.go index 9f7bdbfd..0c0568d8 100644 --- a/pkg/steelthread/selfcheck_test.go +++ b/pkg/steelthread/selfcheck_test.go @@ -467,6 +467,19 @@ func TestSelfCheck_CleanCaptureHasNoFindings(t *testing.T) { assert.Empty(t, steelthread.SelfCheck(cleanCapture(t))) } +// requestNotContains is a claim checkExpect compares, so a step carrying only +// it is guarded. Read as "asserts nothing", it would raise a hard empty-expect +// finding against a perfectly good step. +func TestSelfCheck_RequestNotContainsAloneIsAnAssertion(t *testing.T) { + in := cleanCapture(t) + in.Folded.LLM[0].Expect = bt.Expect{RequestNotContains: "a value the session never read"} + + for _, f := range steelthread.SelfCheck(in) { + assert.NotEqual(t, steelthread.CodeEmptyExpect, f.Code, + "a step asserting requestNotContains is not unguarded: %v", f) + } +} + // TestSelfCheck_UnseededAllowSaysWhichSlotCaseTheReaderIsIn pins the one thing // the finding's key cannot say: WHY this permission survived the declared-slot // exemption. diff --git a/test/e2e/bronzethread/testdata/recreated-session-starts-fresh/bundle.json b/test/e2e/bronzethread/testdata/recreated-session-starts-fresh/bundle.json new file mode 100644 index 00000000..05e0ba26 --- /dev/null +++ b/test/e2e/bronzethread/testdata/recreated-session-starts-fresh/bundle.json @@ -0,0 +1,67 @@ +{ + "name": "recreated-session-starts-fresh", + "description": "A person messages a conversation whose session was deleted. The second message must start a FRESH session that answers it, with none of the deleted session's conversation in its context.\n\nA session's name is derived from (channel, key), and its memory scope is keyed by that name, not the CR's UID. The transcript outlives the CR on purpose: turns are append-only, signed audit evidence. So the second message used to mint a session under the SAME name, onto the deleted one's transcript. The runner read that as a resume: turn 0 already existed, so spec.prompt was never placed. A predecessor ending on the agent's reply parked without a word; one ending on a tool result — as this one does, on agent_work_complete's — ran the OLD conversation again. Either way the person's message was lost, and the deleted session's tool output — never re-authorized for whoever was asking now — became the new session's context. channelsd now mints the successor under the next free name instead (pipeline.freshSessionName).\n\nWhat this pins, step by step. Step 4 is the second session's FIRST model call: userTextContains proves the new message was placed as its opening prompt rather than parked on, and requestNotContains proves nothing the first session read is in its context at all — not merely absent from the latest tool result. Step 5 asserts on the TOOL RESULT the second session's own call returned, not on the reply text, since a scripted reply says the same words either way. If the name is ever reused again, step 4 is answering the old conversation: its latest user text is the FIRST message and Acme Payroll is in its context, so both of its checks fail.", + + "agentDir": "../testdata/agent-centerdot-companies", + "agentClass": "centerdot-companies", + + "toolOutputSequence": { + "list_companies": [ + { "results": [{ "id": "co-7731", "name": "Acme Payroll", "ownerId": "owner-1", "note": "board-only: acquisition target" }] }, + { "results": [{ "id": "co-1002", "name": "Globex", "ownerId": "owner-1" }] } + ] + }, + "toolOutputs": { + "list_contacts_for_company": { "results": [] } + }, + + "userTurns": [ + "list the companies", + { "text": "list the companies again", "deleteSessionFirst": true } + ], + + "llm": [ + { + "expect": { "userTextContains": "list the companies" }, + "reply": [{ "toolUse": { "name": "centerdot_list_companies", "args": { "operation_id": "op-list-first", "_reason": "user asked for the company list", "args": {} } } }] + }, + { + "expect": { + "lastToolResult": "centerdot_list_companies", + "lastToolResultContains": "Acme Payroll", + "lastToolResultIsError": false + }, + "reply": [{ "text": "found Acme Payroll" }] + }, + { + "expect": { "lastToolResult": "respond_to_user" }, + "reply": [{ "toolUse": { "name": "agent_work_complete", "args": { "summary": "listed companies" } } }] + }, + { + "expect": { + "userTextContains": "list the companies again", + "requestNotContains": "Acme Payroll" + }, + "reply": [{ "toolUse": { "name": "centerdot_list_companies", "args": { "operation_id": "op-list-second", "_reason": "user asked for the company list", "args": {} } } }] + }, + { + "expect": { + "lastToolResult": "centerdot_list_companies", + "lastToolResultContains": "Globex", + "lastToolResultNotContains": "Acme Payroll", + "lastToolResultIsError": false, + "requestNotContains": "acquisition target" + }, + "reply": [{ "text": "found Globex" }] + }, + { + "expect": { "lastToolResult": "respond_to_user" }, + "reply": [{ "toolUse": { "name": "agent_work_complete", "args": { "summary": "listed companies" } } }] + } + ], + + "assert": { + "agentReplyContains": ["found Acme Payroll", "found Globex"], + "agentReplyNotContains": ["", "Acme Payroll"] + } +} diff --git a/test/e2e/session_delete.go b/test/e2e/session_delete.go new file mode 100644 index 00000000..f93a660b --- /dev/null +++ b/test/e2e/session_delete.go @@ -0,0 +1,42 @@ +//go:build e2e + +package e2e + +import ( + "context" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" +) + +// DeleteSession deletes the (ns, name) AgentSession and waits until it is gone +// — finalizer run, CR no longer readable — or fails the test after deadline. +// +// What it leaves behind is the point. The session's memory scope is keyed by +// its name and outlives the CR by design (the transcript is append-only audit +// evidence), so a later inbound on the same channel key finds a transcript and +// no session: the state a person returning to a deleted conversation is in. +func (h *Harness) DeleteSession(ns, name string, deadline time.Duration) { + h.t.Helper() + key := client.ObjectKey{Namespace: ns, Name: name} + sess := &spiceboxv1alpha1.AgentSession{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}} + if err := h.K8s.Delete(context.Background(), sess); err != nil && !apierrors.IsNotFound(err) { + h.t.Fatalf("delete AgentSession %s: %v", key, err) + } + cutoff := time.Now().Add(deadline) + var lastErr error + for time.Now().Before(cutoff) { + err := h.K8s.Get(context.Background(), key, &spiceboxv1alpha1.AgentSession{}) + if apierrors.IsNotFound(err) { + return + } + lastErr = err + time.Sleep(100 * time.Millisecond) + } + h.t.Fatalf("AgentSession %s still present %s after delete (its finalizer has not completed; last get err: %v)", + key, deadline, lastErr) +} diff --git a/test/e2e/threadrun/driver.go b/test/e2e/threadrun/driver.go index e4896d0f..d6aa801d 100644 --- a/test/e2e/threadrun/driver.go +++ b/test/e2e/threadrun/driver.go @@ -427,6 +427,9 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { // than a per-bundle hook, and for the no-op it is for every other bundle. workshopSessionBound := false for i, turn := range b.UserTurns { + if turn.DeleteSessionFirst { + deleteSessionBeforeTurn(t, h, i) + } h.SendUserMessage(turn.Text, e2e.InThread("bt-"+b.Name), e2e.WithAttachments(bundleAttachments(t, dir, i, turn)...)) // Contains and NotContains are positional and run on the SAME reply, so a // turn asserting both awaits its reply once and checks presence + absence @@ -475,6 +478,21 @@ func Run(t *testing.T, dir string, b bt.Bundle) *e2e.Harness { return h } +// deleteSessionBeforeTurn carries out a turn's deleteSessionFirst: the +// conversation's one session is deleted, and gone, before the turn is sent. +// +// It waits for the session to come to rest first. The previous turn's reply can +// land before the runner has written Idle, and deleting a session mid-write +// would make the scenario about that race rather than about what the delete +// leaves behind — the session's transcript, which outlives it by design. +func deleteSessionBeforeTurn(t *testing.T, h *e2e.Harness, turnIdx int) { + t.Helper() + ns, name := h.SessionRef() + h.WaitForSessionPhase(ns, name, spiceboxv1alpha1.AgentSessionPhaseIdle, 60*time.Second) + h.DeleteSession(ns, name, 60*time.Second) + t.Logf("bronzethread: before turn %d, deleted session %s/%s (its transcript stays in memory)", turnIdx, ns, name) +} + // checkMetaRepliesApplied proves every canned meta reply was actually installed. // // A canned reply that never reached the assembly is INVISIBLE in every other @@ -1443,6 +1461,10 @@ func checkExpect(t *testing.T, idx int, e bt.Expect, req llm.Request) { } } } + if e.RequestNotContains != "" { + assert.False(t, requestMentions(req, e.RequestNotContains), + "llm step %d: %q reached the model's context", idx, e.RequestNotContains) + } if len(e.ToolResultCounts) > 0 { assert.True(t, toolResultCountsMatch(req, e.ToolResultCounts), "llm step %d: exact result counts: want=%v got=%v", idx, diff --git a/test/e2e/threadrun/helpers.go b/test/e2e/threadrun/helpers.go index 43c834ec..3187f4db 100644 --- a/test/e2e/threadrun/helpers.go +++ b/test/e2e/threadrun/helpers.go @@ -57,6 +57,26 @@ func lastToolResultName(req llm.Request) string { return "" } +// requestMentions reports whether s appears anywhere in the request's messages: +// a text block, a tool call's input, or a tool result, in any message. It backs +// expect.requestNotContains, whose claim is that a value never reached the +// model's context at all, by whatever door. +func requestMentions(req llm.Request, s string) bool { + for _, m := range req.Messages { + for _, c := range m.Content { + switch { + case strings.Contains(c.Text, s): + return true + case c.ToolUse != nil && strings.Contains(string(c.ToolUse.Input), s): + return true + case c.ToolResult != nil && strings.Contains(c.ToolResult.Content, s): + return true + } + } + } + return false +} + // lastToolResultBlock returns the most recent tool_result block, or nil when // the request carries none. func lastToolResultBlock(req llm.Request) *llm.ToolResultBlock { diff --git a/test/e2e/threadrun/helpers_request_test.go b/test/e2e/threadrun/helpers_request_test.go new file mode 100644 index 00000000..b4992982 --- /dev/null +++ b/test/e2e/threadrun/helpers_request_test.go @@ -0,0 +1,34 @@ +//go:build e2e + +package threadrun + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/authzed/openagentprimitives/pkg/agent/llm" +) + +// requestMentions backs expect.requestNotContains, whose claim is "this never +// reached the model". A value can arrive as prose, as a tool's input, or as a +// tool's output, in ANY earlier message — so a check that read only the +// latest message, or only text, would pass a leak that came in by another door. +func TestRequestMentionsSeesEveryMessageAndBlockKind(t *testing.T) { + req := llm.Request{Messages: []llm.Message{ + {Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "prose-needle"}}}, + {Role: "assistant", Content: []llm.ContentBlock{ + {Type: "tool_use", ToolUse: &llm.ToolUseBlock{ID: "u1", Name: "fixture_read", Input: json.RawMessage(`{"q":"input-needle"}`)}}, + }}, + {Role: "user", Content: []llm.ContentBlock{ + {Type: "tool_result", ToolResult: &llm.ToolResultBlock{ToolUseID: "u1", Content: "result-needle"}}, + }}, + {Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "latest message"}}}, + }} + + for _, needle := range []string{"prose-needle", "input-needle", "result-needle", "latest message"} { + assert.True(t, requestMentions(req, needle), "%q is in the request and must be found", needle) + } + assert.False(t, requestMentions(req, "absent-needle"), "a value in no message is not mentioned") +}