From 6919661a7aaca0e06ef429443e99b0788c2a6e7f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 6 Nov 2026 22:09:21 -0500 Subject: [PATCH 1/2] Keep binary tool output from failing a session's memory write A reviewbot session ran `git show :`. The raw bytes carried NUL into the tool_result turn, Postgres jsonb refused it (SQLSTATE 22P05), the runner retried the doomed write eight times, and the session failed as MemoryUnavailable. - The runner replaces binary tool output with a sized notice before it reaches the model or memory, and replaces a stray NUL in text with U+FFFD. - The memory facade refuses, on every backend, entries carrying text Postgres cannot store (NUL, unpaired surrogate escapes, invalid UTF-8 in text fields) as ErrInvalidEntry, a permanent 400 rather than a retried 500. - The Postgres backend tags any data-exception SQLSTATE (class 22) as ErrInvalidEntry as a backstop for shapes the facade does not name. --- pkg/agent/runner/loop_binaryoutput.go | 40 +++++++++ pkg/agent/runner/loop_binaryoutput_test.go | 97 ++++++++++++++++++++++ pkg/agent/runner/loop_dispatch.go | 4 + pkg/memory/entry_storable.go | 94 +++++++++++++++++++++ pkg/memory/entry_storable_internal_test.go | 41 +++++++++ pkg/memory/facade.go | 3 + pkg/memory/facade_put_invalid_test.go | 31 +++++++ pkg/memory/kinds/turn/accessor_test.go | 22 +++++ pkg/memory/postgres/backend.go | 22 ++++- pkg/memory/postgres/backend_test.go | 31 +++++++ 10 files changed, 383 insertions(+), 2 deletions(-) create mode 100644 pkg/agent/runner/loop_binaryoutput.go create mode 100644 pkg/agent/runner/loop_binaryoutput_test.go create mode 100644 pkg/memory/entry_storable.go create mode 100644 pkg/memory/entry_storable_internal_test.go diff --git a/pkg/agent/runner/loop_binaryoutput.go b/pkg/agent/runner/loop_binaryoutput.go new file mode 100644 index 00000000..7547d0d6 --- /dev/null +++ b/pkg/agent/runner/loop_binaryoutput.go @@ -0,0 +1,40 @@ +package runner + +import ( + "fmt" + "strings" + + "github.com/dustin/go-humanize" +) + +// binarySniffLen is how far into a tool's output a NUL byte marks the whole +// output as binary. It is git's own buffer_is_binary heuristic (the first 8000 +// bytes), so a file git would diff as "Binary files differ" is treated as +// binary here too. +const binarySniffLen = 8000 + +// sanitizeToolOutput makes a tool's raw output safe to carry into a +// tool_result. It exists because NUL survives every other step: it is valid +// UTF-8, so JSON encoding keeps it (as \u0000) rather than replacing it, and +// Postgres jsonb then refuses the whole turn (SQLSTATE 22P05). The turn is the +// session's durable record, so one `git show :` failed the +// session outright. +// +// Output that looks binary is replaced with a notice naming its size — the +// model cannot use the bytes, and the notice tells it what to run instead. A +// stray NUL past the sniff window is replaced with U+FFFD, the same +// substitution JSON encoding already makes for invalid UTF-8, so the text +// around it survives. +func sanitizeToolOutput(content string) string { + i := strings.IndexByte(content, 0) + if i < 0 { + return content + } + if i < binarySniffLen { + return fmt.Sprintf("[binary output omitted: %s. The command printed raw binary data, which cannot be shown. "+ + "To inspect a binary file, use a command that prints text about it instead "+ + "(for example `git cat-file -s :` for its size, or `git ls-tree -- ` for its mode).]", + humanize.Bytes(uint64(len(content)))) + } + return strings.ReplaceAll(content, "\x00", "�") +} diff --git a/pkg/agent/runner/loop_binaryoutput_test.go b/pkg/agent/runner/loop_binaryoutput_test.go new file mode 100644 index 00000000..9545c607 --- /dev/null +++ b/pkg/agent/runner/loop_binaryoutput_test.go @@ -0,0 +1,97 @@ +package runner + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/authzed/openagentprimitives/pkg/agent/llm" + "github.com/authzed/openagentprimitives/pkg/agent/tool" + "github.com/authzed/openagentprimitives/pkg/authz" + "github.com/authzed/openagentprimitives/pkg/memory" + "github.com/authzed/openagentprimitives/pkg/platform/pipeline" +) + +func TestSanitizeToolOutput(t *testing.T) { + cases := []struct { + name string + in string + check func(t *testing.T, out string) + }{ + { + name: "text with no NUL: returned unchanged", + in: "diff --git a/x b/x\n+hello\n", + check: func(t *testing.T, out string) { + assert.Equal(t, "diff --git a/x b/x\n+hello\n", out) + }, + }, + { + name: "NUL at offset 0 (an ELF header): replaced with a sized binary notice", + in: "\x7fELF\x02\x01\x01\x00" + strings.Repeat("x", 2040), + check: func(t *testing.T, out string) { + assert.Contains(t, out, "[binary output omitted: 2.0 kB.") + assert.NotContains(t, out, "ELF", "none of the binary bytes may survive") + }, + }, + { + name: "NUL just inside the sniff window: treated as binary", + in: strings.Repeat("a", binarySniffLen-1) + "\x00", + check: func(t *testing.T, out string) { + assert.True(t, strings.HasPrefix(out, "[binary output omitted:"), "got %q…", out[:min(len(out), 40)]) + }, + }, + { + name: "NUL past the sniff window: only the NUL is replaced with U+FFFD", + in: strings.Repeat("a", binarySniffLen) + "\x00tail", + check: func(t *testing.T, out string) { + assert.Equal(t, strings.Repeat("a", binarySniffLen)+"�tail", out) + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out := sanitizeToolOutput(tc.in) + assert.NotContains(t, out, "\x00", "sanitized output must never carry NUL") + tc.check(t, out) + }) + } +} + +// TestDispatch_BinaryToolOutputIsSanitizedOnBothPaths pins the regression where +// a `git show :` result carried NUL into the tool_result turn, +// Postgres jsonb refused it (SQLSTATE 22P05), and the session failed as +// MemoryUnavailable. executeToolRaw is the one place both dispatch paths +// execute through, so both must come back clean. +func TestDispatch_BinaryToolOutputIsSanitizedOnBothPaths(t *testing.T) { + binary := "\x7fELF\x02\x01\x01\x00\x00\x00binary-body" + cases := []struct { + name string + kind tool.Kind + perm authz.Permission + }{ + {name: "gated MCP tool: result sanitized", kind: tool.KindMCP}, + {name: "ungated meta tool: result sanitized", kind: tool.KindMeta, perm: authz.Permission{StateImpact: authz.Stateless}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ft := &fakeDispatchTool{name: "git", kind: tc.kind, perm: tc.perm, result: tool.Result{Content: binary}} + l := &Loop{ + Tools: []tool.Tool{ft}, + SessionKey: memory.NamespacedName{Namespace: "default", Name: "disp"}, + } + loopWithInjectedExecutor(t, l, pipeline.NewRegistry()) + + uses := []llm.ToolUseBlock{{ID: "tu-1", Name: "git", Input: json.RawMessage(`{"args":{}}`)}} + results, _ := l.dispatchToolUses(memory.WithSystemApproval(context.Background(), "test"), uses, dispatchTestSession(), 0, 0, nil, nil) + + require.Len(t, results, 1) + assert.Equal(t, 1, ft.executed()) + assert.NotContains(t, results[0].Content, "\x00") + assert.Contains(t, results[0].Content, "[binary output omitted:") + }) + } +} diff --git a/pkg/agent/runner/loop_dispatch.go b/pkg/agent/runner/loop_dispatch.go index 708aff1c..364c1ede 100644 --- a/pkg/agent/runner/loop_dispatch.go +++ b/pkg/agent/runner/loop_dispatch.go @@ -192,6 +192,10 @@ func (l *Loop) executeToolRaw(ctx context.Context, t tool.Tool, name string, arg Trusted: true, } } + // Both dispatch paths (gated and ungated meta) execute through here, so + // cleaning once covers every consumer downstream: the Post hooks and their + // audit records, the LLM tool_result, and the session's turn record. + res.Content = sanitizeToolOutput(res.Content) // A user-initiated Interrupt fired while this call was in flight: override // whatever Execute returned (success, its own error content, or the // dispatch-failed wrapper above — any of the three, depending on how the diff --git a/pkg/memory/entry_storable.go b/pkg/memory/entry_storable.go new file mode 100644 index 00000000..a77fd04c --- /dev/null +++ b/pkg/memory/entry_storable.go @@ -0,0 +1,94 @@ +package memory + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "unicode/utf8" +) + +// checkStorable refuses an entry carrying text a durable backend cannot hold. +// Postgres rejects each of these with a data-exception SQLSTATE — NUL in jsonb +// (22P05), an unpaired surrogate escape in jsonb (22P02), NUL or invalid UTF-8 +// in a text column (22021) — and the refusal is about the entry, not the store, +// so a retry can never succeed. Checked at the facade rather than in the +// postgres backend so the same entry is refused on every backend, instead of +// being accepted by inmem/sqlite and failing only where it finally lands. +func checkStorable(e Entry) error { + type text struct{ field, value string } + texts := []text{ + {"Scope.Kind", e.Scope.Kind}, + {"Scope.ID", e.Scope.ID}, + {"Kind", e.Kind}, + {"ID", e.ID}, + } + for i, t := range e.Tags { + texts = append(texts, text{fmt.Sprintf("Tags[%d]", i), t}) + } + for i, l := range e.Links { + p := fmt.Sprintf("Links[%d].", i) + texts = append(texts, + text{p + "Relation", l.Relation}, + text{p + "Scope.Kind", l.Scope.Kind}, + text{p + "Scope.ID", l.Scope.ID}, + text{p + "Kind", l.Kind}, + text{p + "ID", l.ID}, + ) + } + for _, t := range texts { + if !utf8.ValidString(t.value) { + return fmt.Errorf("field %s is not valid UTF-8", t.field) + } + if strings.IndexByte(t.value, 0) >= 0 { + return fmt.Errorf("field %s contains a NUL character", t.field) + } + } + return checkStorableJSON(e.Content) +} + +// checkStorableJSON walks raw JSON's string escapes for the two a jsonb column +// refuses: \u0000, and a UTF-16 surrogate escape that is not part of a +// high+low pair. encoding/json accepts both as valid JSON — it checks only that +// an escape has four hex digits — so a RawMessage built by a client reaches the +// store with them intact. Escapes are walked rather than searched for, since +// `\\u0000` is an escaped backslash followed by the literal text "u0000". +func checkStorableJSON(raw []byte) error { + if bytes.IndexByte(raw, 0) >= 0 { + return fmt.Errorf("content contains a NUL byte") + } + for i := 0; i < len(raw); i++ { + if raw[i] != '\\' { + continue + } + cp, ok := unicodeEscapeAt(raw, i) + if !ok { + i++ // skip the escaped character, so `\\` cannot start a second escape + continue + } + switch { + case cp == 0: + return fmt.Errorf("content contains a NUL character (\\u0000)") + case cp >= 0xDC00 && cp <= 0xDFFF: + return fmt.Errorf("content contains an unpaired low surrogate (\\u%04x)", cp) + case cp >= 0xD800 && cp <= 0xDBFF: + lo, ok := unicodeEscapeAt(raw, i+6) + if !ok || lo < 0xDC00 || lo > 0xDFFF { + return fmt.Errorf("content contains an unpaired high surrogate (\\u%04x)", cp) + } + i += 6 // the low half is consumed with its high half + } + i += 5 + } + return nil +} + +// unicodeEscapeAt parses a \uXXXX escape starting at raw[i], reporting false +// when raw[i:] is not one. +func unicodeEscapeAt(raw []byte, i int) (uint64, bool) { + if i+6 > len(raw) || raw[i] != '\\' || raw[i+1] != 'u' { + return 0, false + } + cp, err := strconv.ParseUint(string(raw[i+2:i+6]), 16, 16) + return cp, err == nil +} diff --git a/pkg/memory/entry_storable_internal_test.go b/pkg/memory/entry_storable_internal_test.go new file mode 100644 index 00000000..840ba008 --- /dev/null +++ b/pkg/memory/entry_storable_internal_test.go @@ -0,0 +1,41 @@ +package memory + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestCheckStorableJSON pins the escape walk: the shapes jsonb refuses are +// caught, and the look-alikes it stores fine — an escaped backslash before +// literal "u0000", a correctly paired surrogate — are not. +func TestCheckStorableJSON(t *testing.T) { + cases := []struct { + name string + raw string + wantErr string + }{ + {name: "plain text: storable", raw: `{"text":"hello"}`}, + {name: "empty content: storable", raw: ``}, + {name: "U+FFFD replacement char: storable", raw: `{"text":"a\ufffdb"}`}, + {name: "escaped backslash then literal u0000: storable", raw: `{"text":"C:\\u0000"}`}, + {name: "paired surrogate escape: storable", raw: `{"text":"\ud83d\ude00"}`}, + {name: "\\u0000 escape: refused as NUL", raw: `{"text":"a\u0000b"}`, wantErr: "NUL character"}, + {name: "\\u0000 after an escaped backslash: refused as NUL", raw: `{"text":"\\\u0000"}`, wantErr: "NUL character"}, + {name: "raw NUL byte: refused", raw: "{\"text\":\"a\x00b\"}", wantErr: "NUL byte"}, + {name: "lone high surrogate: refused", raw: `{"text":"\ud83dx"}`, wantErr: "unpaired high surrogate"}, + {name: "high surrogate at end of input: refused", raw: `"\ud83d`, wantErr: "unpaired high surrogate"}, + {name: "high followed by non-low escape: refused", raw: `{"text":"\ud83d\u0041"}`, wantErr: "unpaired high surrogate"}, + {name: "lone low surrogate: refused", raw: `{"text":"\ude00"}`, wantErr: "unpaired low surrogate"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := checkStorableJSON([]byte(tc.raw)) + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + assert.ErrorContains(t, err, tc.wantErr) + }) + } +} diff --git a/pkg/memory/facade.go b/pkg/memory/facade.go index 84aa6e5b..04b624c6 100644 --- a/pkg/memory/facade.go +++ b/pkg/memory/facade.go @@ -288,6 +288,9 @@ func (m *Local) Put(ctx context.Context, e Entry) (Entry, error) { ErrInvalidEntry, i, lnk.ID, lnk.Kind, target.IDPrefix()) } } + if err := checkStorable(e); err != nil { + return Entry{}, fmt.Errorf("memory.Put: %w: %s %s: %v", ErrInvalidEntry, e.Kind, e.ID, err) + } // releaseAppendOnly ends the append-only critical section. It stays a no-op // for a mutable Kind, which never takes the lock: last-writer-wins is that // path's defined behavior, so serializing it would buy nothing and cost diff --git a/pkg/memory/facade_put_invalid_test.go b/pkg/memory/facade_put_invalid_test.go index 72f55cd3..926180d1 100644 --- a/pkg/memory/facade_put_invalid_test.go +++ b/pkg/memory/facade_put_invalid_test.go @@ -82,6 +82,37 @@ func TestPutValidationErrorsAreErrInvalidEntry(t *testing.T) { backend: inmem.NewBackend(), invalid: true, }, + { + name: "NUL escape in Content (jsonb refuses, SQLSTATE 22P05): ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1", Content: []byte(`{"text":"a\u0000b"}`)}, + backend: inmem.NewBackend(), + invalid: true, + }, + { + name: "unpaired surrogate escape in Content: ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1", Content: []byte(`{"text":"\ud83d"}`)}, + backend: inmem.NewBackend(), + invalid: true, + }, + { + name: "NUL in a tag (text column refuses, SQLSTATE 22021): ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1", Tags: []string{"a\x00b"}}, + backend: inmem.NewBackend(), + invalid: true, + }, + { + name: "invalid UTF-8 in Entry.ID: ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-\xff"}, + backend: inmem.NewBackend(), + invalid: true, + }, + { + name: "NUL in a Link field: ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1", + Links: []memory.Link{{Relation: "r\x00", Kind: "other", ID: "other-1"}}}, + backend: inmem.NewBackend(), + invalid: true, + }, { name: "store fault on the durable write: NOT ErrInvalidEntry", entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1"}, diff --git a/pkg/memory/kinds/turn/accessor_test.go b/pkg/memory/kinds/turn/accessor_test.go index 89990d88..a13a0200 100644 --- a/pkg/memory/kinds/turn/accessor_test.go +++ b/pkg/memory/kinds/turn/accessor_test.go @@ -534,3 +534,25 @@ func TestAppender_AttachmentContentRoundTrip(t *testing.T) { assert.Equal(t, in.Content[1].Attachment.TextRef, block.Attachment.TextRef) assert.Equal(t, in.Content[1].Attachment.Pages, block.Attachment.Pages) } + +// TestAppender_NULToolResultIsInvalidEntry pins the second half of the +// demo-reviewbot failure: a tool_result carrying NUL (raw `git show` of a +// binary) used to reach Postgres, fail as a 500 with SQLSTATE 22P05, and be +// retried eight times before the session was marked MemoryUnavailable. The +// facade now refuses it up front as ErrInvalidEntry — a 400 httpclient does +// not retry — on every backend, inmem included. +func TestAppender_NULToolResultIsInvalidEntry(t *testing.T) { + a := newAppender(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + + err := a.Append(ctx, memory.Turn{ + Index: 0, Role: "user", + Content: []memory.ContentBlock{{ + Type: "tool_result", + ToolResult: &memory.ToolResultBlock{ToolUseID: "tu-1", Content: "\x7fELF\x02\x01\x01\x00"}, + }}, + CreatedAt: time.Unix(0, 0).UTC(), + }) + require.ErrorIs(t, err, memory.ErrInvalidEntry) + assert.ErrorContains(t, err, "NUL") +} diff --git a/pkg/memory/postgres/backend.go b/pkg/memory/postgres/backend.go index 4025ff57..68aba6a1 100644 --- a/pkg/memory/postgres/backend.go +++ b/pkg/memory/postgres/backend.go @@ -3,9 +3,12 @@ package postgres import ( "context" "encoding/json" + "errors" "fmt" + "strings" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/authzed/openagentprimitives/pkg/memory" ) @@ -113,6 +116,21 @@ func unmarshalLinks(raw []byte) ([]memory.Link, error) { return links, nil } +// invalidIfDataException tags a Postgres data exception (SQLSTATE class 22 — +// an untranslatable character, an invalid escape, a value out of range) as +// memory.ErrInvalidEntry. Those are refusals of the entry's bytes, so a retry +// cannot succeed; untagged, httpsrv would answer 500 and httpclient would retry +// a doomed write until it gave up. The facade's checkStorable refuses the known +// shapes before they get here; this is the backstop for any it does not name. +// Every other error stays a fault, retryable as before. +func invalidIfDataException(err error) error { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && strings.HasPrefix(pgErr.Code, "22") { + return fmt.Errorf("%w: %w", memory.ErrInvalidEntry, err) + } + return err +} + func (b *Backend) Put(ctx context.Context, e memory.Entry) error { tx, err := b.client.Pool().Begin(ctx) if err != nil { @@ -139,7 +157,7 @@ func (b *Backend) Put(ctx context.Context, e memory.Entry) error { e.Scope.Kind, e.Scope.ID, e.Kind, e.ID, e.CreatedAt, tags, e.Content, linksJSON, provJSON) if err != nil { - return fmt.Errorf("postgres put entry: %w", err) + return fmt.Errorf("postgres put entry: %w", invalidIfDataException(err)) } _, err = tx.Exec(ctx, @@ -164,7 +182,7 @@ func (b *Backend) Put(ctx context.Context, e memory.Entry) error { "relation", "target_scope_kind", "target_scope_id", "target_kind", "target_id"}, pgx.CopyFromRows(rows)) if err != nil { - return fmt.Errorf("postgres put links: %w", err) + return fmt.Errorf("postgres put links: %w", invalidIfDataException(err)) } } diff --git a/pkg/memory/postgres/backend_test.go b/pkg/memory/postgres/backend_test.go index 2f2755bf..32552789 100644 --- a/pkg/memory/postgres/backend_test.go +++ b/pkg/memory/postgres/backend_test.go @@ -102,6 +102,37 @@ func TestPostgres_PutGetWithContent(t *testing.T) { assert.JSONEq(t, string(content), string(got.Content)) } +// TestPostgres_DataExceptionIsInvalidEntry drives the backend directly — +// below the facade's checkStorable — with each value Postgres itself refuses, +// and requires the refusal to come back as ErrInvalidEntry. That tag is what +// turns a doomed write into a 400 instead of a 500 retried eight times. +func TestPostgres_DataExceptionIsInvalidEntry(t *testing.T) { + b := newTestBackend(t) + ctx := context.Background() + scope := memory.Scope{Kind: "session", ID: "ns/a"} + cases := []struct { + name string + entry memory.Entry + }{ + { + name: "NUL escape in jsonb content (22P05): ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "alpha", ID: "a-1", Content: json.RawMessage(`{"text":"a\u0000b"}`)}, + }, + { + name: "NUL in a text[] tag (22021): ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "alpha", ID: "a-2", Tags: []string{"a\x00b"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.entry.CreatedAt = time.Now().UTC().Truncate(time.Microsecond) + err := b.Put(ctx, tc.entry) + require.Error(t, err) + assert.ErrorIs(t, err, memory.ErrInvalidEntry, "got: %v", err) + }) + } +} + func TestPostgres_PutGetWithLinks(t *testing.T) { b := newTestBackend(t) ctx := context.Background() From 34937b6c9a49117629072148f3265977832b9a40 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 10 Nov 2026 22:45:27 -0500 Subject: [PATCH 2/2] Keep readable tool output and secret values intact when cleaning NUL; replace unstorable characters when a turn is recorded Tool output is now treated as binary only when a NUL comes with bytes that do not read as text, so NUL-separated listings keep their content. A secret-output value is passed through untouched, and an error result keeps its text. Altered output is logged. A turn or tool-session event carrying a character the store refuses is recorded with a placeholder instead of failing the write, whichever writer or block it came from. The store's own check now also refuses invalid UTF-8 in content and bad provenance strings, and no longer builds field labels on the success path. --- pkg/agent/runner/loop_binaryoutput.go | 142 ++++++++++++++--- pkg/agent/runner/loop_binaryoutput_test.go | 133 +++++++++++++--- pkg/agent/runner/loop_dispatch.go | 6 +- pkg/memory/entry_storable.go | 144 ++++++++++++++---- pkg/memory/entry_storable_internal_test.go | 29 ++++ pkg/memory/facade_put_invalid_test.go | 13 ++ pkg/memory/kinds/toolsession/accessor.go | 11 +- pkg/memory/kinds/toolsession/accessor_test.go | 16 ++ pkg/memory/kinds/turn/accessor.go | 26 +++- pkg/memory/kinds/turn/accessor_test.go | 100 +++++++++--- 10 files changed, 517 insertions(+), 103 deletions(-) diff --git a/pkg/agent/runner/loop_binaryoutput.go b/pkg/agent/runner/loop_binaryoutput.go index 7547d0d6..5e968f76 100644 --- a/pkg/agent/runner/loop_binaryoutput.go +++ b/pkg/agent/runner/loop_binaryoutput.go @@ -2,39 +2,133 @@ package runner import ( "fmt" + "log/slog" "strings" + "unicode" + "unicode/utf8" "github.com/dustin/go-humanize" + + "github.com/authzed/openagentprimitives/pkg/agent/tool" ) -// binarySniffLen is how far into a tool's output a NUL byte marks the whole -// output as binary. It is git's own buffer_is_binary heuristic (the first 8000 -// bytes), so a file git would diff as "Binary files differ" is treated as -// binary here too. +// binarySniffLen is how much of a tool's output is examined to decide whether +// it is binary. It is the window git's own buffer_is_binary uses (the first +// 8000 bytes). const binarySniffLen = 8000 +// nulReplacement stands in for a NUL that is removed from text. It is the +// substitution JSON encoding already makes for invalid UTF-8. +const nulReplacement = "�" + +// outputCleanup says what sanitizeToolOutput did to a tool's output. +type outputCleanup string + +const ( + outputUnchanged outputCleanup = "" + outputNULReplaced outputCleanup = "nul-replaced" + outputBinaryOmitted outputCleanup = "binary-omitted" +) + // sanitizeToolOutput makes a tool's raw output safe to carry into a // tool_result. It exists because NUL survives every other step: it is valid // UTF-8, so JSON encoding keeps it (as \u0000) rather than replacing it, and -// Postgres jsonb then refuses the whole turn (SQLSTATE 22P05). The turn is the -// session's durable record, so one `git show :` failed the -// session outright. +// Postgres jsonb then refuses the whole turn (SQLSTATE 22P05). +// +// Output with no NUL is returned untouched. Otherwise the rule is: +// +// - Binary — a NUL in the first binarySniffLen bytes AND those bytes do not +// read as text (see looksBinary) — is replaced whole with a notice naming +// its size. The model cannot use the bytes. +// - Anything else keeps its text, with each NUL replaced by U+FFFD. That +// covers a stray NUL anywhere, and output that uses NUL as a separator +// (`git ls-files -z`, `find -print0`), where the NULs are the least +// interesting part of a listing the model asked for. +// +// A NUL alone is deliberately not enough to call output binary. git treats it +// as enough because it is deciding whether to DIFF a file; here the cost of a +// wrong "binary" is the model losing output it could have read. +func sanitizeToolOutput(content string) (string, outputCleanup) { + if strings.IndexByte(content, 0) < 0 { + return content, outputUnchanged + } + if looksBinary(content) { + return fmt.Sprintf("[binary output omitted: %s. The tool returned raw binary data, which cannot be shown. "+ + "To inspect binary content, ask for something that describes it instead of its bytes — "+ + "its size, its type, a checksum.]", + humanize.Bytes(uint64(len(content)))), outputBinaryOmitted + } + return replaceNUL(content), outputNULReplaced +} + +// replaceNUL swaps each NUL in content for U+FFFD and keeps everything else. +func replaceNUL(content string) string { + return strings.ReplaceAll(content, "\x00", nulReplacement) +} + +// looksBinary reports whether the sniff window holds a NUL and does not read +// as text. "Not text" is either of: bytes that are not valid UTF-8, or a +// control character other than NUL itself, whitespace, backspace and ESC (the +// last two so terminal-styled output still counts as text). An executable or +// an image trips one of those within a few bytes of its header; a NUL-separated +// listing trips neither. +func looksBinary(content string) bool { + window := content + if len(window) > binarySniffLen { + // Cut on a rune boundary, so a multi-byte character straddling the + // window's edge is not mistaken for invalid UTF-8. + end := binarySniffLen + for end > binarySniffLen-utf8.UTFMax && !utf8.RuneStart(window[end]) { + end-- + } + window = window[:end] + } + if strings.IndexByte(window, 0) < 0 { + return false + } + if !utf8.ValidString(window) { + return true + } + for _, r := range window { + switch { + case r == 0, r == '\b', r == '\x1b', unicode.IsSpace(r): + case unicode.IsControl(r): + return true + } + } + return false +} + +// cleanToolOutput applies sanitizeToolOutput to a tool's result, with the two +// exceptions that make it safe to run on every result: +// +// - A secret-output result is left alone. Its Content is the credential +// itself, which applySecretOutput diverts into the session's Secret before +// the LLM or memory sees it; replacing it would store the notice, or a +// corrupted value, as the credential and report success. +// - An error result never becomes the binary notice. Its text is what tells +// the model what went wrong, so only the NULs are replaced. // -// Output that looks binary is replaced with a notice naming its size — the -// model cannot use the bytes, and the notice tells it what to run instead. A -// stray NUL past the sniff window is replaced with U+FFFD, the same -// substitution JSON encoding already makes for invalid UTF-8, so the text -// around it survives. -func sanitizeToolOutput(content string) string { - i := strings.IndexByte(content, 0) - if i < 0 { - return content - } - if i < binarySniffLen { - return fmt.Sprintf("[binary output omitted: %s. The command printed raw binary data, which cannot be shown. "+ - "To inspect a binary file, use a command that prints text about it instead "+ - "(for example `git cat-file -s :` for its size, or `git ls-tree -- ` for its mode).]", - humanize.Bytes(uint64(len(content)))) - } - return strings.ReplaceAll(content, "\x00", "�") +// Altered output is logged: the model and the transcript no longer hold what +// the tool returned, and nothing else would say so. +func (l *Loop) cleanToolOutput(name string, res tool.Result) tool.Result { + if res.SecretOutput != nil { + return res + } + raw := res.Content + cleanup := outputUnchanged + if res.IsError { + if strings.IndexByte(raw, 0) >= 0 { + res.Content, cleanup = replaceNUL(raw), outputNULReplaced + } + } else { + res.Content, cleanup = sanitizeToolOutput(raw) + } + if cleanup != outputUnchanged { + slog.Default().Info("tool output carried NUL, which a tool_result cannot hold; altered before use", + "session", l.SessionKey.Namespace+"/"+l.SessionKey.Name, + "tool", name, "action", string(cleanup), + "bytes", len(raw), "nulCount", strings.Count(raw, "\x00"), "isError", res.IsError) + } + return res } diff --git a/pkg/agent/runner/loop_binaryoutput_test.go b/pkg/agent/runner/loop_binaryoutput_test.go index 9545c607..141239b6 100644 --- a/pkg/agent/runner/loop_binaryoutput_test.go +++ b/pkg/agent/runner/loop_binaryoutput_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/authzed/openagentprimitives/pkg/agent/llm" + "github.com/authzed/openagentprimitives/pkg/agent/secretout" "github.com/authzed/openagentprimitives/pkg/agent/tool" "github.com/authzed/openagentprimitives/pkg/authz" "github.com/authzed/openagentprimitives/pkg/memory" @@ -18,43 +19,74 @@ import ( func TestSanitizeToolOutput(t *testing.T) { cases := []struct { - name string - in string - check func(t *testing.T, out string) + name string + in string + cleanup outputCleanup + check func(t *testing.T, out string) }{ { - name: "text with no NUL: returned unchanged", - in: "diff --git a/x b/x\n+hello\n", + name: "text with no NUL: returned unchanged", + in: "diff --git a/x b/x\n+hello\n", + cleanup: outputUnchanged, check: func(t *testing.T, out string) { assert.Equal(t, "diff --git a/x b/x\n+hello\n", out) }, }, { - name: "NUL at offset 0 (an ELF header): replaced with a sized binary notice", - in: "\x7fELF\x02\x01\x01\x00" + strings.Repeat("x", 2040), + name: "an ELF header (NUL plus control bytes): replaced with a sized binary notice", + in: "\x7fELF\x02\x01\x01\x00" + strings.Repeat("x", 2040), + cleanup: outputBinaryOmitted, check: func(t *testing.T, out string) { assert.Contains(t, out, "[binary output omitted: 2.0 kB.") assert.NotContains(t, out, "ELF", "none of the binary bytes may survive") }, }, { - name: "NUL just inside the sniff window: treated as binary", - in: strings.Repeat("a", binarySniffLen-1) + "\x00", + name: "a PNG header (NUL plus invalid UTF-8): replaced with a binary notice", + in: "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR", + cleanup: outputBinaryOmitted, check: func(t *testing.T, out string) { - assert.True(t, strings.HasPrefix(out, "[binary output omitted:"), "got %q…", out[:min(len(out), 40)]) + assert.True(t, strings.HasPrefix(out, "[binary output omitted:"), "got %q", out) }, }, { - name: "NUL past the sniff window: only the NUL is replaced with U+FFFD", - in: strings.Repeat("a", binarySniffLen) + "\x00tail", + name: "NUL-separated listing (git ls-files -z): every name kept, NULs replaced with U+FFFD", + in: "a.go\x00b.go\x00c.go\x00", + cleanup: outputNULReplaced, check: func(t *testing.T, out string) { - assert.Equal(t, strings.Repeat("a", binarySniffLen)+"�tail", out) + assert.Equal(t, "a.go\ufffdb.go\ufffdc.go\ufffd", out) + }, + }, + { + name: "NUL-separated listing with non-ASCII names and terminal colour: still text", + in: "\x1b[32mnaïve.go\x1b[0m\x00日本語.txt\x00", + cleanup: outputNULReplaced, + check: func(t *testing.T, out string) { + assert.Contains(t, out, "naïve.go") + assert.Contains(t, out, "日本語.txt") + }, + }, + { + name: "a multi-byte character straddling the sniff window's edge: not mistaken for invalid UTF-8", + in: "a\x00" + strings.Repeat("b", binarySniffLen-3) + "日tail", + cleanup: outputNULReplaced, + check: func(t *testing.T, out string) { + assert.True(t, strings.HasSuffix(out, "日tail"), "the text must be kept") + }, + }, + { + name: "binary bytes only past the sniff window, a stray NUL before them: text kept, NULs replaced", + in: strings.Repeat("a", binarySniffLen) + "\x00\x01tail", + cleanup: outputNULReplaced, + check: func(t *testing.T, out string) { + assert.Equal(t, strings.Repeat("a", binarySniffLen)+"\ufffd\x01tail", out) }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - out := sanitizeToolOutput(tc.in) + out, cleanup := sanitizeToolOutput(tc.in) + assert.Equal(t, tc.cleanup, cleanup) assert.NotContains(t, out, "\x00", "sanitized output must never carry NUL") tc.check(t, out) }) @@ -78,20 +110,73 @@ func TestDispatch_BinaryToolOutputIsSanitizedOnBothPaths(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - ft := &fakeDispatchTool{name: "git", kind: tc.kind, perm: tc.perm, result: tool.Result{Content: binary}} - l := &Loop{ - Tools: []tool.Tool{ft}, - SessionKey: memory.NamespacedName{Namespace: "default", Name: "disp"}, - } - loopWithInjectedExecutor(t, l, pipeline.NewRegistry()) - - uses := []llm.ToolUseBlock{{ID: "tu-1", Name: "git", Input: json.RawMessage(`{"args":{}}`)}} - results, _ := l.dispatchToolUses(memory.WithSystemApproval(context.Background(), "test"), uses, dispatchTestSession(), 0, 0, nil, nil) + results := dispatchOnce(t, tc.kind, tc.perm, tool.Result{Content: binary}) require.Len(t, results, 1) - assert.Equal(t, 1, ft.executed()) assert.NotContains(t, results[0].Content, "\x00") assert.Contains(t, results[0].Content, "[binary output omitted:") }) } } + +// TestDispatch_OutputSanitizerExceptions pins the two results the sanitizer +// must not treat like ordinary output, on the same dispatch path as above. +func TestDispatch_OutputSanitizerExceptions(t *testing.T) { + cases := []struct { + name string + result tool.Result + check func(t *testing.T, got tool.Result) + }{ + { + // applySecretOutput stores Content as the credential. A sanitized + // value would be captured, reported as success, and be wrong. + name: "secret-output result that looks binary: Content reaches the capture byte-for-byte", + result: tool.Result{ + Content: "\x00\x01\x02key\xff", + SecretOutput: &secretout.Spec{Name: "token"}, + }, + check: func(t *testing.T, got tool.Result) { + require.NotNil(t, got.SecretOutput, "the result must still be marked for capture") + assert.Equal(t, "\x00\x01\x02key\xff", got.Content) + }, + }, + { + name: "error result with a NUL after binary-looking bytes: text kept, only the NUL replaced", + result: tool.Result{Content: "\x01stderr: boom\n\x00", IsError: true}, + check: func(t *testing.T, got tool.Result) { + assert.True(t, got.IsError) + assert.Contains(t, got.Content, "stderr: boom") + assert.NotContains(t, got.Content, "\x00") + assert.NotContains(t, got.Content, "[binary output omitted:") + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Meta + Trusted keeps the content guard out of the way, so what + // comes back is what executeToolRaw produced. + tc.result.Trusted = true + results := dispatchOnce(t, tool.KindMeta, authz.Permission{StateImpact: authz.Stateless}, tc.result) + + require.Len(t, results, 1) + tc.check(t, results[0]) + }) + } +} + +// dispatchOnce runs one call of a fake tool returning result through +// dispatchToolUses and returns what the loop would go on to record. +func dispatchOnce(t *testing.T, kind tool.Kind, perm authz.Permission, result tool.Result) []tool.Result { + t.Helper() + ft := &fakeDispatchTool{name: "git", kind: kind, perm: perm, result: result} + l := &Loop{ + Tools: []tool.Tool{ft}, + SessionKey: memory.NamespacedName{Namespace: "default", Name: "disp"}, + } + loopWithInjectedExecutor(t, l, pipeline.NewRegistry()) + + uses := []llm.ToolUseBlock{{ID: "tu-1", Name: "git", Input: json.RawMessage(`{"args":{}}`)}} + results, _ := l.dispatchToolUses(memory.WithSystemApproval(context.Background(), "test"), uses, dispatchTestSession(), 0, 0, nil, nil) + require.Equal(t, 1, ft.executed()) + return results +} diff --git a/pkg/agent/runner/loop_dispatch.go b/pkg/agent/runner/loop_dispatch.go index 364c1ede..e1753832 100644 --- a/pkg/agent/runner/loop_dispatch.go +++ b/pkg/agent/runner/loop_dispatch.go @@ -194,8 +194,10 @@ func (l *Loop) executeToolRaw(ctx context.Context, t tool.Tool, name string, arg } // Both dispatch paths (gated and ungated meta) execute through here, so // cleaning once covers every consumer downstream: the Post hooks and their - // audit records, the LLM tool_result, and the session's turn record. - res.Content = sanitizeToolOutput(res.Content) + // audit records, the LLM tool_result, and the session's turn record. A + // secret-output value passes through untouched and an error result keeps + // its text — see cleanToolOutput. + res = l.cleanToolOutput(name, res) // A user-initiated Interrupt fired while this call was in flight: override // whatever Execute returned (success, its own error content, or the // dispatch-failed wrapper above — any of the three, depending on how the diff --git a/pkg/memory/entry_storable.go b/pkg/memory/entry_storable.go index a77fd04c..70ce0169 100644 --- a/pkg/memory/entry_storable.go +++ b/pkg/memory/entry_storable.go @@ -2,6 +2,7 @@ package memory import ( "bytes" + "encoding/json" "fmt" "strconv" "strings" @@ -11,52 +12,126 @@ import ( // checkStorable refuses an entry carrying text a durable backend cannot hold. // Postgres rejects each of these with a data-exception SQLSTATE — NUL in jsonb // (22P05), an unpaired surrogate escape in jsonb (22P02), NUL or invalid UTF-8 -// in a text column (22021) — and the refusal is about the entry, not the store, -// so a retry can never succeed. Checked at the facade rather than in the -// postgres backend so the same entry is refused on every backend, instead of -// being accepted by inmem/sqlite and failing only where it finally lands. +// in a text or jsonb column (22021) — and the refusal is about the entry, not +// the store, so a retry can never succeed. Checked at the facade rather than in +// the postgres backend so the same entry is refused on every backend, instead +// of being accepted by inmem/sqlite and failing only where it finally lands. +// +// This runs on every Put, so a field's label is only built once the field has +// failed. func checkStorable(e Entry) error { - type text struct{ field, value string } - texts := []text{ + fixed := [...]struct{ field, value string }{ {"Scope.Kind", e.Scope.Kind}, {"Scope.ID", e.Scope.ID}, {"Kind", e.Kind}, {"ID", e.ID}, } + for _, f := range fixed { + if p := textProblem(f.value); p != "" { + return fmt.Errorf("field %s %s", f.field, p) + } + } for i, t := range e.Tags { - texts = append(texts, text{fmt.Sprintf("Tags[%d]", i), t}) + if p := textProblem(t); p != "" { + return fmt.Errorf("field Tags[%d] %s", i, p) + } } for i, l := range e.Links { - p := fmt.Sprintf("Links[%d].", i) - texts = append(texts, - text{p + "Relation", l.Relation}, - text{p + "Scope.Kind", l.Scope.Kind}, - text{p + "Scope.ID", l.Scope.ID}, - text{p + "Kind", l.Kind}, - text{p + "ID", l.ID}, - ) + link := [...]struct{ field, value string }{ + {"Relation", l.Relation}, + {"Scope.Kind", l.Scope.Kind}, + {"Scope.ID", l.Scope.ID}, + {"Kind", l.Kind}, + {"ID", l.ID}, + } + for _, f := range link { + if p := textProblem(f.value); p != "" { + return fmt.Errorf("field Links[%d].%s %s", i, f.field, p) + } + } } - for _, t := range texts { - if !utf8.ValidString(t.value) { - return fmt.Errorf("field %s is not valid UTF-8", t.field) + if pv := e.Provenance; pv != nil { + // Sig is bytes and travels base64-encoded, so it cannot carry either. + prov := [...]struct{ field, value string }{ + {"Publisher", pv.Publisher}, + {"KeyID", pv.KeyID}, + {"PrevHash", pv.PrevHash}, } - if strings.IndexByte(t.value, 0) >= 0 { - return fmt.Errorf("field %s contains a NUL character", t.field) + for _, f := range prov { + if p := textProblem(f.value); p != "" { + return fmt.Errorf("field Provenance.%s %s", f.field, p) + } } } + if !utf8.Valid(e.Content) { + return fmt.Errorf("content is not valid UTF-8") + } return checkStorableJSON(e.Content) } -// checkStorableJSON walks raw JSON's string escapes for the two a jsonb column -// refuses: \u0000, and a UTF-16 surrogate escape that is not part of a -// high+low pair. encoding/json accepts both as valid JSON — it checks only that -// an escape has four hex digits — so a RawMessage built by a client reaches the -// store with them intact. Escapes are walked rather than searched for, since -// `\\u0000` is an escaped backslash followed by the literal text "u0000". +// textProblem says what makes s unstorable in a text column, or "" when +// nothing does. +func textProblem(s string) string { + switch { + case !utf8.ValidString(s): + return "is not valid UTF-8" + case strings.IndexByte(s, 0) >= 0: + return "contains a NUL character" + default: + return "" + } +} + +// checkStorableJSON refuses raw JSON holding a string escape a jsonb column +// refuses, or a raw NUL byte. func checkStorableJSON(raw []byte) error { if bytes.IndexByte(raw, 0) >= 0 { return fmt.Errorf("content contains a NUL byte") } + var err error + unstorableEscapes(raw, func(_, _ int, problem string) bool { + err = fmt.Errorf("content contains %s", problem) + return false + }) + return err +} + +// StorableJSON returns raw with every string escape a durable backend refuses +// replaced by \ufffd, and how many it replaced. It is for a writer that would +// rather keep a record with a placeholder in it than lose the record: the +// transcript, where one such character in a model's reply or a user's message +// would otherwise fail the whole turn. A caller that replaces anything owes a +// log line saying so. +// +// It must run BEFORE the entry is signed — it builds the bytes that get +// recorded, it does not rewrite a record. raw is expected to be json.Marshal +// output, which never holds a raw NUL byte or invalid UTF-8 (Marshal escapes +// the first and replaces the second), so escapes are the only shape left. +// Returns raw itself when there is nothing to replace. +func StorableJSON(raw json.RawMessage) (json.RawMessage, int) { + var out []byte + replaced, last := 0, 0 + unstorableEscapes(raw, func(start, end int, _ string) bool { + out = append(out, raw[last:start]...) + out = append(out, `\ufffd`...) + last = end + replaced++ + return true + }) + if replaced == 0 { + return raw, 0 + } + return append(out, raw[last:]...), replaced +} + +// unstorableEscapes walks raw JSON's string escapes and calls visit with the +// byte range of each one a jsonb column refuses: \u0000, and a UTF-16 +// surrogate escape that is not part of a high+low pair. encoding/json accepts +// both as valid JSON — it checks only that an escape has four hex digits — so a +// RawMessage built by a client reaches the store with them intact. Escapes are +// walked rather than searched for, since `\\u0000` is an escaped backslash +// followed by the literal text "u0000". visit returning false stops the walk. +func unstorableEscapes(raw []byte, visit func(start, end int, problem string) bool) { for i := 0; i < len(raw); i++ { if raw[i] != '\\' { continue @@ -66,21 +141,24 @@ func checkStorableJSON(raw []byte) error { i++ // skip the escaped character, so `\\` cannot start a second escape continue } + problem := "" switch { case cp == 0: - return fmt.Errorf("content contains a NUL character (\\u0000)") + problem = `a NUL character (\u0000)` case cp >= 0xDC00 && cp <= 0xDFFF: - return fmt.Errorf("content contains an unpaired low surrogate (\\u%04x)", cp) + problem = fmt.Sprintf(`an unpaired low surrogate (\u%04x)`, cp) case cp >= 0xD800 && cp <= 0xDBFF: - lo, ok := unicodeEscapeAt(raw, i+6) - if !ok || lo < 0xDC00 || lo > 0xDFFF { - return fmt.Errorf("content contains an unpaired high surrogate (\\u%04x)", cp) + if lo, ok := unicodeEscapeAt(raw, i+6); ok && lo >= 0xDC00 && lo <= 0xDFFF { + i += 6 // the low half is consumed with its high half + } else { + problem = fmt.Sprintf(`an unpaired high surrogate (\u%04x)`, cp) } - i += 6 // the low half is consumed with its high half + } + if problem != "" && !visit(i, i+6, problem) { + return } i += 5 } - return nil } // unicodeEscapeAt parses a \uXXXX escape starting at raw[i], reporting false diff --git a/pkg/memory/entry_storable_internal_test.go b/pkg/memory/entry_storable_internal_test.go index 840ba008..e38db3d5 100644 --- a/pkg/memory/entry_storable_internal_test.go +++ b/pkg/memory/entry_storable_internal_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestCheckStorableJSON pins the escape walk: the shapes jsonb refuses are @@ -39,3 +40,31 @@ func TestCheckStorableJSON(t *testing.T) { }) } } + +// TestStorableJSON pins the rewrite half of the same walk: every escape +// checkStorableJSON refuses is replaced, the look-alikes are left alone, and +// what comes out is storable and still valid JSON. +func TestStorableJSON(t *testing.T) { + cases := []struct { + name string + raw string + want string + replaced int + }{ + {name: "plain text: returned unchanged", raw: `{"text":"hello"}`, want: `{"text":"hello"}`}, + {name: "escaped backslash then literal u0000: returned unchanged", raw: `{"text":"C:\\u0000"}`, want: `{"text":"C:\\u0000"}`}, + {name: "paired surrogate escape: returned unchanged", raw: `{"text":"\ud83d\ude00"}`, want: `{"text":"\ud83d\ude00"}`}, + {name: "\\u0000 escapes: each replaced", raw: `{"text":"a\u0000b\u0000"}`, want: `{"text":"a\ufffdb\ufffd"}`, replaced: 2}, + {name: "lone high surrogate: replaced, the text after it kept", raw: `{"text":"\ud83dx"}`, want: `{"text":"\ufffdx"}`, replaced: 1}, + {name: "lone low surrogate: replaced", raw: `{"text":"\ude00"}`, want: `{"text":"\ufffd"}`, replaced: 1}, + {name: "high surrogate followed by NUL: both replaced", raw: `{"text":"\ud83d\u0000"}`, want: `{"text":"\ufffd\ufffd"}`, replaced: 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out, replaced := StorableJSON([]byte(tc.raw)) + assert.Equal(t, tc.want, string(out)) + assert.Equal(t, tc.replaced, replaced) + require.NoError(t, checkStorableJSON(out), "the result must be storable") + }) + } +} diff --git a/pkg/memory/facade_put_invalid_test.go b/pkg/memory/facade_put_invalid_test.go index 926180d1..e395da11 100644 --- a/pkg/memory/facade_put_invalid_test.go +++ b/pkg/memory/facade_put_invalid_test.go @@ -113,6 +113,19 @@ func TestPutValidationErrorsAreErrInvalidEntry(t *testing.T) { backend: inmem.NewBackend(), invalid: true, }, + { + name: "invalid UTF-8 in raw Content (jsonb refuses, SQLSTATE 22021): ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1", Content: []byte("{\"a\":\"\xff\"}")}, + backend: inmem.NewBackend(), + invalid: true, + }, + { + name: "NUL in Provenance.Publisher: ErrInvalidEntry", + entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1", + Provenance: &memory.Provenance{Publisher: "system:\x00"}}, + backend: inmem.NewBackend(), + invalid: true, + }, { name: "store fault on the durable write: NOT ErrInvalidEntry", entry: memory.Entry{Scope: scope, Kind: "mk", ID: "mk-1"}, diff --git a/pkg/memory/kinds/toolsession/accessor.go b/pkg/memory/kinds/toolsession/accessor.go index c61b5082..713b65df 100644 --- a/pkg/memory/kinds/toolsession/accessor.go +++ b/pkg/memory/kinds/toolsession/accessor.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "sort" "time" @@ -49,10 +50,18 @@ func Record(ctx context.Context, mem memory.Memory, scope memory.Scope, ev Event if err != nil { return fmt.Errorf("toolsession.Record: marshal: %w", err) } + // ev.Text is whatever the inner tool streamed, so it can carry a character + // the store refuses. Losing one character beats losing the event. + id := memory.NewID(Kind{}) + raw, replaced := memory.StorableJSON(raw) + if replaced > 0 { + slog.Info("toolsession.Record: replaced characters the store cannot hold with U+FFFD", + "scope", scope.ID, "entry", id, "replaced", replaced) + } _, err = mem.Put(ctx, memory.Entry{ Scope: scope, Kind: KindName, - ID: memory.NewID(Kind{}), + ID: id, CreatedAt: time.Now().UTC(), Content: raw, }) diff --git a/pkg/memory/kinds/toolsession/accessor_test.go b/pkg/memory/kinds/toolsession/accessor_test.go index cfc43dc1..39ae5259 100644 --- a/pkg/memory/kinds/toolsession/accessor_test.go +++ b/pkg/memory/kinds/toolsession/accessor_test.go @@ -135,3 +135,19 @@ func mustMarshal(t *testing.T, ev toolsession.Event) []byte { require.NoError(t, err) return raw } + +// TestRecordReplacesUnstorableCharacters: an event's Text is whatever the +// inner tool streamed, and the facade refuses an entry carrying NUL. Record is +// best-effort, so a refusal would drop the event from the replayed stream. +func TestRecordReplacesUnstorableCharacters(t *testing.T) { + ctx := memory.WithSystemApproval(context.Background(), "test") + mem := newMemory(t) + require.NoError(t, toolsession.Record(ctx, mem, testScope, toolsession.Event{ + ToolCallRef: "call-1", EventType: "text_delta", Text: "before\x00after", + })) + + got, err := toolsession.ReadAll(ctx, mem, testScope) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "before�after", got[0].Text) +} diff --git a/pkg/memory/kinds/turn/accessor.go b/pkg/memory/kinds/turn/accessor.go index 2f3218f8..71d450c3 100644 --- a/pkg/memory/kinds/turn/accessor.go +++ b/pkg/memory/kinds/turn/accessor.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "sort" "time" @@ -79,6 +80,16 @@ func NewAppender(mem memory.Memory, scope memory.Scope) *Appender { // writer per session scope (the runner loop) means no two turn-writers race. // A failed conflict-check Query is fatal — returned rather than falling // through to a blind Put, which would drop both the error and the check. +// +// A character the store cannot hold (NUL, in practice) is replaced with U+FFFD +// here rather than left for Put to refuse. A turn is the session's durable +// record, so refusing it fails the session, over one character in a model's +// reply or a user's message. A tool call's arguments are not at risk: they are +// bytes, stored base64-encoded. Every writer of a turn — the runner, and +// channelsd for inbound messages — comes through this method, so replacing +// here covers all of them and every field, and it happens before the entry is +// signed. The live conversation keeps the original; only the record carries +// the placeholder, and the log line below says so. func (a *Appender) Append(ctx context.Context, t memory.Turn) error { id := EntryID(t.Index, t.Role) raw, err := json.Marshal(turnContent{Content: t.Content, Usage: t.Usage, Author: t.Author, Via: t.Via, Refused: t.Refused, Model: t.Model}) @@ -91,15 +102,28 @@ func (a *Appender) Append(ctx context.Context, t memory.Turn) error { if qerr != nil { return fmt.Errorf("turn.Append: conflict check: %w", qerr) } + unclean := raw + raw, replaced := memory.StorableJSON(raw) + if replaced > 0 { + slog.Info("turn.Append: replaced characters the store cannot hold with U+FFFD", + "scope", a.scope.ID, "entry", id, "replaced", replaced) + } if len(existing.Entries) == 1 { // Canonical JSON, not raw bytes: the existing entry came back OUT of a // backend, and postgres stores content in a JSONB column that re-emits // object keys in its own order and whitespace. A byte compare read every // legitimate re-append — a replay after restart, not only a retry after a // lost response — as divergent, failing ErrIndexConflict on postgres only. - if bytes.Equal(memory.CanonicalContent(existing.Entries[0].Content), memory.CanonicalContent(raw)) { + stored := memory.CanonicalContent(existing.Entries[0].Content) + if bytes.Equal(stored, memory.CanonicalContent(raw)) { return nil // idempotent re-append } + // A turn written before the replacement above existed is stored with + // the original characters (inmem and sqlite accepted them), and + // re-appending it is just as much a no-op. + if replaced > 0 && bytes.Equal(stored, memory.CanonicalContent(unclean)) { + return nil + } return memory.ErrIndexConflict } _, err = a.mem.Put(ctx, memory.Entry{ diff --git a/pkg/memory/kinds/turn/accessor_test.go b/pkg/memory/kinds/turn/accessor_test.go index a13a0200..010ba392 100644 --- a/pkg/memory/kinds/turn/accessor_test.go +++ b/pkg/memory/kinds/turn/accessor_test.go @@ -2,6 +2,7 @@ package turn_test import ( "context" + "encoding/json" "errors" "testing" "time" @@ -535,24 +536,87 @@ func TestAppender_AttachmentContentRoundTrip(t *testing.T) { assert.Equal(t, in.Content[1].Attachment.Pages, block.Attachment.Pages) } -// TestAppender_NULToolResultIsInvalidEntry pins the second half of the -// demo-reviewbot failure: a tool_result carrying NUL (raw `git show` of a -// binary) used to reach Postgres, fail as a 500 with SQLSTATE 22P05, and be -// retried eight times before the session was marked MemoryUnavailable. The -// facade now refuses it up front as ErrInvalidEntry — a 400 httpclient does -// not retry — on every backend, inmem included. -func TestAppender_NULToolResultIsInvalidEntry(t *testing.T) { - a := newAppender(t) +// TestAppender_UnstorableCharactersAreReplaced pins the second half of the +// demo-reviewbot failure. A turn carrying NUL used to reach Postgres, fail as +// a 500 with SQLSTATE 22P05, and be retried eight times before the session was +// marked MemoryUnavailable. The facade refuses such an entry on every backend, +// so the appender must hand it one it can store — whichever block the +// character arrived in, since tool output is not the only source: a model's +// reply and a user's message reach here unfiltered. +func TestAppender_UnstorableCharactersAreReplaced(t *testing.T) { + cases := []struct { + name string + block memory.ContentBlock + check func(t *testing.T, got memory.ContentBlock) + }{ + { + name: "NUL in a text block: replaced with U+FFFD, the text around it kept", + block: memory.ContentBlock{Type: "text", Text: "before\x00after"}, + check: func(t *testing.T, got memory.ContentBlock) { + assert.Equal(t, "before\ufffdafter", got.Text) + }, + }, + { + name: "NUL in a tool_result: replaced with U+FFFD", + block: memory.ContentBlock{Type: "tool_result", + ToolResult: &memory.ToolResultBlock{ToolUseID: "tu-1", Content: "stderr: boom\x00"}}, + check: func(t *testing.T, got memory.ContentBlock) { + require.NotNil(t, got.ToolResult) + assert.Equal(t, "stderr: boom\ufffd", got.ToolResult.Content) + }, + }, + { + // Input is bytes, so it is stored base64-encoded and no escape in + // it ever reaches the column: the recorded arguments stay exactly + // the ones the tool was called with. + name: "NUL escape in tool_use input: stored byte-for-byte", + block: memory.ContentBlock{Type: "tool_use", + ToolUse: &memory.ToolUseBlock{ID: "tu-1", Name: "write", Input: []byte(`{"path":"a\u0000b"}`)}}, + check: func(t *testing.T, got memory.ContentBlock) { + require.NotNil(t, got.ToolUse) + assert.Equal(t, `{"path":"a\u0000b"}`, string(got.ToolUse.Input)) + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + a := newAppender(t) + ctx := memory.WithSystemApproval(context.Background(), "test") + in := memory.Turn{ + Index: 0, Role: "user", + Content: []memory.ContentBlock{tc.block}, + CreatedAt: time.Unix(0, 0).UTC(), + } + require.NoError(t, a.Append(ctx, in), "a turn must not be refused over one character") + require.NoError(t, a.Append(ctx, in), "re-appending the same turn stays a no-op") + + turns, err := a.ReadAll(ctx) + require.NoError(t, err) + require.Len(t, turns, 1) + require.Len(t, turns[0].Content, 1) + tc.check(t, turns[0].Content[0]) + }) + } +} + +// TestAppender_ReappendOfTurnStoredWithNULIsNoOp covers a transcript written +// before the appender replaced anything: inmem and sqlite accepted NUL, so such +// a turn is stored with it. Re-appending that turn must stay the no-op it was, +// not turn into an index conflict because the new bytes carry U+FFFD instead. +func TestAppender_ReappendOfTurnStoredWithNULIsNoOp(t *testing.T) { + backend := inmem.NewBackend() + scope := memory.Scope{Kind: "session", ID: "default/s1"} ctx := memory.WithSystemApproval(context.Background(), "test") + in := textTurn(0, "user", "before\x00after") - err := a.Append(ctx, memory.Turn{ - Index: 0, Role: "user", - Content: []memory.ContentBlock{{ - Type: "tool_result", - ToolResult: &memory.ToolResultBlock{ToolUseID: "tu-1", Content: "\x7fELF\x02\x01\x01\x00"}, - }}, - CreatedAt: time.Unix(0, 0).UTC(), - }) - require.ErrorIs(t, err, memory.ErrInvalidEntry) - assert.ErrorContains(t, err, "NUL") + // Straight into the backend, below the facade that now refuses it. + stored, err := json.Marshal(map[string]any{"content": in.Content}) + require.NoError(t, err) + require.NoError(t, backend.Put(ctx, memory.Entry{ + Scope: scope, Kind: turn.KindName, ID: turn.EntryID(0, "user"), + CreatedAt: in.CreatedAt, Content: stored, + })) + + a := turn.NewAppender(memory.NewLocal(backend), scope) + require.NoError(t, a.Append(ctx, in)) }