diff --git a/cmd/eval.go b/cmd/eval.go index 0fe7c7e3..6914c153 100644 --- a/cmd/eval.go +++ b/cmd/eval.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "strings" "time" @@ -73,6 +74,7 @@ func init() { evalCmd.AddCommand(evalListCmd) evalCmd.AddCommand(evalResultsCmd) evalCmd.AddCommand(evalCacheCmd) + evalCmd.AddCommand(evalSmokeCmd) } func runEval(_ *cobra.Command, _ []string) error { @@ -225,6 +227,36 @@ func runEvalList(_ *cobra.Command, _ []string) error { return nil } +var evalSmokeCmd = &cobra.Command{ + Use: "smoke", + Short: "Run headless agent-loop smoke benchmarks (no provider/API key needed)", + RunE: runEvalSmoke, +} + +func runEvalSmoke(_ *cobra.Command, _ []string) error { + suite := eval.SmokeSuite() + + fmt.Printf("Running %d agent-loop smoke tasks...\n", len(suite.Tasks)) + + // Quiet engine INFO logs so the scorecard report is the only stdout. + origLevel := slog.SetLogLoggerLevel(slog.LevelWarn) + defer slog.SetLogLoggerLevel(origLevel) + + runner := eval.NewRunner("smoke", "") + runner.NoCache = true + runner.Filters = nil // no code-block extraction for smoke tasks + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + result, err := runner.Run(ctx, suite) + if err != nil { + return err + } + + fmt.Println(eval.GenerateReport(result)) + return nil +} + func runEvalResults(_ *cobra.Command, _ []string) error { store := eval.DefaultResultStore() files, err := store.List() diff --git a/docs/acp/client.md b/docs/acp/client.md new file mode 100644 index 00000000..20428129 --- /dev/null +++ b/docs/acp/client.md @@ -0,0 +1,279 @@ +# Hawk ACP Client Guide + +Hawk speaks the **Agent Client Protocol (ACP)** — newline-delimited JSON-RPC 2.0 +over stdio — so editors (e.g. Zed) and custom tooling can drive the agent the +same way the TUI does, including the control plane: work modes, isolation +profiles, and background tasks. + +This guide is a worked reference for writing an ACP client. It pairs with the +protocol surface documented in `docs/architecture/control-plane.md`. + +--- + +## Starting the server + +```bash +hawk acp +``` + +Hawk reads newline-delimited JSON-RPC 2.0 requests from stdin and writes +responses and notifications to stdout. **Do not log to stdout** — it is the +protocol channel. All other diagnostics go to stderr. + +A minimal client starts the process and exchanges `initialize`: + +```bash +hawk acp <<'EOF' +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}} +EOF +``` + +Response: + +```json +{"jsonrpc":"2.0","id":1,"result":{ + "protocolVersion": 1, + "agentCapabilities": { + "loadSession": false, + "promptCapabilities": {"image": false, "audio": false} + }, + "hawkCapabilities": { + "workModes": ["plan", "act", "review"], + "isolation": ["dev", "workspace", "strict", "container"], + "folderTrust": true, + "lazyTools": true, + "autoCommit": true, + "spawnController": true + } +}} +``` + +`hawkCapabilities` is hawk-specific metadata IDE clients can use to show the +control plane in their UI. It is additive — ignore it if you do not need it. + +--- + +## Protocol summary + +| Method | Direction | Purpose | +|--------|-----------|---------| +| `initialize` | client → server | handshake, capability negotiation | +| `session/new` | client → server | create a session, get `sessionId` | +| `session/setMode` | client → server | switch work mode (`plan` \| `act` \| `review`) | +| `session/setIsolation` | client → server | apply an isolation profile | +| `session/status` | client → server | control-plane snapshot | +| `session/prompt` | client → server | run a prompt (streams `session/update`) | +| `session/cancel` | client → server | cancel the in-flight prompt | +| `session/update` | server → client | streaming progress notification | +| `session/request_permission` | server → client | ask the client to approve a tool call | + +--- + +## Session lifecycle with the control plane + +The canonical flow for a controlled edit session: + +### 1. Create a session + +```json +{"jsonrpc":"2.0","id":2,"method":"session/new","params":{}} +``` + +Response: + +```json +{"jsonrpc":"2.0","id":2,"result":{ + "sessionId": "sess_1", + "hawk": { + "workMode": "act", + "isolation": "dev", + "autoCommit": false + } +}} +``` + +New sessions default to `act` mode with the `dev` isolation profile — the same +defaults as the interactive TUI. Prompts are a structured array +(`[{"type":"text","text":"..."}]`) so clients can later send non-text blocks; +hawk currently consumes the `text` blocks. + +### 2. Switch to plan mode + +```json +{"jsonrpc":"2.0","id":3,"method":"session/setMode","params":{ + "sessionId": "sess_1", + "mode": "plan" +}} +``` + +Response: + +```json +{"jsonrpc":"2.0","id":3,"result":{"sessionId":"sess_1","workMode":"plan"}} +``` + +`plan` restricts the model surface to read + plan tools and makes Bash +read-only — useful before letting the agent touch files. `act` is the full +surface; `review` is a read-only review surface. + +### 3. Raise the isolation profile + +```json +{"jsonrpc":"2.0","id":4,"method":"session/setIsolation","params":{ + "sessionId": "sess_1", + "profile": "workspace" +}} +``` + +Response: + +```json +{"jsonrpc":"2.0","id":4,"result":{"sessionId":"sess_1","isolation":"workspace"}} +``` + +Profiles: `dev` (no sandbox), `workspace` (sandboxed to the workspace), +`strict` (stricter sandbox), `container` (requires a container). An invalid +profile returns a JSON-RPC error with code `-32602`. + +### 4. Check the snapshot + +```json +{"jsonrpc":"2.0","id":5,"method":"session/status","params":{ + "sessionId": "sess_1" +}} +``` + +Response: + +```json +{"jsonrpc":"2.0","id":5,"result":{ + "sessionId": "sess_1", + "workMode": "plan", + "isolation": "workspace", + "autoCommit": false, + "messages": 0 +}} +``` + +### 5. Run a prompt + +```json +{"jsonrpc":"2.0","id":6,"method":"session/prompt","params":{ + "sessionId": "sess_1", + "prompt": [{"type":"text","text":"Summarize the retry logic in internal/engine/stream.go"}] +}} +``` + +The server streams `session/update` notifications while the agent works and +answers `id:6` when the turn finishes. A prompt that wants to use a tool the +client must approve triggers `session/request_permission`; the client replies +to the server's request with the same `id`. + +### 6. Cancel if needed + +```json +{"jsonrpc":"2.0","id":7,"method":"session/cancel","params":{ + "sessionId": "sess_1" +}} +``` + +--- + +## Reference client (Python, stdlib only) + +A complete, dependency-free client showing the full lifecycle above: + +```python +#!/usr/bin/env python3 +"""Minimal hawk ACP client: initialize → new → setMode → status → prompt.""" +import json +import subprocess +import sys + +proc = subprocess.Popen( + ["hawk", "acp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + bufsize=1, +) + + +def call(method, params=None, rid=None): + """Send one request, return the decoded result (or raise on error).""" + global _rid + rid = rid if rid is not None else _rid + _rid += 1 + body = {"jsonrpc": "2.0", "id": rid, "method": method} + if params is not None: + body["params"] = params + proc.stdin.write(json.dumps(body) + "\n") + proc.stdin.flush() + while True: + line = proc.stdout.readline() + if not line: + raise RuntimeError("hawk acp closed the stream") + msg = json.loads(line) + if msg.get("id") != rid: + continue # notifications / other responses + if "error" in msg: + raise RuntimeError(f"{method}: {msg['error']}") + return msg["result"] + + +_rid = 1 + +call("initialize", {}) +sess = call("session/new", {})["sessionId"] +print("session:", sess) +print("setMode ->", call("session/setMode", {"sessionId": sess, "mode": "plan"})) +print("isolation ->", call("session/setIsolation", {"sessionId": sess, "profile": "workspace"})) +print("status ->", call("session/status", {"sessionId": sess})) + +# Streaming prompt: read notifications until the response for id=6 arrives. +proc.stdin.write(json.dumps({ + "jsonrpc": "2.0", "id": 6, "method": "session/prompt", + "params": {"sessionId": sess, "prompt": [{"type": "text", "text": "Say hello."}]}, +}) + "\n") +proc.stdin.flush() +while True: + line = proc.stdout.readline() + if not line: + break + msg = json.loads(line) + if msg.get("id") == 6: + print("prompt done:", msg["result"]) + break + # session/update notifications stream past here + +proc.stdin.close() +proc.wait() +``` + +--- + +## Errors + +Errors follow JSON-RPC 2.0: + +| Code | Meaning | +|------|---------| +| `-32700` | Parse error | +| `-32600` | Invalid request | +| `-32601` | Method not found | +| `-32602` | Invalid params (e.g. unknown `sessionId`, bad mode/profile) | +| `-32603` | Internal error (e.g. session factory failure) | + +Unknown sessions and invalid `mode`/`profile` values return `-32602` with a +human-readable message in `error.message`. + +--- + +## Where to go next + +| Document | What You Will Learn | +|----------|-------------------| +| [Control plane](../architecture/control-plane.md) | The full control-plane design | +| [Headless mode](../user-guide/14-headless-mode.md) | Scripting without ACP | + +© 2026 GrayCode AI. All rights reserved. diff --git a/docs/architecture/control-plane.md b/docs/architecture/control-plane.md index 607d996a..d4d06a9a 100644 --- a/docs/architecture/control-plane.md +++ b/docs/architecture/control-plane.md @@ -131,16 +131,30 @@ Single entry for subagents + background tasks: - `session/status` — control-plane snapshot (mode, isolation, autoCommit, message count) ### Deprecations -- `BackgroundAgentPool` / `NewBackgroundAgentPool*` / `FormatResults` marked Deprecated - in favor of `Session.SpawnController()` (same taskruntime.Registry). Retained - for compatibility; no production callers found. +- `BackgroundAgentPool` / `NewBackgroundAgentPool*` / `FormatResults` **removed** + in favor of `Session.SpawnController()` (same taskruntime.Registry). No + production callers existed; the type and shims were deleted outright. + +## Iteration 5 + +### ACP client docs +- `docs/acp/client.md` — wire protocol, worked lifecycle example + (new → setMode → setIsolation → status → prompt), reference client. + +### Benchmark scorecard +- `hawk eval smoke` — headless agent-loop smoke benchmark (stub provider, + no API key, CI-safe): drives the real `Session.Stream` loop and scores + steps / tool calls / token usage per task. +- Fixtures: `smoke-read-file` (must emit ≥1 Read tool call), `smoke-no-tools` + (must terminate cleanly). Run with `hawk eval smoke`. ## Not done yet (next iterations) -- True 60s binary install path (packaging/CI) -- Deeper ACP (session/setMode, client fs routing) -- Public Terminal-Bench scorecard -- Optional: deprecate BackgroundAgentPool reexports +- True 60s binary install path (packaging/CI) — Homebrew tap configured in + goreleaser; requires `GrayCodeAI/homebrew-tap` repo + `HOMEBREW_TAP_TOKEN` + secret before the next tagged release. +- Deeper ACP (session/load, client fs routing) +- Full Terminal-Bench scorecard against external agents ## Tests diff --git a/internal/engine/agent/background_agent.go b/internal/engine/agent/background_agent.go deleted file mode 100644 index 77f4bf7f..00000000 --- a/internal/engine/agent/background_agent.go +++ /dev/null @@ -1,208 +0,0 @@ -package agent - -import ( - "context" - "strings" - "sync" - "time" - - agentcontracts "github.com/GrayCodeAI/hawk-core-contracts/agent" - - "github.com/GrayCodeAI/hawk/internal/taskruntime" -) - -// BackgroundAgentPool manages async sub-agents that run in the background. -// PACK-02: backed by taskruntime.Registry (shared with tool.BackgroundAgentManager). -// -// Deprecated: new code should use Session.SpawnController() (Spawn / -// SpawnBackground / Tasks) which shares the same taskruntime.Registry via -// ToolService.EnsureBackgroundManager. This pool is retained for older -// callers and tests. -type BackgroundAgentPool struct { - mu sync.Mutex - reg *taskruntime.Registry - results []BackgroundResult - maxWait time.Duration - // parent is the context every background agent derives its cancellable - // context from. When the owning session ends, call Stop() to cancel all - // in-flight agents. Previously Submit used context.Background(), so - // agents could never be cancelled and leaked past session teardown (C8). - parent context.Context - // cancels tracks the per-agent cancel functions so Stop()/completion can - // release them. Keyed by task ID. - cancels map[string]context.CancelFunc -} - -// BackgroundResult holds the output of a completed background agent. -type BackgroundResult struct { - ID string - Prompt string - Output string - Error error - Elapsed time.Duration -} - -// NewBackgroundAgentPool creates a pool with configurable wait limits. -// Agents derive their contexts from context.Background() unless -// NewBackgroundAgentPoolWithContext is used. Call Stop() when the owning -// session ends to cancel any in-flight agents. -func NewBackgroundAgentPool() *BackgroundAgentPool { - return NewBackgroundAgentPoolWithContext(context.Background()) -} - -// NewBackgroundAgentPoolWithContext creates a pool whose background agents -// derive their cancellable contexts from parent. Cancelling the parent (e.g. -// via session teardown) or calling Stop() cancels every in-flight agent. -func NewBackgroundAgentPoolWithContext(parent context.Context) *BackgroundAgentPool { - return &BackgroundAgentPool{ - reg: taskruntime.New(), - maxWait: 2 * time.Minute, - parent: parent, - cancels: make(map[string]context.CancelFunc), - } -} - -// Submit launches a background sub-agent. The spawn function runs asynchronously. -func (p *BackgroundAgentPool) Submit(id, prompt string, spawn func(ctx context.Context, prompt string) (string, error)) { - if p.parent == nil { - p.parent = context.Background() - } - ctx, cancel := context.WithCancel(p.parent) - p.mu.Lock() - if p.cancels == nil { - p.cancels = make(map[string]context.CancelFunc) - } - p.cancels[id] = cancel - p.mu.Unlock() - req := agentcontracts.SpawnRequest{Prompt: prompt, Background: true} - fn := func(ctx context.Context, r agentcontracts.SpawnRequest) (agentcontracts.SpawnResult, error) { - defer p.releaseCancel(id) - out, err := spawn(ctx, r.Prompt) - if err != nil { - return agentcontracts.SpawnResult{Status: agentcontracts.StatusFailed, Error: err.Error()}, err - } - return agentcontracts.SpawnResult{Status: agentcontracts.StatusCompleted, Output: out}, nil - } - p.reg.SpawnAgent(ctx, id, req, fn) -} - -// releaseCancel cancels and forgets the cancel func for a finished task so -// the pool does not accumulate entries for every completed background agent. -func (p *BackgroundAgentPool) releaseCancel(id string) { - p.mu.Lock() - defer p.mu.Unlock() - if p.cancels != nil { - if c, ok := p.cancels[id]; ok { - c() - delete(p.cancels, id) - } - } -} - -// Stop cancels every in-flight background agent and releases their context -// resources. Safe to call multiple times. Call this during session teardown -// so background agents do not outlive the session that spawned them (C8). -func (p *BackgroundAgentPool) Stop() { - p.mu.Lock() - defer p.mu.Unlock() - for id, c := range p.cancels { - c() - delete(p.cancels, id) - } -} - -// Collect gathers all completed background results without blocking. -func (p *BackgroundAgentPool) Collect() []BackgroundResult { - completed := p.reg.CollectCompleted() - var out []BackgroundResult - for _, t := range completed { - br := toPoolResult(t) - out = append(out, br) - p.mu.Lock() - p.results = append(p.results, br) - p.mu.Unlock() - } - return out -} - -// WaitAll blocks until all pending tasks complete or timeout. -func (p *BackgroundAgentPool) WaitAll() []BackgroundResult { - tasks := p.reg.Wait(p.maxWait) - var all []BackgroundResult - for _, t := range tasks { - // Wait returns done map snapshot; also drain collect - all = append(all, toPoolResult(t)) - } - // Clear done via CollectCompleted so WaitAll is not sticky forever - _ = p.reg.CollectCompleted() - p.mu.Lock() - p.results = append(p.results, all...) - p.mu.Unlock() - return all -} - -// HasPending returns true if background agents are still running. -func (p *BackgroundAgentPool) HasPending() bool { - return p.reg.HasPending() -} - -// PendingCount returns the number of in-flight background agents. -func (p *BackgroundAgentPool) PendingCount() int { - return p.reg.PendingCount() -} - -// AllResults returns all results collected so far (completed background tasks). -func (p *BackgroundAgentPool) AllResults() []BackgroundResult { - p.mu.Lock() - defer p.mu.Unlock() - out := make([]BackgroundResult, len(p.results)) - copy(out, p.results) - return out -} - -// ClearResults clears all collected results to free memory. -func (p *BackgroundAgentPool) ClearResults() { - p.mu.Lock() - defer p.mu.Unlock() - p.results = nil -} - -// FormatResults formats background results for injection into the agent context. -func FormatResults(results []BackgroundResult) string { - if len(results) == 0 { - return "" - } - var b strings.Builder - b.WriteString("Background research completed:\n\n") - for _, r := range results { - b.WriteString("## Task: " + r.Prompt + "\n") - if r.Error != nil { - b.WriteString("Error: " + r.Error.Error() + "\n\n") - } else { - b.WriteString(r.Output + "\n\n") - } - } - return b.String() -} - -func toPoolResult(t *taskruntime.Task) BackgroundResult { - br := BackgroundResult{ - ID: t.ID, - Prompt: t.Prompt, - Output: t.Output, - Elapsed: t.DoneAt.Sub(t.StartedAt), - } - if t.Error != "" { - br.Error = context.DeadlineExceeded - if t.Status != taskruntime.StatusKilled { - br.Error = errString(t.Error) - } - } - return br -} - -type stringError string - -func (e stringError) Error() string { return string(e) } - -func errString(s string) error { return stringError(s) } diff --git a/internal/engine/agent/background_agent_test.go b/internal/engine/agent/background_agent_test.go deleted file mode 100644 index 2c694342..00000000 --- a/internal/engine/agent/background_agent_test.go +++ /dev/null @@ -1,261 +0,0 @@ -package agent - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" -) - -func TestBackgroundAgentPool_NewPool(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - if pool == nil { - t.Fatal("NewBackgroundAgentPool returned nil") - } - if pool.HasPending() { - t.Error("new pool should have no pending tasks") - } - if pool.PendingCount() != 0 { - t.Errorf("PendingCount() = %d, want 0", pool.PendingCount()) - } -} - -// TestBackgroundAgentPool_StopCancelsInFlight verifies that Stop() cancels -// every in-flight background agent (C8 fix). Previously Submit used -// context.Background(), so agents could never be cancelled via the pool. -func TestBackgroundAgentPool_StopCancelsInFlight(t *testing.T) { - t.Parallel() - parent, pcancel := context.WithCancel(context.Background()) - defer pcancel() - pool := NewBackgroundAgentPoolWithContext(parent) - - var started atomic.Bool - var cancelled atomic.Bool - pool.Submit("bg-stop", "wait", func(ctx context.Context, prompt string) (string, error) { - started.Store(true) - <-ctx.Done() - cancelled.Store(true) - return "", ctx.Err() - }) - - // Wait for the agent to actually start before stopping. - deadline := time.Now().Add(2 * time.Second) - for !started.Load() && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !started.Load() { - t.Fatal("background agent did not start") - } - - pool.Stop() - - // The registry only drains a task once its spawn fn returns, so poll - // until the cancellation is observed AND the task has exited. - deadline = time.Now().Add(2 * time.Second) - for (!cancelled.Load() || pool.PendingCount() != 0) && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !cancelled.Load() { - t.Error("Stop() did not cancel the in-flight background agent") - } - if pool.PendingCount() != 0 { - t.Errorf("PendingCount() = %d, want 0 after Stop()", pool.PendingCount()) - } -} - -// TestBackgroundAgentPool_ParentCancellation verifies that cancelling the -// parent context (session teardown) also cancels in-flight agents. -func TestBackgroundAgentPool_ParentCancellation(t *testing.T) { - t.Parallel() - parent, cancel := context.WithCancel(context.Background()) - pool := NewBackgroundAgentPoolWithContext(parent) - - var cancelled atomic.Bool - pool.Submit("bg-parent", "wait", func(ctx context.Context, prompt string) (string, error) { - <-ctx.Done() - cancelled.Store(true) - return "", ctx.Err() - }) - - time.Sleep(50 * time.Millisecond) - cancel() - - deadline := time.Now().Add(2 * time.Second) - for !cancelled.Load() && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !cancelled.Load() { - t.Error("cancelling parent context did not cancel the background agent") - } - pool.Stop() -} - -func TestBackgroundAgentPool_SubmitAndCollect(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - - pool.Submit("task-1", "do something", func(ctx context.Context, prompt string) (string, error) { - time.Sleep(time.Millisecond) - return "result-1", nil - }) - - // Use WaitAll to ensure task completes deterministically - results := pool.WaitAll() - if len(results) != 1 { - t.Fatalf("WaitAll() returned %d results, want 1", len(results)) - } - if results[0].ID != "task-1" { - t.Errorf("ID = %q, want %q", results[0].ID, "task-1") - } - if results[0].Output != "result-1" { - t.Errorf("Output = %q, want %q", results[0].Output, "result-1") - } - if results[0].Error != nil { - t.Errorf("Error = %v, want nil", results[0].Error) - } - if results[0].Elapsed <= 0 { - t.Error("Elapsed should be positive") - } -} - -func TestBackgroundAgentPool_SubmitError(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - expectedErr := errors.New("spawn failed") - - pool.Submit("err-task", "fail", func(ctx context.Context, prompt string) (string, error) { - return "", expectedErr - }) - - time.Sleep(50 * time.Millisecond) - - results := pool.Collect() - if len(results) != 1 { - t.Fatalf("Collect() returned %d results, want 1", len(results)) - } - if results[0].Error == nil || results[0].Error.Error() != expectedErr.Error() { - t.Errorf("Error = %v, want %v", results[0].Error, expectedErr) - } -} - -func TestBackgroundAgentPool_CollectEmpty(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - results := pool.Collect() - if len(results) != 0 { - t.Errorf("Collect() on empty pool returned %d results", len(results)) - } -} - -func TestBackgroundAgentPool_MultipleSubmits(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - - for i := 0; i < 5; i++ { - id := "task-" + string(rune('a'+i)) - pool.Submit(id, "prompt", func(ctx context.Context, prompt string) (string, error) { - time.Sleep(10 * time.Millisecond) - return "done", nil - }) - } - - if pool.PendingCount() != 5 { - t.Errorf("PendingCount() = %d, want 5", pool.PendingCount()) - } - - time.Sleep(100 * time.Millisecond) - - results := pool.Collect() - if len(results) != 5 { - t.Errorf("Collect() returned %d results, want 5", len(results)) - } - - if pool.HasPending() { - t.Error("HasPending() should be false after all collected") - } -} - -func TestBackgroundAgentPool_WaitAll(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - - pool.Submit("slow", "wait", func(ctx context.Context, prompt string) (string, error) { - time.Sleep(100 * time.Millisecond) - return "waited", nil - }) - - results := pool.WaitAll() - if len(results) != 1 { - t.Fatalf("WaitAll() returned %d results, want 1", len(results)) - } - if results[0].Output != "waited" { - t.Errorf("Output = %q, want %q", results[0].Output, "waited") - } -} - -func TestBackgroundAgentPool_AllResults(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - - pool.Submit("r1", "p1", func(ctx context.Context, prompt string) (string, error) { - return "out1", nil - }) - pool.Submit("r2", "p2", func(ctx context.Context, prompt string) (string, error) { - return "out2", nil - }) - - time.Sleep(50 * time.Millisecond) - pool.Collect() - - all := pool.AllResults() - if len(all) != 2 { - t.Errorf("AllResults() returned %d, want 2", len(all)) - } -} - -func TestBackgroundAgentPool_ConcurrentAccess(t *testing.T) { - t.Parallel() - pool := NewBackgroundAgentPool() - var count atomic.Int32 - - for i := 0; i < 20; i++ { - pool.Submit("concurrent", "p", func(ctx context.Context, prompt string) (string, error) { - count.Add(1) - time.Sleep(10 * time.Millisecond) - return "ok", nil - }) - } - - // Concurrent reads while tasks are running - go func() { pool.HasPending() }() - go func() { pool.PendingCount() }() - go func() { pool.Collect() }() - - pool.WaitAll() - - if count.Load() != 20 { - t.Errorf("expected 20 tasks to run, got %d", count.Load()) - } -} - -func TestBackgroundAgentPool_FormatResults_Empty(t *testing.T) { - t.Parallel() - result := FormatResults(nil) - if result != "" { - t.Errorf("FormatResults(nil) = %q, want empty", result) - } -} - -func TestBackgroundAgentPool_FormatResults_WithResults(t *testing.T) { - t.Parallel() - results := []BackgroundResult{ - {ID: "t1", Prompt: "research X", Output: "found Y", Elapsed: time.Second}, - {ID: "t2", Prompt: "check Z", Error: errors.New("failed"), Elapsed: 2 * time.Second}, - } - formatted := FormatResults(results) - if formatted == "" { - t.Error("FormatResults should produce non-empty output") - } -} diff --git a/internal/engine/agent_reexports.go b/internal/engine/agent_reexports.go index 41caf109..591a6a20 100644 --- a/internal/engine/agent_reexports.go +++ b/internal/engine/agent_reexports.go @@ -4,8 +4,6 @@ package engine import ( - "context" - "github.com/GrayCodeAI/hawk/internal/engine/agent" ) @@ -13,11 +11,6 @@ type ( SubAgentMode = agent.SubAgentMode SubAgentConfig = agent.SubAgentConfig SubAgentBudget = agent.SubAgentBudget - // Deprecated: use Session.SpawnController() and BackgroundAgentManager - // (taskruntime-backed) instead. BackgroundAgentPool is retained for - // compatibility with older callers and tests. - BackgroundAgentPool = agent.BackgroundAgentPool - BackgroundResult = agent.BackgroundResult ) const ( @@ -46,16 +39,3 @@ func FilterToolsForMode(mode SubAgentMode, available []string) []string { } func DefaultTurnsForMode(mode SubAgentMode) int { return agent.DefaultTurnsForMode(mode) } func IsReadOnlyMode(mode SubAgentMode) bool { return agent.IsReadOnlyMode(mode) } - -// Deprecated: prefer Session.SpawnController().SpawnBackground for async -// sub-agents. Retained for compatibility. -func NewBackgroundAgentPool() *BackgroundAgentPool { return agent.NewBackgroundAgentPool() } - -// Deprecated: prefer Session.SpawnController().SpawnBackground for async -// sub-agents. Retained for compatibility. -func NewBackgroundAgentPoolWithContext(ctx context.Context) *BackgroundAgentPool { - return agent.NewBackgroundAgentPoolWithContext(ctx) -} - -// Deprecated: prefer SpawnController for background result formatting. -func FormatResults(results []BackgroundResult) string { return agent.FormatResults(results) } diff --git a/internal/feature/eval/headless.go b/internal/feature/eval/headless.go new file mode 100644 index 00000000..74c0272b --- /dev/null +++ b/internal/feature/eval/headless.go @@ -0,0 +1,115 @@ +package eval + +// Headless agent-loop driver used by the smoke benchmarks. It drives the real +// engine.Session.Stream loop with a stub ChatClient (no provider, no API key) +// and reports steps, tool calls, and token usage — the "scorecard" for the +// agent pipeline itself. Used by `hawk eval smoke`. + +import ( + "context" + "encoding/json" + "fmt" + "io" + + "github.com/GrayCodeAI/hawk/internal/engine" + "github.com/GrayCodeAI/hawk/internal/observability/logger" + "github.com/GrayCodeAI/hawk/internal/tool" + "github.com/GrayCodeAI/hawk/internal/types" +) + +// smokeChatClient is a stub ChatClient that replays a canned event stream per +// StreamChatContinue call. It records the tool calls it was asked to emit so +// the driver can score whether the agent attempted them. +type smokeChatClient struct { + events []types.EyrieStreamEvent + calls []string + used bool +} + +func (m *smokeChatClient) Chat(_ context.Context, _ []types.EyrieMessage, _ types.ChatOptions) (*types.EyrieResponse, error) { + return &types.EyrieResponse{Content: "stub", FinishReason: "end_turn"}, nil +} + +func (m *smokeChatClient) StreamChatContinue(_ context.Context, _ []types.EyrieMessage, _ types.ChatOptions, _ types.ContinuationConfig) (*types.StreamResult, error) { + // Emit the scripted stream once; on subsequent turns (tool-result loops) + // fall back to a plain answer so the loop terminates instead of replaying + // the tool call forever. + events := m.events + if m.used { + events = []types.EyrieStreamEvent{ + {Type: "content", Content: "done"}, + {Type: "done", StopReason: "end_turn"}, + } + } + m.used = true + ch := make(chan types.EyrieStreamEvent, len(events)+1) + for _, e := range events { + ch <- e + } + close(ch) + return &types.StreamResult{Events: ch}, nil +} + +// streamHeadless runs the agent loop once and returns (steps, toolCalls, +// tokens). It never contacts a provider — the stub client replays the given +// events, and tools are wired to real registered tools (spawn/background +// through SpawnController) so the loop exercises the real execution path. +func streamHeadless(ctx context.Context, events []json.RawMessage) (int, int, int, error) { + client := &smokeChatClient{} + for _, raw := range events { + var ev types.EyrieStreamEvent + if err := json.Unmarshal(raw, &ev); err != nil { + return 0, 0, 0, fmt.Errorf("bad event json: %w", err) + } + if ev.Type == "tool_call" && ev.ToolCall != nil { + client.calls = append(client.calls, ev.ToolCall.Name) + } + client.events = append(client.events, ev) + } + + // Tool service wires the production essential tool set so the loop + // exercises the real execution path (permissions, sandbox, registry). + s := engine.NewSessionWithClient(client, "smoke", "smoke-model", "smoke system prompt", + tool.NewRegistry(tool.BashTool{}, tool.FileReadTool{}, tool.FileWriteTool{}, tool.FileEditTool{}, + tool.LSTool{}, tool.GrepTool{}, tool.WebFetchTool{}, tool.ToolSearchTool{}, + tool.AgentTool{}, tool.AskUserQuestionTool{}, tool.TodoWriteTool{}, + tool.MonitorTool{}, tool.MultiEditTool{}), false) + // Auto-approve every tool so the loop executes them rather than stalling + // on permission prompts. + s.PermSvc().Memory().AlwaysAllow("*") + s.WireAgentTool() + s.Tools().EnsureBackgroundManager() + + s.AddUser("smoke task") + + // Quiet the engine logger so the scorecard report is the only stdout. + s.SetLogger(logger.New(io.Discard, logger.Error)) + + ch, err := s.Stream(ctx) + if err != nil { + return 0, 0, 0, err + } + + steps, toolCalls, tokens := 0, 0, 0 + terminated := false + for ev := range ch { + steps++ + switch ev.Type { + case "tool_use": + toolCalls++ + case "usage": + if ev.Usage != nil { + tokens += ev.Usage.PromptTokens + ev.Usage.CompletionTokens + } + case "done": + terminated = true + } + } + if !terminated { + return steps, toolCalls, tokens, fmt.Errorf("stream closed without a done event") + } + return steps, toolCalls, tokens, nil +} + +// Verify that the smoke driver's stub client satisfies the engine contract. +var _ engine.ChatClient = (*smokeChatClient)(nil) diff --git a/internal/feature/eval/smoke.go b/internal/feature/eval/smoke.go new file mode 100644 index 00000000..f1219d1a --- /dev/null +++ b/internal/feature/eval/smoke.go @@ -0,0 +1,142 @@ +package eval + +// Smoke benchmarking: drive the real headless agent loop (Session.Stream) +// against a stub provider and score the run on steps and tokens. No API key +// is needed, so this doubles as a CI regression gate for the agent pipeline +// itself — the "scorecard" mode of `hawk eval`. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" +) + +// SmokeSuite returns the headless agent-loop smoke tasks. Each task's +// ValidateFn checks the run's scorecard (steps, tool calls, tokens). +func SmokeSuite() *BenchmarkSuite { + return &BenchmarkSuite{ + Name: "Agent-Loop Smoke (headless)", + Tasks: []BenchmarkTask{ + taskSmokeReadFile(), + taskSmokeNoTools(), + }, + } +} + +// SmokeMode is the duration budget for the smoke tasks. +const SmokeMode = 3 * time.Second + +// SmokeScorecard is the JSON shape of a smoke run, passed to ValidateFn via +// the JSON written into the work directory. +type SmokeScorecard struct { + Steps int `json:"steps"` + ToolCalls int `json:"tool_calls"` + TokensUsed int `json:"tokens_used"` + DurationMS int64 `json:"duration_ms"` + Passed bool `json:"passed"` + Error string `json:"error,omitempty"` +} + +// runSmokeStream drives engine.Session.Stream against a stub provider and +// writes the scorecard JSON into workDir, which the task's ValidateFn reads. +func runSmokeStream(ctx context.Context, workDir string, events []json.RawMessage) { + start := time.Now() + steps, toolCalls, tokens, err := streamHeadless(ctx, events) + card := SmokeScorecard{ + Steps: steps, + ToolCalls: toolCalls, + TokensUsed: tokens, + DurationMS: time.Since(start).Milliseconds(), + Passed: err == nil, + } + if err != nil { + card.Error = err.Error() + } + _ = os.WriteFile(workDir+"/scorecard.json", mustJSON(card), 0o600) +} + +// taskSmokeReadFile scores a read+answer task: the loop must terminate and +// emit at least one Read tool call. +func taskSmokeReadFile() BenchmarkTask { + return BenchmarkTask{ + ID: "smoke-read-file", + Description: "Headless agent loop: read a file via the Read tool", + Prompt: "Read internal/engine/stream.go and describe the retry timer.", + TimeLimit: SmokeMode, + Tags: []string{"smoke", "agent-loop"}, + SetupFn: func(workDir string) error { + // Drive the real agent loop with a stub provider that asks for a + // Read tool call, then answers. + runSmokeStream(context.Background(), workDir, []json.RawMessage{ + json.RawMessage(`{"type":"tool_call","tool_call":{"name":"Read","arguments":{"path":"internal/engine/stream.go"}}}`), + json.RawMessage(`{"type":"content","content":"done"}`), + json.RawMessage(`{"type":"done","stop_reason":"end_turn"}`), + }) + return nil + }, + ValidateFn: func(workDir string) (bool, string) { + card, err := loadSmokeCard(workDir) + if err != nil { + return false, err.Error() + } + if !card.Passed { + return false, "stream did not terminate cleanly: " + card.Error + } + if card.ToolCalls < 1 { + return false, fmt.Sprintf("expected >= 1 tool call, got %d", card.ToolCalls) + } + return true, "" + }, + } +} + +// taskSmokeNoTools scores a no-tool answer task: the loop must terminate. +func taskSmokeNoTools() BenchmarkTask { + return BenchmarkTask{ + ID: "smoke-no-tools", + Description: "Headless agent loop: answer without tools", + Prompt: "What is the capital of France?", + TimeLimit: SmokeMode, + Tags: []string{"smoke", "agent-loop"}, + SetupFn: func(workDir string) error { + // Stub provider answers directly; the loop should end in one turn. + runSmokeStream(context.Background(), workDir, []json.RawMessage{ + json.RawMessage(`{"type":"content","content":"Paris"}`), + json.RawMessage(`{"type":"done","stop_reason":"end_turn"}`), + }) + return nil + }, + ValidateFn: func(workDir string) (bool, string) { + card, err := loadSmokeCard(workDir) + if err != nil { + return false, err.Error() + } + if !card.Passed { + return false, "stream did not terminate cleanly: " + card.Error + } + return true, "" + }, + } +} + +func loadSmokeCard(workDir string) (*SmokeScorecard, error) { + data, err := os.ReadFile(workDir + "/scorecard.json") + if err != nil { + return nil, fmt.Errorf("read scorecard: %w", err) + } + var card SmokeScorecard + if err := json.Unmarshal(data, &card); err != nil { + return nil, fmt.Errorf("parse scorecard: %w", err) + } + return &card, nil +} + +func mustJSON(v any) []byte { + data, err := json.Marshal(v) + if err != nil { + panic(err) + } + return data +} diff --git a/internal/feature/eval/smoke_test.go b/internal/feature/eval/smoke_test.go new file mode 100644 index 00000000..72bc1d1d --- /dev/null +++ b/internal/feature/eval/smoke_test.go @@ -0,0 +1,91 @@ +package eval + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" +) + +// TestSmokeSuiteTasks runs the smoke tasks through the real Runner (stub +// provider, no API key) and asserts both terminate cleanly. This is the CI +// gate for the agent-loop smoke scorecard. +func TestSmokeSuiteTasks(t *testing.T) { + suite := SmokeSuite() + if len(suite.Tasks) != 2 { + t.Fatalf("SmokeSuite has %d tasks, want 2", len(suite.Tasks)) + } + + runner := NewRunner("smoke", "") + runner.NoCache = true + runner.Filters = nil + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + result, err := runner.Run(ctx, suite) + if err != nil { + t.Fatalf("Run: %v", err) + } + if result.Passed != 2 { + for _, r := range result.Results { + if !r.Passed { + t.Logf("FAIL %s: %s", r.TaskID, r.Error) + } + } + t.Fatalf("passed = %d/%d, want 2/2", result.Passed, result.TotalTasks) + } +} + +// TestStreamHeadless_ReadTool ensures the driver counts a tool call and +// terminates when the stub asks for Read. +func TestStreamHeadless_ReadTool(t *testing.T) { + events := []json.RawMessage{ + json.RawMessage(`{"type":"tool_call","tool_call":{"name":"Read","arguments":{"path":"go.mod"}}}`), + json.RawMessage(`{"type":"content","content":"done"}`), + json.RawMessage(`{"type":"done","stop_reason":"end_turn"}`), + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + steps, calls, _, err := streamHeadless(ctx, events) + if err != nil { + t.Fatalf("streamHeadless: %v", err) + } + if steps == 0 { + t.Fatal("no steps emitted") + } + if calls != 1 { + t.Fatalf("tool calls = %d, want 1", calls) + } +} + +// TestSmokeTask_ValidateFn ensures the read-file task's ValidateFn passes +// when the scorecard records a tool call. +func TestSmokeTask_ValidateFn(t *testing.T) { + dir := t.TempDir() + card := SmokeScorecard{Steps: 3, ToolCalls: 1, Passed: true} + if err := os.WriteFile(filepath.Join(dir, "scorecard.json"), mustJSON(card), 0o600); err != nil { + t.Fatal(err) + } + ok, msg := taskSmokeReadFile().ValidateFn(dir) + if !ok { + t.Fatalf("ValidateFn failed: %s", msg) + } +} + +// TestSmokeTask_ValidateFn_NoTools ensures the no-tools task rejects a +// scorecard that did not terminate cleanly. +func TestSmokeTask_ValidateFn_NoTools(t *testing.T) { + dir := t.TempDir() + card := SmokeScorecard{Steps: 5, Passed: false, Error: "stream closed without a done event"} + if err := os.WriteFile(filepath.Join(dir, "scorecard.json"), mustJSON(card), 0o600); err != nil { + t.Fatal(err) + } + ok, _ := taskSmokeNoTools().ValidateFn(dir) + if ok { + t.Fatal("ValidateFn should fail for a non-terminating run") + } +}