feat(bench): answer bench harness and the 5-filing pilot

cmd/navbench/answer.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/hallelx2/vectorless-engine/pkg/retrieval"
+)
+
+// The answer step: one generative call over the evidence pages, and the
+// only generation in the pipeline. The prompt is the engine's
+// synthesiseAnswer (internal/handler/answer.go) with evidence pages in
+// place of sections.
+//
+// It talks to an OpenAI-compatible endpoint directly rather than through
+// llmgate, because llmgate cannot yet turn a reasoning model's thinking
+// off, and GLM-4.6 thinks by default: measured 2026-09-28, 127 output
+// tokens to answer "17*23" against 3 with thinking disabled. Answering
+// from evidence that is already selected needs no deliberation.
+
+type answerer struct {
+	baseURL, key, model string
+	thinking            bool
+	maxPages            int
+	client              *http.Client
+}
+
+func newAnswerer(thinking bool, maxPages int) (*answerer, error) {
+	key := os.Getenv("ANSWER_API_KEY")
+	if key == "" {
+		key = os.Getenv("VLE_LLM_ANTHROPIC_API_KEY")
+	}
+	if key == "" {
+		return nil, fmt.Errorf("answer: set ANSWER_API_KEY (or VLE_LLM_ANTHROPIC_API_KEY for GLM)")
+	}
+	base := os.Getenv("ANSWER_BASE_URL")
+	if base == "" {
+		base = "https://api.z.ai/api/paas/v4"
+	}
+	model := os.Getenv("ANSWER_MODEL")
+	if model == "" {
+		model = "glm-4.6"
+	}
+	return &answerer{baseURL: strings.TrimRight(base, "/"), key: key, model: model, thinking: thinking,
+		maxPages: maxPages, client: &http.Client{Timeout: 3 * time.Minute}}, nil
+}
+
+type answerOut struct {
+	Text      string
+	Seconds   float64
+	InTokens  int
+	OutTokens int
+}
+
+func (a *answerer) answer(ctx context.Context, query string, evidence []retrieval.PageScore) (answerOut, error) {
+	var b strings.Builder
+	b.WriteString("You are answering a user's question using ONLY the evidence below.\n\n")
+	b.WriteString("User query:\n")
+	b.WriteString(query)
+	b.WriteString("\n\nRetrieved evidence (each block is a page of the document):\n")
+	for i, ev := range evidence {
+		if i >= a.maxPages {
+			break
+		}
+		fmt.Fprintf(&b, "\n[%d] page=%d\nPage content:\n%s\n", i+1, ev.Page.Number, ev.Page.Text)
+	}
+	b.WriteString("\nWrite a concise answer to the user's query. ")
+	b.WriteString("If the evidence does not contain an answer, say so. ")
+	b.WriteString("Inline citations should reference the page numbers shown above. ")
+	b.WriteString("Output plain prose; no JSON.")
+
+	body := map[string]any{
+		"model": a.model,
+		"messages": []map[string]string{
+			{"role": "system", "content": "You synthesise grounded answers from retrieved document pages. Never invent facts; only cite what the evidence shows."},
+			{"role": "user", "content": b.String()},
+		},
+		"max_tokens":  1024,
+		"temperature": 0,
+	}
+	if !a.thinking {
+		body["thinking"] = map[string]string{"type": "disabled"}
+	}
+	raw, _ := json.Marshal(body)
+	start := time.Now()
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.baseURL+"/chat/completions", bytes.NewReader(raw))
+	if err != nil {
+		return answerOut{}, err
+	}
+	req.Header.Set("Authorization", "Bearer "+a.key)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return answerOut{}, err
+	}
+	defer resp.Body.Close()
+	data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if resp.StatusCode != http.StatusOK {
+		return answerOut{}, fmt.Errorf("answer: %d %s", resp.StatusCode, strings.TrimSpace(string(data[:min(len(data), 300)])))
+	}
+	var out struct {
+		Choices []struct {
+			Message struct {
+				Content string `json:"content"`
+			} `json:"message"`
+		} `json:"choices"`
+		Usage struct {
+			PromptTokens     int `json:"prompt_tokens"`
+			CompletionTokens int `json:"completion_tokens"`
+		} `json:"usage"`
+	}
+	if err := json.Unmarshal(data, &out); err != nil || len(out.Choices) == 0 {
+		return answerOut{}, fmt.Errorf("answer: unreadable response: %v", err)
+	}
+	return answerOut{
+		Text:      strings.TrimSpace(out.Choices[0].Message.Content),
+		Seconds:   time.Since(start).Seconds(),
+		InTokens:  out.Usage.PromptTokens,
+		OutTokens: out.Usage.CompletionTokens,
+	}, nil
+}

cmd/navbench/main.go

@@ -64,6 +64,11 @@ type outcome struct {
 	InTokens      int       `json:"in_tokens"`
 	CostUSD       float64   `json:"cost_usd"`
 	Seconds       float64   `json:"seconds"`
+	Answer        string    `json:"answer,omitempty"`
+	GoldAnswer    string    `json:"gold_answer,omitempty"`
+	AnswerSeconds float64   `json:"answer_seconds,omitempty"`
+	AnswerIn      int       `json:"answer_in_tokens,omitempty"`
+	AnswerOut     int       `json:"answer_out_tokens,omitempty"`
 	Err           string    `json:"err,omitempty"`
 }
 
@@ -75,10 +80,22 @@ func main() {
 	maxLeaves := flag.Int("leaves", 0, "cap on sections read per question; 0 lets the page budget decide")
 	maxPages := flag.Int("pages", 0, "pages judged in full per question (0 = the engine default)")
 	limitQ := flag.Int("limit", 0, "stop after this many questions (0 = all)")
+	headChars := flag.Int("head", 0, "characters of each page the coarse skim sees (0 = the engine default)")
+	answer := flag.Bool("answer", false, "write an answer from the evidence pages with one generative call (ANSWER_BASE_URL/ANSWER_MODEL, default GLM-4.6 on z.ai)")
+	answerThinking := flag.Bool("answer-thinking", false, "let the answer model reason before answering (GLM-4.6 does by default)")
+	answerPages := flag.Int("answer-pages", 8, "evidence pages given to the answer step")
 	skimAll := flag.Bool("skim-all", false, "skim every page's head alongside the section ranking (HAL-1566), as the persisted-pages path does")
 	parallel := flag.Int("parallel", 1, "questions in flight at once; the provider's adaptive limiter governs requests")
 	flag.Parse()
 	stats := &judgestats.Recorder{}
+	var ans *answerer
+	if *answer {
+		var err error
+		if ans, err = newAnswerer(*answerThinking, *answerPages); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+	}
 	if *qPath == "" || *trees == "" || *pdfs == "" {
 		fmt.Fprintln(os.Stderr, "usage: navbench -questions q.jsonl -trees dir -pdfs dir [-out o.jsonl]")
 		os.Exit(2)
@@ -98,7 +115,7 @@ func main() {
 	lim := limit.New(limit.Config{Initial: 16, OnChange: func(e limit.Event) {
 		fmt.Fprintf(os.Stderr, "  limiter %s %d -> %d %v\n", e.Cause, e.From, e.To, e.Err)
 	}})
-	nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages, SkimAll: *skimAll}
+	nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages, SkimAll: *skimAll, HeadChars: *headChars}
 
 	qs := readQuestions(*qPath)
 	if *limitQ > 0 && len(qs) > *limitQ {
@@ -177,6 +194,17 @@ func main() {
 				o.LeafHit = allInRanges(q.Evidence, o.SelectedPages)
 				o.Recall = recall(q.Evidence, o.Evidence)
 				o.Hit = o.Recall == 1
+				if ans != nil {
+					actx, acancel := context.WithTimeout(context.Background(), 3*time.Minute)
+					a, err := ans.answer(actx, q.Question, res.Evidence)
+					acancel()
+					o.GoldAnswer = q.Answer
+					if err != nil {
+						o.Err = err.Error()
+					} else {
+						o.Answer, o.AnswerSeconds, o.AnswerIn, o.AnswerOut = a.Text, a.Seconds, a.InTokens, a.OutTokens
+					}
+				}
 			}
 			results[i] = o
 			outMu.Lock()

cmd/tocdump/main.go

@@ -42,6 +42,7 @@ type dump struct {
 	Doc        string         `json:"doc"`
 	Pages      int            `json:"pages"`
 	Seconds    float64        `json:"seconds"`
+	ParseSecs  float64        `json:"parse_seconds"` // PDF to per-page text, before Build; ingest end to end is the sum
 	Requests   int            `json:"requests"`
 	InTokens   int            `json:"in_tokens"`
 	CostUSD    float64        `json:"cost_usd"`
@@ -107,7 +108,9 @@ func main() {
 			name := strings.TrimSuffix(filepath.Base(path), ".pdf")
 			d := dump{Doc: name}
 
+			parseStart := time.Now()
 			pages, err := readPages(path)
+			d.ParseSecs = time.Since(parseStart).Seconds()
 			if err != nil {
 				d.Err = "parse: " + err.Error()
 				write(*out, d)