diff --git a/cmd/navbench/answer.go b/cmd/navbench/answer.go new file mode 100644 index 0000000..8a8064c --- /dev/null +++ b/cmd/navbench/answer.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "github.com/hallelx2/vectorless-engine/pkg/retrieval" +) + +// The answer step: one generative call over the evidence pages, and the +// only generation in the pipeline. The prompt is the engine's +// synthesiseAnswer (internal/handler/answer.go) with evidence pages in +// place of sections. +// +// It talks to an OpenAI-compatible endpoint directly rather than through +// llmgate, because llmgate cannot yet turn a reasoning model's thinking +// off, and GLM-4.6 thinks by default: measured 2026-09-28, 127 output +// tokens to answer "17*23" against 3 with thinking disabled. Answering +// from evidence that is already selected needs no deliberation. + +type answerer struct { + baseURL, key, model string + thinking bool + maxPages int + client *http.Client +} + +func newAnswerer(thinking bool, maxPages int) (*answerer, error) { + key := os.Getenv("ANSWER_API_KEY") + if key == "" { + key = os.Getenv("VLE_LLM_ANTHROPIC_API_KEY") + } + if key == "" { + return nil, fmt.Errorf("answer: set ANSWER_API_KEY (or VLE_LLM_ANTHROPIC_API_KEY for GLM)") + } + base := os.Getenv("ANSWER_BASE_URL") + if base == "" { + base = "https://api.z.ai/api/paas/v4" + } + model := os.Getenv("ANSWER_MODEL") + if model == "" { + model = "glm-4.6" + } + return &answerer{baseURL: strings.TrimRight(base, "/"), key: key, model: model, thinking: thinking, + maxPages: maxPages, client: &http.Client{Timeout: 3 * time.Minute}}, nil +} + +type answerOut struct { + Text string + Seconds float64 + InTokens int + OutTokens int +} + +func (a *answerer) answer(ctx context.Context, query string, evidence []retrieval.PageScore) (answerOut, error) { + var b strings.Builder + b.WriteString("You are answering a user's question using ONLY the evidence below.\n\n") + b.WriteString("User query:\n") + b.WriteString(query) + b.WriteString("\n\nRetrieved evidence (each block is a page of the document):\n") + for i, ev := range evidence { + if i >= a.maxPages { + break + } + fmt.Fprintf(&b, "\n[%d] page=%d\nPage content:\n%s\n", i+1, ev.Page.Number, ev.Page.Text) + } + b.WriteString("\nWrite a concise answer to the user's query. ") + b.WriteString("If the evidence does not contain an answer, say so. ") + b.WriteString("Inline citations should reference the page numbers shown above. ") + b.WriteString("Output plain prose; no JSON.") + + body := map[string]any{ + "model": a.model, + "messages": []map[string]string{ + {"role": "system", "content": "You synthesise grounded answers from retrieved document pages. Never invent facts; only cite what the evidence shows."}, + {"role": "user", "content": b.String()}, + }, + "max_tokens": 1024, + "temperature": 0, + } + if !a.thinking { + body["thinking"] = map[string]string{"type": "disabled"} + } + raw, _ := json.Marshal(body) + start := time.Now() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.baseURL+"/chat/completions", bytes.NewReader(raw)) + if err != nil { + return answerOut{}, err + } + req.Header.Set("Authorization", "Bearer "+a.key) + req.Header.Set("Content-Type", "application/json") + resp, err := a.client.Do(req) + if err != nil { + return answerOut{}, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if resp.StatusCode != http.StatusOK { + return answerOut{}, fmt.Errorf("answer: %d %s", resp.StatusCode, strings.TrimSpace(string(data[:min(len(data), 300)]))) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(data, &out); err != nil || len(out.Choices) == 0 { + return answerOut{}, fmt.Errorf("answer: unreadable response: %v", err) + } + return answerOut{ + Text: strings.TrimSpace(out.Choices[0].Message.Content), + Seconds: time.Since(start).Seconds(), + InTokens: out.Usage.PromptTokens, + OutTokens: out.Usage.CompletionTokens, + }, nil +} diff --git a/cmd/navbench/main.go b/cmd/navbench/main.go index 48c4390..d53a0fb 100644 --- a/cmd/navbench/main.go +++ b/cmd/navbench/main.go @@ -64,6 +64,11 @@ type outcome struct { InTokens int `json:"in_tokens"` CostUSD float64 `json:"cost_usd"` Seconds float64 `json:"seconds"` + Answer string `json:"answer,omitempty"` + GoldAnswer string `json:"gold_answer,omitempty"` + AnswerSeconds float64 `json:"answer_seconds,omitempty"` + AnswerIn int `json:"answer_in_tokens,omitempty"` + AnswerOut int `json:"answer_out_tokens,omitempty"` Err string `json:"err,omitempty"` } @@ -75,10 +80,22 @@ func main() { maxLeaves := flag.Int("leaves", 0, "cap on sections read per question; 0 lets the page budget decide") maxPages := flag.Int("pages", 0, "pages judged in full per question (0 = the engine default)") limitQ := flag.Int("limit", 0, "stop after this many questions (0 = all)") + headChars := flag.Int("head", 0, "characters of each page the coarse skim sees (0 = the engine default)") + answer := flag.Bool("answer", false, "write an answer from the evidence pages with one generative call (ANSWER_BASE_URL/ANSWER_MODEL, default GLM-4.6 on z.ai)") + answerThinking := flag.Bool("answer-thinking", false, "let the answer model reason before answering (GLM-4.6 does by default)") + answerPages := flag.Int("answer-pages", 8, "evidence pages given to the answer step") skimAll := flag.Bool("skim-all", false, "skim every page's head alongside the section ranking (HAL-1566), as the persisted-pages path does") parallel := flag.Int("parallel", 1, "questions in flight at once; the provider's adaptive limiter governs requests") flag.Parse() stats := &judgestats.Recorder{} + var ans *answerer + if *answer { + var err error + if ans, err = newAnswerer(*answerThinking, *answerPages); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } if *qPath == "" || *trees == "" || *pdfs == "" { fmt.Fprintln(os.Stderr, "usage: navbench -questions q.jsonl -trees dir -pdfs dir [-out o.jsonl]") os.Exit(2) @@ -98,7 +115,7 @@ func main() { lim := limit.New(limit.Config{Initial: 16, OnChange: func(e limit.Event) { fmt.Fprintf(os.Stderr, " limiter %s %d -> %d %v\n", e.Cause, e.From, e.To, e.Err) }}) - nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages, SkimAll: *skimAll} + nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages, SkimAll: *skimAll, HeadChars: *headChars} qs := readQuestions(*qPath) if *limitQ > 0 && len(qs) > *limitQ { @@ -177,6 +194,17 @@ func main() { o.LeafHit = allInRanges(q.Evidence, o.SelectedPages) o.Recall = recall(q.Evidence, o.Evidence) o.Hit = o.Recall == 1 + if ans != nil { + actx, acancel := context.WithTimeout(context.Background(), 3*time.Minute) + a, err := ans.answer(actx, q.Question, res.Evidence) + acancel() + o.GoldAnswer = q.Answer + if err != nil { + o.Err = err.Error() + } else { + o.Answer, o.AnswerSeconds, o.AnswerIn, o.AnswerOut = a.Text, a.Seconds, a.InTokens, a.OutTokens + } + } } results[i] = o outMu.Lock() diff --git a/cmd/tocdump/main.go b/cmd/tocdump/main.go index 279563e..772acd1 100644 --- a/cmd/tocdump/main.go +++ b/cmd/tocdump/main.go @@ -42,6 +42,7 @@ type dump struct { Doc string `json:"doc"` Pages int `json:"pages"` Seconds float64 `json:"seconds"` + ParseSecs float64 `json:"parse_seconds"` // PDF to per-page text, before Build; ingest end to end is the sum Requests int `json:"requests"` InTokens int `json:"in_tokens"` CostUSD float64 `json:"cost_usd"` @@ -107,7 +108,9 @@ func main() { name := strings.TrimSuffix(filepath.Base(path), ".pdf") d := dump{Doc: name} + parseStart := time.Now() pages, err := readPages(path) + d.ParseSecs = time.Since(parseStart).Seconds() if err != nil { d.Err = "parse: " + err.Error() write(*out, d) diff --git a/navbench b/navbench new file mode 100755 index 0000000..6ba224f Binary files /dev/null and b/navbench differ