diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b523a5..6c73af5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - go: ["1.25"] + go: ["1.25.11"] steps: - uses: actions/checkout@v6 @@ -70,7 +70,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: "1.25" + go-version: "1.25.11" cache: true # MinIO can't run as a `services:` container because GitHub Actions @@ -115,7 +115,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: "1.25" + go-version: "1.25.11" cache: true - name: staticcheck @@ -137,7 +137,7 @@ jobs: - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: - go-version: "1.25" + go-version: "1.25.11" cache: true - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/.golangci.yml b/.golangci.yml index 800f35b..dafca87 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,6 +30,11 @@ linters: - (net/http.ResponseWriter).Write - (github.com/jackc/pgx/v5.Tx).Rollback - (github.com/hallelx2/vectorless-engine/pkg/queue.Queue).Close + misspell: + # "strat" is our abbreviation for "strategy" (variable + key names), + # not a misspelling of "start". + ignore-rules: + - strat exclusions: rules: - path: _test\.go diff --git a/cmd/benchmark/main.go b/cmd/benchmark/main.go index 3bd73fc..0b1fd3e 100644 --- a/cmd/benchmark/main.go +++ b/cmd/benchmark/main.go @@ -77,7 +77,7 @@ func main() { ID string `json:"id"` } `json:"sections"` } - json.Unmarshal(treeResp, &treeData) + _ = json.Unmarshal(treeResp, &treeData) // benchmark best-effort parse sectionID := "" if len(treeData.Sections) > 1 { sectionID = treeData.Sections[1].ID // pick a child section @@ -87,9 +87,9 @@ func main() { fmt.Printf("Using section ID: %s\n\n", sectionID) type result struct { - name string - rest []time.Duration - grpc []time.Duration + name string + rest []time.Duration + grpc []time.Duration } var results []result @@ -101,12 +101,12 @@ func main() { fmt.Print(".") // REST start := time.Now() - restGET(base + "/v1/health") + _, _ = restGET(base + "/v1/health") r.rest = append(r.rest, time.Since(start)) // gRPC (Connect) start = time.Now() - healthClient.Check(ctx, connect.NewRequest(&v1.HealthCheckRequest{})) + _, _ = healthClient.Check(ctx, connect.NewRequest(&v1.HealthCheckRequest{})) r.grpc = append(r.grpc, time.Since(start)) } fmt.Println(" done") @@ -118,11 +118,11 @@ func main() { for i := 0; i < n; i++ { fmt.Print(".") start := time.Now() - restGET(base + "/v1/documents") + _, _ = restGET(base + "/v1/documents") r.rest = append(r.rest, time.Since(start)) start = time.Now() - docsClient.ListDocuments(ctx, connect.NewRequest(&v1.ListDocumentsRequest{Limit: 10})) + _, _ = docsClient.ListDocuments(ctx, connect.NewRequest(&v1.ListDocumentsRequest{Limit: 10})) r.grpc = append(r.grpc, time.Since(start)) } fmt.Println(" done") @@ -134,11 +134,11 @@ func main() { for i := 0; i < n; i++ { fmt.Print(".") start := time.Now() - restGET(base + "/v1/documents/" + *docID + "/tree") + _, _ = restGET(base + "/v1/documents/" + *docID + "/tree") r.rest = append(r.rest, time.Since(start)) start = time.Now() - docsClient.GetDocumentTree(ctx, connect.NewRequest(&v1.GetDocumentTreeRequest{ + _, _ = docsClient.GetDocumentTree(ctx, connect.NewRequest(&v1.GetDocumentTreeRequest{ DocumentId: *docID, })) r.grpc = append(r.grpc, time.Since(start)) @@ -153,11 +153,11 @@ func main() { for i := 0; i < n; i++ { fmt.Print(".") start := time.Now() - restGET(base + "/v1/sections/" + sectionID) + _, _ = restGET(base + "/v1/sections/" + sectionID) r.rest = append(r.rest, time.Since(start)) start = time.Now() - docsClient.GetSection(ctx, connect.NewRequest(&v1.GetSectionRequest{ + _, _ = docsClient.GetSection(ctx, connect.NewRequest(&v1.GetSectionRequest{ SectionId: sectionID, })) r.grpc = append(r.grpc, time.Since(start)) @@ -182,12 +182,12 @@ func main() { "query": q, }) start := time.Now() - restPOST(base+"/v1/query", body) + _, _ = restPOST(base+"/v1/query", body) r.rest = append(r.rest, time.Since(start)) // gRPC (Connect) start = time.Now() - queryClient.Query(ctx, connect.NewRequest(&v1.QueryRequest{ + _, _ = queryClient.Query(ctx, connect.NewRequest(&v1.QueryRequest{ DocumentId: *docID, Query: q, })) @@ -246,7 +246,7 @@ func restGET(url string) ([]byte, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close return io.ReadAll(resp.Body) } @@ -255,7 +255,7 @@ func restPOST(url string, body []byte) ([]byte, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close return io.ReadAll(resp.Body) } diff --git a/cmd/engine/main.go b/cmd/engine/main.go index ed2fed1..e2048ed 100644 --- a/cmd/engine/main.go +++ b/cmd/engine/main.go @@ -90,7 +90,7 @@ func run() error { if err != nil { return fmt.Errorf("init queue: %w", err) } - defer q.Close() + defer func() { _ = q.Close() }() // best-effort close llmClient, err := buildLLM(cfg.LLM) if err != nil { @@ -206,41 +206,41 @@ func run() error { } q.Register(queue.KindIngestDocument, pipeline.Handler()) - // /v1/answer/pageindex gets its OWN PageIndexStrategy instance, + // /v1/answer/treewalk gets its OWN TreeWalkStrategy instance, // independent of whatever selection strategy is configured in // retrieval.strategy. This way the endpoint is always available - // (gated by retrieval.pageindex.enabled), even on a deployment + // (gated by retrieval.treewalk.enabled), even on a deployment // using chunked-tree as its default selection path. - var pageIndexStrategy *retrieval.PageIndexStrategy - if cfg.Retrieval.PageIndex.Enabled && llmClient != nil { - pageIndexStrategy = buildPageIndexStrategy(cfg.Retrieval, llmClient, store) - logger.Info("retrieval: pageindex answer endpoint enabled", - "max_hops", pageIndexStrategy.MaxHops, - "page_content_limit", pageIndexStrategy.PageContentLimit, - "model_override", cfg.Retrieval.PageIndex.Model, + var treeWalkStrategy *retrieval.TreeWalkStrategy + if cfg.Retrieval.TreeWalk.Enabled && llmClient != nil { + treeWalkStrategy = buildTreeWalkStrategy(cfg.Retrieval, llmClient, store) + logger.Info("retrieval: treewalk answer endpoint enabled", + "max_hops", treeWalkStrategy.MaxHops, + "page_content_limit", treeWalkStrategy.PageContentLimit, + "model_override", cfg.Retrieval.TreeWalk.Model, ) } deps := api.Deps{ - Logger: logger, - DB: pool, - Storage: store, - Queue: q, - Strategy: strategy, - Version: version, - MultiDoc: multiDoc, - LLM: llmClient, - LLMModel: modelFor(cfg.LLM), - AnswerSpan: cfg.Retrieval.AnswerSpan, - Answer: cfg.Retrieval.Answer, - Planner: planner, - Planning: cfg.Retrieval.Planning, - ReRanker: reRanker, - ReRank: cfg.Retrieval.ReRank, - Replay: replayStore, - Abstain: cfg.Retrieval.Abstain, - PageIndexStrategy: pageIndexStrategy, - PageIndex: cfg.Retrieval.PageIndex, + Logger: logger, + DB: pool, + Storage: store, + Queue: q, + Strategy: strategy, + Version: version, + MultiDoc: multiDoc, + LLM: llmClient, + LLMModel: modelFor(cfg.LLM), + AnswerSpan: cfg.Retrieval.AnswerSpan, + Answer: cfg.Retrieval.Answer, + Planner: planner, + Planning: cfg.Retrieval.Planning, + ReRanker: reRanker, + ReRank: cfg.Retrieval.ReRank, + Replay: replayStore, + Abstain: cfg.Retrieval.Abstain, + TreeWalkStrategy: treeWalkStrategy, + TreeWalk: cfg.Retrieval.TreeWalk, } srv := &http.Server{ @@ -393,36 +393,38 @@ func buildStrategy(c config.RetrievalConfig, client llmgate.Client, store storag } a.ModelOverride = c.Agentic.Model return a - case "pageindex": - return buildPageIndexStrategy(c, client, store) + case "treewalk": + return buildTreeWalkStrategy(c, client, store) + case "auto": + return retrieval.NewAuto(retrieval.NewSinglePass(client), buildTreeWalkStrategy(c, client, store)) default: return retrieval.NewChunkedTree(client) } } -// buildPageIndexStrategy constructs the page-based agentic +// buildTreeWalkStrategy constructs the page-based agentic // strategy with the storage-backed PageLoader and the configured -// caps. Used by buildStrategy when retrieval.strategy=pageindex AND -// by the /v1/answer/pageindex endpoint setup (which wires its own +// caps. Used by buildStrategy when retrieval.strategy=treewalk AND +// by the /v1/answer/treewalk endpoint setup (which wires its own // instance regardless of the selection strategy). // // The TOCProvider is left nil here. PR-A (toc-tree-builder) adds // documents.toc_tree + a DB-backed provider; until it lands the // strategy degrades to its synthesised view, which is the // documented fallback path. -func buildPageIndexStrategy(c config.RetrievalConfig, client llmgate.Client, store storage.Storage) *retrieval.PageIndexStrategy { - p := retrieval.NewPageIndexStrategy(client) +func buildTreeWalkStrategy(c config.RetrievalConfig, client llmgate.Client, store storage.Storage) *retrieval.TreeWalkStrategy { + p := retrieval.NewTreeWalkStrategy(client) p.PageLoader = storagePageLoader{s: store} - if c.PageIndex.MaxHops > 0 { - p.MaxHops = c.PageIndex.MaxHops + if c.TreeWalk.MaxHops > 0 { + p.MaxHops = c.TreeWalk.MaxHops } - if c.PageIndex.PageContentLimit > 0 { - p.PageContentLimit = c.PageIndex.PageContentLimit + if c.TreeWalk.PageContentLimit > 0 { + p.PageContentLimit = c.TreeWalk.PageContentLimit } - if c.PageIndex.MaxCitations > 0 { - p.MaxCitations = c.PageIndex.MaxCitations + if c.TreeWalk.MaxCitations > 0 { + p.MaxCitations = c.TreeWalk.MaxCitations } - p.ModelOverride = c.PageIndex.Model + p.ModelOverride = c.TreeWalk.Model return p } @@ -437,14 +439,14 @@ func (sf storageFetcher) Get(ctx context.Context, ref string) ([]byte, error) { if err != nil { return nil, err } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close return io.ReadAll(rc) } // storagePageLoader adapts a storage.Storage to // retrieval.PageContentLoader. Mirrors storageFetcher but lives // behind a separate interface so the two callers (agentic / -// pageindex) can be wired independently. The PageIndex strategy +// treewalk) can be wired independently. The TreeWalk strategy // materialises section bodies once per get_pages observation, so // reading the full reader into a []byte is the right shape. type storagePageLoader struct{ s storage.Storage } @@ -454,7 +456,7 @@ func (l storagePageLoader) Load(ctx context.Context, ref string) ([]byte, error) if err != nil { return nil, err } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close return io.ReadAll(rc) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 835414e..129fbeb 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -134,7 +134,7 @@ func run() error { if err != nil { return fmt.Errorf("init queue: %w", err) } - defer q.Close() + defer func() { _ = q.Close() }() // best-effort close // ── LLM + retrieval strategy ────────────────────────────────── llmClient, err := buildLLM(cfg.Engine.LLM) @@ -164,13 +164,13 @@ func run() error { // Pre-built set of selectable strategies, keyed by config name. // Backs the per-request "strategy" override on /v1/query so the - // benchmark can A/B chunked-tree vs pageindex against this same + // benchmark can A/B chunked-tree vs treewalk against this same // running engine without a redeploy. Built from the raw client so // each override behaves identically to booting with that strategy // as the default (no shared cache wrapper across overrides). strategies := buildStrategySet(cfg.Engine.Retrieval, llmClient, store, pool) - // Replay store: every /v1/answer and /v1/answer/pageindex response + // Replay store: every /v1/answer and /v1/answer/treewalk response // is stamped with a deterministic trace_token and its body bytes // persisted here so /v1/replay can return them verbatim. On by // default; operators opt out via retrieval.replay.enabled=false. @@ -186,17 +186,17 @@ func run() error { ) } - // /v1/answer/pageindex gets its OWN PageIndexStrategy instance, + // /v1/answer/treewalk gets its OWN TreeWalkStrategy instance, // independent of whatever selection strategy retrieval.strategy // chose, so the endpoint is always available (gated by - // retrieval.pageindex.enabled) even on a chunked-tree deployment. - var pageIndexStrategy *retrieval.PageIndexStrategy - if cfg.Engine.Retrieval.PageIndex.Enabled && llmClient != nil { - pageIndexStrategy = buildPageIndexStrategy(cfg.Engine.Retrieval, llmClient, store, pool) - logger.Info("retrieval: pageindex answer endpoint enabled", - "max_hops", pageIndexStrategy.MaxHops, - "page_content_limit", pageIndexStrategy.PageContentLimit, - "model_override", cfg.Engine.Retrieval.PageIndex.Model, + // retrieval.treewalk.enabled) even on a chunked-tree deployment. + var treeWalkStrategy *retrieval.TreeWalkStrategy + if cfg.Engine.Retrieval.TreeWalk.Enabled && llmClient != nil { + treeWalkStrategy = buildTreeWalkStrategy(cfg.Engine.Retrieval, llmClient, store, pool) + logger.Info("retrieval: treewalk answer endpoint enabled", + "max_hops", treeWalkStrategy.MaxHops, + "page_content_limit", treeWalkStrategy.PageContentLimit, + "model_override", cfg.Engine.Retrieval.TreeWalk.Model, ) } @@ -250,22 +250,22 @@ func run() error { // Only start the HTTP server in "server" role. if *role == "server" { deps := handler.Deps{ - Logger: logger, - DB: pool, - Storage: store, - Queue: q, - Strategy: strategy, - MultiDoc: multiDoc, - Version: version, - Config: cfg, - Strategies: strategies, - LLM: llmClient, - LLMModel: modelFor(cfg.Engine.LLM), - AnswerSpan: cfg.Engine.Retrieval.AnswerSpan, - Answer: cfg.Engine.Retrieval.Answer, - Replay: replayStore, - PageIndexStrategy: pageIndexStrategy, - PageIndex: cfg.Engine.Retrieval.PageIndex, + Logger: logger, + DB: pool, + Storage: store, + Queue: q, + Strategy: strategy, + MultiDoc: multiDoc, + Version: version, + Config: cfg, + Strategies: strategies, + LLM: llmClient, + LLMModel: modelFor(cfg.Engine.LLM), + AnswerSpan: cfg.Engine.Retrieval.AnswerSpan, + Answer: cfg.Engine.Retrieval.Answer, + Replay: replayStore, + TreeWalkStrategy: treeWalkStrategy, + TreeWalk: cfg.Engine.Retrieval.TreeWalk, } srv := &http.Server{ @@ -383,7 +383,7 @@ func buildQueue(c enginecfg.QueueConfig, dbURL string) (queue.Queue, error) { // modelFor returns the configured chat/general-purpose model name for // the selected LLM driver. Used as the engine-default fallback when an -// API request omits an explicit model (answer + answer/pageindex). +// API request omits an explicit model (answer + answer/treewalk). func modelFor(c enginecfg.LLMConfig) string { switch c.Driver { case "anthropic": @@ -424,7 +424,7 @@ func buildLLM(c enginecfg.LLMConfig) (llmgate.Client, error) { // buildStrategy constructs the retrieval strategy named by // retrieval.strategy. The DB pool is threaded through so the -// pageindex strategy can wire a TOC provider that reads +// treewalk strategy can wire a TOC provider that reads // documents.toc_tree (the other strategies ignore it). func buildStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, store storage.Storage, pool *db.Pool) retrieval.Strategy { switch c.Strategy { @@ -439,8 +439,10 @@ func buildStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, store sto } a.ModelOverride = c.Agentic.Model return a - case "pageindex": - return buildPageIndexStrategy(c, client, store, pool) + case "treewalk": + return buildTreeWalkStrategy(c, client, store, pool) + case "auto": + return retrieval.NewAuto(retrieval.NewSinglePass(client), buildTreeWalkStrategy(c, client, store, pool)) default: return retrieval.NewChunkedTree(client) } @@ -451,9 +453,9 @@ func buildStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, store sto // uses this map to honour a per-request "strategy" override without // rebuilding a strategy on the hot path: selection is a map lookup. // -// This is what lets the benchmark A/B chunked-tree vs pageindex +// This is what lets the benchmark A/B chunked-tree vs treewalk // against the SAME running engine — no redeploy, no config flip. The -// caps (agentic max-hops, pageindex page limits, model overrides) come +// caps (agentic max-hops, treewalk page limits, model overrides) come // from the same config blocks the default builder reads, so an // override behaves identically to booting with that strategy as the // default. @@ -468,44 +470,45 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, store "single-pass": retrieval.NewSinglePass(client), "chunked-tree": retrieval.NewChunkedTree(client), "agentic": agentic, - "pageindex": buildPageIndexStrategy(c, client, store, pool), + "treewalk": buildTreeWalkStrategy(c, client, store, pool), + "auto": retrieval.NewAuto(retrieval.NewSinglePass(client), buildTreeWalkStrategy(c, client, store, pool)), } } -// buildPageIndexStrategy constructs the page-based agentic strategy +// buildTreeWalkStrategy constructs the page-based agentic strategy // with the storage-backed PageLoader, a DB-backed TOC provider, and // the configured caps. Ported from cmd/engine so the DEPLOYED -// cmd/server binary can serve retrieval.strategy=pageindex AND the -// /v1/answer/pageindex endpoint. +// cmd/server binary can serve retrieval.strategy=treewalk AND the +// /v1/answer/treewalk endpoint. // // The TOC provider reads documents.toc_tree via the worker-scoped // document lookup. The strategy degrades to its synthesised view // (built from the loaded section tree) whenever the column is NULL or // the read errors, so a document ingested before the TOC builder ran // still navigates cleanly. -func buildPageIndexStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, store storage.Storage, pool *db.Pool) *retrieval.PageIndexStrategy { - p := retrieval.NewPageIndexStrategy(client) +func buildTreeWalkStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, store storage.Storage, pool *db.Pool) *retrieval.TreeWalkStrategy { + p := retrieval.NewTreeWalkStrategy(client) p.PageLoader = storagePageLoader{s: store} if pool != nil { p.TOC = dbTOCProvider{db: pool} } - if c.PageIndex.MaxHops > 0 { - p.MaxHops = c.PageIndex.MaxHops + if c.TreeWalk.MaxHops > 0 { + p.MaxHops = c.TreeWalk.MaxHops } - if c.PageIndex.PageContentLimit > 0 { - p.PageContentLimit = c.PageIndex.PageContentLimit + if c.TreeWalk.PageContentLimit > 0 { + p.PageContentLimit = c.TreeWalk.PageContentLimit } - if c.PageIndex.MaxCitations > 0 { - p.MaxCitations = c.PageIndex.MaxCitations + if c.TreeWalk.MaxCitations > 0 { + p.MaxCitations = c.TreeWalk.MaxCitations } - p.ModelOverride = c.PageIndex.Model + p.ModelOverride = c.TreeWalk.Model return p } // storagePageLoader adapts a storage.Storage to // retrieval.PageContentLoader. Mirrors storageFetcher but lives behind -// a separate interface so the two callers (agentic / pageindex) can be -// wired independently. The PageIndex strategy materialises section +// a separate interface so the two callers (agentic / treewalk) can be +// wired independently. The TreeWalk strategy materialises section // bodies once per get_pages observation, so reading the full reader // into a []byte is the right shape. type storagePageLoader struct{ s storage.Storage } @@ -515,7 +518,7 @@ func (l storagePageLoader) Load(ctx context.Context, ref string) ([]byte, error) if err != nil { return nil, err } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close return io.ReadAll(rc) } @@ -556,7 +559,7 @@ func (sf storageFetcher) Get(ctx context.Context, ref string) ([]byte, error) { if err != nil { return nil, err } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close return io.ReadAll(rc) } diff --git a/config.example.yaml b/config.example.yaml index 4a91048..977fe24 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -95,7 +95,7 @@ llm: reasoning_model: "gemini-2.5-pro" retrieval: - # strategy: single-pass | chunked-tree | agentic | pageindex + # strategy: single-pass | chunked-tree | agentic | treewalk # # single-pass: whole tree in one LLM call; fastest, smallest docs. # chunked-tree: split the tree, reason over slices in parallel, merge. @@ -103,7 +103,7 @@ retrieval: # context for parallelism. # agentic: iterative outline → expand → read → done loop. # Picks per-section IDs via a tool-using model. - # pageindex: PageIndex-style page-based agentic loop. Three + # treewalk: TreeWalk-style page-based agentic loop. Three # tools (get_document_structure / get_pages / done); # the model navigates by INCLUSIVE PAGE RANGE # rather than by section ID. Best for paginated @@ -245,8 +245,8 @@ retrieval: # audit flows may bump this; tight memory budgets shrink it. ttl_seconds: 86400 - # pageindex: PageIndex-style page-based agentic strategy and its - # dedicated POST /v1/answer/pageindex endpoint. + # treewalk: TreeWalk-style page-based agentic strategy and its + # dedicated POST /v1/answer/treewalk endpoint. # # The strategy runs a three-tool loop: # 1. get_document_structure() — returns the TOC tree (titles + @@ -267,17 +267,17 @@ retrieval: # # OPT-OUT. Default enabled. Disable to unwire the endpoint # (returns 501); the strategy itself can still be selected by - # setting `retrieval.strategy: pageindex` even when this block + # setting `retrieval.strategy: treewalk` even when this block # is disabled. # # Works WITHOUT a persisted TOC tree (pre-PR-A state) — the # strategy synthesises a TOC view from the section list when # documents.toc_tree is NULL. No request fails because of a # missing TOC. - pageindex: + treewalk: enabled: true # Cap on LLM turns per request, including the terminal done - # turn. The reference PageIndex demo converges in 3-5 hops on + # turn. The reference TreeWalk demo converges in 3-5 hops on # typical questions; 8 leaves buffer for retries on parse # failures and the occasional extra get_pages call. max_hops: 8 @@ -294,7 +294,7 @@ retrieval: # miss all" while a single confident pick scores perfectly; # capping the final set tames the spray. 3 keeps a genuinely # multi-location answer (e.g. a figure + its footnote) while - # cutting the low-confidence tail. Env: VLE_/VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS. + # cutting the low-confidence tail. Env: VLE_/VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS. max_citations: 3 # Override the navigation-loop model; empty inherits the # request's model (which itself falls back to the engine @@ -317,7 +317,7 @@ ingest: # per-section LLM enrichment (summarize, HyDE, multi-axis, # TOC build) AND the pdftable table-extraction pass, so a # document becomes queryable in ~parse-speed (seconds). - # The page-based strategy (/v1/answer/pageindex) needs none + # The page-based strategy (/v1/answer/treewalk) needs none # of the skipped work: it navigates a TOC synthesised from # the section tree (documents.toc_tree is left NULL) and # reads raw section/page text at query time — and that raw @@ -428,7 +428,7 @@ ingest: max_entities: 8 max_numbers: 6 - # LLM-built table-of-contents tree (PageIndex-style). Runs after + # LLM-built table-of-contents tree (TreeWalk-style). Runs after # summarize+HyDE on PDF inputs and persists a hierarchical TOC on # documents.toc_tree (JSONB). The tree is small (a few KB even # for 300-page filings) and is intended as a higher-level map @@ -449,7 +449,7 @@ ingest: # (one call per leaf node). concurrency: 4 # The detector scans the first N pages for a table of - # contents. PageIndex defaults this to 20 — financial filings + # contents. TreeWalk defaults this to 20 — financial filings # put their TOC inside the first dozen pages and a document # without one by page 20 almost never has one further in. toc_check_pages: 20 diff --git a/config.server.example.yaml b/config.server.example.yaml index 9e56a7b..02d7835 100644 --- a/config.server.example.yaml +++ b/config.server.example.yaml @@ -105,7 +105,7 @@ engine: # filing. # minimal parse -> persist -> ready. Skips every LLM enrichment # stage AND table extraction — queryable in seconds. The - # page-based strategy (/v1/answer/pageindex) works on it + # page-based strategy (/v1/answer/treewalk) works on it # unchanged (synthesised TOC + raw page reads). # Flip the live service without a secret edit: VLS_INGEST_MODE=minimal. mode: "full" diff --git a/go.mod b/go.mod index 4cd83a3..45659ca 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/hallelx2/vectorless-engine go 1.25.0 +toolchain go1.25.11 + require ( cloud.google.com/go/storage v1.62.1 connectrpc.com/connect v1.20.0 @@ -26,7 +28,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 - golang.org/x/net v0.53.0 + golang.org/x/net v0.55.0 golang.org/x/sync v0.20.0 google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 @@ -118,11 +120,11 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.uber.org/goleak v1.3.0 // indirect - golang.org/x/crypto v0.50.0 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/image v0.39.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.274.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect diff --git a/go.sum b/go.sum index 44dfa72..3b3c895 100644 --- a/go.sum +++ b/go.sum @@ -257,20 +257,20 @@ go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOV go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww= golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= diff --git a/internal/api/replay_test.go b/internal/api/replay_test.go index e90a552..a6e17bd 100644 --- a/internal/api/replay_test.go +++ b/internal/api/replay_test.go @@ -52,7 +52,7 @@ func TestReplayByteExact(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want 200", resp.StatusCode) } @@ -78,7 +78,7 @@ func TestReplayUnknownToken(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode != http.StatusNotFound { t.Errorf("status = %d, want 404", resp.StatusCode) } @@ -105,7 +105,7 @@ func TestReplayDocumentIDMismatch(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode != http.StatusConflict { t.Fatalf("status = %d, want 409", resp.StatusCode) } @@ -140,7 +140,7 @@ func TestReplayQueryMismatch(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode != http.StatusConflict { t.Fatalf("status = %d, want 409", resp.StatusCode) } @@ -168,7 +168,7 @@ func TestReplayDisabled(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode != http.StatusNotImplemented { t.Errorf("status = %d, want 501", resp.StatusCode) } @@ -196,7 +196,7 @@ func TestReplayRequiresFields(t *testing.T) { if err != nil { t.Fatalf("case %d: %v", i, err) } - resp.Body.Close() + _ = resp.Body.Close() // best-effort close if resp.StatusCode != http.StatusBadRequest { t.Errorf("case %d: status = %d, want 400", i, resp.StatusCode) } @@ -214,7 +214,7 @@ func TestReplayBadJSON(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode != http.StatusBadRequest { t.Errorf("status = %d, want 400", resp.StatusCode) } @@ -289,7 +289,7 @@ func TestReplayEndToEndByteExact(t *testing.T) { if err != nil { t.Fatal(err) } - defer got.Body.Close() + defer func() { _ = got.Body.Close() }() // best-effort close if got.StatusCode != http.StatusOK { t.Fatalf("status = %d", got.StatusCode) } @@ -328,7 +328,7 @@ func TestReplayPreservesUnicodeAndWhitespace(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close got, _ := io.ReadAll(resp.Body) if !bytes.Equal(got, want) { t.Errorf("byte drift:\n got %q\n want %q", got, want) diff --git a/internal/api/server.go b/internal/api/server.go index e0dd375..c34fc14 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -99,27 +99,27 @@ type Deps struct { // rather than risk hallucinating an answer from weak evidence. Abstain config.AbstainBlock - // PageIndexStrategy is the dedicated page-based agentic strategy - // instance used by /v1/answer/pageindex. Wired in main.go from + // TreeWalkStrategy is the dedicated page-based agentic strategy + // instance used by /v1/answer/treewalk. Wired in main.go from // the same storage backend the rest of the engine uses, even // when the selection strategy chosen by retrieval.strategy is // something else. Nil disables the endpoint (returns 501) along - // with PageIndex.Enabled=false. - PageIndexStrategy *retrieval.PageIndexStrategy + // with TreeWalk.Enabled=false. + TreeWalkStrategy *retrieval.TreeWalkStrategy - // PageIndex carries the server-side config for the page-based + // TreeWalk carries the server-side config for the page-based // answer endpoint. The body-level fields max_hops / - // max_pages_per_fetch on /v1/answer/pageindex override - // PageIndex.MaxHops / PageIndex.PageContentLimit per request. - PageIndex config.PageIndexBlock + // max_pages_per_fetch on /v1/answer/treewalk override + // TreeWalk.MaxHops / TreeWalk.PageContentLimit per request. + TreeWalk config.TreeWalkBlock - // PageIndexTreeLoader is a test seam that overrides how the - // /v1/answer/pageindex handler resolves the document tree. + // TreeWalkTreeLoader is a test seam that overrides how the + // /v1/answer/treewalk handler resolves the document tree. // Nil routes through d.DB.LoadTree (the production path). // Tests set this to a deterministic in-memory function so the // handler can run end-to-end via httptest without a real // Postgres backend. - PageIndexTreeLoader func(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) + TreeWalkTreeLoader func(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) } // Router builds and returns the chi router wired with v1 routes. @@ -146,7 +146,7 @@ func Router(d Deps) http.Handler { r.Post("/query", d.handleQuery) r.Post("/query/multi", d.handleQueryMulti) r.Post("/answer", d.handleAnswer) - r.Post("/answer/pageindex", d.handleAnswerPageIndex) + r.Post("/answer/treewalk", d.handleAnswerTreeWalk) r.Post("/replay", d.handleReplay) }) @@ -242,7 +242,7 @@ func (d Deps) handleIngestDocument(w http.ResponseWriter, r *http.Request) { writeErr(w, http.StatusBadRequest, `missing form field "file"`) return } - defer file.Close() + defer func() { _ = file.Close() }() // best-effort close filename = header.Filename contentType = header.Header.Get("Content-Type") body = file @@ -395,7 +395,7 @@ func (d Deps) handleGetSection(w http.ResponseWriter, r *http.Request) { rc, _, err := d.Storage.Get(r.Context(), sec.ContentRef) if err == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } @@ -538,7 +538,7 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { rc, _, err := d.Storage.Get(r.Context(), sec.ContentRef) if err == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } @@ -835,7 +835,7 @@ func (d Deps) handleAnswer(w http.ResponseWriter, r *http.Request) { rc, _, err := d.Storage.Get(r.Context(), sec.ContentRef) if err == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } @@ -1077,7 +1077,7 @@ func (d Deps) handleQueryMulti(w http.ResponseWriter, r *http.Request) { rc, _, err := d.Storage.Get(r.Context(), sec.ContentRef) if err == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } diff --git a/internal/api/pageindex.go b/internal/api/treewalk.go similarity index 78% rename from internal/api/pageindex.go rename to internal/api/treewalk.go index 541799a..f6f8a54 100644 --- a/internal/api/pageindex.go +++ b/internal/api/treewalk.go @@ -2,8 +2,6 @@ package api import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -21,28 +19,28 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/tree" ) -// loadTreeForPageIndex resolves the document tree for the -// pageindex answer endpoint. Routes through the optional -// PageIndexTreeLoader hook when set (tests), otherwise falls +// loadTreeForTreeWalk resolves the document tree for the +// treewalk answer endpoint. Routes through the optional +// TreeWalkTreeLoader hook when set (tests), otherwise falls // through to the real DB. // // Kept here rather than inlined in the handler so the test seam is // obvious: production code path goes straight to d.DB.LoadTree; -// tests set d.PageIndexTreeLoader to an in-memory function. -func (d Deps) loadTreeForPageIndex(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) { - if d.PageIndexTreeLoader != nil { - return d.PageIndexTreeLoader(ctx, docID) +// tests set d.TreeWalkTreeLoader to an in-memory function. +func (d Deps) loadTreeForTreeWalk(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) { + if d.TreeWalkTreeLoader != nil { + return d.TreeWalkTreeLoader(ctx, docID) } return d.DB.LoadTree(ctx, docID, standaloneOrgID, "") } -// pageIndexAnswerRequest is the body shape for /v1/answer/pageindex. +// treeWalkAnswerRequest is the body shape for /v1/answer/treewalk. // // The endpoint mirrors /v1/answer's shape but exposes the // page-based loop's specific knobs (max_hops, max_pages_per_fetch) // plus a streaming variant. Per-request fields override the -// PageIndexBlock config when present. -type pageIndexAnswerRequest struct { +// TreeWalkBlock config when present. +type treeWalkAnswerRequest struct { DocumentID tree.DocumentID `json:"document_id"` Query string `json:"query"` Model string `json:"model"` @@ -52,7 +50,7 @@ type pageIndexAnswerRequest struct { IncludeReasoning bool `json:"reasoning"` } -// handleAnswerPageIndex runs the PageIndex agentic loop end-to-end +// handleAnswerTreeWalk runs the TreeWalk agentic loop end-to-end // and returns the model's answer + page-grounded citations in one // round-trip. // @@ -74,23 +72,23 @@ type pageIndexAnswerRequest struct { // // Body shape (canonical, non-streaming): // -// POST /v1/answer/pageindex +// POST /v1/answer/treewalk // { "document_id": "...", "query": "...", "model"?: "...", // "max_hops"?: 8, "max_pages_per_fetch"?: 16000, // "stream"?: false, "reasoning"?: false } // -// Response: see pageIndexAnswerResponse below. -func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { +// Response: see treeWalkAnswerResponse below. +func (d Deps) handleAnswerTreeWalk(w http.ResponseWriter, r *http.Request) { if d.LLM == nil { - writeErr(w, http.StatusNotImplemented, "answer/pageindex endpoint requires an LLM client") + writeErr(w, http.StatusNotImplemented, "answer/treewalk endpoint requires an LLM client") return } - if d.PageIndexStrategy == nil || !d.PageIndex.Enabled { - writeErr(w, http.StatusNotImplemented, "pageindex strategy not configured on this server (retrieval.pageindex.enabled=false)") + if d.TreeWalkStrategy == nil || !d.TreeWalk.Enabled { + writeErr(w, http.StatusNotImplemented, "treewalk strategy not configured on this server (retrieval.treewalk.enabled=false)") return } - var body pageIndexAnswerRequest + var body treeWalkAnswerRequest if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeErr(w, http.StatusBadRequest, "invalid json: "+err.Error()) return @@ -106,7 +104,7 @@ func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { body.IncludeReasoning = true } - t, err := d.loadTreeForPageIndex(r.Context(), body.DocumentID) + t, err := d.loadTreeForTreeWalk(r.Context(), body.DocumentID) if err != nil { if errors.Is(err, db.ErrNotFound) { writeErr(w, http.StatusNotFound, "document not found") @@ -117,11 +115,11 @@ func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { } // Build a per-request strategy that wraps the engine's - // configured PageIndexStrategy. We do this because per-request + // configured TreeWalkStrategy. We do this because per-request // overrides (max_hops, max_pages_per_fetch, model, OnEvent for // streaming) must NOT mutate the shared Deps instance — Deps // is read by many goroutines concurrently. - perReq := *d.PageIndexStrategy + perReq := *d.TreeWalkStrategy if body.MaxHops > 0 { perReq.MaxHops = body.MaxHops } @@ -141,7 +139,7 @@ func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { // Stream variant: hijack the response writer for SSE and emit // one event per tool call. if body.Stream { - d.serveAnswerPageIndexStream(w, r, &perReq, t, body, budget, started) + d.serveAnswerTreeWalkStream(w, r, &perReq, t, body, budget, started) return } @@ -152,7 +150,7 @@ func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { trace []map[string]any ) if body.IncludeReasoning { - perReq.OnEvent = func(ev retrieval.PageIndexEvent) { + perReq.OnEvent = func(ev retrieval.TreeWalkEvent) { traceMu.Lock() defer traceMu.Unlock() trace = append(trace, eventToTraceMap(ev)) @@ -161,12 +159,12 @@ func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { res, err := perReq.SelectWithCost(r.Context(), t, body.Query, budget) if err != nil { - d.Logger.Error("answer/pageindex: strategy failed", "err", err, "document_id", body.DocumentID) - writeErr(w, http.StatusInternalServerError, "pageindex strategy failed: "+err.Error()) + d.Logger.Error("answer/treewalk: strategy failed", "err", err, "document_id", body.DocumentID) + writeErr(w, http.StatusInternalServerError, "treewalk strategy failed: "+err.Error()) return } - citations := d.buildPageIndexCitations(r.Context(), t, res, body.Query, body.Model) + citations := d.buildTreeWalkCitations(r.Context(), t, res, body.Query, body.Model) resp := map[string]any{ "document_id": body.DocumentID, @@ -209,14 +207,14 @@ func (d Deps) handleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { }) } -// serveAnswerPageIndexStream handles the stream=true SSE variant. +// serveAnswerTreeWalkStream handles the stream=true SSE variant. // Each tool call emits one `event:` line so the caller can watch // the navigation in real time. The final event ("answer") carries // the full JSON response so the client doesn't need to make a // second request. // // SSE format: `event: \ndata: \n\n` per the W3C spec. -func (d Deps) serveAnswerPageIndexStream(w http.ResponseWriter, r *http.Request, strat *retrieval.PageIndexStrategy, t *tree.Tree, body pageIndexAnswerRequest, budget retrieval.ContextBudget, started time.Time) { +func (d Deps) serveAnswerTreeWalkStream(w http.ResponseWriter, r *http.Request, strat *retrieval.TreeWalkStrategy, t *tree.Tree, body treeWalkAnswerRequest, budget retrieval.ContextBudget, started time.Time) { flusher, ok := w.(http.Flusher) if !ok { writeErr(w, http.StatusInternalServerError, "streaming requires http.Flusher; response writer does not support it") @@ -239,11 +237,11 @@ func (d Deps) serveAnswerPageIndexStream(w http.ResponseWriter, r *http.Request, } writeMu.Lock() defer writeMu.Unlock() - fmt.Fprintf(w, "event: %s\ndata: %s\n\n", eventType, raw) + _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", eventType, raw) flusher.Flush() } - strat.OnEvent = func(ev retrieval.PageIndexEvent) { + strat.OnEvent = func(ev retrieval.TreeWalkEvent) { emitSSE(ev.Type, ev) } @@ -261,7 +259,7 @@ func (d Deps) serveAnswerPageIndexStream(w http.ResponseWriter, r *http.Request, return } - citations := d.buildPageIndexCitations(r.Context(), t, res, body.Query, body.Model) + citations := d.buildTreeWalkCitations(r.Context(), t, res, body.Query, body.Model) final := map[string]any{ "document_id": body.DocumentID, "query": body.Query, @@ -285,7 +283,7 @@ func (d Deps) serveAnswerPageIndexStream(w http.ResponseWriter, r *http.Request, emitSSE("answer", final) } -// buildPageIndexCitations transforms the strategy's PagesRead + +// buildTreeWalkCitations transforms the strategy's PagesRead + // the section tree into the response's citations array. // // One citation per cited page range (deduplicated). Each citation @@ -296,7 +294,7 @@ func (d Deps) serveAnswerPageIndexStream(w http.ResponseWriter, r *http.Request, // - quote / quote_start / quote_end: pulled via the existing // SpanExtractor over the concatenated cited-page content. If the // extractor finds no match the quote field is empty (offsets -1). -func (d Deps) buildPageIndexCitations(ctx context.Context, t *tree.Tree, res *retrieval.Result, query, requestModel string) []map[string]any { +func (d Deps) buildTreeWalkCitations(ctx context.Context, t *tree.Tree, res *retrieval.Result, query, requestModel string) []map[string]any { if res == nil { return nil } @@ -330,7 +328,7 @@ func (d Deps) buildPageIndexCitations(ctx context.Context, t *tree.Tree, res *re if d.LLM != nil { content := d.materialiseCitedContent(ctx, t, sectionIDs) if strings.TrimSpace(content) != "" { - ext := d.pageIndexSpanExtractor(requestModel) + ext := d.treeWalkSpanExtractor(requestModel) span, _, err := ext.Extract(ctx, content, query) if err == nil && span != nil && span.Text != "" { c["quote"] = span.Text @@ -404,11 +402,11 @@ func (d Deps) materialiseCitedContent(ctx context.Context, t *tree.Tree, section return b.String() } -// pageIndexSpanExtractor builds a SpanExtractor configured for the -// /v1/answer/pageindex endpoint. Same fall-through pattern as the +// treeWalkSpanExtractor builds a SpanExtractor configured for the +// /v1/answer/treewalk endpoint. Same fall-through pattern as the // existing spanExtractor helper (config override → request model → // engine default). -func (d Deps) pageIndexSpanExtractor(requestModel string) *retrieval.SpanExtractor { +func (d Deps) treeWalkSpanExtractor(requestModel string) *retrieval.SpanExtractor { model := d.AnswerSpan.Model if model == "" { model = requestModel @@ -423,10 +421,10 @@ func (d Deps) pageIndexSpanExtractor(requestModel string) *retrieval.SpanExtract return ext } -// eventToTraceMap converts a PageIndexEvent into the +// eventToTraceMap converts a TreeWalkEvent into the // reasoning_trace entry shape. Only documented fields ship — // nothing internal leaks via the trace. -func eventToTraceMap(ev retrieval.PageIndexEvent) map[string]any { +func eventToTraceMap(ev retrieval.TreeWalkEvent) map[string]any { args := map[string]any{} switch ev.Type { case "get_pages": @@ -469,8 +467,8 @@ func eventToTraceMap(ev retrieval.PageIndexEvent) map[string]any { return entry } -// pageIndexTraceTokenFromCitations exposes the same hash a -// PageIndexStrategy emits to callers who want to recompute the +// treeWalkTraceTokenFromCitations exposes the same hash a +// TreeWalkStrategy emits to callers who want to recompute the // token client-side. The page-range string form mirrors the one // the strategy uses internally so the two stay in lock-step. // @@ -478,39 +476,12 @@ func eventToTraceMap(ev retrieval.PageIndexEvent) map[string]any { // the in-response trace_token against the canonical input set — // kept here rather than exported from the retrieval package so // the API layer owns its own input wiring. -func pageIndexTraceTokenFromCitations(docID tree.DocumentID, model string, ranges [][2]int) string { - strs := make([]string, 0, len(ranges)) - for _, r := range ranges { - if r[0] == r[1] { - strs = append(strs, fmt.Sprintf("%d", r[0])) - } else { - strs = append(strs, fmt.Sprintf("%d-%d", r[0], r[1])) - } - } - sort.Strings(strs) - h := sha256.New() - h.Write([]byte(string(docID))) - h.Write([]byte{0}) - h.Write([]byte("1-pages")) - h.Write([]byte{0}) - h.Write([]byte("pageindex:" + model)) - h.Write([]byte{0}) - h.Write([]byte(retrieval.SystemPromptVersion)) - for i, s := range strs { - if i == 0 { - h.Write([]byte{0}) - } else { - h.Write([]byte{0}) - } - h.Write([]byte("p:" + s)) - } - return hex.EncodeToString(h.Sum(nil)) -} +// -// Compile-time guard: the PageIndex strategy must satisfy +// Compile-time guard: the TreeWalk strategy must satisfy // retrieval.CostStrategy so SelectWithCost works without a // type-assert dance. -var _ retrieval.CostStrategy = (*retrieval.PageIndexStrategy)(nil) +var _ retrieval.CostStrategy = (*retrieval.TreeWalkStrategy)(nil) // Compile-time check that the Deps fields the handler reads are // the only API-layer dependencies it pulls in. If a future edit diff --git a/internal/api/pageindex_test.go b/internal/api/treewalk_test.go similarity index 77% rename from internal/api/pageindex_test.go rename to internal/api/treewalk_test.go index ca31cc7..708cfd0 100644 --- a/internal/api/pageindex_test.go +++ b/internal/api/treewalk_test.go @@ -24,15 +24,15 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/tree" ) -// pageIndexScriptedLLM is the same shape as the strategy test's +// treeWalkScriptedLLM is the same shape as the strategy test's // scripted LLM but mirrored here so the api package's tests don't // reach into pkg/retrieval's test file. -type pageIndexScriptedLLM struct { +type treeWalkScriptedLLM struct { replies []string calls int32 } -func (p *pageIndexScriptedLLM) Complete(ctx context.Context, req llmgate.Request) (*llmgate.Response, error) { +func (p *treeWalkScriptedLLM) Complete(ctx context.Context, req llmgate.Request) (*llmgate.Response, error) { i := int(atomic.AddInt32(&p.calls, 1)) - 1 if i >= len(p.replies) { return nil, fmt.Errorf("scripted LLM exhausted at call %d", i+1) @@ -40,12 +40,12 @@ func (p *pageIndexScriptedLLM) Complete(ctx context.Context, req llmgate.Request return &llmgate.Response{Content: p.replies[i]}, nil } -func (p *pageIndexScriptedLLM) CountTokens(ctx context.Context, t string) (int, error) { +func (p *treeWalkScriptedLLM) CountTokens(ctx context.Context, t string) (int, error) { return len(t) / 4, nil } // inMemoryStorage is a minimal storage.Storage backed by a map. -// Only Get is meaningful for the pageindex handler tests. +// Only Get is meaningful for the treewalk handler tests. type inMemoryStorage struct { data map[string][]byte } @@ -81,20 +81,20 @@ func (m *inMemoryStorage) SignedURL(ctx context.Context, key string, expiry time return "", nil } -// pageIndexHandlerRouter wires only the endpoint under test. We +// treeWalkHandlerRouter wires only the endpoint under test. We // don't want middleware noise interfering with the assertion path. -func pageIndexHandlerRouter(d Deps) http.Handler { +func treeWalkHandlerRouter(d Deps) http.Handler { r := chi.NewRouter() r.Route("/v1", func(r chi.Router) { - r.Post("/answer/pageindex", d.handleAnswerPageIndex) + r.Post("/answer/treewalk", d.handleAnswerTreeWalk) }) return r } -// buildPageIndexTestTree mirrors the strategy tests' tree so +// buildTreeWalkTestTree mirrors the strategy tests' tree so // assertions about which section IDs surface in citations stay // consistent across the two suites. -func buildPageIndexTestTree() *tree.Tree { +func buildTreeWalkTestTree() *tree.Tree { a1 := &tree.Section{ID: "sec_a1", ParentID: "sec_a", Title: "Install", Summary: "install steps", ContentRef: "a1_ref", PageStart: 1, PageEnd: 2} a2 := &tree.Section{ID: "sec_a2", ParentID: "sec_a", Title: "Config", Summary: "config keys", ContentRef: "a2_ref", PageStart: 3, PageEnd: 4} b1 := &tree.Section{ID: "sec_b1", ParentID: "sec_b", Title: "Querying", Summary: "how to query", ContentRef: "b1_ref", PageStart: 5, PageEnd: 7} @@ -105,41 +105,41 @@ func buildPageIndexTestTree() *tree.Tree { return &tree.Tree{DocumentID: "doc_x", Title: "Atlas", Root: root} } -// newTestDeps wires the minimum surface for the pageindex handler +// newTestDeps wires the minimum surface for the treewalk handler // to run end-to-end against httptest. The strategy is constructed // directly (no DB / cache wrapper) so per-test LLM scripting // drives behaviour deterministically. -func newTestDeps(t *testing.T, replies ...string) (Deps, *pageIndexScriptedLLM, *inMemoryStorage) { +func newTestDeps(t *testing.T, replies ...string) (Deps, *treeWalkScriptedLLM, *inMemoryStorage) { t.Helper() - llm := &pageIndexScriptedLLM{replies: replies} + llm := &treeWalkScriptedLLM{replies: replies} store := &inMemoryStorage{data: map[string][]byte{ "a1_ref": []byte("Install steps: run vle ingest..."), "a2_ref": []byte("Config keys: VLE_FOO, VLE_BAR."), "b1_ref": []byte("How to query the API."), "b2_ref": []byte("Debt registration is in line items A and B."), }} - strat := retrieval.NewPageIndexStrategy(llm) + strat := retrieval.NewTreeWalkStrategy(llm) strat.PageLoader = pageStorageLoader{s: store} deps := Deps{ - Logger: slog.Default(), - Storage: store, - LLM: llm, - LLMModel: "test-model", - Strategy: strat, // unrelated to /v1/answer/pageindex; populated for sanity - PageIndexStrategy: strat, - PageIndex: config.PageIndexBlock{Enabled: true, MaxHops: 8, PageContentLimit: 16000}, - AnswerSpan: config.AnswerSpanBlock{Enabled: false}, + Logger: slog.Default(), + Storage: store, + LLM: llm, + LLMModel: "test-model", + Strategy: strat, // unrelated to /v1/answer/treewalk; populated for sanity + TreeWalkStrategy: strat, + TreeWalk: config.TreeWalkBlock{Enabled: true, MaxHops: 8, PageContentLimit: 16000}, + AnswerSpan: config.AnswerSpanBlock{Enabled: false}, Replay: retrieval.NewLRUReplayStore(retrieval.LRUReplayConfig{ MaxEntries: 16, TTL: 5 * time.Minute, }), - PageIndexTreeLoader: func(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) { + TreeWalkTreeLoader: func(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) { if docID != "doc_x" { return nil, fmt.Errorf("unknown document %q (test loader only knows doc_x)", docID) } - return buildPageIndexTestTree(), nil + return buildTreeWalkTestTree(), nil }, } return deps, llm, store @@ -160,13 +160,13 @@ func (l pageStorageLoader) Load(ctx context.Context, ref string) ([]byte, error) return io.ReadAll(rc) } -// TestHandleAnswerPageIndexHappyPath: the canonical 3-tool +// TestHandleAnswerTreeWalkHappyPath: the canonical 3-tool // sequence ends with a JSON response carrying answer, citations, // hops_taken, trace_token, pages_read, and a usage block. The // LLM is NOT called for span extraction in this test path because // AnswerSpan.Enabled is false at the config-block level — but the // citations still surface section_ids and page ranges. -func TestHandleAnswerPageIndexHappyPath(t *testing.T) { +func TestHandleAnswerTreeWalkHappyPath(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t, @@ -176,9 +176,9 @@ func TestHandleAnswerPageIndexHappyPath(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"how do I install?","model":"test-model"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) @@ -190,8 +190,8 @@ func TestHandleAnswerPageIndexHappyPath(t *testing.T) { if resp["answer"].(string) != "Run vle ingest." { t.Errorf("answer = %v, want \"Run vle ingest.\"", resp["answer"]) } - if resp["strategy"].(string) != "pageindex" { - t.Errorf("strategy = %v, want pageindex", resp["strategy"]) + if resp["strategy"].(string) != "treewalk" { + t.Errorf("strategy = %v, want treewalk", resp["strategy"]) } if resp["hops_taken"].(float64) != 3 { t.Errorf("hops_taken = %v, want 3", resp["hops_taken"]) @@ -218,10 +218,10 @@ func TestHandleAnswerPageIndexHappyPath(t *testing.T) { } } -// TestHandleAnswerPageIndexReasoningTrace: with reasoning=true, +// TestHandleAnswerTreeWalkReasoningTrace: with reasoning=true, // the response carries a reasoning_trace array describing each // tool call. Each entry must have hop + tool + (optional) args. -func TestHandleAnswerPageIndexReasoningTrace(t *testing.T) { +func TestHandleAnswerTreeWalkReasoningTrace(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t, @@ -231,9 +231,9 @@ func TestHandleAnswerPageIndexReasoningTrace(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"config?","reasoning":true}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) var resp map[string]any _ = json.Unmarshal(rec.Body.Bytes(), &resp) @@ -261,14 +261,14 @@ func TestHandleAnswerPageIndexReasoningTrace(t *testing.T) { } } -// TestHandleAnswerPageIndexDedupAndCapCitations is the end-to-end +// TestHandleAnswerTreeWalkDedupAndCapCitations is the end-to-end // proof of the bench-facing fix: a done that sprays the SAME range // five times plus extras must produce a citations[] array that is // deduped and capped at MaxCitations — no duplicate page ranges, no // repeated section ids across citations, and confidence surfaced. // This is the API-layer mirror of the strategy's dedup test and the // reason precision@5 stops deflating. -func TestHandleAnswerPageIndexDedupAndCapCitations(t *testing.T) { +func TestHandleAnswerTreeWalkDedupAndCapCitations(t *testing.T) { t.Parallel() // Read one range, then a done that cites [1,2] five times plus @@ -279,9 +279,9 @@ func TestHandleAnswerPageIndexDedupAndCapCitations(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"q"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } @@ -321,12 +321,12 @@ func TestHandleAnswerPageIndexDedupAndCapCitations(t *testing.T) { } } -// TestHandleAnswerPageIndexConfidentSingleCitation is the happy half +// TestHandleAnswerTreeWalkConfidentSingleCitation is the happy half // at the API layer: a confident single-range done — even after the // model skimmed several pages — surfaces exactly ONE citation. This // is the f1=1.0 commit case, and the fix that stops a multi-page // navigation footprint from leaking into citations[]. -func TestHandleAnswerPageIndexConfidentSingleCitation(t *testing.T) { +func TestHandleAnswerTreeWalkConfidentSingleCitation(t *testing.T) { t.Parallel() // The model reads pages 1-2 AND 8-9 while searching, but commits @@ -338,9 +338,9 @@ func TestHandleAnswerPageIndexConfidentSingleCitation(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"debt?"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } @@ -362,10 +362,10 @@ func TestHandleAnswerPageIndexConfidentSingleCitation(t *testing.T) { } } -// TestHandleAnswerPageIndexReasoningTraceQueryParam: the +// TestHandleAnswerTreeWalkReasoningTraceQueryParam: the // ?reasoning=true query param is an alternative to the body field. // Some clients prefer it for GET-friendliness when prototyping. -func TestHandleAnswerPageIndexReasoningTraceQueryParam(t *testing.T) { +func TestHandleAnswerTreeWalkReasoningTraceQueryParam(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t, @@ -373,9 +373,9 @@ func TestHandleAnswerPageIndexReasoningTraceQueryParam(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"q"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex?reasoning=true", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk?reasoning=true", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) var resp map[string]any _ = json.Unmarshal(rec.Body.Bytes(), &resp) @@ -384,9 +384,9 @@ func TestHandleAnswerPageIndexReasoningTraceQueryParam(t *testing.T) { } } -// TestHandleAnswerPageIndexBadRequest: missing document_id / +// TestHandleAnswerTreeWalkBadRequest: missing document_id / // query → 400. -func TestHandleAnswerPageIndexBadRequest(t *testing.T) { +func TestHandleAnswerTreeWalkBadRequest(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t) @@ -397,89 +397,89 @@ func TestHandleAnswerPageIndexBadRequest(t *testing.T) { `{"query":"q"}`, // missing document_id `not-json`, } { - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", strings.NewReader(body)) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", strings.NewReader(body)) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusBadRequest { t.Errorf("body %q: status = %d, want 400", body, rec.Code) } } } -// TestHandleAnswerPageIndexDocumentNotFound: a tree-loader that +// TestHandleAnswerTreeWalkDocumentNotFound: a tree-loader that // returns ErrNotFound bubbles up as 404. The test loader rejects // unknown doc IDs. -func TestHandleAnswerPageIndexDocumentNotFound(t *testing.T) { +func TestHandleAnswerTreeWalkDocumentNotFound(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t) // Re-wire the loader to return ErrNotFound for the right error // path. The default test loader returns a generic error // (different status — also valid but less specific). - deps.PageIndexTreeLoader = func(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) { + deps.TreeWalkTreeLoader = func(ctx context.Context, docID tree.DocumentID) (*tree.Tree, error) { return nil, dbNotFoundError() } body := strings.NewReader(`{"document_id":"missing","query":"q"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusNotFound { t.Errorf("status = %d, want 404 (body: %s)", rec.Code, rec.Body.String()) } } -// TestHandleAnswerPageIndexDisabled: when PageIndex.Enabled=false -// or PageIndexStrategy is nil, the endpoint returns 501. Two +// TestHandleAnswerTreeWalkDisabled: when TreeWalk.Enabled=false +// or TreeWalkStrategy is nil, the endpoint returns 501. Two // failure modes, both must produce the same status. -func TestHandleAnswerPageIndexDisabled(t *testing.T) { +func TestHandleAnswerTreeWalkDisabled(t *testing.T) { t.Parallel() // Mode 1: config disabled. deps, _, _ := newTestDeps(t) - deps.PageIndex.Enabled = false + deps.TreeWalk.Enabled = false body := strings.NewReader(`{"document_id":"doc_x","query":"q"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusNotImplemented { t.Errorf("config disabled: status = %d, want 501", rec.Code) } // Mode 2: strategy nil. deps2, _, _ := newTestDeps(t) - deps2.PageIndexStrategy = nil + deps2.TreeWalkStrategy = nil body = strings.NewReader(`{"document_id":"doc_x","query":"q"}`) - req = httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req = httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec = httptest.NewRecorder() - pageIndexHandlerRouter(deps2).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps2).ServeHTTP(rec, req) if rec.Code != http.StatusNotImplemented { t.Errorf("strategy nil: status = %d, want 501", rec.Code) } } -// TestHandleAnswerPageIndexNoLLM: no LLM client → 501. -func TestHandleAnswerPageIndexNoLLM(t *testing.T) { +// TestHandleAnswerTreeWalkNoLLM: no LLM client → 501. +func TestHandleAnswerTreeWalkNoLLM(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t) deps.LLM = nil body := strings.NewReader(`{"document_id":"doc_x","query":"q"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusNotImplemented { t.Errorf("status = %d, want 501", rec.Code) } } -// TestHandleAnswerPageIndexReplayPersisted: the response is +// TestHandleAnswerTreeWalkReplayPersisted: the response is // stored in the replay store under its trace_token, and the // existing /v1/replay handler returns the byte-identical body. -func TestHandleAnswerPageIndexReplayPersisted(t *testing.T) { +func TestHandleAnswerTreeWalkReplayPersisted(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t, @@ -487,9 +487,9 @@ func TestHandleAnswerPageIndexReplayPersisted(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"replay-me"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("first call: status = %d, body = %s", rec.Code, rec.Body.String()) } @@ -514,7 +514,7 @@ func TestHandleAnswerPageIndexReplayPersisted(t *testing.T) { if rec2.Code != http.StatusOK { t.Fatalf("replay status = %d, body = %s", rec2.Code, rec2.Body.String()) } - // The original /v1/answer/pageindex response carries a trailing + // The original /v1/answer/treewalk response carries a trailing // newline from marshalJSONForReplay; the replay path returns // the exact stored bytes, so we compare with the newline. if !bytes.Equal(originalBody, rec2.Body.Bytes()) { @@ -522,10 +522,10 @@ func TestHandleAnswerPageIndexReplayPersisted(t *testing.T) { } } -// TestHandleAnswerPageIndexStreaming: with stream=true, the +// TestHandleAnswerTreeWalkStreaming: with stream=true, the // response is SSE with one event per tool call plus a started + // answer event. The data payloads are JSON. -func TestHandleAnswerPageIndexStreaming(t *testing.T) { +func TestHandleAnswerTreeWalkStreaming(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t, @@ -535,9 +535,9 @@ func TestHandleAnswerPageIndexStreaming(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"q","stream":true}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("stream status = %d", rec.Code) } @@ -558,13 +558,13 @@ func TestHandleAnswerPageIndexStreaming(t *testing.T) { } } -// TestHandleAnswerPageIndexPerRequestOverrides: max_hops and +// TestHandleAnswerTreeWalkPerRequestOverrides: max_hops and // max_pages_per_fetch on the body override the engine's config. // We can't measure max_pages_per_fetch from outside (it shapes // content size, not response shape), but we can verify max_hops // caps the loop. Set max_hops=1 and a script that emits // 5 turns — the strategy must stop after 1. -func TestHandleAnswerPageIndexPerRequestOverrides(t *testing.T) { +func TestHandleAnswerTreeWalkPerRequestOverrides(t *testing.T) { t.Parallel() // 6 replies but max_hops=1 → only the first runs as a normal @@ -580,9 +580,9 @@ func TestHandleAnswerPageIndexPerRequestOverrides(t *testing.T) { ) body := strings.NewReader(`{"document_id":"doc_x","query":"q","max_hops":1}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } @@ -596,25 +596,25 @@ func TestHandleAnswerPageIndexPerRequestOverrides(t *testing.T) { } } -// TestHandleAnswerPageIndexTOCFallback: with a tree that has +// TestHandleAnswerTreeWalkTOCFallback: with a tree that has // page metadata but no persisted TOC, the synthesised TOC drives // the get_document_structure tool. This test runs end-to-end and // asserts the response shape; the strategy-level test covers the // synthesis logic directly. -func TestHandleAnswerPageIndexTOCFallback(t *testing.T) { +func TestHandleAnswerTreeWalkTOCFallback(t *testing.T) { t.Parallel() deps, _, _ := newTestDeps(t, `{"tool":"get_document_structure"}`, `{"tool":"done","answer":"saw the toc","cited_pages":[]}`, ) - // PageIndexStrategy.TOC is left nil — the synthesised path is + // TreeWalkStrategy.TOC is left nil — the synthesised path is // the default for any deployment without PR-A merged. body := strings.NewReader(`{"document_id":"doc_x","query":"what is in the doc?"}`) - req := httptest.NewRequest(http.MethodPost, "/v1/answer/pageindex", body) + req := httptest.NewRequest(http.MethodPost, "/v1/answer/treewalk", body) rec := httptest.NewRecorder() - pageIndexHandlerRouter(deps).ServeHTTP(rec, req) + treeWalkHandlerRouter(deps).ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } diff --git a/internal/config/config.go b/internal/config/config.go index a44d5e8..53d2950 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -326,14 +326,14 @@ func applyEnvOverrides(c *Config) { if v := firstEnv("VLS_INGEST_MODE", "VLE_INGEST_MODE"); v != "" { c.Engine.Ingest.Mode = v } - // PageIndex final-citation cap. Forwarded so the spray backstop can + // TreeWalk final-citation cap. Forwarded so the spray backstop can // be tuned on the live server without a config edit (e.g. tighten // to 1 to force single-citation answers, or relax for a doc set // with many genuinely multi-location questions). VLS_ wins over // VLE_; a garbled or negative value leaves the engine default (3). - if v := firstEnv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS"); v != "" { + if v := firstEnv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS", "VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS"); v != "" { if n, err := strconv.Atoi(v); err == nil && n >= 0 { - c.Engine.Retrieval.PageIndex.MaxCitations = n + c.Engine.Retrieval.TreeWalk.MaxCitations = n } } // Total-parse timeout (seconds). Forwarded so the deployed server diff --git a/internal/config/config_pageindex_test.go b/internal/config/config_pageindex_test.go deleted file mode 100644 index 326d8b9..0000000 --- a/internal/config/config_pageindex_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package config - -import ( - "os" - "testing" -) - -// TestForwardPageIndexMaxCitations covers the server-config -// pass-through for the PageIndex final-citation cap: the operator can -// set it via VLS_ or VLE_ and have it reach the embedded engine -// config, with VLS_ winning when both are present and a garbled value -// preserving the engine default. -func TestForwardPageIndexMaxCitations(t *testing.T) { - prevVLS := os.Getenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - prevVLE := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - defer func() { - os.Setenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", prevVLS) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", prevVLE) - }() - - // Engine default is 3 with no env set. - os.Unsetenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - os.Unsetenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - cfg := Default() - applyEnvOverrides(&cfg) - if cfg.Engine.Retrieval.PageIndex.MaxCitations != 3 { - t.Errorf("default max_citations = %d, want 3", cfg.Engine.Retrieval.PageIndex.MaxCitations) - } - - // VLE_ alone forwards through. - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "2") - cfg2 := Default() - applyEnvOverrides(&cfg2) - if cfg2.Engine.Retrieval.PageIndex.MaxCitations != 2 { - t.Errorf("VLE_ forward: max_citations = %d, want 2", cfg2.Engine.Retrieval.PageIndex.MaxCitations) - } - - // VLS_ wins over VLE_. - os.Setenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "1") - cfg3 := Default() - applyEnvOverrides(&cfg3) - if cfg3.Engine.Retrieval.PageIndex.MaxCitations != 1 { - t.Errorf("VLS_ should win: max_citations = %d, want 1", cfg3.Engine.Retrieval.PageIndex.MaxCitations) - } - - // Garbled value preserves the engine default (3), does not zero it. - os.Unsetenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "heaps") - cfg4 := Default() - applyEnvOverrides(&cfg4) - if cfg4.Engine.Retrieval.PageIndex.MaxCitations != 3 { - t.Errorf("garbage value should preserve default 3, got %d", cfg4.Engine.Retrieval.PageIndex.MaxCitations) - } -} diff --git a/internal/config/config_treewalk_test.go b/internal/config/config_treewalk_test.go new file mode 100644 index 0000000..10143b8 --- /dev/null +++ b/internal/config/config_treewalk_test.go @@ -0,0 +1,54 @@ +package config + +import ( + "os" + "testing" +) + +// TestForwardTreeWalkMaxCitations covers the server-config +// pass-through for the TreeWalk final-citation cap: the operator can +// set it via VLS_ or VLE_ and have it reach the embedded engine +// config, with VLS_ winning when both are present and a garbled value +// preserving the engine default. +func TestForwardTreeWalkMaxCitations(t *testing.T) { + prevVLS := os.Getenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS") + prevVLE := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS") + defer func() { + os.Setenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS", prevVLS) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", prevVLE) + }() + + // Engine default is 3 with no env set. + os.Unsetenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS") + os.Unsetenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS") + cfg := Default() + applyEnvOverrides(&cfg) + if cfg.Engine.Retrieval.TreeWalk.MaxCitations != 3 { + t.Errorf("default max_citations = %d, want 3", cfg.Engine.Retrieval.TreeWalk.MaxCitations) + } + + // VLE_ alone forwards through. + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", "2") + cfg2 := Default() + applyEnvOverrides(&cfg2) + if cfg2.Engine.Retrieval.TreeWalk.MaxCitations != 2 { + t.Errorf("VLE_ forward: max_citations = %d, want 2", cfg2.Engine.Retrieval.TreeWalk.MaxCitations) + } + + // VLS_ wins over VLE_. + os.Setenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS", "1") + cfg3 := Default() + applyEnvOverrides(&cfg3) + if cfg3.Engine.Retrieval.TreeWalk.MaxCitations != 1 { + t.Errorf("VLS_ should win: max_citations = %d, want 1", cfg3.Engine.Retrieval.TreeWalk.MaxCitations) + } + + // Garbled value preserves the engine default (3), does not zero it. + os.Unsetenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", "heaps") + cfg4 := Default() + applyEnvOverrides(&cfg4) + if cfg4.Engine.Retrieval.TreeWalk.MaxCitations != 3 { + t.Errorf("garbage value should preserve default 3, got %d", cfg4.Engine.Retrieval.TreeWalk.MaxCitations) + } +} diff --git a/internal/connecthandler/documents.go b/internal/connecthandler/documents.go index e7f805e..c85b9f3 100644 --- a/internal/connecthandler/documents.go +++ b/internal/connecthandler/documents.go @@ -284,7 +284,7 @@ func (s *DocumentsService) GetDocumentSource( if err != nil { return connect.NewError(connect.CodeInternal, fmt.Errorf("read source: %w", err)) } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close // Stream in 32 KiB chunks. buf := make([]byte, 32*1024) diff --git a/internal/connecthandler/query.go b/internal/connecthandler/query.go index 407d2e6..0490174 100644 --- a/internal/connecthandler/query.go +++ b/internal/connecthandler/query.go @@ -276,12 +276,12 @@ func (s *QueryService) QueryMultiStream( for mevt := range events { evt := mevt.Event protoEvt := &v1.QueryMultiStreamEvent{ - DocumentId: string(mevt.DocumentID), - DocIndex: int32(mevt.DocIndex), - Type: string(evt.Type), - SliceIndex: int32(evt.SliceIndex), + DocumentId: string(mevt.DocumentID), + DocIndex: int32(mevt.DocIndex), + Type: string(evt.Type), + SliceIndex: int32(evt.SliceIndex), TotalSlices: int32(evt.TotalSlices), - Message: evt.Message, + Message: evt.Message, } if evt.Type == retrieval.EventStarted { @@ -309,6 +309,6 @@ func fetchContent(ctx context.Context, store storage.Storage, ref string) string return "" } raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close return string(raw) } diff --git a/internal/handler/answer.go b/internal/handler/answer.go index b9f1626..9f53ab3 100644 --- a/internal/handler/answer.go +++ b/internal/handler/answer.go @@ -169,7 +169,7 @@ func (h *AnswerHandler) HandleAnswer(w http.ResponseWriter, r *http.Request) { rc, _, getErr := h.storage.Get(r.Context(), sec.ContentRef) if getErr == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } diff --git a/internal/handler/answer_pageindex.go b/internal/handler/answer_treewalk.go similarity index 82% rename from internal/handler/answer_pageindex.go rename to internal/handler/answer_treewalk.go index 3d890d5..d917957 100644 --- a/internal/handler/answer_pageindex.go +++ b/internal/handler/answer_treewalk.go @@ -22,15 +22,15 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/tree" ) -// AnswerPageIndexHandler implements POST /v1/answer/pageindex: it runs -// the PageIndex agentic loop end-to-end and returns the model's answer +// AnswerTreeWalkHandler implements POST /v1/answer/treewalk: it runs +// the TreeWalk agentic loop end-to-end and returns the model's answer // plus page-grounded citations in one round-trip. Unlike /v1/answer, // the loop owns the answer — there is no separate synthesis call. // -// Ported from cmd/engine's internal/api.handleAnswerPageIndex, adapted +// Ported from cmd/engine's internal/api.handleAnswerTreeWalk, adapted // to the deployed server's multi-tenant model (org + store from // headers). -type AnswerPageIndexHandler struct { +type AnswerTreeWalkHandler struct { logger *slog.Logger db *db.Pool storage storage.Storage @@ -38,8 +38,8 @@ type AnswerPageIndexHandler struct { llmModel string answerSpan enginecfg.AnswerSpanBlock replay retrieval.ReplayStore - strategy *retrieval.PageIndexStrategy - pageIndex enginecfg.PageIndexBlock + strategy *retrieval.TreeWalkStrategy + treeWalk enginecfg.TreeWalkBlock // treeLoader is a test seam overriding how the handler resolves // the document tree. Nil routes through the org-scoped DB lookup @@ -49,10 +49,10 @@ type AnswerPageIndexHandler struct { treeLoader func(ctx context.Context, orgID, storeID string, docID tree.DocumentID) (*tree.Tree, error) } -// NewAnswerPageIndexHandler creates an AnswerPageIndexHandler. llm, +// NewAnswerTreeWalkHandler creates an AnswerTreeWalkHandler. llm, // replay, and strategy may be nil; a nil llm or strategy (or -// PageIndex.Enabled=false) makes the endpoint return 501. -func NewAnswerPageIndexHandler( +// TreeWalk.Enabled=false) makes the endpoint return 501. +func NewAnswerTreeWalkHandler( logger *slog.Logger, pool *db.Pool, store storage.Storage, @@ -60,10 +60,10 @@ func NewAnswerPageIndexHandler( llmModel string, answerSpan enginecfg.AnswerSpanBlock, replay retrieval.ReplayStore, - strategy *retrieval.PageIndexStrategy, - pageIndex enginecfg.PageIndexBlock, -) *AnswerPageIndexHandler { - return &AnswerPageIndexHandler{ + strategy *retrieval.TreeWalkStrategy, + treeWalk enginecfg.TreeWalkBlock, +) *AnswerTreeWalkHandler { + return &AnswerTreeWalkHandler{ logger: logger, db: pool, storage: store, @@ -72,21 +72,21 @@ func NewAnswerPageIndexHandler( answerSpan: answerSpan, replay: replay, strategy: strategy, - pageIndex: pageIndex, + treeWalk: treeWalk, } } -// loadTree resolves the document tree for the pageindex answer +// loadTree resolves the document tree for the treewalk answer // endpoint, routing through the test seam when set. -func (h *AnswerPageIndexHandler) loadTree(ctx context.Context, orgID, storeID string, docID tree.DocumentID) (*tree.Tree, error) { +func (h *AnswerTreeWalkHandler) loadTree(ctx context.Context, orgID, storeID string, docID tree.DocumentID) (*tree.Tree, error) { if h.treeLoader != nil { return h.treeLoader(ctx, orgID, storeID, docID) } return h.db.LoadTree(ctx, docID, orgID, storeID) } -// pageIndexAnswerRequest is the body shape for /v1/answer/pageindex. -type pageIndexAnswerRequest struct { +// treeWalkAnswerRequest is the body shape for /v1/answer/treewalk. +type treeWalkAnswerRequest struct { DocumentID tree.DocumentID `json:"document_id"` Query string `json:"query"` Model string `json:"model"` @@ -96,25 +96,25 @@ type pageIndexAnswerRequest struct { IncludeReasoning bool `json:"reasoning"` } -// HandleAnswerPageIndex runs the PageIndex agentic loop and returns +// HandleAnswerTreeWalk runs the TreeWalk agentic loop and returns // the answer + page-grounded citations. Supports an SSE streaming // variant (stream=true) and an opt-in reasoning trace // (reasoning=true, or ?reasoning=true). -func (h *AnswerPageIndexHandler) HandleAnswerPageIndex(w http.ResponseWriter, r *http.Request) { +func (h *AnswerTreeWalkHandler) HandleAnswerTreeWalk(w http.ResponseWriter, r *http.Request) { orgID, ok := requireOrgID(w, r) if !ok { return } if h.llm == nil { - writeErr(w, http.StatusNotImplemented, "answer/pageindex endpoint requires an LLM client") + writeErr(w, http.StatusNotImplemented, "answer/treewalk endpoint requires an LLM client") return } - if h.strategy == nil || !h.pageIndex.Enabled { - writeErr(w, http.StatusNotImplemented, "pageindex strategy not configured on this server (retrieval.pageindex.enabled=false)") + if h.strategy == nil || !h.treeWalk.Enabled { + writeErr(w, http.StatusNotImplemented, "treewalk strategy not configured on this server (retrieval.treewalk.enabled=false)") return } - var body pageIndexAnswerRequest + var body treeWalkAnswerRequest if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeErr(w, http.StatusBadRequest, "invalid json: "+err.Error()) return @@ -177,17 +177,17 @@ func (h *AnswerPageIndexHandler) HandleAnswerPageIndex(w http.ResponseWriter, r trace []map[string]any ) if body.IncludeReasoning { - perReq.OnEvent = func(ev retrieval.PageIndexEvent) { + perReq.OnEvent = func(ev retrieval.TreeWalkEvent) { traceMu.Lock() defer traceMu.Unlock() - trace = append(trace, pageIndexEventToTraceMap(ev)) + trace = append(trace, treeWalkEventToTraceMap(ev)) } } res, err := perReq.SelectWithCost(r.Context(), t, body.Query, budget) if err != nil { - h.logger.Error("answer/pageindex: strategy failed", "err", err, "document_id", body.DocumentID) - writeErr(w, http.StatusInternalServerError, "pageindex strategy failed: "+err.Error()) + h.logger.Error("answer/treewalk: strategy failed", "err", err, "document_id", body.DocumentID) + writeErr(w, http.StatusInternalServerError, "treewalk strategy failed: "+err.Error()) return } @@ -234,7 +234,7 @@ func (h *AnswerPageIndexHandler) HandleAnswerPageIndex(w http.ResponseWriter, r // serveStream handles the stream=true SSE variant. Each tool call // emits one event so the caller can watch navigation in real time; // the final "answer" event carries the full JSON response. -func (h *AnswerPageIndexHandler) serveStream(w http.ResponseWriter, r *http.Request, strat *retrieval.PageIndexStrategy, t *tree.Tree, body pageIndexAnswerRequest, budget retrieval.ContextBudget, started time.Time) { +func (h *AnswerTreeWalkHandler) serveStream(w http.ResponseWriter, r *http.Request, strat *retrieval.TreeWalkStrategy, t *tree.Tree, body treeWalkAnswerRequest, budget retrieval.ContextBudget, started time.Time) { flusher, ok := w.(http.Flusher) if !ok { writeErr(w, http.StatusInternalServerError, "streaming requires http.Flusher; response writer does not support it") @@ -253,11 +253,11 @@ func (h *AnswerPageIndexHandler) serveStream(w http.ResponseWriter, r *http.Requ } writeMu.Lock() defer writeMu.Unlock() - fmt.Fprintf(w, "event: %s\ndata: %s\n\n", eventType, raw) + _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", eventType, raw) flusher.Flush() } - strat.OnEvent = func(ev retrieval.PageIndexEvent) { + strat.OnEvent = func(ev retrieval.TreeWalkEvent) { emitSSE(ev.Type, ev) } @@ -305,7 +305,7 @@ func (h *AnswerPageIndexHandler) serveStream(w http.ResponseWriter, r *http.Requ // extracted from the cited content. Falls back to the PagesRead // footprint when nothing was cited (refusal / hop-capped run) so the // caller still sees where the model looked. -func (h *AnswerPageIndexHandler) buildCitations(ctx context.Context, t *tree.Tree, res *retrieval.Result, query, requestModel string) []map[string]any { +func (h *AnswerTreeWalkHandler) buildCitations(ctx context.Context, t *tree.Tree, res *retrieval.Result, query, requestModel string) []map[string]any { if res == nil { return nil } @@ -353,7 +353,7 @@ func (h *AnswerPageIndexHandler) buildCitations(ctx context.Context, t *tree.Tre // materialiseCitedContent loads + concatenates every cited section's // content (capped at 16K chars), used for answer-span extraction over // the pages the model relied on. -func (h *AnswerPageIndexHandler) materialiseCitedContent(ctx context.Context, t *tree.Tree, sectionIDs []tree.SectionID) string { +func (h *AnswerTreeWalkHandler) materialiseCitedContent(ctx context.Context, t *tree.Tree, sectionIDs []tree.SectionID) string { if len(sectionIDs) == 0 { return "" } @@ -394,7 +394,7 @@ func (h *AnswerPageIndexHandler) materialiseCitedContent(ctx context.Context, t // spanExtractor builds a SpanExtractor for citation quoting, using the // same model fall-through as the /v1/answer handler. -func (h *AnswerPageIndexHandler) spanExtractor(requestModel string) *retrieval.SpanExtractor { +func (h *AnswerTreeWalkHandler) spanExtractor(requestModel string) *retrieval.SpanExtractor { model := h.answerSpan.Model if model == "" { model = requestModel @@ -409,9 +409,9 @@ func (h *AnswerPageIndexHandler) spanExtractor(requestModel string) *retrieval.S return ext } -// pageIndexEventToTraceMap converts a PageIndexEvent into the +// treeWalkEventToTraceMap converts a TreeWalkEvent into the // reasoning_trace entry shape. Only documented fields ship. -func pageIndexEventToTraceMap(ev retrieval.PageIndexEvent) map[string]any { +func treeWalkEventToTraceMap(ev retrieval.TreeWalkEvent) map[string]any { args := map[string]any{} switch ev.Type { case "get_pages": diff --git a/internal/handler/documents.go b/internal/handler/documents.go index e1eca7b..f5d7cef 100644 --- a/internal/handler/documents.go +++ b/internal/handler/documents.go @@ -177,7 +177,7 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R writeErr(w, http.StatusBadRequest, `missing form field "file"`) return } - defer file.Close() + defer func() { _ = file.Close() }() // best-effort close filename = header.Filename contentType = header.Header.Get("Content-Type") body = file @@ -347,7 +347,7 @@ func (h *DocumentsHandler) HandleGetDocumentSource(w http.ResponseWriter, r *htt writeErr(w, http.StatusInternalServerError, "read source: "+err.Error()) return } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close ct := doc.ContentType if ct == "" { @@ -361,7 +361,7 @@ func (h *DocumentsHandler) HandleGetDocumentSource(w http.ResponseWriter, r *htt w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, doc.Title)) } w.WriteHeader(http.StatusOK) - io.Copy(w, rc) + _, _ = io.Copy(w, rc) // best-effort write to response } // HandleGetTree returns the compact tree view used for LLM reasoning. @@ -428,7 +428,7 @@ func (h *DocumentsHandler) HandleGetSection(w http.ResponseWriter, r *http.Reque rc, _, getErr := h.storage.Get(r.Context(), sec.ContentRef) if getErr == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } diff --git a/internal/handler/query.go b/internal/handler/query.go index 73621b8..27d6377 100644 --- a/internal/handler/query.go +++ b/internal/handler/query.go @@ -22,7 +22,7 @@ type QueryHandler struct { storage storage.Storage strategy retrieval.Strategy // strategies is the pre-built set of selectable strategies keyed - // by config name (chunked-tree, pageindex, agentic, single-pass). + // by config name (chunked-tree, treewalk, agentic, single-pass). // A per-request "strategy" field selects one of these; an absent // or empty field falls back to the configured default (strategy). // Nil/empty disables the override entirely — every request uses @@ -76,8 +76,8 @@ type queryRequest struct { MaxParallelCalls int `json:"max_parallel_calls"` MaxSections int `json:"max_sections"` // Strategy optionally overrides the configured retrieval strategy - // for THIS request only. One of: chunked-tree, pageindex, agentic, - // single-pass. Empty uses the server default. This lets a caller + // for THIS request only. One of: auto, single-pass, chunked-tree, + // agentic, treewalk. Empty uses the server default. This lets a caller // (e.g. the benchmark harness) A/B strategies against the same // running engine without a redeploy. Unknown values return 400. Strategy string `json:"strategy"` @@ -200,7 +200,7 @@ func (h *QueryHandler) HandleQuery(w http.ResponseWriter, r *http.Request) { rc, _, getErr := h.storage.Get(r.Context(), sec.ContentRef) if getErr == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } diff --git a/internal/handler/query_multi.go b/internal/handler/query_multi.go index bf0595a..dd4a96a 100644 --- a/internal/handler/query_multi.go +++ b/internal/handler/query_multi.go @@ -106,7 +106,7 @@ func (h *QueryMultiHandler) HandleQueryMulti(w http.ResponseWriter, r *http.Requ rc, _, getErr := h.storage.Get(r.Context(), sec.ContentRef) if getErr == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } diff --git a/internal/handler/query_strategy_test.go b/internal/handler/query_strategy_test.go index 1bce66e..d74634a 100644 --- a/internal/handler/query_strategy_test.go +++ b/internal/handler/query_strategy_test.go @@ -105,41 +105,41 @@ func doQuery(t *testing.T, h *QueryHandler, jsonBody string) (*httptest.Response return rec, resp } -// TestHandleQueryStrategyOverrideRoutesToPageIndex is the Task-2 -// acceptance gate: a /v1/query body carrying {"strategy":"pageindex"} +// TestHandleQueryStrategyOverrideRoutesToTreeWalk is the Task-2 +// acceptance gate: a /v1/query body carrying {"strategy":"treewalk"} // must route to the page-based strategy, NOT the configured default. -func TestHandleQueryStrategyOverrideRoutesToPageIndex(t *testing.T) { +func TestHandleQueryStrategyOverrideRoutesToTreeWalk(t *testing.T) { t.Parallel() def := &labeledStrategy{name: "chunked-tree", picks: []tree.SectionID{"sec_a"}} - page := &labeledStrategy{name: "pageindex", picks: []tree.SectionID{"sec_b"}} + page := &labeledStrategy{name: "treewalk", picks: []tree.SectionID{"sec_b"}} set := map[string]retrieval.Strategy{ "chunked-tree": def, - "pageindex": page, + "treewalk": page, } h := newQueryStrategyHandler(def, set) - rec, resp := doQuery(t, h, `{"document_id":"doc_x","query":"q","strategy":"pageindex"}`) + rec, resp := doQuery(t, h, `{"document_id":"doc_x","query":"q","strategy":"treewalk"}`) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } if page.wasCalled() == false { - t.Error("pageindex strategy was NOT called for {\"strategy\":\"pageindex\"}") + t.Error("treewalk strategy was NOT called for {\"strategy\":\"treewalk\"}") } if def.wasCalled() { - t.Error("default (chunked-tree) strategy was called despite a pageindex override") + t.Error("default (chunked-tree) strategy was called despite a treewalk override") } - if got := resp["strategy"]; got != "pageindex" { - t.Errorf("response strategy = %v, want pageindex", got) + if got := resp["strategy"]; got != "treewalk" { + t.Errorf("response strategy = %v, want treewalk", got) } - // The pageindex mock picks sec_b; prove the override's result (not + // The treewalk mock picks sec_b; prove the override's result (not // the default's sec_a) is what surfaced. secs, _ := resp["sections"].([]any) if len(secs) != 1 { - t.Fatalf("sections = %v, want 1 (sec_b from pageindex)", resp["sections"]) + t.Fatalf("sections = %v, want 1 (sec_b from treewalk)", resp["sections"]) } if id := secs[0].(map[string]any)["id"]; id != "sec_b" { - t.Errorf("section id = %v, want sec_b (pageindex's pick)", id) + t.Errorf("section id = %v, want sec_b (treewalk's pick)", id) } } @@ -149,8 +149,8 @@ func TestHandleQueryDefaultStrategyWhenAbsent(t *testing.T) { t.Parallel() def := &labeledStrategy{name: "chunked-tree", picks: []tree.SectionID{"sec_a"}} - page := &labeledStrategy{name: "pageindex", picks: []tree.SectionID{"sec_b"}} - set := map[string]retrieval.Strategy{"chunked-tree": def, "pageindex": page} + page := &labeledStrategy{name: "treewalk", picks: []tree.SectionID{"sec_b"}} + set := map[string]retrieval.Strategy{"chunked-tree": def, "treewalk": page} h := newQueryStrategyHandler(def, set) rec, resp := doQuery(t, h, `{"document_id":"doc_x","query":"q"}`) @@ -195,7 +195,7 @@ func TestHandleQueryOverrideWithNilSet(t *testing.T) { def := &labeledStrategy{name: "chunked-tree", picks: []tree.SectionID{"sec_a"}} h := newQueryStrategyHandler(def, nil) - rec, _ := doQuery(t, h, `{"document_id":"doc_x","query":"q","strategy":"pageindex"}`) + rec, _ := doQuery(t, h, `{"document_id":"doc_x","query":"q","strategy":"treewalk"}`) if rec.Code != http.StatusBadRequest { t.Errorf("nil set + override: status = %d, want 400", rec.Code) } diff --git a/internal/handler/query_stream.go b/internal/handler/query_stream.go index 4702cee..8b5f5a0 100644 --- a/internal/handler/query_stream.go +++ b/internal/handler/query_stream.go @@ -139,7 +139,7 @@ func (h *QueryStreamHandler) HandleQueryStream(w http.ResponseWriter, r *http.Re rc, _, getErr := h.storage.Get(ctx, sec.ContentRef) if getErr == nil { raw, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close content = string(raw) } } @@ -163,7 +163,7 @@ func (h *QueryStreamHandler) HandleQueryStream(w http.ResponseWriter, r *http.Re } data, _ := json.Marshal(sse) - fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data) + _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data) // best-effort stream write if canFlush { flusher.Flush() } diff --git a/internal/handler/query_stream_multi.go b/internal/handler/query_stream_multi.go index d2d01a1..cfa1063 100644 --- a/internal/handler/query_stream_multi.go +++ b/internal/handler/query_stream_multi.go @@ -127,7 +127,7 @@ func (h *QueryStreamMultiHandler) HandleQueryStreamMulti(w http.ResponseWriter, } data, _ := json.Marshal(sse) - fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data) + _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data) // best-effort stream write if canFlush { flusher.Flush() } diff --git a/internal/handler/router.go b/internal/handler/router.go index b14fca6..f9efceb 100644 --- a/internal/handler/router.go +++ b/internal/handler/router.go @@ -34,12 +34,12 @@ type Deps struct { // Strategies is the pre-built set of selectable retrieval // strategies keyed by config name. It backs the per-request // "strategy" override on /v1/query (the benchmark uses it to A/B - // chunked-tree vs pageindex against one running engine). Nil + // chunked-tree vs treewalk against one running engine). Nil // disables the override — every /v1/query uses Strategy. Strategies map[string]retrieval.Strategy // LLM is the shared llmgate client used by the answer endpoints - // (/v1/answer, /v1/answer/pageindex) for span extraction and + // (/v1/answer, /v1/answer/treewalk) for span extraction and // synthesis. Nil makes those endpoints return 501. LLM llmgate.Client @@ -52,19 +52,19 @@ type Deps struct { Answer enginecfg.AnswerBlock // Replay is the replay-trace store. Every /v1/answer and - // /v1/answer/pageindex response is stamped with a trace_token and + // /v1/answer/treewalk response is stamped with a trace_token and // persisted here. Nil skips replay capture for those endpoints. Replay retrieval.ReplayStore - // PageIndexStrategy is the dedicated page-based agentic strategy - // instance used by /v1/answer/pageindex, independent of whichever + // TreeWalkStrategy is the dedicated page-based agentic strategy + // instance used by /v1/answer/treewalk, independent of whichever // selection strategy retrieval.strategy chose. Nil (or - // PageIndex.Enabled=false) makes the endpoint return 501. - PageIndexStrategy *retrieval.PageIndexStrategy + // TreeWalk.Enabled=false) makes the endpoint return 501. + TreeWalkStrategy *retrieval.TreeWalkStrategy - // PageIndex carries the page-based answer endpoint's config. The + // TreeWalk carries the page-based answer endpoint's config. The // per-request max_hops / max_pages_per_fetch fields override it. - PageIndex enginecfg.PageIndexBlock + TreeWalk enginecfg.TreeWalkBlock } // Router builds the chi router with all v1 routes and the full @@ -146,7 +146,7 @@ func Router(d Deps) http.Handler { queryMulti := NewQueryMultiHandler(d.Logger, d.Storage, d.Strategy, d.MultiDoc) queryStreamMulti := NewQueryStreamMultiHandler(d.Logger, d.Storage, d.MultiDoc) answer := NewAnswerHandler(d.Logger, d.DB, d.Storage, d.Strategy, d.LLM, d.LLMModel, d.AnswerSpan, d.Answer, d.Replay) - answerPageIndex := NewAnswerPageIndexHandler(d.Logger, d.DB, d.Storage, d.LLM, d.LLMModel, d.AnswerSpan, d.Replay, d.PageIndexStrategy, d.PageIndex) + answerTreeWalk := NewAnswerTreeWalkHandler(d.Logger, d.DB, d.Storage, d.LLM, d.LLMModel, d.AnswerSpan, d.Replay, d.TreeWalkStrategy, d.TreeWalk) webhook := NewWebhookHandler(d.Logger, d.Queue) // ── Connect-RPC Handlers (generated stubs, three-transport) ─── @@ -200,10 +200,10 @@ func Router(d Deps) http.Handler { }) // Answer: retrieval + synthesis in one round-trip. /answer - // uses the configured selection strategy; /answer/pageindex + // uses the configured selection strategy; /answer/treewalk // runs the page-based agentic loop end-to-end. r.Post("/answer", answer.HandleAnswer) - r.Post("/answer/pageindex", answerPageIndex.HandleAnswerPageIndex) + r.Post("/answer/treewalk", answerTreeWalk.HandleAnswerTreeWalk) }) // Internal: queue webhook (QStash). diff --git a/internal/handler/router_parity_test.go b/internal/handler/router_parity_test.go index d920b34..7e9140e 100644 --- a/internal/handler/router_parity_test.go +++ b/internal/handler/router_parity_test.go @@ -10,8 +10,8 @@ import ( // TestRouterParity is a divergence guard. The deployed cmd/server // binary and the standalone cmd/engine binary serve overlapping route // sets from two different routers (internal/handler vs internal/api). -// They have silently diverged before — the PageIndex redesign landed -// only on cmd/engine, leaving /v1/answer and /v1/answer/pageindex +// They have silently diverged before — the TreeWalk redesign landed +// only on cmd/engine, leaving /v1/answer and /v1/answer/treewalk // unreachable in production. // // This test walks the mounted chi router and asserts that the routes @@ -46,7 +46,7 @@ func TestRouterParity(t *testing.T) { want := []string{ "POST /v1/query/", "POST /v1/answer", - "POST /v1/answer/pageindex", + "POST /v1/answer/treewalk", } for _, route := range want { if !got[route] { @@ -56,7 +56,7 @@ func TestRouterParity(t *testing.T) { } // TestRouterMountsAnswerEndpoints is the focused assertion the task -// calls out explicitly: /v1/answer and /v1/answer/pageindex must be +// calls out explicitly: /v1/answer and /v1/answer/treewalk must be // mounted on the deployed router. Kept separate from the broad parity // set so a failure points straight at the answer-endpoint regression. func TestRouterMountsAnswerEndpoints(t *testing.T) { @@ -73,7 +73,7 @@ func TestRouterMountsAnswerEndpoints(t *testing.T) { return nil }) - for _, route := range []string{"/v1/answer", "/v1/answer/pageindex"} { + for _, route := range []string{"/v1/answer", "/v1/answer/treewalk"} { if !found[route] { t.Errorf("deployed router must mount %q but does not", route) } diff --git a/internal/middleware/cors.go b/internal/middleware/cors.go index 4e4e062..6f3913a 100644 --- a/internal/middleware/cors.go +++ b/internal/middleware/cors.go @@ -3,7 +3,6 @@ package middleware import ( "net/http" "strconv" - "strings" ) // CORSConfig controls Cross-Origin Resource Sharing behaviour. @@ -88,11 +87,4 @@ func CORS(cfg CORSConfig) func(http.Handler) http.Handler { // originMatches is a helper that checks if an origin matches any of the // patterns in the allowed list. It supports exact matches only; for // wildcard sub-domain patterns extend this function. -func originMatches(origin string, patterns []string) bool { - for _, p := range patterns { - if strings.EqualFold(origin, p) { - return true - } - } - return false -} +// diff --git a/internal/middleware/idempotency.go b/internal/middleware/idempotency.go index 19c70b2..fdba808 100644 --- a/internal/middleware/idempotency.go +++ b/internal/middleware/idempotency.go @@ -31,8 +31,7 @@ type cachedResponse struct { // idempotencyCache is a thread-safe TTL map backed by sync.Map. type idempotencyCache struct { - mu sync.Mutex // guards reap; reads/writes use sync.Map - entries sync.Map // key -> *cachedResponse + entries sync.Map // key -> *cachedResponse } // get returns the cached response if it exists and has not expired. @@ -134,7 +133,7 @@ func Idempotency(cfg IdempotencyConfig) func(http.Handler) http.Handler { if cr, ok := cache.get(key); ok { w.Header().Set("X-Idempotency-Replayed", "true") w.WriteHeader(cr.statusCode) - w.Write(cr.body) //nolint:errcheck + _, _ = w.Write(cr.body) //nolint:errcheck // best-effort write to response return } diff --git a/openapi.yaml b/openapi.yaml index 511ab1b..b05ac9f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -279,11 +279,11 @@ paths: "501": description: Endpoint not available — no LLM client configured - /v1/answer/pageindex: + /v1/answer/treewalk: post: tags: [Query] - summary: PageIndex-style page-based agentic answer - operationId: answerPageIndex + summary: TreeWalk-style page-based agentic answer + operationId: answerTreeWalk description: | Quote-grounded answer endpoint backed by the page-based agentic strategy. The model navigates the document via a @@ -333,7 +333,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PageIndexAnswerRequest" + $ref: "#/components/schemas/TreeWalkAnswerRequest" responses: "200": description: | @@ -344,7 +344,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PageIndexAnswerResponse" + $ref: "#/components/schemas/TreeWalkAnswerResponse" text/event-stream: schema: type: string @@ -353,7 +353,7 @@ paths: order: `started`, one event per tool call (`get_document_structure`, `get_pages`, `done`), and finally `answer` carrying the - full PageIndexAnswerResponse payload. On + full TreeWalkAnswerResponse payload. On error a single `error` event is emitted. "400": description: Missing or invalid request body. @@ -363,8 +363,8 @@ paths: description: | Endpoint not available. Either the server has no LLM client configured, or - `retrieval.pageindex.enabled=false`, or the - PageIndexStrategy instance was not wired at boot. + `retrieval.treewalk.enabled=false`, or the + TreeWalkStrategy instance was not wired at boot. /v1/replay: post: @@ -665,7 +665,7 @@ components: type: string strategy: type: string - enum: [single-pass, chunked-tree, agentic, pageindex] + enum: [single-pass, chunked-tree, agentic, treewalk] model: type: string sections: @@ -1035,10 +1035,10 @@ components: re-rank ran. Higher means the section is more directly relevant to the query. - PageIndexAnswerRequest: + TreeWalkAnswerRequest: type: object description: | - Body for POST /v1/answer/pageindex. The endpoint exposes + Body for POST /v1/answer/treewalk. The endpoint exposes per-request overrides for the page-based loop's caps (max_hops, max_pages_per_fetch) alongside the standard document_id / query / model fields. @@ -1053,14 +1053,14 @@ components: type: string description: | Override the LLM model used by the navigation loop. - Falls back to `retrieval.pageindex.model`, then to the + Falls back to `retrieval.treewalk.model`, then to the engine's default model. max_hops: type: integer description: | Cap on the number of LLM turns the loop consumes, counting the terminal `done` turn. Overrides - `retrieval.pageindex.max_hops` for this request only. + `retrieval.treewalk.max_hops` for this request only. Default at the server level is 8. max_pages_per_fetch: type: integer @@ -1068,7 +1068,7 @@ components: Cap on the characters one `get_pages` tool call may return. Keeps a stray full-document fetch from torching the model's context window. Overrides - `retrieval.pageindex.page_content_limit`. Default + `retrieval.treewalk.page_content_limit`. Default 16000. stream: type: boolean @@ -1076,7 +1076,7 @@ components: When true, the response is Server-Sent Events: one event per tool call (get_document_structure / get_pages / done) followed by a terminal `answer` - event carrying the full PageIndexAnswerResponse + event carrying the full TreeWalkAnswerResponse payload. Lets the caller WATCH the agent navigate in real time rather than waiting for the final answer. @@ -1089,7 +1089,7 @@ components: result-chars metadata). Equivalent to the `?reasoning=true` query parameter. - PageIndexAnswerResponse: + TreeWalkAnswerResponse: type: object description: | Non-streaming response shape. The `answer` field carries @@ -1112,10 +1112,10 @@ components: citations: type: array items: - $ref: "#/components/schemas/PageIndexCitation" + $ref: "#/components/schemas/TreeWalkCitation" strategy: type: string - enum: [pageindex] + enum: [treewalk] model: type: string confidence: @@ -1151,7 +1151,7 @@ components: type: string description: | Deterministic 64-char hex sha256 token over the - document, system-prompt version, "pageindex:" model + document, system-prompt version, "treewalk:" model tag, and sorted cited page ranges (e.g. ["1-2","5-7"]). Two runs that cite the same pages — even via different navigation paths — collapse to the same @@ -1170,13 +1170,13 @@ components: reasoning_trace: type: array items: - $ref: "#/components/schemas/PageIndexTraceEntry" + $ref: "#/components/schemas/TreeWalkTraceEntry" description: | Per-hop tool calls + arg summaries. Present only when the request opted in via `reasoning:true` or `?reasoning=true`. - PageIndexCitation: + TreeWalkCitation: type: object description: | One citation behind the agent's answer. The pages @@ -1239,7 +1239,7 @@ components: clipping at `page_content_limit`. Bytes-on-the-wire, not bytes-requested. - PageIndexTraceEntry: + TreeWalkTraceEntry: type: object description: | One tool call in the navigation timeline. Lets a diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 4498395..ccd5cce 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -198,8 +198,8 @@ func TestKeyDeterministic(t *testing.T) { func TestKeyDiffersOnInput(t *testing.T) { t.Parallel() - k1 := Key("doc1", "query", "strat", "model") - k2 := Key("doc2", "query", "strat", "model") + k1 := Key("doc1", "query", "strategy", "model") + k2 := Key("doc2", "query", "strategy", "model") if k1 == k2 { t.Error("different doc IDs should produce different keys") } diff --git a/pkg/config/config.go b/pkg/config/config.go index bfca023..70cf5e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -48,7 +48,7 @@ type IngestConfig struct { // AND the pdftable table-finding pass, so a // document becomes queryable in ~parse-speed // (seconds). The page-based retrieval strategy - // (/v1/answer/pageindex) needs none of the + // (/v1/answer/treewalk) needs none of the // skipped enrichment: it navigates a TOC tree // (synthesised from the section tree when // documents.toc_tree is NULL) and reads raw @@ -77,7 +77,7 @@ type IngestConfig struct { // populate it). SummaryAxes SummaryAxesBlock `yaml:"summary_axes"` - // TOC configures the PageIndex-style LLM-built table-of-contents + // TOC configures the TreeWalk-style LLM-built table-of-contents // tree stage. Enabled by default for PDF inputs; the resulting // tree is persisted on documents.toc_tree (JSONB). Failures are // non-fatal — they leave the column NULL and the document fully @@ -151,7 +151,7 @@ type IngestConfig struct { // TOCBlock configures the LLM-driven table-of-contents tree // builder. The builder reads page-by-page text from a freshly- -// ingested PDF and emits a hierarchical TOC (PageIndex-style), +// ingested PDF and emits a hierarchical TOC (TreeWalk-style), // persisted on documents.toc_tree (JSONB). // // Enabled by default for PDF inputs; non-PDF documents skip the @@ -417,38 +417,38 @@ type RetrievalConfig struct { ReRank ReRankBlock `yaml:"rerank"` Replay ReplayBlock `yaml:"replay"` Abstain AbstainBlock `yaml:"abstain"` - PageIndex PageIndexBlock `yaml:"pageindex"` + TreeWalk TreeWalkBlock `yaml:"treewalk"` } -// PageIndexBlock configures the PageIndex page-based agentic -// strategy and its dedicated /v1/answer/pageindex endpoint. +// TreeWalkBlock configures the TreeWalk page-based agentic +// strategy and its dedicated /v1/answer/treewalk endpoint. // // The strategy is registered as a Strategy choice -// (retrieval.strategy: pageindex) AND is wired into the -// /v1/answer/pageindex endpoint regardless of which selection +// (retrieval.strategy: treewalk) AND is wired into the +// /v1/answer/treewalk endpoint regardless of which selection // strategy the server runs by default. The Enabled flag controls // the endpoint only — flipping it off does not unregister the // strategy, so a deployment that wants the strategy available // to /v1/query but not the dedicated answer endpoint can still // disable the endpoint here. // -// Defaults are tuned to match the reference PageIndex demo: 8 +// Defaults are tuned to match the reference TreeWalk demo: 8 // hops covers structure → 3 navigation calls → done + buffer, // and 16,000 chars of get_pages content fits a 5-7 page excerpt // comfortably under any flagship model's context window. // -// Per-request overrides on /v1/answer/pageindex (max_hops, +// Per-request overrides on /v1/answer/treewalk (max_hops, // max_pages_per_fetch) win over this block; this block is the // server-side default. -type PageIndexBlock struct { - // Enabled toggles the /v1/answer/pageindex endpoint. Default: +type TreeWalkBlock struct { + // Enabled toggles the /v1/answer/treewalk endpoint. Default: // true. When false, the endpoint returns 501. The - // PageIndexStrategy itself stays registered as a selection + // TreeWalkStrategy itself stays registered as a selection // strategy regardless — disabling here only unwires the // dedicated answer surface. Enabled bool `yaml:"enabled"` - // MaxHops caps the number of LLM turns one /v1/answer/pageindex + // MaxHops caps the number of LLM turns one /v1/answer/treewalk // request consumes, including the terminal done turn. Default: // 8. Set to 0 to use the strategy's built-in default. MaxHops int `yaml:"max_hops"` @@ -472,7 +472,7 @@ type PageIndexBlock struct { // Empty means use the request's model (which itself falls back // to the engine default). Useful when navigation should run on // a fast/cheap model while answering benefits from a stronger - // one — though the PageIndex protocol does both in the same + // one — though the TreeWalk protocol does both in the same // loop, so most deployments leave this blank. Model string `yaml:"model"` } @@ -707,7 +707,7 @@ func Default() Config { }, LLM: LLMConfig{Driver: "anthropic"}, Retrieval: RetrievalConfig{ - Strategy: "chunked-tree", + Strategy: "auto", ChunkedTree: ChunkedTreeBlock{ MaxTokensPerCall: 60000, MaxParallelCalls: 8, @@ -749,7 +749,7 @@ func Default() Config { Enabled: true, Below: 0.4, }, - PageIndex: PageIndexBlock{ + TreeWalk: TreeWalkBlock{ Enabled: true, MaxHops: 8, PageContentLimit: 16000, @@ -1001,7 +1001,7 @@ func applyEnvOverrides(c *Config) { c.Ingest.SummaryAxes.MaxNumbers = n } } - // LLM-built TOC tree (PageIndex-style). Same truthy-string set + // LLM-built TOC tree (TreeWalk-style). Same truthy-string set // as the other ingest toggles; numeric overrides require a // positive int so a typo doesn't silently flip the default. if v := os.Getenv("VLE_INGEST_TOC_ENABLED"); v != "" { @@ -1125,34 +1125,34 @@ func applyEnvOverrides(c *Config) { c.Retrieval.Abstain.Below = f } } - if v := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED"); v != "" { + if v := os.Getenv("VLE_RETRIEVAL_TREEWALK_ENABLED"); v != "" { switch strings.ToLower(strings.TrimSpace(v)) { case "1", "true", "yes", "on": - c.Retrieval.PageIndex.Enabled = true + c.Retrieval.TreeWalk.Enabled = true case "0", "false", "no", "off": - c.Retrieval.PageIndex.Enabled = false + c.Retrieval.TreeWalk.Enabled = false } } - if v := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS"); v != "" { + if v := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS"); v != "" { if n, err := strconv.Atoi(v); err == nil && n >= 0 { - c.Retrieval.PageIndex.MaxHops = n + c.Retrieval.TreeWalk.MaxHops = n } } - if v := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT"); v != "" { + if v := os.Getenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT"); v != "" { if n, err := strconv.Atoi(v); err == nil && n >= 0 { - c.Retrieval.PageIndex.PageContentLimit = n + c.Retrieval.TreeWalk.PageContentLimit = n } } // MaxCitations accepts both the VLE_ and VLS_ prefixes so the // deploy layer (which forwards VLS_*) and local VLE_* envs both // reach it. A garbled or negative value preserves the default. - if v := firstEnv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS"); v != "" { + if v := firstEnv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", "VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS"); v != "" { if n, err := strconv.Atoi(v); err == nil && n >= 0 { - c.Retrieval.PageIndex.MaxCitations = n + c.Retrieval.TreeWalk.MaxCitations = n } } - if v := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MODEL"); v != "" { - c.Retrieval.PageIndex.Model = v + if v := os.Getenv("VLE_RETRIEVAL_TREEWALK_MODEL"); v != "" { + c.Retrieval.TreeWalk.Model = v } } @@ -1207,7 +1207,7 @@ func (c Config) Validate() error { } switch c.Retrieval.Strategy { - case "single-pass", "chunked-tree", "agentic", "pageindex": + case "auto", "single-pass", "chunked-tree", "agentic", "treewalk": default: return fmt.Errorf("unknown retrieval.strategy: %q", c.Retrieval.Strategy) } @@ -1309,14 +1309,14 @@ func (c Config) Validate() error { return fmt.Errorf("retrieval.abstain.below must be in [0.0, 1.0], got %v", c.Retrieval.Abstain.Below) } - if c.Retrieval.PageIndex.MaxHops < 0 { - return fmt.Errorf("retrieval.pageindex.max_hops must be >= 0, got %d", c.Retrieval.PageIndex.MaxHops) + if c.Retrieval.TreeWalk.MaxHops < 0 { + return fmt.Errorf("retrieval.treewalk.max_hops must be >= 0, got %d", c.Retrieval.TreeWalk.MaxHops) } - if c.Retrieval.PageIndex.PageContentLimit < 0 { - return fmt.Errorf("retrieval.pageindex.page_content_limit must be >= 0, got %d", c.Retrieval.PageIndex.PageContentLimit) + if c.Retrieval.TreeWalk.PageContentLimit < 0 { + return fmt.Errorf("retrieval.treewalk.page_content_limit must be >= 0, got %d", c.Retrieval.TreeWalk.PageContentLimit) } - if c.Retrieval.PageIndex.MaxCitations < 0 { - return fmt.Errorf("retrieval.pageindex.max_citations must be >= 0, got %d", c.Retrieval.PageIndex.MaxCitations) + if c.Retrieval.TreeWalk.MaxCitations < 0 { + return fmt.Errorf("retrieval.treewalk.max_citations must be >= 0, got %d", c.Retrieval.TreeWalk.MaxCitations) } return nil diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 5d4e920..5c64db3 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -25,8 +25,8 @@ func TestDefaultValues(t *testing.T) { if cfg.LLM.Driver != "anthropic" { t.Errorf("llm.driver = %q, want anthropic", cfg.LLM.Driver) } - if cfg.Retrieval.Strategy != "chunked-tree" { - t.Errorf("retrieval.strategy = %q, want chunked-tree", cfg.Retrieval.Strategy) + if cfg.Retrieval.Strategy != "auto" { + t.Errorf("retrieval.strategy = %q, want auto", cfg.Retrieval.Strategy) } if !cfg.Retrieval.Cache.Enabled { t.Error("retrieval.cache.enabled should be true by default") @@ -95,9 +95,9 @@ func TestDefaultValues(t *testing.T) { // parser without a config-file edit. func TestIngestParseTimeoutEnvOverride(t *testing.T) { prev := os.Getenv("VLE_INGEST_PARSE_TIMEOUT_SECONDS") - defer os.Setenv("VLE_INGEST_PARSE_TIMEOUT_SECONDS", prev) + defer func() { _ = os.Setenv("VLE_INGEST_PARSE_TIMEOUT_SECONDS", prev) }() // best-effort setenv - os.Setenv("VLE_INGEST_PARSE_TIMEOUT_SECONDS", "300") + _ = os.Setenv("VLE_INGEST_PARSE_TIMEOUT_SECONDS", "300") cfg := Default() applyEnvOverrides(&cfg) if cfg.Ingest.ParseTimeoutSeconds != 300 { @@ -131,9 +131,9 @@ func TestIngestModeDefault(t *testing.T) { // single env var that flips the engine into fast/minimal ingest. func TestIngestModeEnvOverride(t *testing.T) { prev := os.Getenv("VLE_INGEST_MODE") - defer os.Setenv("VLE_INGEST_MODE", prev) + defer func() { _ = os.Setenv("VLE_INGEST_MODE", prev) }() // best-effort setenv - os.Setenv("VLE_INGEST_MODE", "minimal") + _ = os.Setenv("VLE_INGEST_MODE", "minimal") cfg := Default() applyEnvOverrides(&cfg) if cfg.Ingest.Mode != "minimal" { @@ -176,14 +176,14 @@ func TestTOCEnvOverride(t *testing.T) { } defer func() { for k, v := range prev { - os.Setenv(k, v) + _ = os.Setenv(k, v) } }() - os.Setenv("VLE_INGEST_TOC_ENABLED", "false") - os.Setenv("VLE_INGEST_TOC_MODEL", "gemini-2.5-pro") - os.Setenv("VLE_INGEST_TOC_CONCURRENCY", "12") - os.Setenv("VLE_INGEST_TOC_TOC_CHECK_PAGES", "30") + _ = os.Setenv("VLE_INGEST_TOC_ENABLED", "false") + _ = os.Setenv("VLE_INGEST_TOC_MODEL", "gemini-2.5-pro") + _ = os.Setenv("VLE_INGEST_TOC_CONCURRENCY", "12") + _ = os.Setenv("VLE_INGEST_TOC_TOC_CHECK_PAGES", "30") cfg := Default() applyEnvOverrides(&cfg) @@ -207,12 +207,12 @@ func TestAbstainEnvOverride(t *testing.T) { prevEnabled := os.Getenv("VLE_RETRIEVAL_ABSTAIN_ENABLED") prevBelow := os.Getenv("VLE_RETRIEVAL_ABSTAIN_BELOW") defer func() { - os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", prevEnabled) - os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", prevBelow) + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", prevEnabled) + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", prevBelow) }() - os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", "false") - os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", "0.6") + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", "false") + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", "0.6") cfg := Default() applyEnvOverrides(&cfg) @@ -228,11 +228,11 @@ func TestAbstainEnvOverride(t *testing.T) { func TestAbstainEnvOverrideEnable(t *testing.T) { // Toggle on via env from an explicitly-disabled starting state. prev := os.Getenv("VLE_RETRIEVAL_ABSTAIN_ENABLED") - defer os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", prev) + defer func() { _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", prev) }() // best-effort setenv cfg := Default() cfg.Retrieval.Abstain.Enabled = false - os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", "true") + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_ENABLED", "true") applyEnvOverrides(&cfg) if !cfg.Retrieval.Abstain.Enabled { t.Error("VLE_RETRIEVAL_ABSTAIN_ENABLED=true should enable abstention even when previously disabled") @@ -245,11 +245,11 @@ func TestAbstainEnvOverrideEnable(t *testing.T) { // (Below must be in [0,1]). func TestAbstainEnvOverrideRejectsBad(t *testing.T) { prev := os.Getenv("VLE_RETRIEVAL_ABSTAIN_BELOW") - defer os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", prev) + defer func() { _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", prev) }() // best-effort setenv cases := []string{"not-a-float", "1.5", "-0.1", "abc"} for _, v := range cases { - os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", v) + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", v) cfg := Default() applyEnvOverrides(&cfg) if cfg.Retrieval.Abstain.Below != 0.4 { @@ -264,7 +264,7 @@ func TestAbstainEnvOverrideRejectsBad(t *testing.T) { // be accepted. func TestAbstainEnvOverrideParsesEdgeCases(t *testing.T) { prev := os.Getenv("VLE_RETRIEVAL_ABSTAIN_BELOW") - defer os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", prev) + defer func() { _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", prev) }() // best-effort setenv cases := map[string]float64{ "0": 0.0, @@ -274,7 +274,7 @@ func TestAbstainEnvOverrideParsesEdgeCases(t *testing.T) { "0.5": 0.5, } for raw, want := range cases { - os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", raw) + _ = os.Setenv("VLE_RETRIEVAL_ABSTAIN_BELOW", raw) cfg := Default() applyEnvOverrides(&cfg) if cfg.Retrieval.Abstain.Below != want { @@ -317,14 +317,14 @@ func TestReplayEnvOverride(t *testing.T) { prevMax := os.Getenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES") prevTTL := os.Getenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS") defer func() { - os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", prevEnabled) - os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", prevMax) - os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", prevTTL) + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", prevEnabled) + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", prevMax) + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", prevTTL) }() - os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", "false") - os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", "256") - os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", "3600") + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", "false") + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", "256") + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", "3600") cfg := Default() applyEnvOverrides(&cfg) @@ -343,11 +343,11 @@ func TestReplayEnvOverride(t *testing.T) { func TestReplayEnvOverrideEnable(t *testing.T) { // Toggle on via env from an explicitly-disabled starting state. prev := os.Getenv("VLE_RETRIEVAL_REPLAY_ENABLED") - defer os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", prev) + defer func() { _ = os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", prev) }() // best-effort setenv cfg := Default() cfg.Retrieval.Replay.Enabled = false - os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", "true") + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_ENABLED", "true") applyEnvOverrides(&cfg) if !cfg.Retrieval.Replay.Enabled { t.Error("VLE_RETRIEVAL_REPLAY_ENABLED=true should enable replay even when disabled in YAML") @@ -358,12 +358,12 @@ func TestReplayEnvOverrideRejectsBad(t *testing.T) { prevMax := os.Getenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES") prevTTL := os.Getenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS") defer func() { - os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", prevMax) - os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", prevTTL) + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", prevMax) + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", prevTTL) }() - os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", "not-a-number") - os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", "wat") + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_MAX_ENTRIES", "not-a-number") + _ = os.Setenv("VLE_RETRIEVAL_REPLAY_TTL_SECONDS", "wat") cfg := Default() applyEnvOverrides(&cfg) @@ -408,16 +408,16 @@ func TestReRankEnvOverride(t *testing.T) { prevMax := os.Getenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS") prevTopK := os.Getenv("VLE_RETRIEVAL_RERANK_TOP_K") defer func() { - os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", prevEnabled) - os.Setenv("VLE_RETRIEVAL_RERANK_MODEL", prevModel) - os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", prevMax) - os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", prevTopK) + _ = os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", prevEnabled) + _ = os.Setenv("VLE_RETRIEVAL_RERANK_MODEL", prevModel) + _ = os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", prevMax) + _ = os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", prevTopK) }() - os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", "true") - os.Setenv("VLE_RETRIEVAL_RERANK_MODEL", "gemini-2.0-flash") - os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", "1500") - os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", "3") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", "true") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_MODEL", "gemini-2.0-flash") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", "1500") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", "3") cfg := Default() applyEnvOverrides(&cfg) @@ -441,11 +441,11 @@ func TestReRankEnvOverrideDisable(t *testing.T) { // then set =false explicitly; verify the path executes (not just // that the default value is preserved). prev := os.Getenv("VLE_RETRIEVAL_RERANK_ENABLED") - defer os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", prev) + defer func() { _ = os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", prev) }() // best-effort setenv cfg := Default() cfg.Retrieval.ReRank.Enabled = true // simulate a YAML-set true - os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", "false") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_ENABLED", "false") applyEnvOverrides(&cfg) if cfg.Retrieval.ReRank.Enabled { t.Error("VLE_RETRIEVAL_RERANK_ENABLED=false should disable rerank even when YAML set it true") @@ -457,12 +457,12 @@ func TestReRankEnvOverrideRejectsBad(t *testing.T) { prevMax := os.Getenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS") prevTopK := os.Getenv("VLE_RETRIEVAL_RERANK_TOP_K") defer func() { - os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", prevMax) - os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", prevTopK) + _ = os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", prevMax) + _ = os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", prevTopK) }() - os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", "not-a-number") - os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", "abc") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_MAX_CONTENT_CHARS", "not-a-number") + _ = os.Setenv("VLE_RETRIEVAL_RERANK_TOP_K", "abc") cfg := Default() applyEnvOverrides(&cfg) @@ -511,16 +511,16 @@ func TestPlanningEnvOverride(t *testing.T) { prevCache := os.Getenv("VLE_RETRIEVAL_PLANNING_CACHE_SIZE") prevDecompose := os.Getenv("VLE_RETRIEVAL_PLANNING_DECOMPOSE") defer func() { - os.Setenv("VLE_RETRIEVAL_PLANNING_ENABLED", prevEnabled) - os.Setenv("VLE_RETRIEVAL_PLANNING_MODEL", prevModel) - os.Setenv("VLE_RETRIEVAL_PLANNING_CACHE_SIZE", prevCache) - os.Setenv("VLE_RETRIEVAL_PLANNING_DECOMPOSE", prevDecompose) + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_ENABLED", prevEnabled) + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_MODEL", prevModel) + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_CACHE_SIZE", prevCache) + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_DECOMPOSE", prevDecompose) }() - os.Setenv("VLE_RETRIEVAL_PLANNING_ENABLED", "true") - os.Setenv("VLE_RETRIEVAL_PLANNING_MODEL", "gemini-2.0-flash") - os.Setenv("VLE_RETRIEVAL_PLANNING_CACHE_SIZE", "256") - os.Setenv("VLE_RETRIEVAL_PLANNING_DECOMPOSE", "false") + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_ENABLED", "true") + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_MODEL", "gemini-2.0-flash") + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_CACHE_SIZE", "256") + _ = os.Setenv("VLE_RETRIEVAL_PLANNING_DECOMPOSE", "false") cfg := Default() applyEnvOverrides(&cfg) @@ -652,7 +652,7 @@ func TestValidateLLMDrivers(t *testing.T) { func TestValidateRetrievalStrategy(t *testing.T) { t.Parallel() - for _, s := range []string{"single-pass", "chunked-tree", "agentic", "pageindex"} { + for _, s := range []string{"auto", "single-pass", "chunked-tree", "agentic", "treewalk"} { cfg := Default() cfg.Database.URL = "postgres://localhost/test" cfg.Retrieval.Strategy = s @@ -669,173 +669,173 @@ func TestValidateRetrievalStrategy(t *testing.T) { } } -// TestPageIndexDefaults locks in the PageIndex block's defaults so +// TestTreeWalkDefaults locks in the TreeWalk block's defaults so // a regression on shipping values is loud. Endpoint enabled by // default, 8 hops, 16K char limit. -func TestPageIndexDefaults(t *testing.T) { +func TestTreeWalkDefaults(t *testing.T) { t.Parallel() cfg := Default() - if !cfg.Retrieval.PageIndex.Enabled { - t.Error("retrieval.pageindex.enabled should default to true (opt-out)") + if !cfg.Retrieval.TreeWalk.Enabled { + t.Error("retrieval.treewalk.enabled should default to true (opt-out)") } - if cfg.Retrieval.PageIndex.MaxHops != 8 { - t.Errorf("max_hops = %d, want 8", cfg.Retrieval.PageIndex.MaxHops) + if cfg.Retrieval.TreeWalk.MaxHops != 8 { + t.Errorf("max_hops = %d, want 8", cfg.Retrieval.TreeWalk.MaxHops) } - if cfg.Retrieval.PageIndex.PageContentLimit != 16000 { - t.Errorf("page_content_limit = %d, want 16000", cfg.Retrieval.PageIndex.PageContentLimit) + if cfg.Retrieval.TreeWalk.PageContentLimit != 16000 { + t.Errorf("page_content_limit = %d, want 16000", cfg.Retrieval.TreeWalk.PageContentLimit) } - if cfg.Retrieval.PageIndex.MaxCitations != 3 { - t.Errorf("max_citations = %d, want 3", cfg.Retrieval.PageIndex.MaxCitations) + if cfg.Retrieval.TreeWalk.MaxCitations != 3 { + t.Errorf("max_citations = %d, want 3", cfg.Retrieval.TreeWalk.MaxCitations) } - if cfg.Retrieval.PageIndex.Model != "" { - t.Errorf("model default should be empty (inherit), got %q", cfg.Retrieval.PageIndex.Model) + if cfg.Retrieval.TreeWalk.Model != "" { + t.Errorf("model default should be empty (inherit), got %q", cfg.Retrieval.TreeWalk.Model) } } -// TestPageIndexEnvOverride exercises every env knob the PageIndex +// TestTreeWalkEnvOverride exercises every env knob the TreeWalk // block exposes. -func TestPageIndexEnvOverride(t *testing.T) { - prevEnabled := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED") - prevHops := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS") - prevLimit := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT") - prevCits := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - prevModel := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MODEL") +func TestTreeWalkEnvOverride(t *testing.T) { + prevEnabled := os.Getenv("VLE_RETRIEVAL_TREEWALK_ENABLED") + prevHops := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS") + prevLimit := os.Getenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT") + prevCits := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS") + prevModel := os.Getenv("VLE_RETRIEVAL_TREEWALK_MODEL") defer func() { - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED", prevEnabled) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS", prevHops) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT", prevLimit) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", prevCits) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MODEL", prevModel) + os.Setenv("VLE_RETRIEVAL_TREEWALK_ENABLED", prevEnabled) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS", prevHops) + os.Setenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT", prevLimit) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", prevCits) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MODEL", prevModel) }() - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED", "false") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS", "12") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT", "32000") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "5") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MODEL", "gemini-2.0-flash") + os.Setenv("VLE_RETRIEVAL_TREEWALK_ENABLED", "false") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS", "12") + os.Setenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT", "32000") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", "5") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MODEL", "gemini-2.0-flash") cfg := Default() applyEnvOverrides(&cfg) - if cfg.Retrieval.PageIndex.Enabled { - t.Error("VLE_RETRIEVAL_PAGEINDEX_ENABLED=false should disable") + if cfg.Retrieval.TreeWalk.Enabled { + t.Error("VLE_RETRIEVAL_TREEWALK_ENABLED=false should disable") } - if cfg.Retrieval.PageIndex.MaxHops != 12 { - t.Errorf("max_hops = %d, want 12", cfg.Retrieval.PageIndex.MaxHops) + if cfg.Retrieval.TreeWalk.MaxHops != 12 { + t.Errorf("max_hops = %d, want 12", cfg.Retrieval.TreeWalk.MaxHops) } - if cfg.Retrieval.PageIndex.PageContentLimit != 32000 { - t.Errorf("page_content_limit = %d, want 32000", cfg.Retrieval.PageIndex.PageContentLimit) + if cfg.Retrieval.TreeWalk.PageContentLimit != 32000 { + t.Errorf("page_content_limit = %d, want 32000", cfg.Retrieval.TreeWalk.PageContentLimit) } - if cfg.Retrieval.PageIndex.MaxCitations != 5 { - t.Errorf("max_citations = %d, want 5", cfg.Retrieval.PageIndex.MaxCitations) + if cfg.Retrieval.TreeWalk.MaxCitations != 5 { + t.Errorf("max_citations = %d, want 5", cfg.Retrieval.TreeWalk.MaxCitations) } - if cfg.Retrieval.PageIndex.Model != "gemini-2.0-flash" { - t.Errorf("model = %q, want gemini-2.0-flash", cfg.Retrieval.PageIndex.Model) + if cfg.Retrieval.TreeWalk.Model != "gemini-2.0-flash" { + t.Errorf("model = %q, want gemini-2.0-flash", cfg.Retrieval.TreeWalk.Model) } } -// TestPageIndexMaxCitationsVLSAlias: the VLS_ prefix reaches +// TestTreeWalkMaxCitationsVLSAlias: the VLS_ prefix reaches // MaxCitations too (the deploy layer forwards VLS_*), and VLE_ wins // when both are set. -func TestPageIndexMaxCitationsVLSAlias(t *testing.T) { - prevVLE := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - prevVLS := os.Getenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") +func TestTreeWalkMaxCitationsVLSAlias(t *testing.T) { + prevVLE := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS") + prevVLS := os.Getenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS") defer func() { - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", prevVLE) - os.Setenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", prevVLS) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", prevVLE) + os.Setenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS", prevVLS) }() // VLS_ alone reaches the field. - os.Unsetenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") - os.Setenv("VLS_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "2") + os.Unsetenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS") + os.Setenv("VLS_RETRIEVAL_TREEWALK_MAX_CITATIONS", "2") cfg := Default() applyEnvOverrides(&cfg) - if cfg.Retrieval.PageIndex.MaxCitations != 2 { - t.Errorf("VLS_ alias: max_citations = %d, want 2", cfg.Retrieval.PageIndex.MaxCitations) + if cfg.Retrieval.TreeWalk.MaxCitations != 2 { + t.Errorf("VLS_ alias: max_citations = %d, want 2", cfg.Retrieval.TreeWalk.MaxCitations) } // VLE_ wins when both are set. - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "4") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", "4") cfg2 := Default() applyEnvOverrides(&cfg2) - if cfg2.Retrieval.PageIndex.MaxCitations != 4 { - t.Errorf("VLE_ should win over VLS_: max_citations = %d, want 4", cfg2.Retrieval.PageIndex.MaxCitations) + if cfg2.Retrieval.TreeWalk.MaxCitations != 4 { + t.Errorf("VLE_ should win over VLS_: max_citations = %d, want 4", cfg2.Retrieval.TreeWalk.MaxCitations) } } -// TestPageIndexEnvOverrideEnable: toggle on from an explicitly +// TestTreeWalkEnvOverrideEnable: toggle on from an explicitly // disabled state. -func TestPageIndexEnvOverrideEnable(t *testing.T) { - prev := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED") - defer os.Setenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED", prev) +func TestTreeWalkEnvOverrideEnable(t *testing.T) { + prev := os.Getenv("VLE_RETRIEVAL_TREEWALK_ENABLED") + defer os.Setenv("VLE_RETRIEVAL_TREEWALK_ENABLED", prev) cfg := Default() - cfg.Retrieval.PageIndex.Enabled = false - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_ENABLED", "true") + cfg.Retrieval.TreeWalk.Enabled = false + os.Setenv("VLE_RETRIEVAL_TREEWALK_ENABLED", "true") applyEnvOverrides(&cfg) - if !cfg.Retrieval.PageIndex.Enabled { - t.Error("VLE_RETRIEVAL_PAGEINDEX_ENABLED=true should enable from disabled") + if !cfg.Retrieval.TreeWalk.Enabled { + t.Error("VLE_RETRIEVAL_TREEWALK_ENABLED=true should enable from disabled") } } -// TestPageIndexEnvOverrideRejectsBad: garbled numerics preserve the +// TestTreeWalkEnvOverrideRejectsBad: garbled numerics preserve the // default rather than silently zeroing the cap. -func TestPageIndexEnvOverrideRejectsBad(t *testing.T) { - prevHops := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS") - prevLimit := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT") - prevCits := os.Getenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS") +func TestTreeWalkEnvOverrideRejectsBad(t *testing.T) { + prevHops := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS") + prevLimit := os.Getenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT") + prevCits := os.Getenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS") defer func() { - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS", prevHops) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT", prevLimit) - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", prevCits) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS", prevHops) + os.Setenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT", prevLimit) + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", prevCits) }() - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_HOPS", "abc") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_PAGE_CONTENT_LIMIT", "not-a-number") - os.Setenv("VLE_RETRIEVAL_PAGEINDEX_MAX_CITATIONS", "lots") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_HOPS", "abc") + os.Setenv("VLE_RETRIEVAL_TREEWALK_PAGE_CONTENT_LIMIT", "not-a-number") + os.Setenv("VLE_RETRIEVAL_TREEWALK_MAX_CITATIONS", "lots") cfg := Default() applyEnvOverrides(&cfg) - if cfg.Retrieval.PageIndex.MaxHops != 8 { - t.Errorf("garbage max_hops env should preserve default 8, got %d", cfg.Retrieval.PageIndex.MaxHops) + if cfg.Retrieval.TreeWalk.MaxHops != 8 { + t.Errorf("garbage max_hops env should preserve default 8, got %d", cfg.Retrieval.TreeWalk.MaxHops) } - if cfg.Retrieval.PageIndex.PageContentLimit != 16000 { - t.Errorf("garbage page_content_limit env should preserve default, got %d", cfg.Retrieval.PageIndex.PageContentLimit) + if cfg.Retrieval.TreeWalk.PageContentLimit != 16000 { + t.Errorf("garbage page_content_limit env should preserve default, got %d", cfg.Retrieval.TreeWalk.PageContentLimit) } - if cfg.Retrieval.PageIndex.MaxCitations != 3 { - t.Errorf("garbage max_citations env should preserve default 3, got %d", cfg.Retrieval.PageIndex.MaxCitations) + if cfg.Retrieval.TreeWalk.MaxCitations != 3 { + t.Errorf("garbage max_citations env should preserve default 3, got %d", cfg.Retrieval.TreeWalk.MaxCitations) } } -// TestValidatePageIndexNegatives: negatives rejected by Validate. -func TestValidatePageIndexNegatives(t *testing.T) { +// TestValidateTreeWalkNegatives: negatives rejected by Validate. +func TestValidateTreeWalkNegatives(t *testing.T) { t.Parallel() cfg := Default() cfg.Database.URL = "postgres://localhost/test" - cfg.Retrieval.PageIndex.MaxHops = -1 + cfg.Retrieval.TreeWalk.MaxHops = -1 if err := cfg.Validate(); err == nil { t.Error("negative max_hops should fail validation") } cfg2 := Default() cfg2.Database.URL = "postgres://localhost/test" - cfg2.Retrieval.PageIndex.PageContentLimit = -1 + cfg2.Retrieval.TreeWalk.PageContentLimit = -1 if err := cfg2.Validate(); err == nil { t.Error("negative page_content_limit should fail validation") } cfgCits := Default() cfgCits.Database.URL = "postgres://localhost/test" - cfgCits.Retrieval.PageIndex.MaxCitations = -1 + cfgCits.Retrieval.TreeWalk.MaxCitations = -1 if err := cfgCits.Validate(); err == nil { t.Error("negative max_citations should fail validation") } cfg3 := Default() cfg3.Database.URL = "postgres://localhost/test" - cfg3.Retrieval.PageIndex.MaxHops = 0 - cfg3.Retrieval.PageIndex.PageContentLimit = 0 - cfg3.Retrieval.PageIndex.MaxCitations = 0 + cfg3.Retrieval.TreeWalk.MaxHops = 0 + cfg3.Retrieval.TreeWalk.PageContentLimit = 0 + cfg3.Retrieval.TreeWalk.MaxCitations = 0 if err := cfg3.Validate(); err != nil { t.Errorf("zero values should pass (defaults applied at runtime): %v", err) } @@ -885,8 +885,8 @@ func TestValidateTLS(t *testing.T) { // Neither set → OK (no TLS). cfg5 := Default() cfg5.Database.URL = "postgres://localhost/test" - if !cfg5.Server.TLS.Enabled() == true { - // should be disabled + if cfg5.Server.TLS.Enabled() { + t.Error("TLS should be disabled when neither cert nor key is set") } } @@ -1045,18 +1045,18 @@ func TestTablesEnvOverride(t *testing.T) { prevRows := os.Getenv("VLE_INGEST_TABLES_MIN_ROWS") prevCols := os.Getenv("VLE_INGEST_TABLES_MIN_COLS") defer func() { - os.Setenv("VLE_INGEST_TABLES_ENABLED", prevEnabled) - os.Setenv("VLE_INGEST_TABLES_VERTICAL_STRATEGY", prevV) - os.Setenv("VLE_INGEST_TABLES_HORIZONTAL_STRATEGY", prevH) - os.Setenv("VLE_INGEST_TABLES_MIN_ROWS", prevRows) - os.Setenv("VLE_INGEST_TABLES_MIN_COLS", prevCols) + _ = os.Setenv("VLE_INGEST_TABLES_ENABLED", prevEnabled) + _ = os.Setenv("VLE_INGEST_TABLES_VERTICAL_STRATEGY", prevV) + _ = os.Setenv("VLE_INGEST_TABLES_HORIZONTAL_STRATEGY", prevH) + _ = os.Setenv("VLE_INGEST_TABLES_MIN_ROWS", prevRows) + _ = os.Setenv("VLE_INGEST_TABLES_MIN_COLS", prevCols) }() - os.Setenv("VLE_INGEST_TABLES_ENABLED", "false") - os.Setenv("VLE_INGEST_TABLES_VERTICAL_STRATEGY", "text") - os.Setenv("VLE_INGEST_TABLES_HORIZONTAL_STRATEGY", "lines_strict") - os.Setenv("VLE_INGEST_TABLES_MIN_ROWS", "4") - os.Setenv("VLE_INGEST_TABLES_MIN_COLS", "3") + _ = os.Setenv("VLE_INGEST_TABLES_ENABLED", "false") + _ = os.Setenv("VLE_INGEST_TABLES_VERTICAL_STRATEGY", "text") + _ = os.Setenv("VLE_INGEST_TABLES_HORIZONTAL_STRATEGY", "lines_strict") + _ = os.Setenv("VLE_INGEST_TABLES_MIN_ROWS", "4") + _ = os.Setenv("VLE_INGEST_TABLES_MIN_COLS", "3") cfg := Default() applyEnvOverrides(&cfg) @@ -1127,16 +1127,16 @@ func TestSummaryAxesEnvOverride(t *testing.T) { prevEntities := os.Getenv("VLE_INGEST_SUMMARY_AXES_MAX_ENTITIES") prevNumbers := os.Getenv("VLE_INGEST_SUMMARY_AXES_MAX_NUMBERS") defer func() { - os.Setenv("VLE_INGEST_SUMMARY_AXES_ENABLED", prevEnabled) - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", prevTopics) - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_ENTITIES", prevEntities) - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_NUMBERS", prevNumbers) + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_ENABLED", prevEnabled) + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", prevTopics) + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_ENTITIES", prevEntities) + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_NUMBERS", prevNumbers) }() - os.Setenv("VLE_INGEST_SUMMARY_AXES_ENABLED", "false") - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", "10") - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_ENTITIES", "20") - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_NUMBERS", "15") + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_ENABLED", "false") + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", "10") + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_ENTITIES", "20") + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_NUMBERS", "15") cfg := Default() applyEnvOverrides(&cfg) @@ -1159,8 +1159,8 @@ func TestSummaryAxesEnvOverride(t *testing.T) { // trim model output). func TestSummaryAxesEnvOverrideRejectsBad(t *testing.T) { prevTopics := os.Getenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS") - defer os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", prevTopics) - os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", "not-a-number") + defer func() { _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", prevTopics) }() // best-effort setenv + _ = os.Setenv("VLE_INGEST_SUMMARY_AXES_MAX_TOPICS", "not-a-number") cfg := Default() applyEnvOverrides(&cfg) if cfg.Ingest.SummaryAxes.MaxTopics != 4 { diff --git a/pkg/db/db.go b/pkg/db/db.go index 4c47c4f..1eec601 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -90,7 +90,7 @@ func (p *Pool) runMigration(ctx context.Context, f migrationFile) error { if err != nil { return err } - defer tx.Rollback(ctx) + defer func() { _ = tx.Rollback(ctx) }() // best-effort rollback if _, err := tx.Exec(ctx, f.sql); err != nil { return err } diff --git a/pkg/ingest/hyde.go b/pkg/ingest/hyde.go index 06a7b86..c64b10a 100644 --- a/pkg/ingest/hyde.go +++ b/pkg/ingest/hyde.go @@ -119,7 +119,7 @@ func (p *Pipeline) candidateQuestionsFor(ctx context.Context, s db.Section, prof if err != nil { return nil, err } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close raw, err := io.ReadAll(io.LimitReader(rc, int64(p.SummaryMaxChars))) if err != nil { return nil, err diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index a587ab6..8f24105 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -88,7 +88,7 @@ type Pipeline struct { // pass. Anything else (including the empty Go zero value used by // Pipeline literals in tests) runs the full enrichment pipeline. // - // The page-based retrieval strategy (/v1/answer/pageindex) needs none + // The page-based retrieval strategy (/v1/answer/treewalk) needs none // of the skipped enrichment — it navigates a synthesised-from-sections // TOC and reads raw section/page text at query time — so a // minimal-ingested document is immediately queryable through it. @@ -299,9 +299,7 @@ func completeWithTimeout(ctx context.Context, client llmgate.Client, req llmgate // stop retrying immediately on a timeout: re-issuing a call that just hung // would only multiply the wall-time cost (N retries × the timeout) without // changing the outcome, so a timeout is terminal, not retryable. -func isTimeout(err error) bool { - return errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) -} +// // acquireGlobalLLM blocks until a global-LLM-concurrency slot is free, // or returns false if ctx is canceled first. Returns a release func the @@ -389,7 +387,7 @@ func (p *Pipeline) Run(ctx context.Context, pl Payload) error { } log.Info("ingest: summarize+hyde complete", "elapsed", time.Since(stageStart)) - // LLM-built TOC tree (PageIndex-style). PDF-only because it + // LLM-built TOC tree (TreeWalk-style). PDF-only because it // relies on the parser's PageStart/PageEnd attribution to // reconstruct per-page text. Non-fatal: a builder failure // leaves documents.toc_tree NULL and the document remains @@ -546,7 +544,7 @@ func (p *Pipeline) parse(ctx context.Context, parsers *parser.Registry, pl Paylo if err != nil { return nil, fmt.Errorf("fetch source: %w", err) } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close return parsers.Parse(ctx, pl.ContentType, pl.Filename, rc) } @@ -836,7 +834,7 @@ func (p *Pipeline) summaryFor(ctx context.Context, s db.Section, childLines []st if err != nil { return nil, err } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close raw, err := io.ReadAll(io.LimitReader(rc, int64(p.SummaryMaxChars))) if err != nil { return nil, err diff --git a/pkg/ingest/minimal_mode_test.go b/pkg/ingest/minimal_mode_test.go index 34a96c1..e753b1b 100644 --- a/pkg/ingest/minimal_mode_test.go +++ b/pkg/ingest/minimal_mode_test.go @@ -187,8 +187,8 @@ func TestMinimalModeZeroLLMCalls(t *testing.T) { // must return the persisted text. // // It reconstructs the tree from exactly what runMinimal persisted, so it -// exercises the real post-ingest shape. The end-to-end PageIndexStrategy -// loop is covered in pkg/retrieval (TestPageIndexMinimalIngestedDoc). +// exercises the real post-ingest shape. The end-to-end TreeWalkStrategy +// loop is covered in pkg/retrieval (TestTreeWalkMinimalIngestedDoc). func TestMinimalModeReadyIsQueryable(t *testing.T) { t.Parallel() @@ -268,7 +268,7 @@ func TestMinimalModeReadyIsQueryable(t *testing.T) { t.Fatalf("load section %s content: %v", s.ID, err) } body, _ := io.ReadAll(rc) - rc.Close() + _ = rc.Close() // best-effort close if len(bytes.TrimSpace(body)) > 0 { loadedSomeBody = true } diff --git a/pkg/ingest/toc_builder.go b/pkg/ingest/toc_builder.go index b183a66..5d40527 100644 --- a/pkg/ingest/toc_builder.go +++ b/pkg/ingest/toc_builder.go @@ -28,21 +28,21 @@ type PageText struct { } // TOCBuilder builds an LLM-derived table-of-contents tree for a -// document. The shape mirrors PageIndex's three-phase pipeline: +// document. The shape mirrors TreeWalk's three-phase pipeline: // // 1. detect — scan the first TOCCheckPages pages and ask the LLM -// whether any of them looks like a real TOC. +// whether any of them looks like a real TOC. // 2. extract — if a TOC page was found, ask the LLM to parse it -// into structured nodes; otherwise call the no-TOC -// path that generates a TOC straight from body -// text (the LLM is given the full page text tagged -// with markers it copies back as -// the start page). +// into structured nodes; otherwise call the no-TOC +// path that generates a TOC straight from body +// text (the LLM is given the full page text tagged +// with markers it copies back as +// the start page). // 3. verify — concurrently re-check each leaf node: does its -// title actually appear at the start of the claimed -// page? Mismatches are repaired by clearing the -// page back to zero; downstream readers treat zero -// as "open / unknown" rather than a wrong answer. +// title actually appear at the start of the claimed +// page? Mismatches are repaired by clearing the +// page back to zero; downstream readers treat zero +// as "open / unknown" rather than a wrong answer. // // EndPage is derived from sibling ordering once verification is // done. The builder is deliberately tolerant of LLM parse blips @@ -62,7 +62,7 @@ type TOCBuilder struct { Concurrency int // TOCCheckPages bounds the prefix the detector scans for a - // table of contents. PageIndex defaults this to 20 — financial + // table of contents. TreeWalk defaults this to 20 — financial // filings put their TOC inside the first dozen pages and a // document with no TOC by page 20 almost never has one // further in. Default: 20. @@ -163,12 +163,12 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod } // detectTOCPages scans the first tocCheck pages with the -// PageIndex-style single-page detector. Returns the 1-indexed page +// TreeWalk-style single-page detector. Returns the 1-indexed page // numbers (in order) the LLM judged as table-of-contents pages. // // Detection failures (transport / parse) silently fall back to // "no TOC found here" so the caller transitions to the no-TOC path. -// This matches the PageIndex contract — the no-TOC generator is +// This matches the TreeWalk contract — the no-TOC generator is // strictly more general than the TOC-extraction path. func (b *TOCBuilder) detectTOCPages(ctx context.Context, pages []PageText, tocCheck int, usage *Usage) []int { limit := tocCheck @@ -199,7 +199,7 @@ func (b *TOCBuilder) detectTOCPages(ctx context.Context, pages []PageText, tocCh } // runTOCDetector asks the LLM whether the supplied page text reads -// like a table of contents. Mirrors PageIndex's +// like a table of contents. Mirrors TreeWalk's // toc_detector_single_page. func (b *TOCBuilder) runTOCDetector(ctx context.Context, pageText string, usage *Usage) (bool, error) { prompt := fmt.Sprintf(`Your job is to detect if there is a table of contents provided in the given text. @@ -289,7 +289,7 @@ Return ONLY a JSON object: {"nodes": [{"structure": "1", "title": "...", "physic return assembleHierarchy(flat), nil } -// generateNoTOC is the PageIndex-style process_no_toc driver: when +// generateNoTOC is the TreeWalk-style process_no_toc driver: when // no TOC page was found, page content (tagged with // markers) is fed to the LLM with instructions // to emit a TOC straight from headings in the body. @@ -327,7 +327,7 @@ Return ONLY a JSON object: {"nodes": [{"structure": "1", "title": "...", "physic return assembleHierarchy(flat), nil } -// verifyTitlesConcurrent runs PageIndex's check_title_appearance_in_start +// verifyTitlesConcurrent runs TreeWalk's check_title_appearance_in_start // over every node whose StartPage is set, with bounded concurrency. // Mismatches set StartPage back to zero — the downstream contract // is "zero means unknown / open" — so a misclaimed page never @@ -370,7 +370,18 @@ func (b *TOCBuilder) verifyTitlesConcurrent(ctx context.Context, nodes []tree.TO case <-gctx.Done(): return nil } - startsHere, err := b.runVerifyTitleAtPageStart(gctx, n.Title, pageText, &localUse) + // Accumulate this call's usage into a goroutine-local Usage, + // then fold it into the shared total under the lock — passing + // &localUse directly would race (concurrent usage.add writes). + var u Usage + startsHere, err := b.runVerifyTitleAtPageStart(gctx, n.Title, pageText, &u) + mu.Lock() + localUse.InputTokens += u.InputTokens + localUse.OutputTokens += u.OutputTokens + localUse.TotalTokens += u.TotalTokens + localUse.CostUSD += u.CostUSD + localUse.LLMCalls += u.LLMCalls + mu.Unlock() if err != nil { // Transport / stub LLM — treat as "not verified" but // don't clear the page; the LLM never weighed in. @@ -403,7 +414,7 @@ func (b *TOCBuilder) verifyTitlesConcurrent(ctx context.Context, nodes []tree.TO } } -// runVerifyTitleAtPageStart mirrors PageIndex's +// runVerifyTitleAtPageStart mirrors TreeWalk's // check_title_appearance_in_start: does this section's title appear // at the beginning of the supplied page? func (b *TOCBuilder) runVerifyTitleAtPageStart(ctx context.Context, title, pageText string, usage *Usage) (bool, error) { @@ -454,18 +465,18 @@ Directly return the final JSON structure. Do not output anything else.`, title, // --- prompt + schema constants --- const ( - tocDetectorSystemPrompt = "You are a precise document-structure analyser. Decide whether a single page of text is a table of contents." - tocExtractorSystemPrompt = "You are an expert in extracting hierarchical tree structures from documents. You output strict JSON only." - tocVerifySystemPrompt = "You are a precise verifier. Decide whether a section title starts a page's text." - defaultTOCRetries = 2 - tocDetectorMaxChars = 12000 - tocExtractorMaxChars = 16000 - tocExtractorMaxBody = 60000 - noTOCMaxBody = 80000 - verifyMaxChars = 4000 - tocDetectorJSONSchema = `{"type":"object","properties":{"thinking":{"type":"string"},"toc_detected":{"type":"string"}},"required":["toc_detected"]}` - tocVerifyJSONSchema = `{"type":"object","properties":{"thinking":{"type":"string"},"start_begin":{"type":"string"}},"required":["start_begin"]}` - tocNodesJSONSchema = `{"type":"object","properties":{"nodes":{"type":"array","items":{"type":"object","properties":{"structure":{"type":"string"},"title":{"type":"string"},"physical_index":{"type":["string","null"]}},"required":["title"]}}},"required":["nodes"]}` + tocDetectorSystemPrompt = "You are a precise document-structure analyser. Decide whether a single page of text is a table of contents." + tocExtractorSystemPrompt = "You are an expert in extracting hierarchical tree structures from documents. You output strict JSON only." + tocVerifySystemPrompt = "You are a precise verifier. Decide whether a section title starts a page's text." + defaultTOCRetries = 2 + tocDetectorMaxChars = 12000 + tocExtractorMaxChars = 16000 + tocExtractorMaxBody = 60000 + noTOCMaxBody = 80000 + verifyMaxChars = 4000 + tocDetectorJSONSchema = `{"type":"object","properties":{"thinking":{"type":"string"},"toc_detected":{"type":"string"}},"required":["toc_detected"]}` + tocVerifyJSONSchema = `{"type":"object","properties":{"thinking":{"type":"string"},"start_begin":{"type":"string"}},"required":["start_begin"]}` + tocNodesJSONSchema = `{"type":"object","properties":{"nodes":{"type":"array","items":{"type":"object","properties":{"structure":{"type":"string"},"title":{"type":"string"},"physical_index":{"type":["string","null"]}},"required":["title"]}}},"required":["nodes"]}` ) // --- JSON payload types --- diff --git a/pkg/parser/chunk_test.go b/pkg/parser/chunk_test.go index 6565ee8..503905a 100644 --- a/pkg/parser/chunk_test.go +++ b/pkg/parser/chunk_test.go @@ -47,8 +47,8 @@ func TestChunkOversizedLeavesSplits(t *testing.T) { func TestChunkOversizedLeavesLeavesSmallSectionsAlone(t *testing.T) { in := []Section{ - {Level: 1, Title: "Intro", Content: strings.Repeat("a b c d e f ", 50)}, // ~600 chars - {Level: 1, Title: "Methods", Content: strings.Repeat("x y z ", 200)}, // ~1200 chars + {Level: 1, Title: "Intro", Content: strings.Repeat("a b c d e f ", 50)}, // ~600 chars + {Level: 1, Title: "Methods", Content: strings.Repeat("x y z ", 200)}, // ~1200 chars } out := chunkOversizedLeaves(in) if len(out) != 2 { diff --git a/pkg/parser/docx.go b/pkg/parser/docx.go index c8c55d0..cac6dc8 100644 --- a/pkg/parser/docx.go +++ b/pkg/parser/docx.go @@ -123,7 +123,7 @@ func readZipFile(zr *zip.Reader, name string) []byte { if err != nil { return nil } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close data, err := io.ReadAll(rc) if err != nil { return nil @@ -164,11 +164,11 @@ func extractBlocks(body []byte) ([]docxBlock, error) { var out []docxBlock var ( - tblDepth int - rows [][]string // current outermost table - inRow bool - inCell bool - cellBuf strings.Builder + tblDepth int + rows [][]string // current outermost table + inRow bool + inCell bool + cellBuf strings.Builder inPara bool level int diff --git a/pkg/parser/pdf.go b/pkg/parser/pdf.go index 6a14b95..5b615fd 100644 --- a/pkg/parser/pdf.go +++ b/pkg/parser/pdf.go @@ -10,6 +10,7 @@ import ( "regexp" "sort" "strings" + "sync" "time" "github.com/hallelx2/pdftable" @@ -250,6 +251,22 @@ func runParseWithDeadline(ctx context.Context, timeout time.Duration, work func( } } +// pdftableOpenMu serializes pdftable.OpenBytes. pdftable mutates package-level +// state while opening a document and is not safe for concurrent callers — the +// race detector flags concurrent OpenBytes calls, and this is real in +// production because ingest workers parse documents in parallel. Serializing +// just the open (the only racing call) keeps correctness at a small cost. +// Stopgap: the proper fix is to make pdftable itself concurrency-safe +// (tracked in the Foundational Libraries project). +var pdftableOpenMu sync.Mutex + +// openPDFBytes is the concurrency-safe wrapper around pdftable.OpenBytes. +func openPDFBytes(b []byte) (pdftable.Document, error) { + pdftableOpenMu.Lock() + defer pdftableOpenMu.Unlock() + return pdftable.OpenBytes(b) +} + // parseDoc is the real parse implementation. It is bounded by Parse's // deadline wrapper; on its own it has no time bound beyond the per-stage // table-extraction budgets. @@ -274,7 +291,7 @@ func (p *PDF) parseDoc(_ context.Context, buf []byte) (*ParsedDoc, error) { // (empty password) and retry — this is the path that lets us index // "owner-password" PDFs whose only restriction is print/copy. docBytes := buf - pdoc, err := pdftable.OpenBytes(docBytes) + pdoc, err := openPDFBytes(docBytes) if err != nil { if isPdftableEncryptedErr(err) { cleaned, decErr := decryptPDFWithEmptyPassword(buf) @@ -282,13 +299,13 @@ func (p *PDF) parseDoc(_ context.Context, buf []byte) (*ParsedDoc, error) { return nil, fmt.Errorf("pdf: open: encrypted and could not be unlocked with empty password: %w", decErr) } docBytes = cleaned - pdoc, err = pdftable.OpenBytes(docBytes) + pdoc, err = openPDFBytes(docBytes) } if err != nil { return nil, fmt.Errorf("pdf: open: %w", err) } } - defer pdoc.Close() + defer func() { _ = pdoc.Close() }() // best-effort close reader, err := pdflib.NewReader(bytes.NewReader(docBytes), int64(len(docBytes))) if err != nil { @@ -1429,15 +1446,7 @@ func hasRepeatedAdjacentChars(s string) bool { // isEncryptedPDFError reports whether the given error from // ledongthuc/pdf indicates the document is encrypted. The library // has no proper error type for this, so we match on the message. -func isEncryptedPDFError(err error) bool { - if err == nil { - return false - } - msg := strings.ToLower(err.Error()) - return strings.Contains(msg, "encryption key") || - strings.Contains(msg, "encrypted") || - strings.Contains(msg, "/encrypt") -} +// // decryptPDFWithEmptyPassword strips the encryption dict from a PDF // using pdfcpu, assuming an empty user password (the common case for @@ -1601,7 +1610,12 @@ func safeExtractTables(page pdftable.Page, settings pdftable.TableSettings, page done <- result{} } }() + // pdftable mutates package-level state during table extraction too, + // not just OpenBytes — serialize it on the same mutex. Stopgap until + // pdftable is made concurrency-safe (HAL-118). + pdftableOpenMu.Lock() t, err := page.ExtractTables(settings) + pdftableOpenMu.Unlock() done <- result{tables: t, err: err} }() diff --git a/pkg/queue/asynq.go b/pkg/queue/asynq.go index e09a748..81a028a 100644 --- a/pkg/queue/asynq.go +++ b/pkg/queue/asynq.go @@ -31,7 +31,7 @@ type AsynqConfig struct { // payload is the engine Job marshalled as JSON. Handlers are dispatched // via an asynq.ServeMux at work time. type Asynq struct { - cfg AsynqConfig + cfg AsynqConfig redisOpt asynq.RedisClientOpt mu sync.RWMutex diff --git a/pkg/queue/asynq_test.go b/pkg/queue/asynq_test.go index 6f98fdc..e8f4ac8 100644 --- a/pkg/queue/asynq_test.go +++ b/pkg/queue/asynq_test.go @@ -72,7 +72,7 @@ func TestAsynqNewValidation(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - defer a.Close() + defer func() { _ = a.Close() }() // best-effort close if a.cfg.Concurrency != 20 { t.Errorf("default concurrency: got %d, want 20", a.cfg.Concurrency) @@ -87,7 +87,7 @@ func TestAsynqConcurrencyOverride(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - defer a.Close() + defer func() { _ = a.Close() }() // best-effort close if a.cfg.Concurrency != 50 { t.Errorf("concurrency: got %d, want 50", a.cfg.Concurrency) @@ -115,7 +115,7 @@ func TestAsynqIntegration(t *testing.T) { if err != nil { t.Fatalf("new asynq: %v", err) } - defer q.Close() + defer func() { _ = q.Close() }() // best-effort close done := make(chan Job, 1) q.Register(KindIngestDocument, func(_ context.Context, j Job) error { diff --git a/pkg/queue/qstash.go b/pkg/queue/qstash.go index 1bc1800..5ed739a 100644 --- a/pkg/queue/qstash.go +++ b/pkg/queue/qstash.go @@ -137,7 +137,7 @@ func (q *QStash) Enqueue(ctx context.Context, j Job) error { if err != nil { return fmt.Errorf("qstash publish: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // best-effort close if resp.StatusCode >= 300 { b, _ := io.ReadAll(resp.Body) return fmt.Errorf("qstash publish: status=%d body=%s", resp.StatusCode, string(b)) diff --git a/pkg/queue/river_test.go b/pkg/queue/river_test.go index 29ba8f2..13f39bc 100644 --- a/pkg/queue/river_test.go +++ b/pkg/queue/river_test.go @@ -64,7 +64,7 @@ func TestRiverIntegration(t *testing.T) { if err != nil { t.Fatalf("new river: %v", err) } - defer q.Close() + defer func() { _ = q.Close() }() // best-effort close done := make(chan Job, 1) q.Register(KindIngestDocument, func(_ context.Context, j Job) error { diff --git a/pkg/retrieval/agentic.go b/pkg/retrieval/agentic.go index ea1d367..a5494a1 100644 --- a/pkg/retrieval/agentic.go +++ b/pkg/retrieval/agentic.go @@ -36,7 +36,7 @@ type ContentFetcher interface { // trees that don't fit in any single context window, with reading // behaviour that adapts to each query. // -// Protocol choice +// # Protocol choice // // The strategy uses a JSON-action text protocol rather than llmgate's // Tools field. The provider adapters in llmgate v0.2.0 declare diff --git a/pkg/retrieval/auto.go b/pkg/retrieval/auto.go new file mode 100644 index 0000000..db23ec2 --- /dev/null +++ b/pkg/retrieval/auto.go @@ -0,0 +1,142 @@ +package retrieval + +import ( + "context" + + "github.com/hallelx2/vectorless-engine/pkg/tree" +) + +// AutoStrategy is the default Vectorless strategy: it routes each query to +// the cheapest strategy that fits the document. +// +// Small documents — those whose total leaf content fits comfortably in one +// context window — are answered with Small (single-pass): one fast LLM call +// that picks section IDs straight from the outline. Larger or more complex +// documents fall through to Large (treewalk): the agentic tree-navigation +// strategy that expands and reads sections as it goes. +// +// This keeps the common case (small docs) fast and cheap while preserving +// Vectorless's tree-navigation behaviour where it actually earns its extra +// latency. +type AutoStrategy struct { + // Small handles documents at or below SinglePassMaxTokens. Conceptually + // SinglePass. + Small Strategy + // Large handles documents above the threshold. Conceptually TreeWalk. + Large Strategy + // SinglePassMaxTokens is the total leaf-content token count at or below + // which Small is chosen. Zero means "derive from the call budget" + // (budget.Available()) — i.e. if the whole document could fit in one + // context window, treat it as small. + SinglePassMaxTokens int +} + +// NewAuto constructs an AutoStrategy from a small-document strategy and a +// large-document strategy. +func NewAuto(small, large Strategy) *AutoStrategy { + return &AutoStrategy{Small: small, Large: large} +} + +// Compile-time interface checks. +var ( + _ Strategy = (*AutoStrategy)(nil) + _ CostStrategy = (*AutoStrategy)(nil) +) + +// Name implements Strategy. +func (a *AutoStrategy) Name() string { return "auto" } + +// pick chooses the sub-strategy for a tree given the call budget. It never +// returns nil as long as both Small and Large are set; if a chosen branch +// is nil it falls back to the other so a half-configured Auto still runs. +func (a *AutoStrategy) pick(t *tree.Tree, budget ContextBudget) Strategy { + // TreeWalk (the Large branch) navigates by PAGE RANGE, so it only works + // when the document carries real page metadata. Non-paged formats + // (Markdown / HTML / DOCX / TXT) leave every PageStart/PageEnd at 0, so + // routing them to TreeWalk produces a page-navigation loop over a document + // with no pages. Force the Small (outline-based) branch in that case. + if !hasPageMetadata(t) { + if a.Small != nil { + return a.Small + } + return a.Large + } + + threshold := a.SinglePassMaxTokens + if threshold <= 0 { + threshold = budget.Available() + } + small := threshold > 0 && sumLeafTokens(t) <= threshold + if small { + if a.Small != nil { + return a.Small + } + return a.Large + } + if a.Large != nil { + return a.Large + } + return a.Small +} + +// Select implements Strategy. +func (a *AutoStrategy) Select(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) ([]tree.SectionID, error) { + s := a.pick(t, budget) + if s == nil { + // Misconfigured Auto (both Small and Large nil): an empty selection + // is safer than a nil-pointer panic. + return nil, nil + } + return s.Select(ctx, t, query, budget) +} + +// SelectWithCost implements CostStrategy. It delegates to the chosen +// sub-strategy's CostStrategy when available so token usage + cost flow +// through, and falls back to Select otherwise. +func (a *AutoStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) (*Result, error) { + s := a.pick(t, budget) + if s == nil { + return &Result{}, nil + } + if cs, ok := s.(CostStrategy); ok { + return cs.SelectWithCost(ctx, t, query, budget) + } + ids, err := s.Select(ctx, t, query, budget) + if err != nil { + return nil, err + } + return &Result{SelectedIDs: ids}, nil +} + +// sumLeafTokens sums the approximate token counts of every leaf section in +// the tree — a proxy for how much real content a query would have to reason +// over. Internal nodes are skipped to avoid double-counting their children. +func sumLeafTokens(t *tree.Tree) int { + if t == nil { + return 0 + } + view := t.BuildView() + total := 0 + for _, sv := range view.Sections { + if len(sv.Children) == 0 { + total += sv.Tokens + } + } + return total +} + +// hasPageMetadata reports whether any section in the tree carries a non-zero +// page range. TreeWalk navigation is only meaningful for paged documents; +// non-paged formats (Markdown / HTML / DOCX / TXT) leave PageStart/PageEnd at +// 0, so Auto must not route them to TreeWalk. +func hasPageMetadata(t *tree.Tree) bool { + if t == nil { + return false + } + for _, sv := range t.BuildView().Sections { + if sv.PageStart > 0 || sv.PageEnd > 0 { + return true + } + } + return false +} diff --git a/pkg/retrieval/auto_test.go b/pkg/retrieval/auto_test.go new file mode 100644 index 0000000..4fca924 --- /dev/null +++ b/pkg/retrieval/auto_test.go @@ -0,0 +1,149 @@ +package retrieval + +import ( + "context" + "testing" + + "github.com/hallelx2/vectorless-engine/pkg/tree" +) + +// recordingStrategy records its name into *called when invoked so a test +// can assert which branch AutoStrategy routed to. +type recordingStrategy struct { + name string + called *string +} + +func (r recordingStrategy) Name() string { return r.name } + +func (r recordingStrategy) Select(_ context.Context, _ *tree.Tree, _ string, _ ContextBudget) ([]tree.SectionID, error) { + *r.called = r.name + return []tree.SectionID{"sec"}, nil +} + +func (r recordingStrategy) SelectWithCost(_ context.Context, _ *tree.Tree, _ string, _ ContextBudget) (*Result, error) { + *r.called = r.name + return &Result{SelectedIDs: []tree.SectionID{"sec"}}, nil +} + +// treeWithLeafTokens builds a PAGED root with two leaf children whose token +// counts sum to total. Pages are set so Auto exercises the size-based routing +// (TreeWalk only applies to paged documents). +func treeWithLeafTokens(total int) *tree.Tree { + half := total / 2 + return &tree.Tree{ + Root: &tree.Section{ + ID: "root", + Title: "Doc", + Children: []*tree.Section{ + {ID: "a", ParentID: "root", Title: "A", TokenCount: half, ContentRef: "a.txt", PageStart: 1, PageEnd: 1}, + {ID: "b", ParentID: "root", Title: "B", TokenCount: total - half, ContentRef: "b.txt", PageStart: 2, PageEnd: 2}, + }, + }, + } +} + +// treeNoPages builds the same shape but with NO page metadata (every +// PageStart/PageEnd is 0) — i.e. a non-paged format (Markdown/HTML/DOCX/TXT). +func treeNoPages(total int) *tree.Tree { + half := total / 2 + return &tree.Tree{ + Root: &tree.Section{ + ID: "root", + Title: "Doc", + Children: []*tree.Section{ + {ID: "a", ParentID: "root", Title: "A", TokenCount: half, ContentRef: "a.txt"}, + {ID: "b", ParentID: "root", Title: "B", TokenCount: total - half, ContentRef: "b.txt"}, + }, + }, + } +} + +func TestAutoStrategyRoutesByExplicitThreshold(t *testing.T) { + var called string + small := recordingStrategy{name: "small", called: &called} + large := recordingStrategy{name: "large", called: &called} + auto := NewAuto(small, large) + auto.SinglePassMaxTokens = 100 + + // Below threshold → small. + called = "" + if _, err := auto.SelectWithCost(context.Background(), treeWithLeafTokens(40), "q", ContextBudget{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if called != "small" { + t.Fatalf("small doc routed to %q, want small", called) + } + + // Above threshold → large. + called = "" + if _, err := auto.SelectWithCost(context.Background(), treeWithLeafTokens(1000), "q", ContextBudget{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if called != "large" { + t.Fatalf("large doc routed to %q, want large", called) + } +} + +func TestAutoStrategyDerivesThresholdFromBudget(t *testing.T) { + var called string + small := recordingStrategy{name: "small", called: &called} + large := recordingStrategy{name: "large", called: &called} + auto := NewAuto(small, large) // SinglePassMaxTokens == 0 → use budget.Available() + + budget := ContextBudget{MaxTokens: 1000, ReservedForPrompt: 200} // Available()==800 + + called = "" + if _, err := auto.Select(context.Background(), treeWithLeafTokens(500), "q", budget); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if called != "small" { + t.Fatalf("doc within budget routed to %q, want small", called) + } + + called = "" + if _, err := auto.Select(context.Background(), treeWithLeafTokens(2000), "q", budget); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if called != "large" { + t.Fatalf("doc exceeding budget routed to %q, want large", called) + } +} + +func TestAutoStrategyName(t *testing.T) { + if got := NewAuto(nil, nil).Name(); got != "auto" { + t.Fatalf("Name()=%q, want auto", got) + } +} + +// A large but NON-PAGED document must NOT route to TreeWalk (which navigates +// by page range) — it falls back to Small. +func TestAutoStrategyNonPagedFallsBackToSmall(t *testing.T) { + var called string + small := recordingStrategy{name: "small", called: &called} + large := recordingStrategy{name: "large", called: &called} + auto := NewAuto(small, large) + auto.SinglePassMaxTokens = 100 + + called = "" + // 1000 tokens (>> threshold) but no page metadata → must pick small. + if _, err := auto.SelectWithCost(context.Background(), treeNoPages(1000), "q", ContextBudget{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if called != "small" { + t.Fatalf("non-paged large doc routed to %q, want small", called) + } +} + +// A misconfigured Auto (both sub-strategies nil) must not panic. +func TestAutoStrategyNilSafe(t *testing.T) { + auto := NewAuto(nil, nil) + ids, err := auto.Select(context.Background(), treeWithLeafTokens(10), "q", ContextBudget{}) + if err != nil || ids != nil { + t.Fatalf("Select on nil Auto: got (%v, %v), want (nil, nil)", ids, err) + } + res, err := auto.SelectWithCost(context.Background(), treeWithLeafTokens(10), "q", ContextBudget{}) + if err != nil || res == nil || len(res.SelectedIDs) != 0 { + t.Fatalf("SelectWithCost on nil Auto: got (%v, %v), want empty Result", res, err) + } +} diff --git a/pkg/retrieval/decompose_test.go b/pkg/retrieval/decompose_test.go index b1ff623..f762104 100644 --- a/pkg/retrieval/decompose_test.go +++ b/pkg/retrieval/decompose_test.go @@ -54,13 +54,13 @@ type costStrategyAdapter struct { } func (c *costStrategyAdapter) SelectWithCost(ctx context.Context, t *tree.Tree, query string, budget retrieval.ContextBudget) (*retrieval.Result, error) { - ids, err := c.scriptedStrategy.Select(ctx, t, query, budget) + ids, err := c.Select(ctx, t, query, budget) if err != nil { return nil, err } return &retrieval.Result{ SelectedIDs: ids, - Usage: c.scriptedStrategy.usage, + Usage: c.usage, }, nil } @@ -161,9 +161,9 @@ func TestDecomposerPerSubQuestionDispatch(t *testing.T) { tr := buildTree() s := &scriptedStrategy{ picks: map[string][]tree.SectionID{ - "What is the setup?": {"sec_a"}, - "What is the usage?": {"sec_b"}, - "What's in the FAQ?": {"sec_c"}, + "What is the setup?": {"sec_a"}, + "What is the usage?": {"sec_b"}, + "What's in the FAQ?": {"sec_c"}, }, usage: retrieval.Usage{InputTokens: 10, OutputTokens: 4, TotalTokens: 14, LLMCalls: 1, CostUSD: 0.001}, } diff --git a/pkg/retrieval/retrieval_test.go b/pkg/retrieval/retrieval_test.go index 54cd917..c2db4fe 100644 --- a/pkg/retrieval/retrieval_test.go +++ b/pkg/retrieval/retrieval_test.go @@ -536,7 +536,7 @@ func TestSinglePassStampsTraceToken(t *testing.T) { t.Fatalf("trace_token must be 64 chars, got %d (%q)", len(res.TraceToken), res.TraceToken) } for _, r := range res.TraceToken { - if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { t.Fatalf("trace_token must be lowercase hex, got %q", res.TraceToken) } } diff --git a/pkg/retrieval/single_pass.go b/pkg/retrieval/single_pass.go index 7b676aa..41a06bd 100644 --- a/pkg/retrieval/single_pass.go +++ b/pkg/retrieval/single_pass.go @@ -348,14 +348,14 @@ type selectionPayload struct { // Returns: // // - ids: the section IDs the model picked, in the order the -// model returned them. +// model returned them. // - confidences: map[id]float64 of per-pick confidences in [0.0, 1.0], -// populated only when the model returned the new-shape -// `picks` array. Returns nil (not an empty map) when -// the response was the legacy shape OR when every pick -// omitted its confidence — the distinction matters for -// abstention, which fires only when confidence signal -// is explicitly present. +// populated only when the model returned the new-shape +// `picks` array. Returns nil (not an empty map) when +// the response was the legacy shape OR when every pick +// omitted its confidence — the distinction matters for +// abstention, which fires only when confidence signal +// is explicitly present. // - err: non-nil only when the JSON cannot be decoded at all. func ParseSelection(raw string) ([]tree.SectionID, map[tree.SectionID]float64, error) { raw = strings.TrimSpace(raw) diff --git a/pkg/retrieval/strategy.go b/pkg/retrieval/strategy.go index 0aa428d..2ddd526 100644 --- a/pkg/retrieval/strategy.go +++ b/pkg/retrieval/strategy.go @@ -91,7 +91,7 @@ type Result struct { TraceToken string `json:"trace_token,omitempty"` // PagesRead records the page ranges the strategy actually fetched - // during navigation. Page-based strategies (e.g. pageindex) + // during navigation. Page-based strategies (e.g. treewalk) // populate this; section-by-section strategies leave it nil. // Useful for the API layer's reasoning-trace surfaces and for // cost/coverage debugging: a 10-K answer that read pages 50-55 + diff --git a/pkg/retrieval/pageindex_strategy.go b/pkg/retrieval/treewalk_strategy.go similarity index 89% rename from pkg/retrieval/pageindex_strategy.go rename to pkg/retrieval/treewalk_strategy.go index 6cff87e..d3600b9 100644 --- a/pkg/retrieval/pageindex_strategy.go +++ b/pkg/retrieval/treewalk_strategy.go @@ -14,8 +14,8 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/tree" ) -// PageIndexStrategy is a page-based agentic retrieval loop modelled on -// PageIndex's three-tool reasoning protocol. +// TreeWalkStrategy is a page-based agentic retrieval loop modelled on +// TreeWalk's three-tool reasoning protocol. // // The model navigates by PAGE RANGE rather than by section ID. Each // turn it emits one of: @@ -33,20 +33,20 @@ import ( // loop owns the answer, not just the selection. SelectWithCost // surfaces both the picked section IDs (the intersection of every // cited page range with the document's section map) and the literal -// answer string via Result.Reasoning. The /v1/answer/pageindex +// answer string via Result.Reasoning. The /v1/answer/treewalk // endpoint reads the answer; the legacy /v1/query callers still get // a section list. // // # Protocol choice // -// PageIndex's original demo wires the model via the OpenAI Agents +// TreeWalk's original demo wires the model via the OpenAI Agents // SDK's native tool-calling surface. llmgate v0.2.0 declares ToolDef // / ToolCall as scaffolding but does not populate ToolCalls on // responses, so this strategy uses the same JSON-action text // protocol AgenticStrategy already proved (see pkg/retrieval/agentic.go). // When llmgate wires native tool calling the surface here is the // same — only the request/response plumbing changes. -type PageIndexStrategy struct { +type TreeWalkStrategy struct { // LLM is the shared client used for every turn. LLM llmgate.Client @@ -63,7 +63,7 @@ type PageIndexStrategy struct { // MaxHops caps the number of LLM turns one Select consumes, // including the terminal "done" turn. Zero means use - // defaultPageIndexMaxHops. + // defaultTreeWalkMaxHops. MaxHops int // PageContentLimit caps how many chars a single get_pages @@ -74,7 +74,7 @@ type PageIndexStrategy struct { PageContentLimit int // MaxCitations caps how many distinct page ranges the FINAL done - // action may cite. Zero means use defaultPageIndexMaxCitations. + // action may cite. Zero means use defaultTreeWalkMaxCitations. // // This is a confidence backstop, not a navigation limit: the // model is free to read as many pages as MaxHops allows, but the @@ -92,15 +92,15 @@ type PageIndexStrategy struct { ModelOverride string // OnEvent, when non-nil, is invoked synchronously once per - // tool call so callers (e.g. the /v1/answer/pageindex SSE + // tool call so callers (e.g. the /v1/answer/treewalk SSE // handler) can stream the navigation in real time. The hook // runs inside the loop, after the tool result is computed but // before the next LLM hop. Implementations MUST be cheap and // MUST NOT block; a blocked hook stalls retrieval. - OnEvent func(PageIndexEvent) + OnEvent func(TreeWalkEvent) } -// PageIndexEvent is a single observable step in the strategy's +// TreeWalkEvent is a single observable step in the strategy's // navigation loop. Consumers convert these to whatever wire format // they need (SSE, gRPC stream, console log). // @@ -109,7 +109,7 @@ type PageIndexStrategy struct { // populated; for done, Answer + CitedPages are populated. The Hop // field is the 1-indexed turn number so consumers can interleave // hops from concurrent requests. -type PageIndexEvent struct { +type TreeWalkEvent struct { Hop int `json:"hop"` Type string `json:"type"` Reasoning string `json:"reasoning,omitempty"` @@ -128,19 +128,19 @@ type PageIndexEvent struct { Note string `json:"note,omitempty"` } -// defaultPageIndexMaxHops bounds the loop. Eight turns is enough for +// defaultTreeWalkMaxHops bounds the loop. Eight turns is enough for // structure → 3 get_pages → done with two retry hops on stray bad // JSON, while keeping latency and cost predictable. The reference -// PageIndex demo converges in 3-5 hops on typical questions. -const defaultPageIndexMaxHops = 8 +// TreeWalk demo converges in 3-5 hops on typical questions. +const defaultTreeWalkMaxHops = 8 // defaultPageContentLimit is the per-call chars cap. 16,000 chars // is roughly 4K tokens at GPT/Claude tokenisers — comfortably below // any flagship model's context but enough text for a 5-7 page -// excerpt. Matches PageIndex's reference behaviour. +// excerpt. Matches TreeWalk's reference behaviour. const defaultPageContentLimit = 16000 -// defaultPageIndexMaxCitations bounds the FINAL cited-range set. Three +// defaultTreeWalkMaxCitations bounds the FINAL cited-range set. Three // is generous for the answer-spans-one-place common case (where ONE is // ideal) while still allowing a genuinely multi-location answer (e.g. a // 10-K figure cross-referenced between the income statement and a @@ -148,16 +148,16 @@ const defaultPageContentLimit = 16000 // signal that motivated the cap: confident single-pick = f1 1.0, // 5-range spray = f1 0. Capping at 3 keeps the legitimate multi-range // case while removing the long tail of low-confidence noise. -const defaultPageIndexMaxCitations = 3 +const defaultTreeWalkMaxCitations = 3 -// strategyNamePageIndex is the stable identifier for config -// (retrieval.strategy: pageindex) and telemetry. -const strategyNamePageIndex = "pageindex" +// strategyNameTreeWalk is the stable identifier for config +// (retrieval.strategy: treewalk) and telemetry. +const strategyNameTreeWalk = "treewalk" // Compile-time interface checks. var ( - _ Strategy = (*PageIndexStrategy)(nil) - _ CostStrategy = (*PageIndexStrategy)(nil) + _ Strategy = (*TreeWalkStrategy)(nil) + _ CostStrategy = (*TreeWalkStrategy)(nil) ) // TOCProvider returns a JSON document-structure tree for the LLM's @@ -177,7 +177,7 @@ type TOCProvider interface { // keyed by its ContentRef. Strategies that need to materialise text // at run-time depend on this rather than on a concrete storage // driver — same shape as ContentFetcher; we keep them distinct so -// the two callers (agentic / pageindex) can be wired independently +// the two callers (agentic / treewalk) can be wired independently // in main.go. type PageContentLoader interface { Load(ctx context.Context, ref string) ([]byte, error) @@ -190,24 +190,24 @@ type PageContentLoader interface { // documents.toc_tree) every request will degrade through this path. var ErrNoTOC = fmt.Errorf("retrieval: no TOC tree persisted for document") -// NewPageIndexStrategy constructs a PageIndexStrategy with sensible +// NewTreeWalkStrategy constructs a TreeWalkStrategy with sensible // defaults. The TOC + PageLoader are nil here; the engine wires them // in main.go from the DB pool + storage backend. Tests pass scripted // implementations directly. -func NewPageIndexStrategy(client llmgate.Client) *PageIndexStrategy { - return &PageIndexStrategy{ +func NewTreeWalkStrategy(client llmgate.Client) *TreeWalkStrategy { + return &TreeWalkStrategy{ LLM: client, - MaxHops: defaultPageIndexMaxHops, + MaxHops: defaultTreeWalkMaxHops, PageContentLimit: defaultPageContentLimit, - MaxCitations: defaultPageIndexMaxCitations, + MaxCitations: defaultTreeWalkMaxCitations, } } // Name implements Strategy. -func (s *PageIndexStrategy) Name() string { return strategyNamePageIndex } +func (s *TreeWalkStrategy) Name() string { return strategyNameTreeWalk } // Select implements Strategy. -func (s *PageIndexStrategy) Select(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) ([]tree.SectionID, error) { +func (s *TreeWalkStrategy) Select(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) ([]tree.SectionID, error) { r, err := s.SelectWithCost(ctx, t, query, budget) if err != nil { return nil, err @@ -223,11 +223,11 @@ func (s *PageIndexStrategy) Select(ctx context.Context, t *tree.Tree, query stri // cited page range. This keeps the per-section-id contract for // callers (/v1/query, /v1/answer) that don't yet know about pages. // - Reasoning: the agent's final answer string (the "answer" field -// of the done action). /v1/answer/pageindex reads this directly +// of the done action). /v1/answer/treewalk reads this directly // and skips synthesis. // - PagesRead: an entry per get_pages call. // - HopsTaken / Usage / TraceToken: standard. -func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) (*Result, error) { +func (s *TreeWalkStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) (*Result, error) { if t == nil || t.Root == nil { return &Result{}, nil } @@ -238,7 +238,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu } maxHops := s.MaxHops if maxHops <= 0 { - maxHops = defaultPageIndexMaxHops + maxHops = defaultTreeWalkMaxHops } pageLimit := s.PageContentLimit if pageLimit <= 0 { @@ -246,7 +246,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu } maxCitations := s.MaxCitations if maxCitations <= 0 { - maxCitations = defaultPageIndexMaxCitations + maxCitations = defaultTreeWalkMaxCitations } // Pre-flatten the tree into an ordinal section list ordered by @@ -256,7 +256,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu maxPage := maxKnownPage(sections) msgs := []llmgate.Message{ - {Role: llmgate.RoleSystem, Content: pageIndexSystemPrompt}, + {Role: llmgate.RoleSystem, Content: treeWalkSystemPrompt}, {Role: llmgate.RoleUser, Content: s.initialUserPrompt(t, query, maxPage)}, } @@ -282,7 +282,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu } resp, err := s.LLM.Complete(ctx, req) if err != nil { - return nil, fmt.Errorf("pageindex hop %d: %w", hop+1, err) + return nil, fmt.Errorf("treewalk hop %d: %w", hop+1, err) } hopsTaken++ totalUsage.Add(Usage{ @@ -300,12 +300,12 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu Content: resp.Content, }) - action, parseErr := ParsePageIndexAction(resp.Content) + action, parseErr := ParseTreeWalkAction(resp.Content) if parseErr != nil { - log.Printf("retrieval: pageindex hop %d action parse failed: %v", hop+1, parseErr) + log.Printf("retrieval: treewalk hop %d action parse failed: %v", hop+1, parseErr) msgs = append(msgs, llmgate.Message{ Role: llmgate.RoleUser, - Content: pageIndexParseRetryPrompt, + Content: treeWalkParseRetryPrompt, }) continue } @@ -323,7 +323,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu citedRanges = selectCitedRanges(action.CitedPages, action.CitedConfidences, maxPage, maxCitations) confidence := clampConfidence(action.Confidence) selectedIDs := sectionsOverlapping(sections, citedRanges) - s.emit(PageIndexEvent{ + s.emit(TreeWalkEvent{ Hop: hopsTaken, Type: pageActionDone, Reasoning: finalReasoning, @@ -336,12 +336,12 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu Confidences: confidenceMap(selectedIDs, confidence), Confidence: confidence, CitedPages: rangesToPairs(citedRanges), - Reasoning: finalAnswer, // /v1/answer/pageindex reads this + Reasoning: finalAnswer, // /v1/answer/treewalk reads this ModelUsed: model, Usage: totalUsage, HopsTaken: hopsTaken, PagesRead: pagesRead, - TraceToken: computePageIndexTraceToken(t.DocumentID, model, citedRanges), + TraceToken: computeTreeWalkTraceToken(t.DocumentID, model, citedRanges), }, nil case pageActionStructure: @@ -350,7 +350,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu Role: llmgate.RoleUser, Content: wrapPageObservation("get_document_structure", obs), }) - s.emit(PageIndexEvent{ + s.emit(TreeWalkEvent{ Hop: hopsTaken, Type: pageActionStructure, Reasoning: action.Reasoning, @@ -366,7 +366,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu fmt.Sprintf("invalid range start=%d end=%d (document has %d pages). Pages are 1-indexed inclusive.", action.StartPage, action.EndPage, maxPage)), }) - s.emit(PageIndexEvent{ + s.emit(TreeWalkEvent{ Hop: hopsTaken, Type: pageActionGetPages, Reasoning: action.Reasoning, @@ -388,7 +388,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu Content: wrapPageObservation("get_pages", fmt.Sprintf("pages %d-%d (%d sections, %d chars):\n%s", start, end, len(sectionIDs), len(text), text)), }) - s.emit(PageIndexEvent{ + s.emit(TreeWalkEvent{ Hop: hopsTaken, Type: pageActionGetPages, Reasoning: action.Reasoning, @@ -404,7 +404,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu Content: wrapPageObservation(action.Action, fmt.Sprintf("unsupported tool %q. Use one of: get_document_structure, get_pages, done.", action.Action)), }) - s.emit(PageIndexEvent{ + s.emit(TreeWalkEvent{ Hop: hopsTaken, Type: action.Action, Note: "unsupported tool", @@ -421,7 +421,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu var forcedConfidence float64 finalAnswer, finalReasoning, citedRanges, forcedConfidence = s.forceDone(ctx, &msgs, &totalUsage, &hopsTaken, model, maxPage, maxCitations) selectedIDs := sectionsOverlapping(sections, citedRanges) - log.Printf("retrieval: pageindex strategy hit max_hops=%d; forced done", maxHops) + log.Printf("retrieval: treewalk strategy hit max_hops=%d; forced done", maxHops) _ = finalReasoning return &Result{ SelectedIDs: selectedIDs, @@ -433,7 +433,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu Usage: totalUsage, HopsTaken: hopsTaken, PagesRead: pagesRead, - TraceToken: computePageIndexTraceToken(t.DocumentID, model, citedRanges), + TraceToken: computeTreeWalkTraceToken(t.DocumentID, model, citedRanges), }, nil } @@ -441,7 +441,7 @@ func (s *PageIndexStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, qu // Hooks run synchronously inside the navigation loop and MUST be // cheap; callers that need to do I/O should buffer first and write // outside the strategy's critical path. -func (s *PageIndexStrategy) emit(ev PageIndexEvent) { +func (s *TreeWalkStrategy) emit(ev TreeWalkEvent) { if s.OnEvent != nil { s.OnEvent(ev) } @@ -451,7 +451,7 @@ func (s *PageIndexStrategy) emit(ev PageIndexEvent) { // task, tells the model which page range exists ("the document has N // pages"), and reminds it of the action protocol. Mirrors // AgenticStrategy.initialUserPrompt. -func (s *PageIndexStrategy) initialUserPrompt(t *tree.Tree, query string, maxPage int) string { +func (s *TreeWalkStrategy) initialUserPrompt(t *tree.Tree, query string, maxPage int) string { var b strings.Builder if t.Title != "" { b.WriteString("Document: ") @@ -466,7 +466,7 @@ func (s *PageIndexStrategy) initialUserPrompt(t *tree.Tree, query string, maxPag b.WriteString("\nUser query:\n") b.WriteString(query) b.WriteString("\n\nReply with a JSON action. The tools you may use are:\n") - b.WriteString(pageIndexActionHelp) + b.WriteString(treeWalkActionHelp) return b.String() } @@ -475,7 +475,7 @@ func (s *PageIndexStrategy) initialUserPrompt(t *tree.Tree, query string, maxPag // JSONB); if that's nil or errors, falls back to a synthesised view // derived from the section list. The fallback keeps this strategy // useful even before PR-A merges. -func (s *PageIndexStrategy) renderStructure(ctx context.Context, t *tree.Tree) string { +func (s *TreeWalkStrategy) renderStructure(ctx context.Context, t *tree.Tree) string { if s.TOC != nil { raw, err := s.TOC.GetTOC(ctx, t.DocumentID) if err == nil && len(raw) > 0 { @@ -483,7 +483,7 @@ func (s *PageIndexStrategy) renderStructure(ctx context.Context, t *tree.Tree) s } // Log and degrade — the strategy must keep going. if err != nil { - log.Printf("retrieval: pageindex TOC fetch failed (degrading to synthesised view): %v", err) + log.Printf("retrieval: treewalk TOC fetch failed (degrading to synthesised view): %v", err) } } return synthesiseTOC(t) @@ -495,7 +495,7 @@ func (s *PageIndexStrategy) renderStructure(ctx context.Context, t *tree.Tree) s // section IDs that contributed, in page order. SectionIDs feeds back // into the PageReadEntry so callers can audit which sections the // model actually read. -func (s *PageIndexStrategy) renderPages(ctx context.Context, sections []sectionPageEntry, start, end, pageLimit int) (string, []tree.SectionID) { +func (s *TreeWalkStrategy) renderPages(ctx context.Context, sections []sectionPageEntry, start, end, pageLimit int) (string, []tree.SectionID) { if s.PageLoader == nil { // Without a loader we can still emit a useful observation // from titles + summaries, so the model can keep navigating. @@ -522,7 +522,6 @@ func (s *PageIndexStrategy) renderPages(ctx context.Context, sections []sectionP } if len(header) > remaining { b.WriteString(header[:remaining]) - written += remaining break } b.WriteString(header) @@ -538,7 +537,6 @@ func (s *PageIndexStrategy) renderPages(ctx context.Context, sections []sectionP } if len(body) > remaining { b.WriteString(body[:remaining]) - written += remaining break } b.WriteString(body) @@ -551,7 +549,7 @@ func (s *PageIndexStrategy) renderPages(ctx context.Context, sections []sectionP // used when the strategy has no PageLoader (e.g. in tests, or when // storage is wired but momentarily unavailable). Titles + summaries // still let the model triangulate which range to ask about next. -func (s *PageIndexStrategy) renderPagesNoLoader(sections []sectionPageEntry, start, end, pageLimit int) (string, []tree.SectionID) { +func (s *TreeWalkStrategy) renderPagesNoLoader(sections []sectionPageEntry, start, end, pageLimit int) (string, []tree.SectionID) { var ( b strings.Builder sectionIDs []tree.SectionID @@ -573,7 +571,7 @@ func (s *PageIndexStrategy) renderPagesNoLoader(sections []sectionPageEntry, sta return out, sectionIDs } -func (s *PageIndexStrategy) loadSectionBody(ctx context.Context, sec sectionPageEntry) string { +func (s *TreeWalkStrategy) loadSectionBody(ctx context.Context, sec sectionPageEntry) string { if sec.contentRef == "" { if sec.summary != "" { return fmt.Sprintf("(summary, no content loaded)\n%s", sec.summary) @@ -582,7 +580,7 @@ func (s *PageIndexStrategy) loadSectionBody(ctx context.Context, sec sectionPage } data, err := s.PageLoader.Load(ctx, sec.contentRef) if err != nil { - log.Printf("retrieval: pageindex load failed for section %s: %v", sec.id, err) + log.Printf("retrieval: treewalk load failed for section %s: %v", sec.id, err) if sec.summary != "" { return fmt.Sprintf("(content load failed: %v; using summary)\n%s", err, sec.summary) } @@ -597,7 +595,7 @@ func (s *PageIndexStrategy) loadSectionBody(ctx context.Context, sec sectionPage // doesn't emit a valid done action, the empty values flow back and the // caller sees a hop-capped Result. The forced citation set is gated // through the same dedup + cap as the normal done path. -func (s *PageIndexStrategy) forceDone(ctx context.Context, msgs *[]llmgate.Message, totalUsage *Usage, hopsTaken *int, model string, maxPage, maxCitations int) (string, string, []pageRange, float64) { +func (s *TreeWalkStrategy) forceDone(ctx context.Context, msgs *[]llmgate.Message, totalUsage *Usage, hopsTaken *int, model string, maxPage, maxCitations int) (string, string, []pageRange, float64) { *msgs = append(*msgs, llmgate.Message{ Role: llmgate.RoleUser, Content: "You have used your tool-call budget. Reply NOW with one JSON object: {\"tool\":\"done\",\"answer\":\"= 0 { @@ -1182,10 +1180,10 @@ func ParsePageIndexAction(raw string) (PageIndexAction, error) { // invalidate the rest of the JSON. var fields map[string]json.RawMessage if err := json.Unmarshal([]byte(raw), &fields); err != nil { - return PageIndexAction{}, fmt.Errorf("decode pageindex action: %w", err) + return TreeWalkAction{}, fmt.Errorf("decode treewalk action: %w", err) } - var a PageIndexAction + var a TreeWalkAction if v, ok := fields["tool"]; ok { _ = json.Unmarshal(v, &a.Action) } @@ -1196,7 +1194,7 @@ func ParsePageIndexAction(raw string) (PageIndexAction, error) { } a.Action = strings.ToLower(strings.TrimSpace(a.Action)) if a.Action == "" { - return PageIndexAction{}, fmt.Errorf("missing 'tool' or 'action' field") + return TreeWalkAction{}, fmt.Errorf("missing 'tool' or 'action' field") } if v, ok := fields["start_page"]; ok { @@ -1382,9 +1380,9 @@ func wrapPageObservation(tool, body string) string { // --- system prompt --- -// pageIndexSystemPrompt instructs the model on the navigation loop. -// The wording is a faithful port of the reference PageIndex demo's -// AGENT_SYSTEM_PROMPT (see PageIndex/examples/agentic_vectorless_rag_demo.py:44-52), +// treeWalkSystemPrompt instructs the model on the navigation loop. +// The wording is a faithful port of the reference TreeWalk demo's +// AGENT_SYSTEM_PROMPT (see TreeWalk/examples/agentic_vectorless_rag_demo.py:44-52), // adapted to the JSON-action protocol vle uses in lieu of native // llmgate tool calling. // @@ -1396,7 +1394,7 @@ func wrapPageObservation(tool, body string) string { // - End with a done action carrying answer + cited_pages + confidence. // // CITATION DISCIPLINE is the load-bearing addition over the reference -// PageIndex prompt. FinanceBench measurements show two opposite failure +// TreeWalk prompt. FinanceBench measurements show two opposite failure // modes: (1) when unsure the model sprays ~5 hedged ranges and misses on // all of them; (2) over-correcting to "always exactly ONE" makes it drop a // genuinely-needed second range and commit to the wrong single pick. The @@ -1405,7 +1403,7 @@ func wrapPageObservation(tool, body string) string { // location — and bans hedging (padding with low-relevance maybes) rather // than banning multi-citation per se. Confidence is reported separately so // a low-confidence answer is still a committed set, not a spray. -const pageIndexSystemPrompt = `You are a document QA assistant navigating a paginated document. +const treeWalkSystemPrompt = `You are a document QA assistant navigating a paginated document. TOOL USE PROTOCOL: - Reply with EXACTLY one JSON object per turn. No prose, no markdown fences. @@ -1427,13 +1425,13 @@ RULES: - Be concise. Single-paragraph answers when possible. - If nothing in the document answers the query, emit done with answer="The document does not address this query.", an empty cited_pages array, and confidence 0.` -// pageIndexActionHelp is the one-shot reminder appended to the +// treeWalkActionHelp is the one-shot reminder appended to the // initial user prompt so the model gets concrete examples without us // needing to maintain a separate few-shot block. Both the one-range // (common) and two-range (answer spans separate locations) done forms are // modelled so the example doesn't bias the model toward one when two are // genuinely needed. -const pageIndexActionHelp = `- {"tool":"get_document_structure","reasoning":"orient by titles"} — fetch the TOC tree (titles + page ranges, no body text) +const treeWalkActionHelp = `- {"tool":"get_document_structure","reasoning":"orient by titles"} — fetch the TOC tree (titles + page ranges, no body text) - {"tool":"get_pages","start_page":5,"end_page":7,"reasoning":"section on debt"} — fetch text covering pages 5-7 - {"tool":"done","answer":"...","cited_pages":[[5,7]],"confidence":0.9,"reasoning":"grounded on one range"} — final answer when a single range suffices (the common case) - {"tool":"done","answer":"...","cited_pages":[[12,12],[31,31]],"confidence":0.8,"reasoning":"value on p12, scoping note on p31"} — final answer when it genuinely draws on two separate locations diff --git a/pkg/retrieval/pageindex_strategy_test.go b/pkg/retrieval/treewalk_strategy_test.go similarity index 88% rename from pkg/retrieval/pageindex_strategy_test.go rename to pkg/retrieval/treewalk_strategy_test.go index cbc569d..82997c4 100644 --- a/pkg/retrieval/pageindex_strategy_test.go +++ b/pkg/retrieval/treewalk_strategy_test.go @@ -14,7 +14,7 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/tree" ) -// pageScriptedLLM is a scriptedLLM for the PageIndex strategy. +// pageScriptedLLM is a scriptedLLM for the TreeWalk strategy. // Each Complete call returns the next canned response. When the // script is exhausted, loopReply (if set) is returned on every // subsequent call — the hop-cap test uses this to simulate a model @@ -85,7 +85,7 @@ func (pageErroringTOC) GetTOC(ctx context.Context, _ tree.DocumentID) ([]byte, e } // buildPagedTree mirrors buildAgenticTree but stamps page_start / -// page_end on every section so PageIndexStrategy can navigate. The +// page_end on every section so TreeWalkStrategy can navigate. The // shape: // // sec_root → [sec_a (1-4), sec_b (5-9)] @@ -102,14 +102,14 @@ func buildPagedTree() *tree.Tree { return &tree.Tree{DocumentID: "doc_x", Title: "Atlas", Root: root} } -// TestPageIndexHappyPath drives the canonical 3-tool sequence: +// TestTreeWalkHappyPath drives the canonical 3-tool sequence: // structure → get_pages → done. We assert the strategy: // - returns the answer string in Result.Reasoning // - lists the section IDs whose page range overlaps the citation // - records the get_pages call in PagesRead // - tracks HopsTaken correctly // - computes a non-empty TraceToken keyed by the cited pages -func TestPageIndexHappyPath(t *testing.T) { +func TestTreeWalkHappyPath(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -127,7 +127,7 @@ func TestPageIndexHappyPath(t *testing.T) { "b2_ref": "Debt registration is in line items A and B.", }} - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = loader res, err := s.SelectWithCost(context.Background(), tr, "how do I install?", retrieval.ContextBudget{MaxTokens: 100000}) @@ -206,7 +206,7 @@ func buildMinimalIngestedTree() *tree.Tree { return &tree.Tree{DocumentID: "doc_minimal", Title: "Rust", Root: root} } -// TestPageIndexMinimalIngestedDoc is the cross-package guarantee for the +// TestTreeWalkMinimalIngestedDoc is the cross-package guarantee for the // minimal ingest mode: a document ingested with NO LLM enrichment (no // summaries, no HyDE, NULL toc_tree) is still fully answerable through // the page-based strategy. It drives the canonical structure → get_pages @@ -220,7 +220,7 @@ func buildMinimalIngestedTree() *tree.Tree { // // No summaries are present anywhere in the tree, so this also proves the // strategy does not hard-require a summary to navigate or answer. -func TestPageIndexMinimalIngestedDoc(t *testing.T) { +func TestTreeWalkMinimalIngestedDoc(t *testing.T) { t.Parallel() tr := buildMinimalIngestedTree() @@ -237,7 +237,7 @@ func TestPageIndexMinimalIngestedDoc(t *testing.T) { "b1_ref": "Lifetimes ensure references are valid.", }} - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = loader // s.TOC intentionally left nil — models the NULL documents.toc_tree // state minimal ingest leaves behind. The strategy must synthesise. @@ -271,11 +271,11 @@ func TestPageIndexMinimalIngestedDoc(t *testing.T) { } } -// TestPageIndexMultiRangeDone covers a done with two cited ranges: +// TestTreeWalkMultiRangeDone covers a done with two cited ranges: // the strategy must surface every section that overlaps EITHER // range. This is the FinanceBench-shaped pattern: an answer that // pulls evidence from two unrelated parts of a 10-K. -func TestPageIndexMultiRangeDone(t *testing.T) { +func TestTreeWalkMultiRangeDone(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -287,7 +287,7 @@ func TestPageIndexMultiRangeDone(t *testing.T) { `{"tool":"done","answer":"Config is X. Debt is Y.","cited_pages":[[3,4],[8,9]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{ "a2_ref": "Config keys: VLE_*", "b2_ref": "Debt registration is in line items A and B.", @@ -325,12 +325,12 @@ func TestPageIndexMultiRangeDone(t *testing.T) { } } -// TestPageIndexMaxHopsForcesDone confirms a runaway loop is killed: +// TestTreeWalkMaxHopsForcesDone confirms a runaway loop is killed: // the model emits get_pages on every turn but never done. The // strategy must cap at MaxHops, force a done on the last hop, and // surface a Result with HopsTaken == MaxHops+1 (the +1 for the // forced terminal call). -func TestPageIndexMaxHopsForcesDone(t *testing.T) { +func TestTreeWalkMaxHopsForcesDone(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -338,7 +338,7 @@ func TestPageIndexMaxHopsForcesDone(t *testing.T) { // Every loop reply is a fresh get_pages — never done. loopReply: `{"tool":"get_pages","start_page":1,"end_page":2}`, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{"a1_ref": "install"}} s.MaxHops = 3 @@ -362,11 +362,11 @@ func TestPageIndexMaxHopsForcesDone(t *testing.T) { } } -// TestPageIndexMaxHopsForceDoneSucceeds covers the recovery path: +// TestTreeWalkMaxHopsForceDoneSucceeds covers the recovery path: // the loop hit MaxHops, but on the forced-done turn the model // actually emits a valid done. The strategy must collect the // answer + citations from that final turn rather than dropping them. -func TestPageIndexMaxHopsForceDoneSucceeds(t *testing.T) { +func TestTreeWalkMaxHopsForceDoneSucceeds(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -378,7 +378,7 @@ func TestPageIndexMaxHopsForceDoneSucceeds(t *testing.T) { `{"tool":"done","answer":"forced answer","cited_pages":[[1,2]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{"a1_ref": "install", "a2_ref": "config"}} s.MaxHops = 2 @@ -394,11 +394,11 @@ func TestPageIndexMaxHopsForceDoneSucceeds(t *testing.T) { } } -// TestPageIndexTOCFallback exercises the graceful-degradation path: +// TestTreeWalkTOCFallback exercises the graceful-degradation path: // when the persisted TOC provider returns ErrNoTOC (pre-PR-A // state), the strategy synthesises a TOC view from the section // tree. The model must still receive section titles + page ranges. -func TestPageIndexTOCFallback(t *testing.T) { +func TestTreeWalkTOCFallback(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -408,7 +408,7 @@ func TestPageIndexTOCFallback(t *testing.T) { `{"tool":"done","answer":"see structure","cited_pages":[]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} s.TOC = pageErroringTOC{} // mimic documents.toc_tree IS NULL @@ -434,10 +434,10 @@ func TestPageIndexTOCFallback(t *testing.T) { } } -// TestPageIndexTOCFromProvider asserts the persisted TOC wins over +// TestTreeWalkTOCFromProvider asserts the persisted TOC wins over // the synthesised view: when the provider returns bytes, those // bytes are surfaced verbatim. -func TestPageIndexTOCFromProvider(t *testing.T) { +func TestTreeWalkTOCFromProvider(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -447,7 +447,7 @@ func TestPageIndexTOCFromProvider(t *testing.T) { `{"tool":"done","answer":"from persisted TOC","cited_pages":[]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.TOC = pageStaticTOC{blob: []byte(`[{"title":"OVERRIDDEN","page_start":1,"page_end":99}]`)} _, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -465,16 +465,16 @@ func TestPageIndexTOCFromProvider(t *testing.T) { } } -// TestPageIndexBadJSONGraceful: persistent prose responses must +// TestTreeWalkBadJSONGraceful: persistent prose responses must // trigger a retry prompt and then bail cleanly at MaxHops. -func TestPageIndexBadJSONGraceful(t *testing.T) { +func TestTreeWalkBadJSONGraceful(t *testing.T) { t.Parallel() tr := buildPagedTree() llm := &pageScriptedLLM{ loopReply: "I think the answer is on page 5.", // never JSON } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} s.MaxHops = 3 @@ -490,10 +490,10 @@ func TestPageIndexBadJSONGraceful(t *testing.T) { } } -// TestPageIndexClampInvalidRange: a model that asks for pages past +// TestTreeWalkClampInvalidRange: a model that asks for pages past // the document's end gets a recoverable error observation and can // keep going. The strategy must NOT crash on out-of-range input. -func TestPageIndexClampInvalidRange(t *testing.T) { +func TestTreeWalkClampInvalidRange(t *testing.T) { t.Parallel() tr := buildPagedTree() // max page is 9 @@ -503,7 +503,7 @@ func TestPageIndexClampInvalidRange(t *testing.T) { `{"tool":"done","answer":"recovered","cited_pages":[[1,1]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{"a1_ref": "install"}} res, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -523,10 +523,10 @@ func TestPageIndexClampInvalidRange(t *testing.T) { } } -// TestPageIndexClampPartialOverlap: a range that overlaps the +// TestTreeWalkClampPartialOverlap: a range that overlaps the // document but extends past the end is silently clamped — the // model gets useful content (not an error) for the in-range part. -func TestPageIndexClampPartialOverlap(t *testing.T) { +func TestTreeWalkClampPartialOverlap(t *testing.T) { t.Parallel() tr := buildPagedTree() // max page is 9 @@ -536,7 +536,7 @@ func TestPageIndexClampPartialOverlap(t *testing.T) { `{"tool":"done","answer":"got it","cited_pages":[[8,9]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{"b2_ref": "Debt content."}} res, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -551,12 +551,12 @@ func TestPageIndexClampPartialOverlap(t *testing.T) { } } -// TestPageIndexEmptyTree exercises the early-return guard. -func TestPageIndexEmptyTree(t *testing.T) { +// TestTreeWalkEmptyTree exercises the early-return guard. +func TestTreeWalkEmptyTree(t *testing.T) { t.Parallel() llm := &pageScriptedLLM{} - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) res, err := s.SelectWithCost(context.Background(), &tree.Tree{}, "q", retrieval.ContextBudget{}) if err != nil { @@ -570,10 +570,10 @@ func TestPageIndexEmptyTree(t *testing.T) { } } -// TestPageIndexNoLoaderFallback: PageLoader=nil falls back to a +// TestTreeWalkNoLoaderFallback: PageLoader=nil falls back to a // title+summary rendering of get_pages. The model still gets a // useful observation so it can keep navigating. -func TestPageIndexNoLoaderFallback(t *testing.T) { +func TestTreeWalkNoLoaderFallback(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -583,7 +583,7 @@ func TestPageIndexNoLoaderFallback(t *testing.T) { `{"tool":"done","answer":"titles only","cited_pages":[[1,2]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) // no PageLoader + s := retrieval.NewTreeWalkStrategy(llm) // no PageLoader _, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) if err != nil { @@ -598,9 +598,9 @@ func TestPageIndexNoLoaderFallback(t *testing.T) { } } -// TestPageIndexContentClippedAtLimit: a get_pages call that would +// TestTreeWalkContentClippedAtLimit: a get_pages call that would // produce more chars than PageContentLimit must be clipped. -func TestPageIndexContentClippedAtLimit(t *testing.T) { +func TestTreeWalkContentClippedAtLimit(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -614,7 +614,7 @@ func TestPageIndexContentClippedAtLimit(t *testing.T) { `{"tool":"done","answer":"big","cited_pages":[[1,1]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = loader s.PageContentLimit = 1000 @@ -627,11 +627,11 @@ func TestPageIndexContentClippedAtLimit(t *testing.T) { } } -// TestPageIndexNoCitationsClearsSelection: an empty cited_pages +// TestTreeWalkNoCitationsClearsSelection: an empty cited_pages // list must produce an empty SelectedIDs (no implicit "default to // everything we visited"). This is the "no useful evidence found" // path the system prompt prescribes. -func TestPageIndexNoCitationsClearsSelection(t *testing.T) { +func TestTreeWalkNoCitationsClearsSelection(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -641,7 +641,7 @@ func TestPageIndexNoCitationsClearsSelection(t *testing.T) { `{"tool":"done","answer":"The document does not address this query.","cited_pages":[]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{"a1_ref": "install"}} res, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -656,9 +656,9 @@ func TestPageIndexNoCitationsClearsSelection(t *testing.T) { } } -// TestPageIndexTraceTokenStable: two runs that emit identical +// TestTreeWalkTraceTokenStable: two runs that emit identical // cited_pages produce identical trace tokens. Replay's substrate. -func TestPageIndexTraceTokenStable(t *testing.T) { +func TestTreeWalkTraceTokenStable(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -668,7 +668,7 @@ func TestPageIndexTraceTokenStable(t *testing.T) { `{"tool":"done","answer":"X","cited_pages":[[1,2],[8,9]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{} res, _ := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{ModelName: "gpt-4o-mini"}) return res.TraceToken @@ -680,15 +680,15 @@ func TestPageIndexTraceTokenStable(t *testing.T) { } } -// TestPageIndexTraceTokenOrderInvariant: two runs that cite the +// TestTreeWalkTraceTokenOrderInvariant: two runs that cite the // same pages in different orders must produce identical tokens. -func TestPageIndexTraceTokenOrderInvariant(t *testing.T) { +func TestTreeWalkTraceTokenOrderInvariant(t *testing.T) { t.Parallel() tr := buildPagedTree() mkRun := func(reply string) string { llm := &pageScriptedLLM{replies: []string{reply}} - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) res, _ := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{ModelName: "gpt-4o-mini"}) return res.TraceToken } @@ -699,7 +699,7 @@ func TestPageIndexTraceTokenOrderInvariant(t *testing.T) { } } -// TestParsePageIndexActionTolerance covers the input shapes the +// TestParseTreeWalkActionTolerance covers the input shapes the // parser accepts: // - "tool" key (canonical) // - "action" key (alt) @@ -707,7 +707,7 @@ func TestPageIndexTraceTokenOrderInvariant(t *testing.T) { // - cited_pages as string list ["5-7","10"] // - markdown fences + prose prefix // - case-insensitive tool tag -func TestParsePageIndexActionTolerance(t *testing.T) { +func TestParseTreeWalkActionTolerance(t *testing.T) { t.Parallel() cases := []struct { name string @@ -779,7 +779,7 @@ func TestParsePageIndexActionTolerance(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - got, err := retrieval.ParsePageIndexAction(c.in) + got, err := retrieval.ParseTreeWalkAction(c.in) if err != nil { t.Fatalf("parse: %v", err) } @@ -804,14 +804,14 @@ func TestParsePageIndexActionTolerance(t *testing.T) { } } -func TestParsePageIndexActionRejectsGarbage(t *testing.T) { +func TestParseTreeWalkActionRejectsGarbage(t *testing.T) { t.Parallel() for _, in := range []string{ "", "I think it's page 5.", `{"reasoning":"no tool field"}`, } { - _, err := retrieval.ParsePageIndexAction(in) + _, err := retrieval.ParseTreeWalkAction(in) if err == nil { t.Errorf("want error parsing %q", in) } @@ -841,13 +841,13 @@ func distinctRangeCount(pairs [][2]int) int { return len(seen) } -// TestPageIndexDedupCollapsesRepeatedRange is the core regression for +// TestTreeWalkDedupCollapsesRepeatedRange is the core regression for // the FinanceBench "same id ×5" miss (id_00499 returned sec_363... five // times). A done that cites the SAME range five times, plus two more // distinct ranges, must collapse to distinct ranges only AND respect // the MaxCitations cap. With MaxCitations=3 the three distinct ranges // survive; the four duplicate repeats are gone. -func TestPageIndexDedupCollapsesRepeatedRange(t *testing.T) { +func TestTreeWalkDedupCollapsesRepeatedRange(t *testing.T) { t.Parallel() tr := buildPagedTree() // pages 1-9 @@ -858,7 +858,7 @@ func TestPageIndexDedupCollapsesRepeatedRange(t *testing.T) { `{"tool":"done","answer":"sprayed","cited_pages":[[1,2],[1,2],[1,2],[1,2],[1,2],[3,4],[8,9]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} // Default MaxCitations is 3. @@ -901,12 +901,12 @@ func TestPageIndexDedupCollapsesRepeatedRange(t *testing.T) { } } -// TestPageIndexCapKeepsHighestConfidence proves the cap is +// TestTreeWalkCapKeepsHighestConfidence proves the cap is // confidence-aware: when more than MaxCitations distinct ranges are // cited WITH per-range confidence, the highest-confidence ranges win // (not the first-listed). Here five ranges are cited; the cap is 2; // the two with the top confidence must be the ones kept. -func TestPageIndexCapKeepsHighestConfidence(t *testing.T) { +func TestTreeWalkCapKeepsHighestConfidence(t *testing.T) { t.Parallel() tr := buildPagedTree() // pages 1-9 @@ -922,7 +922,7 @@ func TestPageIndexCapKeepsHighestConfidence(t *testing.T) { `{"pages":[1,1],"confidence":0.05}]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} s.MaxCitations = 2 @@ -946,11 +946,11 @@ func TestPageIndexCapKeepsHighestConfidence(t *testing.T) { } } -// TestPageIndexConfidentSinglePreserved is the happy half of the +// TestTreeWalkConfidentSinglePreserved is the happy half of the // signal: a confident single citation must pass through untouched and // the confidence must surface on both Result.Confidence and the // per-section Confidences map (so the abstain machinery can read it). -func TestPageIndexConfidentSinglePreserved(t *testing.T) { +func TestTreeWalkConfidentSinglePreserved(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -959,7 +959,7 @@ func TestPageIndexConfidentSinglePreserved(t *testing.T) { `{"tool":"done","answer":"Install on pages 1-2.","cited_pages":[[1,2]],"confidence":0.92,"reasoning":"clear"}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} res, err := s.SelectWithCost(context.Background(), tr, "how do I install?", retrieval.ContextBudget{MaxTokens: 100000}) @@ -981,11 +981,11 @@ func TestPageIndexConfidentSinglePreserved(t *testing.T) { } } -// TestPageIndexLowConfidenceStillCommitsSingle guards the over- +// TestTreeWalkLowConfidenceStillCommitsSingle guards the over- // suppression risk: even when the model reports LOW confidence, it must // still return its single best pick — never an empty citation set. A // low confidence annotates the answer; it does not delete it. -func TestPageIndexLowConfidenceStillCommitsSingle(t *testing.T) { +func TestTreeWalkLowConfidenceStillCommitsSingle(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -994,7 +994,7 @@ func TestPageIndexLowConfidenceStillCommitsSingle(t *testing.T) { `{"tool":"done","answer":"Probably debt on 8-9.","cited_pages":[[8,9]],"confidence":0.15,"reasoning":"unsure"}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} res, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -1012,9 +1012,9 @@ func TestPageIndexLowConfidenceStillCommitsSingle(t *testing.T) { } } -// TestPageIndexConfidenceClamped: out-of-range confidence values clamp +// TestTreeWalkConfidenceClamped: out-of-range confidence values clamp // into [0,1] rather than propagating absurd numbers. -func TestPageIndexConfidenceClamped(t *testing.T) { +func TestTreeWalkConfidenceClamped(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -1023,7 +1023,7 @@ func TestPageIndexConfidenceClamped(t *testing.T) { `{"tool":"done","answer":"x","cited_pages":[[1,2]],"confidence":7.5}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} res, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -1035,9 +1035,9 @@ func TestPageIndexConfidenceClamped(t *testing.T) { } } -// TestPageIndexCapConfigurable: a custom MaxCitations is honoured. Six +// TestTreeWalkCapConfigurable: a custom MaxCitations is honoured. Six // distinct ranges cited, cap=1 → exactly one survives. -func TestPageIndexCapConfigurable(t *testing.T) { +func TestTreeWalkCapConfigurable(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -1046,7 +1046,7 @@ func TestPageIndexCapConfigurable(t *testing.T) { `{"tool":"done","answer":"x","cited_pages":[[1,1],[2,2],[3,3],[4,4],[5,5],[6,6]]}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} s.MaxCitations = 1 @@ -1064,11 +1064,11 @@ func TestPageIndexCapConfigurable(t *testing.T) { } } -// TestPageIndexEmptyCitationsNoConfidence: a refusal (empty cited_pages) +// TestTreeWalkEmptyCitationsNoConfidence: a refusal (empty cited_pages) // leaves CitedPages nil, Confidences nil, and Confidence 0 — so the // API layer's abstain check (which fires only on a non-empty // Confidences map) behaves exactly as before for refusals. -func TestPageIndexEmptyCitationsNoConfidence(t *testing.T) { +func TestTreeWalkEmptyCitationsNoConfidence(t *testing.T) { t.Parallel() tr := buildPagedTree() @@ -1077,7 +1077,7 @@ func TestPageIndexEmptyCitationsNoConfidence(t *testing.T) { `{"tool":"done","answer":"The document does not address this query.","cited_pages":[],"confidence":0}`, }, } - s := retrieval.NewPageIndexStrategy(llm) + s := retrieval.NewTreeWalkStrategy(llm) s.PageLoader = pageMapLoader{data: map[string]string{}} res, err := s.SelectWithCost(context.Background(), tr, "q", retrieval.ContextBudget{MaxTokens: 100000}) @@ -1095,14 +1095,14 @@ func TestPageIndexEmptyCitationsNoConfidence(t *testing.T) { } } -// TestParsePageIndexConfidenceAndRichCitations covers the new parser +// TestParseTreeWalkConfidenceAndRichCitations covers the new parser // surfaces: a top-level confidence number, and the rich cited_pages // object form carrying per-range confidence. -func TestParsePageIndexConfidenceAndRichCitations(t *testing.T) { +func TestParseTreeWalkConfidenceAndRichCitations(t *testing.T) { t.Parallel() t.Run("top_level_confidence", func(t *testing.T) { - a, err := retrieval.ParsePageIndexAction(`{"tool":"done","answer":"x","cited_pages":[[1,2]],"confidence":0.8}`) + a, err := retrieval.ParseTreeWalkAction(`{"tool":"done","answer":"x","cited_pages":[[1,2]],"confidence":0.8}`) if err != nil { t.Fatalf("parse: %v", err) } @@ -1112,7 +1112,7 @@ func TestParsePageIndexConfidenceAndRichCitations(t *testing.T) { }) t.Run("confidence_as_string", func(t *testing.T) { - a, err := retrieval.ParsePageIndexAction(`{"tool":"done","answer":"x","cited_pages":[[1,2]],"confidence":"0.6"}`) + a, err := retrieval.ParseTreeWalkAction(`{"tool":"done","answer":"x","cited_pages":[[1,2]],"confidence":"0.6"}`) if err != nil { t.Fatalf("parse: %v", err) } @@ -1122,7 +1122,7 @@ func TestParsePageIndexConfidenceAndRichCitations(t *testing.T) { }) t.Run("rich_cited_pages_objects", func(t *testing.T) { - a, err := retrieval.ParsePageIndexAction(`{"tool":"done","answer":"x","cited_pages":[{"pages":[5,7],"confidence":0.9},{"pages":[12,12],"confidence":0.4}]}`) + a, err := retrieval.ParseTreeWalkAction(`{"tool":"done","answer":"x","cited_pages":[{"pages":[5,7],"confidence":0.9},{"pages":[12,12],"confidence":0.4}]}`) if err != nil { t.Fatalf("parse: %v", err) } @@ -1138,7 +1138,7 @@ func TestParsePageIndexConfidenceAndRichCitations(t *testing.T) { }) t.Run("rich_cited_pages_start_end", func(t *testing.T) { - a, err := retrieval.ParsePageIndexAction(`{"tool":"done","answer":"x","cited_pages":[{"start":3,"end":4,"confidence":0.7}]}`) + a, err := retrieval.ParseTreeWalkAction(`{"tool":"done","answer":"x","cited_pages":[{"start":3,"end":4,"confidence":0.7}]}`) if err != nil { t.Fatalf("parse: %v", err) } diff --git a/pkg/storage/local.go b/pkg/storage/local.go index 9be4767..440c767 100644 --- a/pkg/storage/local.go +++ b/pkg/storage/local.go @@ -39,7 +39,7 @@ func (l *Local) Put(ctx context.Context, key string, r io.Reader, _ Metadata) er if err != nil { return err } - defer f.Close() + defer func() { _ = f.Close() }() // best-effort close _, err = io.Copy(f, r) return err } diff --git a/pkg/storage/s3_test.go b/pkg/storage/s3_test.go index 8a437c6..4a99a42 100644 --- a/pkg/storage/s3_test.go +++ b/pkg/storage/s3_test.go @@ -118,7 +118,7 @@ func TestS3Integration(t *testing.T) { if err != nil { t.Fatalf("get: %v", err) } - defer rc.Close() + defer func() { _ = rc.Close() }() // best-effort close got, err := io.ReadAll(rc) if err != nil { t.Fatalf("read: %v", err) diff --git a/pkg/tree/compact_test.go b/pkg/tree/compact_test.go index 23bd668..5491cf6 100644 --- a/pkg/tree/compact_test.go +++ b/pkg/tree/compact_test.go @@ -15,8 +15,8 @@ func buildLargeTestTree() *Tree { ID: "ch1", ParentID: "root", Title: "Chapter 1", Children: []*Section{ {ID: "ch1a", ParentID: "ch1", Title: "Section 1A", TokenCount: 200, ContentRef: "a.txt"}, - {ID: "ch1b", ParentID: "ch1", Title: "Section 1B", TokenCount: 30, ContentRef: "b.txt"}, // small - {ID: "ch1c", ParentID: "ch1", Title: "Section 1C", TokenCount: 10, ContentRef: "c.txt"}, // tiny + {ID: "ch1b", ParentID: "ch1", Title: "Section 1B", TokenCount: 30, ContentRef: "b.txt"}, // small + {ID: "ch1c", ParentID: "ch1", Title: "Section 1C", TokenCount: 10, ContentRef: "c.txt"}, // tiny }, }, { @@ -92,7 +92,7 @@ func TestCompactMaxDepth(t *testing.T) { compacted := tr.Compact(CompactOpts{ MinTokens: 0, // no pruning MergeSingleChild: false, - MaxDepth: 2, // root(0) + chapters(1) only + MaxDepth: 2, // root(0) + chapters(1) only }) // All sections at depth >= 2 should be flattened. diff --git a/pkg/tree/tree.go b/pkg/tree/tree.go index 9df9f74..7193835 100644 --- a/pkg/tree/tree.go +++ b/pkg/tree/tree.go @@ -114,10 +114,10 @@ func (s *Section) IsLeaf() bool { // persisted on Document.toc_tree. Distinct from Section because // it represents the document's logical outline (headings the LLM // recovered or invented from body text) rather than the parser's -// chunked content tree. Used by the PageIndex-style retrieval +// chunked content tree. Used by the TreeWalk-style retrieval // strategy that reasons over the TOC before drilling into sections. // -// Structure carries the PageIndex-style hierarchical index ("1", +// Structure carries the TreeWalk-style hierarchical index ("1", // "1.1", "1.1.2"). Title is the original heading verbatim (spacing // fixed). StartPage is 1-indexed and refers to the source PDF's // physical page. EndPage is derived from the next sibling's @@ -125,7 +125,7 @@ func (s *Section) IsLeaf() bool { // and downstream readers should treat the node as running until // either the next sibling at the same depth or the document end. // -// The shape mirrors PageIndex's tree-output JSON (start_page / +// The shape mirrors TreeWalk's tree-output JSON (start_page / // end_page / nodes) so external tooling that expects that // vocabulary can interop without translation. type TOCNode struct {