diff --git a/packages/ragmir-core/src/query.test.ts b/packages/ragmir-core/src/query.test.ts index bd2927c..96c41a3 100644 --- a/packages/ragmir-core/src/query.test.ts +++ b/packages/ragmir-core/src/query.test.ts @@ -408,6 +408,53 @@ describe("search", () => { expect(results[0]?.relativePath).toBe(".ragmir/raw/zeta.md") }) + it("should anchor compound identifiers before broad lexical backfill", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-query-identifier-anchor-")) + tempDirs.push(root) + await initProject(root) + await mkdir(path.join(root, ".ragmir", "raw"), { recursive: true }) + await writeFile( + path.join(root, ".ragmir", "config.json"), + JSON.stringify({ retrievalProfile: "fast", topK: 2 }), + ) + await Promise.all([ + ...Array.from({ length: 8 }, (_entry, index) => + writeFile( + path.join(root, ".ragmir", "raw", `target-${String(index).padStart(3, "0")}.md`), + `Find evidence for compound identifier BENCH-IDENTIFIER-14 target ${index}.\n`, + ), + ), + ...Array.from({ length: 12 }, (_entry, index) => + writeFile( + path.join(root, ".ragmir", "raw", `distractor-${String(index).padStart(3, "0")}.md`), + `Find broad evidence for an unrelated routine ${index}.\n`, + ), + ), + ]) + await ingest({ cwd: root }) + + const exact = await search("Find evidence for BENCH-IDENTIFIER-14", { + cwd: root, + topK: 2, + explain: true, + }) + const fuzzy = await search("Find evidence for BENCH-IDENTIFIXR-14", { + cwd: root, + topK: 1, + explain: true, + }) + + expect(exact).toHaveLength(2) + expect(exact.every((result) => result.text.includes("BENCH-IDENTIFIER-14"))).toBe(true) + expect(exact[0]?.score).toMatchObject({ + lexicalBackend: "fts", + lexicalCandidatesMaterialized: 8, + lexicalQueryVariants: 1, + }) + expect(fuzzy[0]?.text).toContain("BENCH-IDENTIFIER-14") + expect(fuzzy[0]?.score?.lexicalQueryVariants).toBeGreaterThan(1) + }, 15_000) + it("should scan a complete lexical fallback in bounded batches", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-query-truncated-fallback-")) tempDirs.push(root) @@ -445,7 +492,7 @@ describe("search", () => { "Routine evidence without query terms.\n", ) await Promise.all( - Array.from({ length: 120 }, (_entry, index) => + Array.from({ length: 90 }, (_entry, index) => writeFile( path.join(root, ".ragmir", "raw", `distractor-${String(index).padStart(3, "0")}.md`), `Ragmir raw policy md distractor evidence ${index}.\n`, @@ -467,7 +514,7 @@ describe("search", () => { lexicalBackend: "fallback", lexicalExactPathMatch: true, }) - expect(vectorCandidateLimit(1)).toBeLessThan(120) + expect(vectorCandidateLimit(1)).toBeLessThan(90) }, 10_000) it("should explain complete lexical fallback activation and coverage", async () => { diff --git a/packages/ragmir-core/src/query.ts b/packages/ragmir-core/src/query.ts index 4493381..6a92c31 100644 --- a/packages/ragmir-core/src/query.ts +++ b/packages/ragmir-core/src/query.ts @@ -1,8 +1,10 @@ import { channel } from "node:diagnostics_channel" import { + BooleanQuery, type Connection, type FullTextQuery, MatchQuery, + Occur, Operator, PhraseQuery, } from "@lancedb/lancedb" @@ -977,20 +979,47 @@ function lexicalQuery( return null } const joined = tokens.join(" ") + const broadQuery = new MatchQuery(joined, "searchText", { operator: Operator.Or }) const supplemental: FullTextQuery[] = [] - if (tokens.length > 1) { - supplemental.push(new PhraseQuery(joined, "searchText")) - } const identifierTerms = [...query.matchAll(LEXICAL_IDENTIFIER_PATTERN)] .map((match) => match[0]) .filter(Boolean) - for (const identifier of [...new Set(identifierTerms)]) { - supplemental.push( - new MatchQuery(identifier, "searchText", { - boost: 2, - ...(isFuzzyLexicalTerm(identifier) ? { fuzziness: 1, prefixLength: 3 } : {}), - }), + const identifiers = [...new Set(identifierTerms)] + const firstIdentifier = identifiers[0] + if (firstIdentifier !== undefined) { + const firstExactIdentifierQuery = new PhraseQuery(firstIdentifier, "searchText") + const exactIdentifierQueries = [ + firstExactIdentifierQuery, + ...identifiers.slice(1).map((identifier) => new PhraseQuery(identifier, "searchText")), + ] + const exactIdentifierQuery: FullTextQuery = + exactIdentifierQueries.length === 1 + ? firstExactIdentifierQuery + : new BooleanQuery( + exactIdentifierQueries.map((item): [Occur, FullTextQuery] => [Occur.Should, item]), + ) + const fuzzyIdentifierQuery = new BooleanQuery( + identifiers.map((identifier): [Occur, FullTextQuery] => [ + Occur.Should, + new MatchQuery(identifier, "searchText", { + boost: 2, + fuzziness: 1, + operator: Operator.And, + prefixLength: 3, + }), + ]), ) + supplemental.push(fuzzyIdentifierQuery, broadQuery) + return { + primary: new BooleanQuery([ + [Occur.Must, exactIdentifierQuery], + [Occur.Should, broadQuery], + ]), + supplemental, + } + } + if (tokens.length > 1) { + supplemental.push(new PhraseQuery(joined, "searchText")) } const rareTerms = tokens .filter(isFuzzyLexicalTerm) @@ -1006,7 +1035,7 @@ function lexicalQuery( ) } return { - primary: new MatchQuery(joined, "searchText", { operator: Operator.Or }), + primary: broadQuery, supplemental, } }