diff --git a/src/benchmarks/README.md b/src/benchmarks/README.md index 8d43cf9..4e26d1a 100644 --- a/src/benchmarks/README.md +++ b/src/benchmarks/README.md @@ -36,6 +36,7 @@ interface Benchmark { | `locomo` | GitHub snap-research/locomo | Long context memory benchmark | | `longmemeval` | HuggingFace xiaowu0162/longmemeval-cleaned | Long-term memory evaluation | | `convomem` | HuggingFace Salesforce/ConvoMem | Conversational memory benchmark | +| `babilong` | HuggingFace RMT-team/babilong | Long-context retrieval and reasoning benchmark | ## Question Types diff --git a/src/benchmarks/babilong/README.md b/src/benchmarks/babilong/README.md new file mode 100644 index 0000000..858f885 --- /dev/null +++ b/src/benchmarks/babilong/README.md @@ -0,0 +1,120 @@ +# BABILong + +This benchmark adapter loads the BABILong evaluation dataset from local JSON files and feeds it into the standard MemoryBench pipeline. + +## Data layout + +Download the BABILong dataset from Hugging Face: + +https://huggingface.co/datasets/RMT-team/babilong + +Extract the dataset so the directory structure looks like: + +```text +data/benchmarks/babilong/ + qa1/ + 0k.json + qa2/ + 0k.json + qa3/ + 0k.json + ... + qa20/ + 0k.json +``` + +The adapter currently loads the `0k.json` evaluation split from every `qa1`–`qa20` task. + +Each task contains 100 evaluation samples. + +## Dataset format + +Each sample follows the native BABILong format: + +```json +{ + "input": "John travelled to the hallway. Mary journeyed to the bathroom...", + "question": "Where is Mary?", + "target": "bathroom" +} +``` + +The adapter maps each sample to: + +- one `UnifiedSession` containing the document (`input`) +- one benchmark question (`question`) +- one ground-truth answer (`target`) + +## List questions + +```bash +bun run src/index.ts list-questions -b babilong -l 10 +``` + +Filter by task: + +```bash +bun run src/index.ts list-questions -b babilong -t qa7 -l 10 +``` + +## Ingest into a provider + +Example using Supermemory: + +PowerShell: + +```powershell +$env:SUPERMEMORY_API_KEY = "sm_xxx" +bun run src/index.ts ingest -p supermemory -b babilong -r babilong-smoke --force +``` + +Bash: + +```bash +SUPERMEMORY_API_KEY=sm_xxx \ +bun run src/index.ts ingest \ + -p supermemory \ + -b babilong \ + -r babilong-smoke \ + --force +``` + +## Search after ingest + +```bash +bun run src/index.ts search -r babilong-smoke +``` + +Search results are written to: + +```text +data/runs/babilong-smoke/results/ +``` + +## What gets ingested + +For every BABILong sample, the adapter creates: + +- one session containing the complete document (`input`) +- one benchmark question (`question`) +- one ground-truth answer (`target`) + +This preserves the original long-context retrieval task while using the standard MemoryBench evaluation pipeline. + +## Benchmark + +BABILong evaluates long-context retrieval and reasoning by embedding a small set of supporting facts inside large amounts of irrelevant text. + +The benchmark contains 20 reasoning tasks (`qa1`–`qa20`) covering skills such as: + +- single supporting fact +- multi-hop reasoning +- counting +- negation +- coreference +- deduction +- induction +- positional reasoning +- path finding + +The adapter keeps the original task names as MemoryBench question types, allowing individual reasoning capabilities to be benchmarked independently. \ No newline at end of file diff --git a/src/benchmarks/babilong/index.ts b/src/benchmarks/babilong/index.ts new file mode 100644 index 0000000..6017fcc --- /dev/null +++ b/src/benchmarks/babilong/index.ts @@ -0,0 +1,153 @@ +import { existsSync, readFileSync } from "fs" +import { join } from "path" +import type { Benchmark, BenchmarkConfig, QuestionFilter } from "../../types/benchmark" +import type { + UnifiedQuestion, + UnifiedSession, + UnifiedMessage, + QuestionTypeRegistry, +} from "../../types/unified" +import type { BABILongItem } from "./types" +import { logger } from "../../utils/logger" + +const DEFAULT_DATA_PATH = "./data/benchmarks/babilong" +const DEFAULT_CONTEXT_LENGTH = "0k" + +// qa subfolders (all use 0k format) +const QA_FOLDERS = Array.from({ length: 20 }, (_, i) => `qa${i + 1}`) + +/** + * BABILong question types - native task types from the dataset. + */ +export const BABILONG_QUESTION_TYPES: QuestionTypeRegistry = { + qa1: { id: "qa1", alias: "single", description: "Single supporting fact" }, + qa2: { id: "qa2", alias: "two", description: "Two supporting facts" }, + qa3: { id: "qa3", alias: "three", description: "Three supporting facts" }, + qa4: { id: "qa4", alias: "two-arg", description: "Two argument relations" }, + qa5: { id: "qa5", alias: "three-arg", description: "Three argument relations" }, + qa6: { id: "qa6", alias: "yes-no", description: "Yes/no questions" }, + qa7: { id: "qa7", alias: "counting", description: "Counting" }, + qa8: { id: "qa8", alias: "lists-sets", description: "Lists/sets" }, + qa9: { id: "qa9", alias: "simple-neg", description: "Simple negation" }, + qa10: { id: "qa10", alias: "indef-know", description: "Indefinite knowledge" }, + qa11: { id: "qa11", alias: "basic-coref", description: "Basic coreference" }, + qa12: { id: "qa12", alias: "conjunction", description: "Conjunction" }, + qa13: { id: "qa13", alias: "compound-coref", description: "Compound coreference" }, + qa14: { id: "qa14", alias: "time-reasoning", description: "Time reasoning" }, + qa15: { id: "qa15", alias: "basic-deduction", description: "Basic deduction" }, + qa16: { id: "qa16", alias: "basic-induction", description: "Basic induction" }, + qa17: { id: "qa17", alias: "positional-reasoning", description: "Positional reasoning" }, + qa18: { id: "qa18", alias: "size-reasoning", description: "Size reasoning" }, + qa19: { id: "qa19", alias: "path-finding", description: "Path finding" }, + qa20: { id: "qa20", alias: "qa20", description: "Task 20" }, +} + +export class BABILongBenchmark implements Benchmark { + name = "babilong" + + private questions: UnifiedQuestion[] = [] + private sessionsMap: Map = new Map() + private dataPath: string = "" + + async load(config?: BenchmarkConfig): Promise { + this.dataPath = config?.dataPath || DEFAULT_DATA_PATH + const fullPath = join(process.cwd(), this.dataPath) + + if (!existsSync(fullPath)) { + throw new Error( + `BABILong dataset not found at ${fullPath}. Download the dataset from https://huggingface.co/datasets/RMT-team/babilong and place the qa1-qa20 folders under ${this.dataPath}.` + ) + } + + this.loadQuestions(fullPath) + } + + private loadQuestions(fullPath: string): void { + for (const qaFolder of QA_FOLDERS) { + const filePath = join(fullPath, qaFolder, `${DEFAULT_CONTEXT_LENGTH}.json`) + + if (!existsSync(filePath)) { + logger.warn(`Missing file for ${qaFolder}: ${filePath}`) + continue + } + + try { + const content = readFileSync(filePath, "utf8") + const items: BABILongItem[] = JSON.parse(content) + + for (let i = 0; i < items.length; i++) { + this.processItem(items[i], qaFolder, i) + } + } catch (e) { + logger.error(`Failed to load BABILong data for ${qaFolder}: ${e}`) + } + } + + logger.info(`Loaded ${this.questions.length} questions from BABILong`) + } + + private processItem(item: BABILongItem, qaFolder: string, index: number): void { + const questionId = `babilong-${qaFolder}-${String(index).padStart(3, "0")}` + + const session = this.extractSession(item, questionId) + + this.questions.push({ + questionId, + question: item.question, + questionType: qaFolder, + groundTruth: item.target, + haystackSessionIds: [session.sessionId], + metadata: { + task: qaFolder, + contextLength: DEFAULT_CONTEXT_LENGTH, + }, + }) + + this.sessionsMap.set(questionId, [session]) + } + + private extractSession(item: BABILongItem, questionId: string): UnifiedSession { + const message: UnifiedMessage = { + role: "user", + content: item.input, + } + + return { + sessionId: `${questionId}-session-0`, + messages: [message], + } + } + + getQuestions(filter?: QuestionFilter): UnifiedQuestion[] { + let result = [...this.questions] + + if (filter?.questionTypes?.length) { + result = result.filter((q) => filter.questionTypes!.includes(q.questionType)) + } + + if (filter?.offset) { + result = result.slice(filter.offset) + } + + if (filter?.limit) { + result = result.slice(0, filter.limit) + } + + return result + } + + getHaystackSessions(questionId: string): UnifiedSession[] { + return this.sessionsMap.get(questionId) || [] + } + + getGroundTruth(questionId: string): string { + const question = this.questions.find((q) => q.questionId === questionId) + return question?.groundTruth || "" + } + + getQuestionTypes(): QuestionTypeRegistry { + return BABILONG_QUESTION_TYPES + } +} + +export default BABILongBenchmark \ No newline at end of file diff --git a/src/benchmarks/babilong/types.ts b/src/benchmarks/babilong/types.ts new file mode 100644 index 0000000..d0bb451 --- /dev/null +++ b/src/benchmarks/babilong/types.ts @@ -0,0 +1,5 @@ +export interface BABILongItem { + input: string + question: string + target: string +} \ No newline at end of file diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index b790e87..5848744 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -2,11 +2,13 @@ import type { Benchmark, BenchmarkName } from "../types/benchmark" import { LoCoMoBenchmark } from "./locomo" import { LongMemEvalBenchmark } from "./longmemeval" import { ConvoMemBenchmark } from "./convomem" +import { BABILongBenchmark } from "./babilong" const benchmarks: Record Benchmark> = { locomo: LoCoMoBenchmark, longmemeval: LongMemEvalBenchmark, convomem: ConvoMemBenchmark, + babilong: BABILongBenchmark, } export function createBenchmark(name: BenchmarkName): Benchmark { @@ -21,4 +23,4 @@ export function getAvailableBenchmarks(): BenchmarkName[] { return Object.keys(benchmarks) as BenchmarkName[] } -export { LoCoMoBenchmark, LongMemEvalBenchmark, ConvoMemBenchmark } +export { LoCoMoBenchmark, LongMemEvalBenchmark, ConvoMemBenchmark, BABILongBenchmark } diff --git a/src/cli/index.ts b/src/cli/index.ts index b3c29d6..89f33d5 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -150,11 +150,16 @@ Available benchmark datasets for evaluation: convomem ConvoMem - Conversational memory benchmark Tests: user facts, assistant facts, preferences, implicit connections Source: HuggingFace Salesforce/ConvoMem (downloaded on first use) + + babilong BABILong - Long-context retrieval benchmark + Tests: retrieval, multi-hop reasoning, deduction, counting, coreference + Source: HuggingFace RMT-team/babilong (local data required) Usage: -b locomo Run LoCoMo benchmark -b longmemeval Run LongMemEval benchmark -b convomem Run ConvoMem benchmark + -b babilong Run BABILong benchmark `) } diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index 07961e4..fe53c68 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -20,4 +20,4 @@ export interface Benchmark { getQuestionTypes(): QuestionTypeRegistry } -export type BenchmarkName = "locomo" | "longmemeval" | "convomem" +export type BenchmarkName = "locomo" | "longmemeval" | "convomem" | "babilong"