From d04e130b4815daa1de354d21ee1cbbd61abe5664 Mon Sep 17 00:00:00 2001 From: Tim Wang Date: Tue, 18 Aug 2026 20:21:55 -0700 Subject: [PATCH 1/6] feat: Add support for eval case level timeouts --- evalbench/evaluator/agentevaluator.py | 84 +++- evalbench/evaluator/evaluator.py | 9 +- evalbench/generators/models/agent_cli.py | 6 +- evalbench/generators/models/agy_cli.py | 38 +- evalbench/generators/models/claude_code.py | 29 +- evalbench/generators/models/codex_cli.py | 49 ++- evalbench/generators/models/gemini_cli.py | 30 +- evalbench/generators/models/noop_agent.py | 6 +- evalbench/test/test_eval_case_timeout.py | 472 +++++++++++++++++++++ evalbench/util/config.py | 73 +++- evalbench/work/agentscorework.py | 2 +- 11 files changed, 728 insertions(+), 70 deletions(-) create mode 100644 evalbench/test/test_eval_case_timeout.py diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index f464a83b..448f4ce9 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -4,12 +4,14 @@ import logging import os import shutil +import time import threading from dataset.evalgeminicliinput import EvalGeminiCliRequest from generators.models import get_generator from generators.models.agent_cli import AgentCliGenerator from mp import mprunner +from util.config import get_eval_case_timeout from work.agentgenwork import AgentGenWork from evaluator.simulateduser import SimulatedUser from work.agentscorework import AgentScoreWork @@ -24,6 +26,7 @@ def __init__( config, ): self.config = config + self.eval_case_timeout_seconds = get_eval_case_timeout(config) model_config_path = config.get("model_config") if not isinstance(model_config_path, str): @@ -142,7 +145,46 @@ def process_scenario( ) session_id = None + + # Dynamically calculate the effective timeout, taking into account + # both the scenario timeout and the global timeout. + start_time = time.monotonic() + scenario_timeout = get_eval_case_timeout(scenario) + effective_case_timeout_seconds = ( + scenario_timeout + if scenario_timeout is not None + else self.eval_case_timeout_seconds + ) + last_result = None + for turn in range(max_turns): + remaining_timeout_seconds = None + if effective_case_timeout_seconds is not None: + elapsed_seconds = time.monotonic() - start_time + remaining_timeout_seconds = effective_case_timeout_seconds - elapsed_seconds + if remaining_timeout_seconds <= 0: + logging.warning( + f"Eval case timeout ({effective_case_timeout_seconds}s) reached before turn {turn + 1}." + ) + timeout_msg = f"TimeoutError: Scenario timed out after {effective_case_timeout_seconds}s before turn {turn + 1}" + if last_result is None: + last_result = subprocess.CompletedProcess( + args=[self.agent_version], + returncode=124, + stdout="", + stderr=timeout_msg, + ) + else: + current_stderr = getattr(last_result, "stderr", "") or "" + new_stderr = f"{current_stderr}\n{timeout_msg}".strip() + last_result = subprocess.CompletedProcess( + args=getattr(last_result, "args", [self.agent_version]), + returncode=124, + stdout=getattr(last_result, "stdout", "") or "", + stderr=new_stderr, + ) + break + logging.info( f"Turn {turn + 1}/{max_turns} - Prompt: {current_prompt}") if isinstance(self.generator, AgentCliGenerator): @@ -155,7 +197,9 @@ def process_scenario( cwd=resolved_work_dir, ) try: - result = self.generator.safe_generate(cli_cmd) + result = self.generator.safe_generate( + cli_cmd, timeout_seconds=remaining_timeout_seconds + ) if result.stdout: parsed = self.generator.parse_response(result.stdout) if parsed.get("session_id"): @@ -205,18 +249,22 @@ def process_scenario( else: break - if last_result: - self._finalize_scenario( - scenario, - last_result, - conversation_history, - accumulated_tools, - accumulated_skills, - eval_result, - job_id, - metadata + if last_result is None: + last_result = subprocess.CompletedProcess( + args=[self.agent_version], returncode=0, stdout="", stderr="" ) + self._finalize_scenario( + scenario, + last_result, + conversation_history, + accumulated_tools, + accumulated_skills, + eval_result, + job_id, + metadata + ) + def _log_cli_result(self, turn: int, max_turns: int, result: subprocess.CompletedProcess): generator_name = self.generator.name logging.info( @@ -238,14 +286,20 @@ def _finalize_scenario( metadata: Dict[str, Any] ): """Finalizes the scenario by scoring and appending results.""" + generated_error = None + if hasattr(last_result, "returncode") and last_result.returncode != 0: + generated_error = getattr(last_result, "stderr", None) or f"Command failed with exit code {last_result.returncode}" + elif hasattr(last_result, "stderr") and "TimeoutError" in (last_result.stderr or ""): + generated_error = last_result.stderr + # Prepare intermediate eval_output with all necessary data for scoring eval_output_data = { "eval_id": scenario["id"], - "stdout": last_result.stdout, - "stderr": last_result.stderr, - "returncode": last_result.returncode, + "stdout": getattr(last_result, "stdout", "") or "", + "stderr": getattr(last_result, "stderr", "") or "", + "returncode": getattr(last_result, "returncode", 0), "prompt_generator_error": None, - "generated_error": None, + "generated_error": generated_error, "sql_generator_error": None, "golden_error": None, "generated_sql": "skipped", diff --git a/evalbench/evaluator/evaluator.py b/evalbench/evaluator/evaluator.py index 8d6b9de0..24c24623 100644 --- a/evalbench/evaluator/evaluator.py +++ b/evalbench/evaluator/evaluator.py @@ -17,6 +17,7 @@ ) from mp import mprunner from util import truncateExecutionOutputs +from util.config import get_eval_case_timeout from work import multi_trial_scorework from work import promptgenwork from work import scorework @@ -71,8 +72,12 @@ def __init__( self.sqlgen_runners = runner_config.get("sqlgen_runners", 10) self.sqlexec_runners = runner_config.get("sqlexec_runners", 10) self.scoring_runners = runner_config.get("scoring_runners", 10) - self.task_timeout_seconds = runner_config.get( - "task_timeout_seconds", 600) + eval_timeout = get_eval_case_timeout(self.config) + self.task_timeout_seconds = ( + eval_timeout + if eval_timeout is not None + else runner_config.get("task_timeout_seconds", 600) + ) self.num_trials = self.config.get("num_trials", 1) def evaluate( diff --git a/evalbench/generators/models/agent_cli.py b/evalbench/generators/models/agent_cli.py index 41010e6c..ddee1b4a 100644 --- a/evalbench/generators/models/agent_cli.py +++ b/evalbench/generators/models/agent_cli.py @@ -1,4 +1,6 @@ from abc import abstractmethod +import subprocess +from typing import Optional from mcp import types as mcp_types @@ -96,7 +98,9 @@ def create_command( raise NotImplementedError("Subclasses must implement this method") @abstractmethod - def safe_generate(self, cli_cmd): + def safe_generate( + self, cli_cmd, timeout_seconds: Optional[float] = None + ) -> subprocess.CompletedProcess: raise NotImplementedError("Subclasses must implement this method") @abstractmethod diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index 37eee849..f5575ea4 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -9,7 +9,9 @@ import sys import tempfile import weakref +from typing import Optional, Union, Dict, List from util.context import rpc_id_var +from util.config import parse_timeout_seconds # Default CLI label reported in metadata. The executed binary is installed # per-session at self.agy_bin (see _ensure_agy_installed). @@ -101,17 +103,7 @@ def __init__(self, querygenerator_config): @staticmethod def _validate_timeout(timeout): if timeout is not None: - if not isinstance(timeout, str): - raise TypeError( - "timeout must be a string (e.g., '20m', '1h30m', '300s')" - ) - # Strict regex for common units (s, m, h). - # Allows things like "20m", "1h30m", "300s". - if not re.match(r'^(\d+(s|m|h))+$', timeout): - raise ValueError( - f"Invalid timeout format: '{timeout}'. " - "Must be a valid duration string (e.g., '20m', '1h30m', '300s')." - ) + parse_timeout_seconds(timeout) def _init_paths(self, querygenerator_config): """Resolves the sandbox ``HOME`` and all derived agy paths, and @@ -866,13 +858,13 @@ def _base_agy_command( command.append("--continue") return command - def generate_internal(self, cli_cmd): + def generate_internal(self, cli_cmd, timeout_seconds=None): if not isinstance(cli_cmd, CLICommand): cli_cmd = CLICommand(self.agy_bin, str(cli_cmd)) - return self._run_agy_cli(cli_cmd) + return self._run_agy_cli(cli_cmd, timeout_seconds=timeout_seconds) def _execute_cli_command( - self, command, env=None, cwd=None + self, command, env=None, cwd=None, timeout_seconds=None ) -> subprocess.CompletedProcess: try: return subprocess.run( @@ -882,6 +874,16 @@ def _execute_cli_command( check=False, env=env, cwd=cwd if cwd else self.fake_home, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired as e: + stdout_str = e.stdout if isinstance(e.stdout, str) else (e.stdout.decode() if e.stdout else "") + stderr_str = f"TimeoutError: Command timed out after {timeout_seconds} seconds" + if e.stderr: + err_text = e.stderr if isinstance(e.stderr, str) else e.stderr.decode() + stderr_str = f"{stderr_str}\n{err_text}" + return subprocess.CompletedProcess( + command, 124, stdout_str, stderr_str ) except FileNotFoundError: return subprocess.CompletedProcess( @@ -893,7 +895,7 @@ def _execute_cli_command( command, 1, "", f"An unexpected error occurred: {e}" ) - def _run_agy_cli(self, cli_cmd: CLICommand): + def _run_agy_cli(self, cli_cmd: CLICommand, timeout_seconds=None): env = self._merged_env(cli_cmd.env) # The executable is always this session's sandbox binary, regardless of # the label carried on cli_cmd.cli (the evaluator passes agent_version, @@ -904,7 +906,7 @@ def _run_agy_cli(self, cli_cmd: CLICommand): timeout=self.timeout, ) cwd = cli_cmd.cwd if cli_cmd.cwd else self.fake_home - result = self._execute_cli_command(command, env=env, cwd=cwd) + result = self._execute_cli_command(command, env=env, cwd=cwd, timeout_seconds=timeout_seconds) # Parse whenever agy emitted a stream, even on a non-zero exit: a # timed-out/errored run still ends in a ``result`` event carrying real @@ -1198,9 +1200,9 @@ def extract_skills(self, stdout: str) -> list: return [] def safe_generate( - self, cli_cmd: CLICommand + self, cli_cmd: CLICommand, timeout_seconds: Optional[float] = None ) -> subprocess.CompletedProcess: - result = self.generate_internal(cli_cmd) + result = self.generate_internal(cli_cmd, timeout_seconds=timeout_seconds) if isinstance(result, str): return subprocess.CompletedProcess( args=[], returncode=0, stdout=result diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index ef10983d..ebcad6de 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -8,6 +8,7 @@ import sys import re import shutil +from typing import Optional, Union, Dict, List from util.context import rpc_id_var @@ -597,21 +598,31 @@ def _install_plugin(self, plugin_id: str, env: dict | None = None): else: logging.info(f"Successfully installed plugin '{plugin_id}'") - def generate_internal(self, cli_cmd): + def generate_internal(self, cli_cmd, timeout_seconds=None): if not isinstance(cli_cmd, CLICommand): cli_cmd = CLICommand(self.claude_code_version, str(cli_cmd)) - return self._run_claude_code(cli_cmd) + return self._run_claude_code(cli_cmd, timeout_seconds=timeout_seconds) def _execute_cli_command( self, command: list[str], env: dict[str, str] | None = None, - cwd: str | None = None, + cwd: str | None = None, timeout_seconds: float | int | None = None, ) -> subprocess.CompletedProcess: try: result = subprocess.run( command, capture_output=True, text=True, check=False, env=env, - cwd=cwd if cwd else self.fake_home, stdin=subprocess.DEVNULL + cwd=cwd if cwd else self.fake_home, stdin=subprocess.DEVNULL, + timeout=timeout_seconds, ) return result + except subprocess.TimeoutExpired as e: + stdout_str = e.stdout if isinstance(e.stdout, str) else (e.stdout.decode() if e.stdout else "") + stderr_str = f"TimeoutError: Command timed out after {timeout_seconds} seconds" + if e.stderr: + err_text = e.stderr if isinstance(e.stderr, str) else e.stderr.decode() + stderr_str = f"{stderr_str}\n{err_text}" + return subprocess.CompletedProcess( + command, 124, stdout_str, stderr_str + ) except FileNotFoundError: return subprocess.CompletedProcess( command, 127, "", f"Error: Command not found: {command[0]}" @@ -621,7 +632,7 @@ def _execute_cli_command( command, 1, "", f"An unexpected error occurred: {e}" ) - def _run_claude_code(self, cli_cmd: CLICommand): + def _run_claude_code(self, cli_cmd: CLICommand, timeout_seconds=None): env = os.environ.copy() env.update(self.env) env.update(cli_cmd.env) @@ -673,7 +684,7 @@ def _run_claude_code(self, cli_cmd: CLICommand): logging.info(f"Running Claude Code CLI: {' '.join(command)}") - result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd) + result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd, timeout_seconds=timeout_seconds) if result.stdout: result.stdout = self._parse_stream_json(result.stdout) @@ -1054,8 +1065,10 @@ def extract_skill_scripts(self, stdout: str) -> list[str]: return [] return self._extract_script_names(by_name) - def safe_generate(self, cli_cmd: CLICommand) -> subprocess.CompletedProcess: - result = self.generate_internal(cli_cmd) + def safe_generate( + self, cli_cmd: CLICommand, timeout_seconds: Optional[float] = None + ) -> subprocess.CompletedProcess: + result = self.generate_internal(cli_cmd, timeout_seconds=timeout_seconds) if isinstance(result, str): return subprocess.CompletedProcess(args=[], returncode=0, stdout=result) diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index 35f6e4ab..a6bcece2 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -9,6 +9,7 @@ import sys import threading import time +from typing import Optional, Union, Dict, List from util.context import rpc_id_var @@ -703,10 +704,10 @@ def _toml_value(cls, value) -> str: return "{ " + inner + " }" return json.dumps(str(value)) - def generate_internal(self, cli_cmd): + def generate_internal(self, cli_cmd, timeout_seconds=None): if not isinstance(cli_cmd, CLICommand): cli_cmd = CLICommand(self.codex_cli_version, str(cli_cmd)) - return self._run_codex_cli(cli_cmd) + return self._run_codex_cli(cli_cmd, timeout_seconds=timeout_seconds) _EV_ITEM_STARTED = "item.started" _EV_ITEM_UPDATED = "item.updated" @@ -719,6 +720,7 @@ def generate_internal(self, cli_cmd): def _execute_cli_command( self, command: list[str], env: dict[str, str] | None = None, cwd: str | None = None, + timeout_seconds: float | int | None = None, ) -> tuple[subprocess.CompletedProcess, dict[str, int]]: """Runs the Codex CLI with line-streamed stdout so we can stamp the wall-clock time at which each NDJSON event arrives. @@ -762,17 +764,32 @@ def _drain_stderr(): started_at_ms: dict[str, float] = {} tool_durations: dict[str, int] = {} + def _drain_stdout(): + try: + for line in proc.stdout: + arrival_ms = time.monotonic() * 1000 + stdout_lines.append(line) + self._stamp_tool_event( + line, arrival_ms, started_at_ms, tool_durations, + ) + except Exception as e: + logging.warning(f"stdout stream read failed: {e}") + + stdout_thread = threading.Thread(target=_drain_stdout, daemon=True) + stdout_thread.start() + try: - for line in proc.stdout: - arrival_ms = time.monotonic() * 1000 - stdout_lines.append(line) - self._stamp_tool_event( - line, arrival_ms, started_at_ms, tool_durations, - ) - except Exception as e: - logging.warning(f"stdout stream read failed: {e}") + proc.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + stdout_thread.join(timeout=5) + stderr_thread.join(timeout=5) + return subprocess.CompletedProcess( + command, 124, "".join(stdout_lines), f"TimeoutError: Command timed out after {timeout_seconds} seconds" + ), tool_durations - proc.wait() + stdout_thread.join(timeout=5) stderr_thread.join(timeout=5) completed = subprocess.CompletedProcess( @@ -813,7 +830,7 @@ def _stamp_tool_event( if t0 is not None: tool_durations[item_id] = max(0, int(arrival_ms - t0)) - def _run_codex_cli(self, cli_cmd: CLICommand): + def _run_codex_cli(self, cli_cmd: CLICommand, timeout_seconds=None): env = os.environ.copy() env.update(self.env) env.update(cli_cmd.env) @@ -866,7 +883,7 @@ def _run_codex_cli(self, cli_cmd: CLICommand): logging.info(f"Running Codex CLI: {' '.join(command)}") start_ms = time.monotonic() - result, tool_durations = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd) + result, tool_durations = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd, timeout_seconds=timeout_seconds) duration_ms = int((time.monotonic() - start_ms) * 1000) if result.stdout: result.stdout = self._parse_stream_json( @@ -1294,8 +1311,10 @@ def add_skill(name: str): return items - def safe_generate(self, cli_cmd: CLICommand) -> subprocess.CompletedProcess: - result = self.generate_internal(cli_cmd) + def safe_generate( + self, cli_cmd: CLICommand, timeout_seconds: Optional[float] = None + ) -> subprocess.CompletedProcess: + result = self.generate_internal(cli_cmd, timeout_seconds=timeout_seconds) if isinstance(result, str): return subprocess.CompletedProcess(args=[], returncode=0, stdout=result) if not result.stdout and result.returncode != 0: diff --git a/evalbench/generators/models/gemini_cli.py b/evalbench/generators/models/gemini_cli.py index a72e1231..0f0d7757 100644 --- a/evalbench/generators/models/gemini_cli.py +++ b/evalbench/generators/models/gemini_cli.py @@ -7,6 +7,7 @@ import re import shutil import sys +from typing import Optional, Union, Dict, List from util.context import rpc_id_var @@ -784,16 +785,16 @@ def _patch_manifest_sensitive(self, ext_path): except Exception as e: logging.error(f"Failed to patch manifest at {manifest_path}: {e}") - def generate_internal(self, cli_cmd: CLICommand | str): + def generate_internal(self, cli_cmd: CLICommand | str, timeout_seconds: float | int | None = None): if not isinstance(cli_cmd, CLICommand): cli_cmd = CLICommand(self.gemini_cli_version, str(cli_cmd)) - return self._run_gemini_cli(cli_cmd) + return self._run_gemini_cli(cli_cmd, timeout_seconds=timeout_seconds) def _execute_cli_command( - self, command: list[str], env: dict[str, str] | None = None, cwd: str | None = None + self, command: list[str], env: dict[str, str] | None = None, cwd: str | None = None, timeout_seconds: float | int | None = None ) -> subprocess.CompletedProcess: try: - result = subprocess.run(command, capture_output=True, text=True, check=False, env=env, cwd=cwd if cwd else self.fake_home) + result = subprocess.run(command, capture_output=True, text=True, check=False, env=env, cwd=cwd if cwd else self.fake_home, timeout=timeout_seconds) # Filter out benign schema warnings from json decoder from stderr to reduce noise if result.stderr: result.stderr = "\n".join( @@ -804,6 +805,15 @@ def _execute_cli_command( ] ) return result + except subprocess.TimeoutExpired as e: + stdout_str = e.stdout if isinstance(e.stdout, str) else (e.stdout.decode() if e.stdout else "") + stderr_str = f"TimeoutError: Command timed out after {timeout_seconds} seconds" + if e.stderr: + err_text = e.stderr if isinstance(e.stderr, str) else e.stderr.decode() + stderr_str = f"{stderr_str}\n{err_text}" + return subprocess.CompletedProcess( + command, 124, stdout_str, stderr_str + ) except FileNotFoundError: return subprocess.CompletedProcess( command, 127, "", f"Error: Command not found: {command[0]}" @@ -813,7 +823,7 @@ def _execute_cli_command( command, 1, "", f"An unexpected error occurred: {e}" ) - def _run_gemini_cli(self, cli_cmd: CLICommand): + def _run_gemini_cli(self, cli_cmd: CLICommand, timeout_seconds: float | int | None = None): gemini_settings_path = os.path.join(self.gemini_home, "settings.json") if not os.path.exists(gemini_settings_path): @@ -847,7 +857,9 @@ def _run_gemini_cli(self, cli_cmd: CLICommand): ] ) - result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd) + result = self._execute_cli_command( + command, env=env, cwd=cli_cmd.cwd, timeout_seconds=timeout_seconds + ) if result.returncode == 0 and result.stdout: result.stdout = self._parse_stream_json(result.stdout) @@ -1072,8 +1084,10 @@ def extract_skills(self, stdout: str) -> list[str]: except (KeyError, TypeError): return [] - def safe_generate(self, cli_cmd: CLICommand) -> subprocess.CompletedProcess: - result = self.generate_internal(cli_cmd) + def safe_generate( + self, cli_cmd: CLICommand, timeout_seconds: Optional[float] = None + ) -> subprocess.CompletedProcess: + result = self.generate_internal(cli_cmd, timeout_seconds=timeout_seconds) if isinstance(result, str): return subprocess.CompletedProcess(args=[], returncode=0, stdout=result) diff --git a/evalbench/generators/models/noop_agent.py b/evalbench/generators/models/noop_agent.py index f43e29cb..6bb76443 100644 --- a/evalbench/generators/models/noop_agent.py +++ b/evalbench/generators/models/noop_agent.py @@ -9,6 +9,8 @@ import subprocess +from typing import Optional + from .agent_cli import AgentCliGenerator @@ -31,7 +33,9 @@ def create_command( ): return [] - def safe_generate(self, cli_cmd): + def safe_generate( + self, cli_cmd, timeout_seconds: Optional[float] = None + ) -> subprocess.CompletedProcess: return subprocess.CompletedProcess( args=[], returncode=0, stdout="", stderr="" ) diff --git a/evalbench/test/test_eval_case_timeout.py b/evalbench/test/test_eval_case_timeout.py new file mode 100644 index 00000000..065edf10 --- /dev/null +++ b/evalbench/test/test_eval_case_timeout.py @@ -0,0 +1,472 @@ +import concurrent.futures +import subprocess +import time +import unittest +from unittest.mock import MagicMock, patch + +from evaluator.agentevaluator import AgentEvaluator +from evaluator.evaluator import Evaluator, _process_futures_with_timeout +from generators.models.agent_cli import AgentCliGenerator +from generators.models.gemini_cli import GeminiCliGenerator +from generators.models.codex_cli import CodexCliGenerator +from generators.models.claude_code import ClaudeCodeGenerator +from generators.models.agy_cli import AgyCliGenerator +from util.config import get_eval_case_timeout, parse_timeout_seconds +from work.agentscorework import AgentScoreWork + + +class TestEvalCaseTimeoutConfig(unittest.TestCase): + """Unit tests for parse_timeout_seconds and get_eval_case_timeout.""" + + def test_parse_timeout_seconds_numeric(self): + self.assertEqual(parse_timeout_seconds(300), 300.0) + self.assertEqual(parse_timeout_seconds(12.5), 12.5) + self.assertEqual(parse_timeout_seconds(0), 0.0) + with self.assertRaises(ValueError): + parse_timeout_seconds(-10) + self.assertIsNone(parse_timeout_seconds(None)) + + def test_parse_timeout_seconds_strings(self): + self.assertEqual(parse_timeout_seconds("300"), 300.0) + self.assertEqual(parse_timeout_seconds("45.5"), 45.5) + self.assertEqual(parse_timeout_seconds("300s"), 300.0) + self.assertEqual(parse_timeout_seconds("5m"), 300.0) + self.assertEqual(parse_timeout_seconds("1h"), 3600.0) + self.assertEqual(parse_timeout_seconds("1h30m"), 5400.0) + self.assertEqual(parse_timeout_seconds("1h\t30m"), 5400.0) + self.assertEqual(parse_timeout_seconds("5m 30s"), 330.0) + self.assertEqual(parse_timeout_seconds("10 s"), 10.0) + self.assertEqual(parse_timeout_seconds("1d"), 86400.0) + self.assertEqual(parse_timeout_seconds("500ms"), 0.5) + self.assertEqual(parse_timeout_seconds(" 10m "), 600.0) + + def test_parse_timeout_seconds_invalid(self): + with self.assertRaises(ValueError): + parse_timeout_seconds("invalid") + self.assertIsNone(parse_timeout_seconds("")) + with self.assertRaises(TypeError): + parse_timeout_seconds([]) + with self.assertRaises(TypeError): + parse_timeout_seconds({}) + with self.assertRaises(TypeError): + parse_timeout_seconds(True) + with self.assertRaises(ValueError): + parse_timeout_seconds("1h and 30m") + + def test_get_eval_case_timeout_top_level(self): + self.assertEqual( + get_eval_case_timeout({"eval_case_timeout": 300}), 300.0 + ) + self.assertEqual( + get_eval_case_timeout({"eval_case_timeout": "5m"}), 300.0 + ) + self.assertEqual( + get_eval_case_timeout({"eval_case_timeout": "1h30m"}), 5400.0 + ) + + def test_get_eval_case_timeout_default_none(self): + self.assertIsNone(get_eval_case_timeout({})) + self.assertIsNone(get_eval_case_timeout({"runners": {}})) + self.assertIsNone(get_eval_case_timeout({"runners": {"task_timeout_seconds": 400}})) + self.assertIsNone(get_eval_case_timeout({"orchestrator": "geminicli"})) + + +class TestAgentEvaluatorTimeout(unittest.TestCase): + """Unit tests for AgentEvaluator timeout enforcement.""" + + @patch("evaluator.agentevaluator.get_generator") + def test_agent_evaluator_initializes_timeout(self, mock_get_generator): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_get_generator.return_value = mock_generator + + # With timeout in run_config + config_with_timeout = { + "model_config": "dummy.yaml", + "eval_case_timeout": "300s", + } + evaluator = AgentEvaluator(config_with_timeout) + self.assertEqual(evaluator.eval_case_timeout_seconds, 300.0) + + # Without timeout (default) + config_without_timeout = { + "model_config": "dummy.yaml", + } + evaluator_default = AgentEvaluator(config_without_timeout) + self.assertIsNone(evaluator_default.eval_case_timeout_seconds) + + @patch("evaluator.agentevaluator.get_generator") + def test_process_scenario_times_out_multi_turn(self, mock_get_generator): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_generator.fake_home = "/tmp/fake_home" + mock_generator.create_command.return_value = MagicMock() + mock_generator.parse_response.return_value = {"session_id": "s1"} + mock_generator.extract_tools.return_value = [] + mock_generator.extract_skills.return_value = [] + mock_get_generator.return_value = mock_generator + + config = { + "model_config": "dummy.yaml", + "eval_case_timeout": 0.05, # 50ms timeout + } + evaluator = AgentEvaluator(config) + evaluator._finalize_scenario = MagicMock() + + # Generator simulates a slow execution + def slow_safe_generate(cli_cmd, timeout_seconds=None): + time.sleep(0.08) + return subprocess.CompletedProcess( + args=["mock"], returncode=0, stdout='{"response": "ok"}', stderr="" + ) + + mock_generator.safe_generate.side_effect = slow_safe_generate + + simulated_user = MagicMock() + simulated_user.get_next_response.return_value = "next question" + + scenario = { + "id": "scenario_timeout_test", + "starting_prompt": "Hello", + "max_turns": 3, + } + + evaluator.process_scenario( + scenario=scenario, + eval_result=MagicMock(), + job_id="job_123", + metadata={}, + simulated_user=simulated_user, + ) + + # Verify finalize_scenario was called with the result + evaluator._finalize_scenario.assert_called_once() + args, kwargs = evaluator._finalize_scenario.call_args + last_result = args[1] + self.assertIsNotNone(last_result) + self.assertEqual(last_result.returncode, 124) + self.assertIn("TimeoutError", last_result.stderr) + # Should have only executed 1 turn due to timeout + self.assertEqual(mock_generator.safe_generate.call_count, 1) + + @patch("evaluator.agentevaluator.get_generator") + def test_process_scenario_times_out_before_turn_1(self, mock_get_generator): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_get_generator.return_value = mock_generator + + config = { + "model_config": "dummy.yaml", + "eval_case_timeout": 0.0, # 0s timeout -> immediately expired + } + evaluator = AgentEvaluator(config) + evaluator._finalize_scenario = MagicMock() + + scenario = { + "id": "scenario_immediate_timeout", + "starting_prompt": "Hello", + "max_turns": 3, + } + + evaluator.process_scenario( + scenario=scenario, + eval_result=MagicMock(), + job_id="job_123", + metadata={}, + ) + + # Finalize scenario must be called even if timed out before turn 1 + evaluator._finalize_scenario.assert_called_once() + args, kwargs = evaluator._finalize_scenario.call_args + last_result = args[1] + self.assertEqual(last_result.returncode, 124) + self.assertIn("TimeoutError", last_result.stderr) + self.assertEqual(mock_generator.safe_generate.call_count, 0) + + @patch("evaluator.agentevaluator.get_generator") + def test_process_scenario_scenario_level_timeout_override(self, mock_get_generator): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_generator.fake_home = "/tmp/fake_home" + mock_generator.create_command.return_value = MagicMock() + mock_generator.parse_response.return_value = {"session_id": "s1"} + mock_generator.extract_tools.return_value = [] + mock_generator.extract_skills.return_value = [] + mock_get_generator.return_value = mock_generator + + # Run config has NO timeout set + config = {"model_config": "dummy.yaml"} + evaluator = AgentEvaluator(config) + evaluator._finalize_scenario = MagicMock() + + def slow_safe_generate(cli_cmd, timeout_seconds=None): + time.sleep(0.08) + return subprocess.CompletedProcess( + args=["mock"], returncode=0, stdout='{"response": "ok"}', stderr="" + ) + + mock_generator.safe_generate.side_effect = slow_safe_generate + + simulated_user = MagicMock() + simulated_user.get_next_response.return_value = "next question" + + # Scenario overrides timeout locally to 50ms + scenario = { + "id": "scenario_override_test", + "starting_prompt": "Hello", + "max_turns": 3, + "eval_case_timeout": "50ms", + } + + evaluator.process_scenario( + scenario=scenario, + eval_result=MagicMock(), + job_id="job_123", + metadata={}, + simulated_user=simulated_user, + ) + + # Should have only executed 1 turn due to scenario-level 50ms timeout + self.assertEqual(mock_generator.safe_generate.call_count, 1) + + @patch("evaluator.agentevaluator.get_generator") + def test_process_scenario_records_timeout_error_in_result(self, mock_get_generator): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_generator.fake_home = "/tmp/fake_home" + mock_generator.create_command.return_value = MagicMock() + mock_generator.parse_response.return_value = {} + mock_generator.extract_tools.return_value = [] + mock_generator.extract_skills.return_value = [] + mock_get_generator.return_value = mock_generator + + config = { + "model_config": "dummy.yaml", + "eval_case_timeout": 0.05, + } + evaluator = AgentEvaluator(config) + + # Simulate safe_generate returning a TimeoutError + mock_generator.safe_generate.return_value = subprocess.CompletedProcess( + args=["mock"], + returncode=124, + stdout="", + stderr="TimeoutError: Command timed out after 0.05 seconds", + ) + + with patch("evaluator.agentevaluator.AgentScoreWork") as mock_score_work: + mock_instance = MagicMock() + mock_score_work.return_value = mock_instance + + scenario = { + "id": "scenario_timeout_err", + "starting_prompt": "Execute task", + "max_turns": 2, + } + + evaluator.process_scenario( + scenario=scenario, + eval_result=MagicMock(scoring_results=[]), + job_id="job_123", + metadata={}, + ) + + # Check that AgentScoreWork received eval_output with job_id, generated_error, and artifacts populated + mock_score_work.assert_called_once() + eval_output = mock_score_work.call_args[1]["eval_output"] + self.assertEqual(eval_output["job_id"], "job_123") + self.assertIn("TimeoutError", eval_output["stderr"]) + self.assertIn("TimeoutError", eval_output["generated_error"]) + self.assertEqual(eval_output["returncode"], 124) + + def test_agent_score_work_propagates_generated_error(self): + eval_output = { + "eval_id": "test_err_prop", + "generated_error": "TimeoutError: Command timed out after 10s", + "accumulated_tools": [], + "scenario": {"starting_prompt": "prompt", "expected_trajectory": []}, + "metadata": {}, + "job_id": "job_123", + } + score_work = AgentScoreWork( + config={}, + eval_output=eval_output, + scoring_results=[], + ) + with patch("scorers.score.compare") as mock_compare: + score_work.run() + mock_compare.assert_called_once() + eval_output_item = mock_compare.call_args[1]["eval_output_item"] + self.assertEqual( + eval_output_item["generated_error"], + "TimeoutError: Command timed out after 10s", + ) + + @patch("evaluator.agentevaluator.SimulatedUser") + @patch("evaluator.agentevaluator.get_generator") + def test_evaluate_agent_cli_returns_job_id_and_outputs_on_timeout(self, mock_get_generator, mock_sim_user): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_generator.fake_home = "/tmp/fake_home" + mock_generator.create_command.return_value = MagicMock() + mock_generator.parse_response.return_value = {} + mock_generator.extract_tools.return_value = [] + mock_generator.extract_skills.return_value = [] + mock_get_generator.return_value = mock_generator + + # Subprocess times out + mock_generator.safe_generate.return_value = subprocess.CompletedProcess( + args=["mock"], + returncode=124, + stdout="partial stdout output", + stderr="TimeoutError: Command timed out after 0.05 seconds", + ) + + config = { + "model_config": "dummy.yaml", + "simulated_user_model_config": "dummy_user.yaml", + "eval_case_timeout": 0.05, + "runners": {"agent_runners": 1}, + } + evaluator = AgentEvaluator(config) + + import json + import types + mock_request = types.SimpleNamespace( + payload=json.dumps({ + "scenarios": [{ + "id": "scenario_timeout_upload", + "starting_prompt": "Run long task", + "max_turns": 3, + }] + }), + agent_results=[], + scoring_results=[], + ) + + mock_sim_user.return_value.get_next_response.return_value = "TERMINATE" + + with patch("evaluator.agentevaluator.AgentScoreWork"): + eval_outputs, scoring_results = evaluator._evaluate_agent_cli( + dataset=[mock_request], + job_id="job_timeout_456", + run_time=MagicMock(), + ) + + # Validate that outputs returned for reporting/upload retain job_id and artifacts + self.assertEqual(len(eval_outputs), 1) + output = eval_outputs[0] + self.assertEqual(output["job_id"], "job_timeout_456") + self.assertEqual(output["eval_id"], "scenario_timeout_upload") + self.assertEqual(output["returncode"], 124) + self.assertIn("TimeoutError", output["generated_error"]) + self.assertEqual(output["stdout"], "partial stdout output") + + +class TestEvaluatorFuturesTimeout(unittest.TestCase): + """Unit tests for OneShot Evaluator task timeout handling.""" + + def test_evaluator_configures_timeout(self): + config_with_timeout = { + "eval_case_timeout": "2m", + } + evaluator = Evaluator(config_with_timeout) + self.assertEqual(evaluator.task_timeout_seconds, 120.0) + self.assertEqual(evaluator.scoring_runners, 10) + self.assertEqual(evaluator.num_trials, 1) + + config_default = { + "runners": {"task_timeout_seconds": 450, "scoring_runners": 5}, + "num_trials": 3, + } + evaluator_default = Evaluator(config_default) + self.assertEqual(evaluator_default.task_timeout_seconds, 450.0) + self.assertEqual(evaluator_default.scoring_runners, 5) + self.assertEqual(evaluator_default.num_trials, 3) + + def test_process_futures_with_timeout_times_out(self): + mock_future = concurrent.futures.Future() + future_map = {mock_future: {"id": "test_1"}} + + # Generator that yields with timeout + results = list( + _process_futures_with_timeout( + [mock_future], + future_map, + timeout=0.05, + ) + ) + + self.assertEqual(len(results), 1) + future, eval_out, timed_out = results[0] + self.assertTrue(timed_out) + self.assertEqual(eval_out["id"], "test_1") + + +class TestGeneratorsTimeout(unittest.TestCase): + """Unit tests for CLI generators timeout execution.""" + + @patch("subprocess.run") + def test_gemini_cli_run_timeout(self, mock_subprocess_run): + mock_subprocess_run.side_effect = subprocess.TimeoutExpired( + cmd=["npm", "exec"], timeout=10.0, output="partial", stderr="err" + ) + gen = GeminiCliGenerator.__new__(GeminiCliGenerator) + gen.fake_home = "/tmp/fake_home" + gen.gemini_home = "/tmp/fake_home/.gemini" + gen.gemini_cli_version = "@google/gemini-cli@0.36.0" + gen.env = {} + + result = gen._execute_cli_command(["npm", "exec"], timeout_seconds=10.0) + self.assertEqual(result.returncode, 124) + self.assertIn("TimeoutError", result.stderr) + + @patch("subprocess.Popen") + def test_codex_cli_run_timeout(self, mock_popen): + mock_proc = MagicMock() + mock_proc.stdout = iter(["item 1\n", "item 2\n"]) + mock_proc.stderr = iter([]) + mock_proc.wait.side_effect = subprocess.TimeoutExpired(cmd=["codex"], timeout=5.0) + mock_popen.return_value = mock_proc + + gen = CodexCliGenerator.__new__(CodexCliGenerator) + gen.fake_home = "/tmp/fake_home" + + result, durations = gen._execute_cli_command(["codex"], timeout_seconds=5.0) + self.assertEqual(result.returncode, 124) + self.assertIn("TimeoutError", result.stderr) + mock_proc.kill.assert_called_once() + + @patch("subprocess.run") + def test_claude_code_run_timeout(self, mock_subprocess_run): + mock_subprocess_run.side_effect = subprocess.TimeoutExpired( + cmd=["claude"], timeout=5.0 + ) + gen = ClaudeCodeGenerator.__new__(ClaudeCodeGenerator) + gen.fake_home = "/tmp/fake_home" + + result = gen._execute_cli_command(["claude"], timeout_seconds=5.0) + self.assertEqual(result.returncode, 124) + self.assertIn("TimeoutError", result.stderr) + + @patch("subprocess.run") + def test_agy_cli_run_timeout(self, mock_subprocess_run): + mock_subprocess_run.side_effect = subprocess.TimeoutExpired( + cmd=["agy"], timeout=5.0 + ) + gen = AgyCliGenerator.__new__(AgyCliGenerator) + gen.fake_home = "/tmp/fake_home" + + result = gen._execute_cli_command(["agy"], timeout_seconds=5.0) + self.assertEqual(result.returncode, 124) + self.assertIn("TimeoutError", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/evalbench/util/config.py b/evalbench/util/config.py index df6a8b01..a65a0253 100644 --- a/evalbench/util/config.py +++ b/evalbench/util/config.py @@ -4,8 +4,9 @@ import os import csv import random +import re import string -from typing import List +from typing import List, Optional from pyaml_env import parse_config import pandas as pd @@ -240,3 +241,73 @@ def breakdown_db_configs_by_dialect(db_configs: list[dict]): else: db_configs_by_dialect[dialect] = [db_config] return db_configs_by_dialect + + +def parse_timeout_seconds(val) -> Optional[float]: + """Parses a timeout duration (int, float, or string like '300', '5m', '1h', '500ms') into seconds as float. + + Returns None if val is None or empty string. + Raises ValueError if val is negative or has an invalid format string. + Raises TypeError if val is not int, float, str, or None. + """ + if val is None: + return None + if isinstance(val, bool): + raise TypeError(f"Timeout cannot be a boolean: {val}") + if isinstance(val, (int, float)): + if val < 0: + raise ValueError(f"Timeout cannot be negative: {val}") + return float(val) + if not isinstance(val, str): + raise TypeError( + f"Timeout must be int, float, str, or None, got {type(val).__name__}" + ) + + s = val.strip() + if not s: + return None + + try: + f = float(s) + if f < 0: + raise ValueError(f"Timeout cannot be negative: {val}") + return f + except ValueError: + pass + + matches = list(re.finditer(r"(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)", s)) + if not matches: + raise ValueError(f"Invalid timeout format: '{val}'") + + matched_len = sum(len(m.group(1)) + len(m.group(2)) for m in matches) + if matched_len != len(re.sub(r"\s+", "", s)): + raise ValueError(f"Invalid timeout format: '{val}'") + + unit_multipliers = { + "ms": 0.001, + "s": 1.0, + "m": 60.0, + "h": 3600.0, + "d": 86400.0, + } + + total = 0.0 + for m in matches: + amount = float(m.group(1)) + unit = m.group(2) + total += amount * unit_multipliers[unit] + + if total < 0: + raise ValueError(f"Timeout cannot be negative: {val}") + + return total + + +def get_eval_case_timeout(config: dict) -> Optional[float]: + """Extracts and parses eval_case_timeout setting from run config or scenario.""" + if not isinstance(config, dict): + return None + + val = config.get("eval_case_timeout") + return parse_timeout_seconds(val) if val is not None else None + diff --git a/evalbench/work/agentscorework.py b/evalbench/work/agentscorework.py index d3ec0d44..6ffb6947 100644 --- a/evalbench/work/agentscorework.py +++ b/evalbench/work/agentscorework.py @@ -51,7 +51,7 @@ def run(self, work_config: Any = None) -> Any: "generated_sql": "skipped", "generated_result": self.eval_output.get("accumulated_tools", []), "eval_results": self.eval_output, - "generated_error": None, + "generated_error": self.eval_output.get("generated_error"), "dialects": metadata.get("dialects", []), "database": metadata.get("database", "unknown"), "job_id": self.eval_output.get("job_id"), From 92d9f0abdcba8ff56d2f90224b60c121006d5e9f Mon Sep 17 00:00:00 2001 From: Tim Wang Date: Wed, 19 Aug 2026 16:05:43 +0000 Subject: [PATCH 2/6] Fix pycodestyle and unit tests --- evalbench/test/test_eval_case_timeout.py | 5 ++++- evalbench/util/config.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/evalbench/test/test_eval_case_timeout.py b/evalbench/test/test_eval_case_timeout.py index 065edf10..7ce8afe6 100644 --- a/evalbench/test/test_eval_case_timeout.py +++ b/evalbench/test/test_eval_case_timeout.py @@ -432,7 +432,10 @@ def test_codex_cli_run_timeout(self, mock_popen): mock_proc = MagicMock() mock_proc.stdout = iter(["item 1\n", "item 2\n"]) mock_proc.stderr = iter([]) - mock_proc.wait.side_effect = subprocess.TimeoutExpired(cmd=["codex"], timeout=5.0) + mock_proc.wait.side_effect = [ + subprocess.TimeoutExpired(cmd=["codex"], timeout=5.0), + 0, + ] mock_popen.return_value = mock_proc gen = CodexCliGenerator.__new__(CodexCliGenerator) diff --git a/evalbench/util/config.py b/evalbench/util/config.py index a65a0253..67f83950 100644 --- a/evalbench/util/config.py +++ b/evalbench/util/config.py @@ -310,4 +310,3 @@ def get_eval_case_timeout(config: dict) -> Optional[float]: val = config.get("eval_case_timeout") return parse_timeout_seconds(val) if val is not None else None - From ecba3cebce168b6f329d4e87e13b71f21b283478 Mon Sep 17 00:00:00 2001 From: Tim Wang Date: Wed, 19 Aug 2026 16:17:26 +0000 Subject: [PATCH 3/6] Fix pytests --- evalbench/generators/models/agy_cli.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index f5575ea4..15de1de9 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -103,7 +103,17 @@ def __init__(self, querygenerator_config): @staticmethod def _validate_timeout(timeout): if timeout is not None: - parse_timeout_seconds(timeout) + if not isinstance(timeout, str): + raise TypeError( + "timeout must be a string (e.g., '20m', '1h30m', '300s')" + ) + # Strict regex for common units (s, m, h). + # Allows things like "20m", "1h30m", "300s". + if not re.match(r"^(\d+(s|m|h))+$", timeout): + raise ValueError( + f"Invalid timeout format: '{timeout}'. " + "Must be a valid duration string (e.g., '20m', '1h30m', '300s')." + ) def _init_paths(self, querygenerator_config): """Resolves the sandbox ``HOME`` and all derived agy paths, and From 1abb46a8d8b9d67edbedc4f54994bc74f6a37ed0 Mon Sep 17 00:00:00 2001 From: Tim Wang Date: Thu, 20 Aug 2026 16:10:08 +0000 Subject: [PATCH 4/6] Do not update generated_error on timeuts, this will block scornig --- evalbench/evaluator/agentevaluator.py | 12 ++++++------ evalbench/test/test_eval_case_timeout.py | 17 ++++++++--------- evalbench/work/agentscorework.py | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index 448f4ce9..0272badc 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -286,11 +286,10 @@ def _finalize_scenario( metadata: Dict[str, Any] ): """Finalizes the scenario by scoring and appending results.""" - generated_error = None - if hasattr(last_result, "returncode") and last_result.returncode != 0: - generated_error = getattr(last_result, "stderr", None) or f"Command failed with exit code {last_result.returncode}" - elif hasattr(last_result, "stderr") and "TimeoutError" in (last_result.stderr or ""): - generated_error = last_result.stderr + timed_out = ( + getattr(last_result, "returncode", 0) == 124 + or "TimeoutError" in (getattr(last_result, "stderr", "") or "") + ) # Prepare intermediate eval_output with all necessary data for scoring eval_output_data = { @@ -298,8 +297,9 @@ def _finalize_scenario( "stdout": getattr(last_result, "stdout", "") or "", "stderr": getattr(last_result, "stderr", "") or "", "returncode": getattr(last_result, "returncode", 0), + "timed_out": timed_out, "prompt_generator_error": None, - "generated_error": generated_error, + "generated_error": None, "sql_generator_error": None, "golden_error": None, "generated_sql": "skipped", diff --git a/evalbench/test/test_eval_case_timeout.py b/evalbench/test/test_eval_case_timeout.py index 7ce8afe6..a3cca1cd 100644 --- a/evalbench/test/test_eval_case_timeout.py +++ b/evalbench/test/test_eval_case_timeout.py @@ -276,18 +276,19 @@ def test_process_scenario_records_timeout_error_in_result(self, mock_get_generat metadata={}, ) - # Check that AgentScoreWork received eval_output with job_id, generated_error, and artifacts populated + # Check that AgentScoreWork received eval_output with job_id, timed_out, and artifacts populated mock_score_work.assert_called_once() eval_output = mock_score_work.call_args[1]["eval_output"] self.assertEqual(eval_output["job_id"], "job_123") self.assertIn("TimeoutError", eval_output["stderr"]) - self.assertIn("TimeoutError", eval_output["generated_error"]) + self.assertIsNone(eval_output["generated_error"]) + self.assertTrue(eval_output["timed_out"]) self.assertEqual(eval_output["returncode"], 124) - def test_agent_score_work_propagates_generated_error(self): + def test_agent_score_work_keeps_generated_error_none(self): eval_output = { "eval_id": "test_err_prop", - "generated_error": "TimeoutError: Command timed out after 10s", + "timed_out": True, "accumulated_tools": [], "scenario": {"starting_prompt": "prompt", "expected_trajectory": []}, "metadata": {}, @@ -302,10 +303,7 @@ def test_agent_score_work_propagates_generated_error(self): score_work.run() mock_compare.assert_called_once() eval_output_item = mock_compare.call_args[1]["eval_output_item"] - self.assertEqual( - eval_output_item["generated_error"], - "TimeoutError: Command timed out after 10s", - ) + self.assertIsNone(eval_output_item["generated_error"]) @patch("evaluator.agentevaluator.SimulatedUser") @patch("evaluator.agentevaluator.get_generator") @@ -365,7 +363,8 @@ def test_evaluate_agent_cli_returns_job_id_and_outputs_on_timeout(self, mock_get self.assertEqual(output["job_id"], "job_timeout_456") self.assertEqual(output["eval_id"], "scenario_timeout_upload") self.assertEqual(output["returncode"], 124) - self.assertIn("TimeoutError", output["generated_error"]) + self.assertTrue(output["timed_out"]) + self.assertIsNone(output["generated_error"]) self.assertEqual(output["stdout"], "partial stdout output") diff --git a/evalbench/work/agentscorework.py b/evalbench/work/agentscorework.py index 6ffb6947..d3ec0d44 100644 --- a/evalbench/work/agentscorework.py +++ b/evalbench/work/agentscorework.py @@ -51,7 +51,7 @@ def run(self, work_config: Any = None) -> Any: "generated_sql": "skipped", "generated_result": self.eval_output.get("accumulated_tools", []), "eval_results": self.eval_output, - "generated_error": self.eval_output.get("generated_error"), + "generated_error": None, "dialects": metadata.get("dialects", []), "database": metadata.get("database", "unknown"), "job_id": self.eval_output.get("job_id"), From 4edde2e71479f158a57a8ef2e919d9d5b5b2dd9f Mon Sep 17 00:00:00 2001 From: Tim Wang Date: Thu, 20 Aug 2026 16:24:04 +0000 Subject: [PATCH 5/6] Separate out task_timeout_seconds and eval case timeout --- evalbench/evaluator/evaluator.py | 8 ++------ evalbench/test/test_eval_case_timeout.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/evalbench/evaluator/evaluator.py b/evalbench/evaluator/evaluator.py index 24c24623..0b89ee35 100644 --- a/evalbench/evaluator/evaluator.py +++ b/evalbench/evaluator/evaluator.py @@ -17,7 +17,6 @@ ) from mp import mprunner from util import truncateExecutionOutputs -from util.config import get_eval_case_timeout from work import multi_trial_scorework from work import promptgenwork from work import scorework @@ -72,11 +71,8 @@ def __init__( self.sqlgen_runners = runner_config.get("sqlgen_runners", 10) self.sqlexec_runners = runner_config.get("sqlexec_runners", 10) self.scoring_runners = runner_config.get("scoring_runners", 10) - eval_timeout = get_eval_case_timeout(self.config) - self.task_timeout_seconds = ( - eval_timeout - if eval_timeout is not None - else runner_config.get("task_timeout_seconds", 600) + self.task_timeout_seconds = runner_config.get( + "task_timeout_seconds", 600 ) self.num_trials = self.config.get("num_trials", 1) diff --git a/evalbench/test/test_eval_case_timeout.py b/evalbench/test/test_eval_case_timeout.py index a3cca1cd..2e4d01ba 100644 --- a/evalbench/test/test_eval_case_timeout.py +++ b/evalbench/test/test_eval_case_timeout.py @@ -372,22 +372,23 @@ class TestEvaluatorFuturesTimeout(unittest.TestCase): """Unit tests for OneShot Evaluator task timeout handling.""" def test_evaluator_configures_timeout(self): - config_with_timeout = { + # eval_case_timeout does NOT override task_timeout_seconds (kept separate) + config_with_eval_case_timeout = { "eval_case_timeout": "2m", } - evaluator = Evaluator(config_with_timeout) - self.assertEqual(evaluator.task_timeout_seconds, 120.0) + evaluator = Evaluator(config_with_eval_case_timeout) + self.assertEqual(evaluator.task_timeout_seconds, 600) self.assertEqual(evaluator.scoring_runners, 10) self.assertEqual(evaluator.num_trials, 1) - config_default = { + config_with_runner_timeout = { "runners": {"task_timeout_seconds": 450, "scoring_runners": 5}, "num_trials": 3, } - evaluator_default = Evaluator(config_default) - self.assertEqual(evaluator_default.task_timeout_seconds, 450.0) - self.assertEqual(evaluator_default.scoring_runners, 5) - self.assertEqual(evaluator_default.num_trials, 3) + evaluator_runner = Evaluator(config_with_runner_timeout) + self.assertEqual(evaluator_runner.task_timeout_seconds, 450) + self.assertEqual(evaluator_runner.scoring_runners, 5) + self.assertEqual(evaluator_runner.num_trials, 3) def test_process_futures_with_timeout_times_out(self): mock_future = concurrent.futures.Future() From b46cf3d4194288bebf36e6fd054003980dbf92e4 Mon Sep 17 00:00:00 2001 From: Tim Wang Date: Thu, 20 Aug 2026 17:31:22 +0000 Subject: [PATCH 6/6] Update docs and add sample timeout config --- .../example_run_timeout_config.yaml | 30 +++++++++++++++++++ docs/configs/run-config.md | 1 + 2 files changed, 31 insertions(+) create mode 100644 datasets/gemini-cli-tools/example_run_timeout_config.yaml diff --git a/datasets/gemini-cli-tools/example_run_timeout_config.yaml b/datasets/gemini-cli-tools/example_run_timeout_config.yaml new file mode 100644 index 00000000..8932efb8 --- /dev/null +++ b/datasets/gemini-cli-tools/example_run_timeout_config.yaml @@ -0,0 +1,30 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/gemini-cli-tools/gemini-cli-fake.evalset.json +dataset_format: gemini-cli-format + +# Orchestrator Configuration +orchestrator: geminicli +model_config: datasets/model_configs/gemini_cli_fake_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +# Per-scenario evaluation timeout duration (e.g., '300s', '5m', '1h30m') +eval_case_timeout: 5m + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + trajectory_matcher: {} + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/docs/configs/run-config.md b/docs/configs/run-config.md index 0c5fcfd6..670b3b1e 100644 --- a/docs/configs/run-config.md +++ b/docs/configs/run-config.md @@ -19,6 +19,7 @@ This section defines the primary resources used during evaluation, including the | `num_trials` | Optional | Number of trials to run for each prompt. | | `scenarios` | Optional | A list of specific scenario IDs to run (only applies to scenario-based agentic datasets like `gemini-cli-format` or `cortado-format`). Defaults to empty (runs all scenarios). | | `scenario_pattern` | Optional | A glob pattern of scenario IDs to run (only applies to scenario-based agentic datasets). Defaults to None (runs all scenarios). | +| `eval_case_timeout` | Optional | Maximum execution duration allowed per individual scenario (e.g. `'300s'`, `'5m'`, `'1h30m'`). Applicable to scenario-based agent evaluations. | --- ## 2. Prompt and Generation Modules