From e04f1689f32cb5884bd5238cf43e97b30e1d7794 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 26 Aug 2026 11:37:28 -0700 Subject: [PATCH] feat: execute() stamps its traces as eval traffic automatically The burden used to sit on every caller: remember monitor=False on every tracer.trace() inside an evaluation, or the engine double-judged each case, raised signals on synthetic questions, and counted the run's latencies into production KPIs. Nobody remembered - including our own samples, including the sample written by the person who wrote the rule. execute() now enters an eval-run scope (tracing/eval_scope.py, a contextvar so it nests and cannot leak across concurrent runs). While it is active every trace the tracer sends - roots and child spans - is stamped source="eval-run", monitor=False, and metadata.evalRunId, so a trace can always be walked back to the run that produced it. An explicit monitor=True is respected: deliberately pointing checks at eval traffic is a choice, not a mistake. Known limit, documented in the module: a bare Thread() the agent function spawns itself does not inherit the contextvar - same posture as the tracer's existing user-managed-thread caveats. Six tests pin the contract: no-op outside the scope, full stamp inside, explicit monitor=True wins, child spans stamped, no leak after exit, caller metadata survives the merge. Co-Authored-By: Claude Fable 5 --- agentx/evaluations/runner.py | 16 ++++- agentx/tracing/eval_scope.py | 44 +++++++++++++ agentx/tracing/tracer.py | 22 ++++++- tests/test_eval_scope.py | 116 +++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 agentx/tracing/eval_scope.py create mode 100644 tests/test_eval_scope.py diff --git a/agentx/evaluations/runner.py b/agentx/evaluations/runner.py index ef33aa0..5e743ae 100644 --- a/agentx/evaluations/runner.py +++ b/agentx/evaluations/runner.py @@ -119,7 +119,21 @@ def __init__( # ------------------------------------------------------------------ def execute(self, adapter: AdapterLike) -> "EvaluationRunContext": - """Run all cases locally and submit batches to AgentX.""" + """Run all cases locally and submit batches to AgentX. + + The whole loop runs inside the eval-run scope (tracing/eval_scope.py): any trace the + agent function creates is stamped source="eval-run" + monitor=False automatically, so + eval traffic never skews production monitoring and no one has to remember a flag. + """ + from agentx.tracing.eval_scope import enter_eval_run, exit_eval_run + + scope_token = enter_eval_run(self._run.run_id) + try: + return self._execute_inner(adapter) + finally: + exit_eval_run(scope_token) + + def _execute_inner(self, adapter: AdapterLike) -> "EvaluationRunContext": normalized = _wrap_adapter(adapter) cases = _build_cases(self._dataset, self._run, self._evaluation_settings) max_batch = self._run.limits.max_batch_size diff --git a/agentx/tracing/eval_scope.py b/agentx/tracing/eval_scope.py new file mode 100644 index 0000000..05377f0 --- /dev/null +++ b/agentx/tracing/eval_scope.py @@ -0,0 +1,44 @@ +"""The eval-run scope: how traces created inside an evaluation stop passing as production. + +An offline run executes the user's own agent function, and an instrumented agent traces itself - +which is exactly what makes trajectory matching and retrieval-context extraction work. But those +traces are not production traffic, and before this scope existed the burden of saying so sat on +every caller: remember ``monitor=False`` on every ``tracer.trace(...)`` inside an eval, or the +engine would double-judge each case, raise signals on synthetic questions, and count the run's +latencies into production KPIs. Nobody remembered - including our own samples. + +``EvaluationRunContext.execute()`` enters this scope around the whole run. While it is active, +every trace the tracer sends is stamped: + + - ``source="eval-run"`` - the engine files it as eval traffic (excluded from monitor KPIs, + metrics, sessions and the Live Traces default view; cost keeps it, split out) + - ``monitor=False`` - unless the caller explicitly passed ``monitor=True``, which is + respected as a deliberate choice + - ``metadata.evalRunId`` - so a trace can always be walked back to the run that produced it + +A ``contextvars.ContextVar`` rather than tracer state: it nests correctly, cannot leak across +concurrent runs in async code, and costs nothing when no run is active. The one known limit is +threads the agent function spawns itself - a context var does not cross a bare ``Thread()`` - +which matches the tracer's existing documented posture for user-managed threads. +""" + +from contextvars import ContextVar +from typing import Optional + +EVAL_RUN_SOURCE = "eval-run" + +_current_eval_run_id: ContextVar[Optional[str]] = ContextVar("agentx_eval_run_id", default=None) + + +def enter_eval_run(run_id: str): + """Mark the current context as inside an eval run. Returns the token for ``exit_eval_run``.""" + return _current_eval_run_id.set(run_id) + + +def exit_eval_run(token) -> None: + _current_eval_run_id.reset(token) + + +def current_eval_run_id() -> Optional[str]: + """The run id when inside ``execute()``, else None.""" + return _current_eval_run_id.get() diff --git a/agentx/tracing/tracer.py b/agentx/tracing/tracer.py index bf33311..b6eb24e 100644 --- a/agentx/tracing/tracer.py +++ b/agentx/tracing/tracer.py @@ -13,6 +13,7 @@ from agentx.exceptions import CIGateFailure from agentx.tracing.ingest_client import IngestClient from agentx.tracing.ci_types import CIRun, CIRunResult, CIRunStatus, CIQuestionScore +from agentx.tracing.eval_scope import EVAL_RUN_SOURCE, current_eval_run_id F = TypeVar("F", bound=Callable[..., Any]) @@ -167,9 +168,21 @@ def __exit__(self, exc_type, exc_val, tb): # flush() uses; child-only spans keep their async fire-and-forget behavior. self._tracer.flush(timeout=5.0) + # Inside an eval run (evaluations' execute()), every trace states what it is: eval + # traffic. monitor=False unless the caller explicitly said True; the run id rides in + # metadata so the trace can be walked back to its run. See tracing/eval_scope.py. + eval_run_id = current_eval_run_id() + monitor = self._monitor + metadata = self._metadata + source = None + if eval_run_id is not None: + source = EVAL_RUN_SOURCE + if monitor is not True: + monitor = False + metadata = {**(metadata or {}), "evalRunId": eval_run_id} self._trace_id = self._tracer._send( sync=self._sync, - monitor=self._monitor, + monitor=monitor, pattern_ids=self._pattern_ids, name=self.name, agent_id=self._agent_id, @@ -177,7 +190,7 @@ def __exit__(self, exc_type, exc_val, tb): output=_safe_serialize(self.output) if self.output is not None else None, latency_ms=latency_ms, error=self._error, - metadata=self._metadata, + metadata=metadata, framework=self._framework or self._captured_framework, model=self._model or self._captured_model, tool_calls=self.tool_calls or None, @@ -189,6 +202,7 @@ def __exit__(self, exc_type, exc_val, tb): span_id=self._span_id, parent_span_id=self._parent_span_id, span_kind=self._span_kind, + source=source, started_at_unix_nano=str(int(self._start * 1_000_000_000)) if self._start else None, ) return False # never suppress exceptions @@ -332,6 +346,8 @@ def child_span( wire["span_kind"] = span_kind if child._session_id: wire["session_id"] = child._session_id + if current_eval_run_id() is not None: + wire["source"] = EVAL_RUN_SOURCE wire["span_id"] = child._span_id if child._parent_span_id: wire["parent_span_id"] = child._parent_span_id @@ -1148,6 +1164,8 @@ def _send(self, sync: bool = False, **kwargs) -> Optional[str]: wire["started_at_unix_nano"] = payload["started_at_unix_nano"] if "agent_id" in payload: wire["agent_id"] = payload["agent_id"] + if "source" in payload: + wire["source"] = payload["source"] if "span_kind" in payload: wire["span_kind"] = payload["span_kind"] diff --git a/tests/test_eval_scope.py b/tests/test_eval_scope.py new file mode 100644 index 0000000..ac9a7fd --- /dev/null +++ b/tests/test_eval_scope.py @@ -0,0 +1,116 @@ +"""The eval-run scope: traces created inside execute() stop passing as production. + +Before this, the burden sat on every caller: remember monitor=False on every trace inside an +eval or the engine double-judges, raises signals on synthetic questions, and counts eval +latencies into production KPIs. These pin that the stamping is automatic, respects an explicit +monitor=True, and vanishes completely outside the scope. +""" + +from agentx.tracing.eval_scope import current_eval_run_id, enter_eval_run, exit_eval_run +from agentx.tracing.tracer import Tracer + + +class _CaptureTracer(Tracer): + """A tracer whose network is a list.""" + + def __init__(self): # noqa: D401 - bypass real client setup + self.sent = [] + self._active_spans = [] + self._pending_tool_calls = [] + self._pending_retrievals = [] + + def _send(self, sync=False, **kwargs): + self.sent.append({k: v for k, v in kwargs.items() if v is not None}) + return "trace-1" + + def _dispatch(self, wire, *, sync=False): + self.sent.append(wire) + return "trace-child" + + # The bits of Tracer the span touches. + def _push_active_span(self, span): + self._active_spans.append(span) + + def _pop_active_span(self, span): + if span in self._active_spans: + self._active_spans.remove(span) + + @property + def current_span(self): + return self._active_spans[-1] if self._active_spans else None + + def flush(self, timeout=5.0): + return True + + +def test_outside_the_scope_nothing_changes(): + tracer = _CaptureTracer() + with tracer.trace("prod-agent", input={"q": "hi"}) as span: + span.output = "hello" + wire = tracer.sent[-1] + assert "source" not in wire + assert "monitor" not in wire # None is filtered out, same as before + assert current_eval_run_id() is None + + +def test_inside_the_scope_traces_are_stamped(): + tracer = _CaptureTracer() + token = enter_eval_run("run-42") + try: + with tracer.trace("agent-under-eval", input={"q": "case 1"}) as span: + span.output = "answer" + finally: + exit_eval_run(token) + wire = tracer.sent[-1] + assert wire["source"] == "eval-run" + assert wire["monitor"] is False + assert wire["metadata"]["evalRunId"] == "run-42" + + +def test_explicit_monitor_true_is_respected(): + # Someone deliberately pointing checks at eval traffic is a choice, not a mistake. + tracer = _CaptureTracer() + token = enter_eval_run("run-42") + try: + with tracer.trace("agent-under-eval", monitor=True) as span: + span.output = "x" + finally: + exit_eval_run(token) + wire = tracer.sent[-1] + assert wire["source"] == "eval-run" + assert wire["monitor"] is True + + +def test_child_spans_are_eval_traffic_too(): + tracer = _CaptureTracer() + token = enter_eval_run("run-42") + try: + with tracer.trace("agent-under-eval") as span: + span.child_span("kb_search", output=["chunk"]) + span.output = "x" + finally: + exit_eval_run(token) + child = next(w for w in tracer.sent if w.get("name") == "kb_search") + assert child["source"] == "eval-run" + + +def test_the_scope_does_not_leak(): + tracer = _CaptureTracer() + token = enter_eval_run("run-42") + exit_eval_run(token) + with tracer.trace("prod-again") as span: + span.output = "y" + assert "source" not in tracer.sent[-1] + + +def test_caller_metadata_survives_the_stamp(): + tracer = _CaptureTracer() + token = enter_eval_run("run-42") + try: + with tracer.trace("agent", metadata={"promptName": "support-v3"}) as span: + span.output = "x" + finally: + exit_eval_run(token) + md = tracer.sent[-1]["metadata"] + assert md["promptName"] == "support-v3" + assert md["evalRunId"] == "run-42"