feat: execute() stamps its traces as eval traffic automatically

agentx/evaluations/runner.py

@@ -119,7 +119,21 @@ def __init__(
     # ------------------------------------------------------------------
 
     def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
-        """Run all cases locally and submit batches to AgentX."""
+        """Run all cases locally and submit batches to AgentX.
+
+        The whole loop runs inside the eval-run scope (tracing/eval_scope.py): any trace the
+        agent function creates is stamped source="eval-run" + monitor=False automatically, so
+        eval traffic never skews production monitoring and no one has to remember a flag.
+        """
+        from agentx.tracing.eval_scope import enter_eval_run, exit_eval_run
+
+        scope_token = enter_eval_run(self._run.run_id)
+        try:
+            return self._execute_inner(adapter)
+        finally:
+            exit_eval_run(scope_token)
+
+    def _execute_inner(self, adapter: AdapterLike) -> "EvaluationRunContext":
         normalized = _wrap_adapter(adapter)
         cases = _build_cases(self._dataset, self._run, self._evaluation_settings)
         max_batch = self._run.limits.max_batch_size

agentx/tracing/eval_scope.py

@@ -0,0 +1,44 @@
+"""The eval-run scope: how traces created inside an evaluation stop passing as production.
+
+An offline run executes the user's own agent function, and an instrumented agent traces itself -
+which is exactly what makes trajectory matching and retrieval-context extraction work. But those
+traces are not production traffic, and before this scope existed the burden of saying so sat on
+every caller: remember ``monitor=False`` on every ``tracer.trace(...)`` inside an eval, or the
+engine would double-judge each case, raise signals on synthetic questions, and count the run's
+latencies into production KPIs. Nobody remembered - including our own samples.
+
+``EvaluationRunContext.execute()`` enters this scope around the whole run. While it is active,
+every trace the tracer sends is stamped:
+
+  - ``source="eval-run"``  - the engine files it as eval traffic (excluded from monitor KPIs,
+    metrics, sessions and the Live Traces default view; cost keeps it, split out)
+  - ``monitor=False``      - unless the caller explicitly passed ``monitor=True``, which is
+    respected as a deliberate choice
+  - ``metadata.evalRunId`` - so a trace can always be walked back to the run that produced it
+
+A ``contextvars.ContextVar`` rather than tracer state: it nests correctly, cannot leak across
+concurrent runs in async code, and costs nothing when no run is active. The one known limit is
+threads the agent function spawns itself - a context var does not cross a bare ``Thread()`` -
+which matches the tracer's existing documented posture for user-managed threads.
+"""
+
+from contextvars import ContextVar
+from typing import Optional
+
+EVAL_RUN_SOURCE = "eval-run"
+
+_current_eval_run_id: ContextVar[Optional[str]] = ContextVar("agentx_eval_run_id", default=None)
+
+
+def enter_eval_run(run_id: str):
+    """Mark the current context as inside an eval run. Returns the token for ``exit_eval_run``."""
+    return _current_eval_run_id.set(run_id)
+
+
+def exit_eval_run(token) -> None:
+    _current_eval_run_id.reset(token)
+
+
+def current_eval_run_id() -> Optional[str]:
+    """The run id when inside ``execute()``, else None."""
+    return _current_eval_run_id.get()

agentx/tracing/tracer.py

@@ -13,6 +13,7 @@
 from agentx.exceptions import CIGateFailure
 from agentx.tracing.ingest_client import IngestClient
 from agentx.tracing.ci_types import CIRun, CIRunResult, CIRunStatus, CIQuestionScore
+from agentx.tracing.eval_scope import EVAL_RUN_SOURCE, current_eval_run_id
 
 F = TypeVar("F", bound=Callable[..., Any])
 
@@ -167,17 +168,29 @@ def __exit__(self, exc_type, exc_val, tb):
             # flush() uses; child-only spans keep their async fire-and-forget behavior.
             self._tracer.flush(timeout=5.0)
 
+        # Inside an eval run (evaluations' execute()), every trace states what it is: eval
+        # traffic. monitor=False unless the caller explicitly said True; the run id rides in
+        # metadata so the trace can be walked back to its run. See tracing/eval_scope.py.
+        eval_run_id = current_eval_run_id()
+        monitor = self._monitor
+        metadata = self._metadata
+        source = None
+        if eval_run_id is not None:
+            source = EVAL_RUN_SOURCE
+            if monitor is not True:
+                monitor = False
+            metadata = {**(metadata or {}), "evalRunId": eval_run_id}
         self._trace_id = self._tracer._send(
             sync=self._sync,
-            monitor=self._monitor,
+            monitor=monitor,
             pattern_ids=self._pattern_ids,
             name=self.name,
             agent_id=self._agent_id,
             input=_safe_serialize(self.input) if self.input is not None else None,
             output=_safe_serialize(self.output) if self.output is not None else None,
             latency_ms=latency_ms,
             error=self._error,
-            metadata=self._metadata,
+            metadata=metadata,
             framework=self._framework or self._captured_framework,
             model=self._model or self._captured_model,
             tool_calls=self.tool_calls or None,
@@ -189,6 +202,7 @@ def __exit__(self, exc_type, exc_val, tb):
             span_id=self._span_id,
             parent_span_id=self._parent_span_id,
             span_kind=self._span_kind,
+            source=source,
             started_at_unix_nano=str(int(self._start * 1_000_000_000)) if self._start else None,
         )
         return False  # never suppress exceptions
@@ -332,6 +346,8 @@ def child_span(
             wire["span_kind"] = span_kind
         if child._session_id:
             wire["session_id"] = child._session_id
+        if current_eval_run_id() is not None:
+            wire["source"] = EVAL_RUN_SOURCE
         wire["span_id"] = child._span_id
         if child._parent_span_id:
             wire["parent_span_id"] = child._parent_span_id
@@ -1148,6 +1164,8 @@ def _send(self, sync: bool = False, **kwargs) -> Optional[str]:
             wire["started_at_unix_nano"] = payload["started_at_unix_nano"]
         if "agent_id" in payload:
             wire["agent_id"] = payload["agent_id"]
+        if "source" in payload:
+            wire["source"] = payload["source"]
         if "span_kind" in payload:
             wire["span_kind"] = payload["span_kind"]
 

tests/test_eval_scope.py

@@ -0,0 +1,116 @@
+"""The eval-run scope: traces created inside execute() stop passing as production.
+
+Before this, the burden sat on every caller: remember monitor=False on every trace inside an
+eval or the engine double-judges, raises signals on synthetic questions, and counts eval
+latencies into production KPIs. These pin that the stamping is automatic, respects an explicit
+monitor=True, and vanishes completely outside the scope.
+"""
+
+from agentx.tracing.eval_scope import current_eval_run_id, enter_eval_run, exit_eval_run
+from agentx.tracing.tracer import Tracer
+
+
+class _CaptureTracer(Tracer):
+    """A tracer whose network is a list."""
+
+    def __init__(self):  # noqa: D401 - bypass real client setup
+        self.sent = []
+        self._active_spans = []
+        self._pending_tool_calls = []
+        self._pending_retrievals = []
+
+    def _send(self, sync=False, **kwargs):
+        self.sent.append({k: v for k, v in kwargs.items() if v is not None})
+        return "trace-1"
+
+    def _dispatch(self, wire, *, sync=False):
+        self.sent.append(wire)
+        return "trace-child"
+
+    # The bits of Tracer the span touches.
+    def _push_active_span(self, span):
+        self._active_spans.append(span)
+
+    def _pop_active_span(self, span):
+        if span in self._active_spans:
+            self._active_spans.remove(span)
+
+    @property
+    def current_span(self):
+        return self._active_spans[-1] if self._active_spans else None
+
+    def flush(self, timeout=5.0):
+        return True
+
+
+def test_outside_the_scope_nothing_changes():
+    tracer = _CaptureTracer()
+    with tracer.trace("prod-agent", input={"q": "hi"}) as span:
+        span.output = "hello"
+    wire = tracer.sent[-1]
+    assert "source" not in wire
+    assert "monitor" not in wire  # None is filtered out, same as before
+    assert current_eval_run_id() is None
+
+
+def test_inside_the_scope_traces_are_stamped():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent-under-eval", input={"q": "case 1"}) as span:
+            span.output = "answer"
+    finally:
+        exit_eval_run(token)
+    wire = tracer.sent[-1]
+    assert wire["source"] == "eval-run"
+    assert wire["monitor"] is False
+    assert wire["metadata"]["evalRunId"] == "run-42"
+
+
+def test_explicit_monitor_true_is_respected():
+    # Someone deliberately pointing checks at eval traffic is a choice, not a mistake.
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent-under-eval", monitor=True) as span:
+            span.output = "x"
+    finally:
+        exit_eval_run(token)
+    wire = tracer.sent[-1]
+    assert wire["source"] == "eval-run"
+    assert wire["monitor"] is True
+
+
+def test_child_spans_are_eval_traffic_too():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent-under-eval") as span:
+            span.child_span("kb_search", output=["chunk"])
+            span.output = "x"
+    finally:
+        exit_eval_run(token)
+    child = next(w for w in tracer.sent if w.get("name") == "kb_search")
+    assert child["source"] == "eval-run"
+
+
+def test_the_scope_does_not_leak():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    exit_eval_run(token)
+    with tracer.trace("prod-again") as span:
+        span.output = "y"
+    assert "source" not in tracer.sent[-1]
+
+
+def test_caller_metadata_survives_the_stamp():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent", metadata={"promptName": "support-v3"}) as span:
+            span.output = "x"
+    finally:
+        exit_eval_run(token)
+    md = tracer.sent[-1]["metadata"]
+    assert md["promptName"] == "support-v3"
+    assert md["evalRunId"] == "run-42"