From 80fd4e46351a4640970d53212eac62579a61ff66 Mon Sep 17 00:00:00 2001 From: "doneill@launchdarkly.com" Date: Thu, 13 Aug 2026 07:07:08 +0000 Subject: [PATCH 01/32] feat: add evaluations module scaffold, credentials, LD API client, result types --- .../evaluations/__init__.py | 28 +++ .../launchdarkly_ai_server/evaluations/api.py | 130 ++++++++++++ .../evaluations/module.py | 77 +++++++ .../evaluations/types.py | 58 +++++ packages/client/tests/test_evaluations.py | 199 ++++++++++++++++++ 5 files changed, 492 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/__init__.py create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/api.py create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/module.py create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/types.py create mode 100644 packages/client/tests/test_evaluations.py diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py new file mode 100644 index 0000000..99340b9 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -0,0 +1,28 @@ +"""Run LaunchDarkly evaluations from your own environment.""" + +from .api import ( + DEFAULT_BASE_URI, + EvaluationsError, + HttpResponse, + LDApiClient, + LDApiError, + Transport, + urllib_transport, +) +from .module import EvaluationsModule, init_evaluations +from .types import EvalRunResult, RunSummary, Usage + +__all__ = [ + "DEFAULT_BASE_URI", + "EvalRunResult", + "EvaluationsError", + "EvaluationsModule", + "HttpResponse", + "LDApiClient", + "LDApiError", + "RunSummary", + "Transport", + "Usage", + "init_evaluations", + "urllib_transport", +] diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/api.py b/packages/client/src/launchdarkly_ai_server/evaluations/api.py new file mode 100644 index 0000000..f87f8f0 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/api.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Protocol + +DEFAULT_BASE_URI = "https://app.launchdarkly.com" + + +class EvaluationsError(Exception): + """Base error for the evaluations harness.""" + + +class LDApiError(EvaluationsError): + """A non-2xx response from the LaunchDarkly API.""" + + def __init__(self, status: int, method: str, path: str, body: str) -> None: + super().__init__( + f"LaunchDarkly API {method} {path} failed with {status}: {body}" + ) + self.status = status + self.method = method + self.path = path + self.body = body + + +@dataclass +class HttpResponse: + status: int + body: str + headers: dict[str, str] = field(default_factory=dict) + + +class Transport(Protocol): + """Seam the API client sends requests through; replaced in tests.""" + + def __call__( + self, + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, + ) -> HttpResponse: ... + + +def urllib_transport( + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, +) -> HttpResponse: + request = urllib.request.Request(url, data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return HttpResponse( + status=response.status, + body=response.read().decode("utf-8"), + headers={k.lower(): v for k, v in response.headers.items()}, + ) + except urllib.error.HTTPError as error: + return HttpResponse( + status=error.code, + body=error.read().decode("utf-8"), + headers={k.lower(): v for k, v in error.headers.items()}, + ) + + +class LDApiClient: + """ + Minimal client for the LaunchDarkly public ``/api/v2`` surface used by the + evaluations harness. Every request carries the API access token; the base + URI is overridable for non-default instances. + """ + + def __init__( + self, + api_token: str, + base_uri: str = DEFAULT_BASE_URI, + transport: Transport = urllib_transport, + timeout: float = 30.0, + ) -> None: + self.api_token = api_token + self.base_uri = base_uri.rstrip("/") + self._transport = transport + self._timeout = timeout + + def url_for(self, path: str, params: dict[str, Any] | None = None) -> str: + url = f"{self.base_uri}/api/v2/{path.lstrip('/')}" + if params: + query = {k: str(v) for k, v in params.items() if v is not None} + if query: + url = f"{url}?{urllib.parse.urlencode(query)}" + return url + + def request( + self, + method: str, + path: str, + body: Any = None, + params: dict[str, Any] | None = None, + ) -> Any: + headers = { + "Authorization": self.api_token, + "Accept": "application/json", + "User-Agent": "launchdarkly-ai-evaluations-python", + } + payload: bytes | None = None + if body is not None: + headers["Content-Type"] = "application/json" + payload = json.dumps(body).encode("utf-8") + + response = self._transport( + method, self.url_for(path, params), headers, payload, self._timeout + ) + if response.status < 200 or response.status >= 300: + raise LDApiError(response.status, method, path, response.body) + if not response.body: + return None + return json.loads(response.body) + + def get(self, path: str, params: dict[str, Any] | None = None) -> Any: + return self.request("GET", path, params=params) + + def post(self, path: str, body: Any = None) -> Any: + return self.request("POST", path, body=body) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py new file mode 100644 index 0000000..c73c703 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import logging +import os + +from .api import ( + DEFAULT_BASE_URI, + EvaluationsError, + LDApiClient, + Transport, + urllib_transport, +) + +logger = logging.getLogger(__name__) + + +def _env(name: str) -> str | None: + """Read an env var, treating blank/whitespace-only values as unset.""" + value = os.environ.get(name, "").strip() + return value if value else None + + +class EvaluationsModule: + """ + Entry point for running LaunchDarkly evaluations from code. Holds the + resolved credentials and the LaunchDarkly API client; ``run()`` arrives with + the harness. + """ + + def __init__(self, api_client: LDApiClient, sdk_key: str | None = None) -> None: + self._api = api_client + self._sdk_key = sdk_key + + @property + def api(self) -> LDApiClient: + return self._api + + @property + def sdk_key(self) -> str | None: + """SDK key used for observability traces; ``None`` disables tracing.""" + return self._sdk_key + + +def init_evaluations( + api_token: str | None = None, + sdk_key: str | None = None, + base_uri: str | None = None, + transport: Transport = urllib_transport, +) -> EvaluationsModule: + """ + Resolves credentials and builds the evaluations module. + + ``api_token`` (``LD_API_TOKEN``) authenticates every ``/api/v2`` call and is + required — a missing token raises before any network I/O rather than + surfacing as an opaque 401 mid-run. ``sdk_key`` (``LD_SDK_KEY``) is optional + and only makes handler calls emit observability traces. Both credentials + must point at the same project. + """ + token = api_token or _env("LD_API_TOKEN") + if not token: + raise EvaluationsError( + "No LaunchDarkly API access token provided. Set the LD_API_TOKEN " + "environment variable or pass api_token to init_evaluations()." + ) + + resolved_sdk_key = sdk_key or _env("LD_SDK_KEY") + if not resolved_sdk_key: + logger.info( + "No LaunchDarkly SDK key provided; evaluation runs will not emit traces." + ) + + api_client = LDApiClient( + api_token=token, + base_uri=base_uri or _env("LD_BASE_URI") or DEFAULT_BASE_URI, + transport=transport, + ) + return EvaluationsModule(api_client=api_client, sdk_key=resolved_sdk_key) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py new file mode 100644 index 0000000..3b5fdb9 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class Usage: + """ + Token counts for a single generation, in the ingest wire shape. Handler + results carry this dict verbatim, so nothing on the eval path adapts it. + """ + + input_tokens: int + output_tokens: int + + def to_wire(self) -> dict[str, int]: + return { + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + } + + @classmethod + def from_wire(cls, data: dict[str, Any]) -> Usage: + return cls( + input_tokens=int(data.get("input_tokens") or 0), + output_tokens=int(data.get("output_tokens") or 0), + ) + + +@dataclass +class RunSummary: + """Row counts for a finished evaluation run.""" + + total_rows: int = 0 + passed_rows: int = 0 + failed_rows: int = 0 + error_rows: int = 0 + + @classmethod + def from_wire(cls, data: dict[str, Any] | None) -> RunSummary: + data = data or {} + return cls( + total_rows=int(data.get("total_rows") or 0), + passed_rows=int(data.get("passed_rows") or 0), + failed_rows=int(data.get("failed_rows") or 0), + error_rows=int(data.get("error_rows") or 0), + ) + + +@dataclass +class EvalRunResult: + """The verdict of an evaluation run, as computed and stored by LaunchDarkly.""" + + passed: bool + url: str + run_id: str + summary: RunSummary diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py new file mode 100644 index 0000000..fcf7dca --- /dev/null +++ b/packages/client/tests/test_evaluations.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from launchdarkly_ai_server.evaluations import ( + DEFAULT_BASE_URI, + EvalRunResult, + EvaluationsError, + HttpResponse, + LDApiClient, + LDApiError, + RunSummary, + Usage, + init_evaluations, +) + + +class RecordingTransport: + """Mocked LD API — records requests and replays canned responses.""" + + def __init__(self, responses: list[HttpResponse] | None = None) -> None: + self.requests: list[dict[str, Any]] = [] + self.responses = responses or [HttpResponse(status=200, body="{}")] + + def __call__( + self, + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, + ) -> HttpResponse: + self.requests.append( + { + "method": method, + "url": url, + "headers": headers, + "body": json.loads(body) if body else None, + "timeout": timeout, + } + ) + index = min(len(self.requests) - 1, len(self.responses) - 1) + return self.responses[index] + + +def failing_transport( + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, +) -> HttpResponse: + raise AssertionError("no network I/O expected") + + +def test_init_resolves_credentials_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token-from-env") + monkeypatch.setenv("LD_SDK_KEY", "sdk-key-from-env") + + evals = init_evaluations(transport=RecordingTransport()) + + assert evals.api.api_token == "api-token-from-env" + assert evals.sdk_key == "sdk-key-from-env" + assert evals.api.base_uri == DEFAULT_BASE_URI + + +def test_init_prefers_explicit_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token-from-env") + monkeypatch.setenv("LD_SDK_KEY", "sdk-key-from-env") + + evals = init_evaluations( + api_token="explicit-token", + sdk_key="explicit-sdk-key", + transport=RecordingTransport(), + ) + + assert evals.api.api_token == "explicit-token" + assert evals.sdk_key == "explicit-sdk-key" + + +def test_missing_api_token_raises_before_network_io( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LD_API_TOKEN", raising=False) + monkeypatch.setenv("LD_SDK_KEY", "sdk-key") + + with pytest.raises(EvaluationsError, match="LD_API_TOKEN"): + init_evaluations(transport=failing_transport) + + +def test_blank_api_token_env_is_treated_as_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LD_API_TOKEN", " ") + + with pytest.raises(EvaluationsError): + init_evaluations(transport=failing_transport) + + +def test_missing_sdk_key_is_allowed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.delenv("LD_SDK_KEY", raising=False) + + evals = init_evaluations(transport=RecordingTransport()) + + assert evals.sdk_key is None + + +def test_base_uri_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.setenv("LD_BASE_URI", "https://ld.internal.example.com/") + + from_env = init_evaluations(transport=RecordingTransport()) + explicit = init_evaluations( + base_uri="https://other.example.com", transport=RecordingTransport() + ) + + assert from_env.api.base_uri == "https://ld.internal.example.com" + assert explicit.api.base_uri == "https://other.example.com" + + +def test_requests_carry_token_auth_and_json_body() -> None: + transport = RecordingTransport([HttpResponse(status=201, body='{"key": "run-1"}')]) + client = LDApiClient(api_token="api-token", transport=transport) + + result = client.post("projects/proj/evaluations", body={"key": "support-qa"}) + + assert result == {"key": "run-1"} + request = transport.requests[0] + assert request["method"] == "POST" + assert request["url"] == f"{DEFAULT_BASE_URI}/api/v2/projects/proj/evaluations" + assert request["headers"]["Authorization"] == "api-token" + assert request["headers"]["Content-Type"] == "application/json" + assert request["body"] == {"key": "support-qa"} + + +def test_get_encodes_query_params_and_omits_none() -> None: + transport = RecordingTransport([HttpResponse(status=200, body='{"items": []}')]) + client = LDApiClient( + api_token="api-token", base_uri="https://ld.example.com", transport=transport + ) + + client.get("projects/proj/datasets/golden", params={"limit": 50, "offset": None}) + + request = transport.requests[0] + assert ( + request["url"] + == "https://ld.example.com/api/v2/projects/proj/datasets/golden?limit=50" + ) + assert "Content-Type" not in request["headers"] + + +def test_error_response_raises_ld_api_error() -> None: + transport = RecordingTransport( + [HttpResponse(status=404, body='{"message": "nope"}')] + ) + client = LDApiClient(api_token="api-token", transport=transport) + + with pytest.raises(LDApiError) as excinfo: + client.get("projects/proj/ai-tools/missing") + + assert excinfo.value.status == 404 + assert excinfo.value.path == "projects/proj/ai-tools/missing" + + +def test_empty_response_body_is_none() -> None: + transport = RecordingTransport([HttpResponse(status=204, body="")]) + client = LDApiClient(api_token="api-token", transport=transport) + + assert client.post("projects/proj/evaluations/support-qa/runs") is None + + +def test_usage_matches_ingest_wire_shape() -> None: + usage = Usage(input_tokens=812, output_tokens=96) + + assert usage.to_wire() == {"input_tokens": 812, "output_tokens": 96} + assert Usage.from_wire({"input_tokens": 1, "output_tokens": 2}) == Usage(1, 2) + assert Usage.from_wire({}) == Usage(0, 0) + + +def test_run_summary_and_result() -> None: + summary = RunSummary.from_wire( + {"total_rows": 500, "passed_rows": 498, "failed_rows": 1, "error_rows": 1} + ) + result = EvalRunResult( + passed=False, + url="https://app.launchdarkly.com/run", + run_id="run-1", + summary=summary, + ) + + assert summary.total_rows == 500 + assert summary.error_rows == 1 + assert RunSummary.from_wire(None) == RunSummary() + assert result.passed is False + assert result.run_id == "run-1" From 50fee37c88b2bc207544bd7a28bf3d4d00a34875 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 12:15:50 -0700 Subject: [PATCH 02/32] feat: run client-side evaluations from the SDK --- packages/ai/README.md | 19 + packages/client/README.md | 36 ++ packages/client/agents.md | 11 +- .../src/launchdarkly_ai_server/__init__.py | 15 + .../evaluations/__init__.py | 3 +- .../launchdarkly_ai_server/evaluations/api.py | 70 ++- .../evaluations/module.py | 126 ++++- .../evaluations/runner.py | 445 ++++++++++++++++++ .../evaluations/types.py | 77 ++- packages/client/tests/test_evaluations.py | 47 +- packages/client/tests/test_evaluations_run.py | 390 +++++++++++++++ 11 files changed, 1198 insertions(+), 41 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/runner.py create mode 100644 packages/client/tests/test_evaluations_run.py diff --git a/packages/ai/README.md b/packages/ai/README.md index e35135b..e1539fd 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -52,6 +52,25 @@ if result["enabled"]: Never raises. Returns `{"enabled": bool, "config": dict | None, "meta": dict | None}`. +## Evaluations from code + +`init_evaluations` and the evaluations result types are also re-exported: + +```python +from launchdarkly_ai_python import init_evaluations + +evals = init_evaluations() +result = await evals.run( + project_key="my-project", + key="unique-evaluation-key", + dataset="golden-dataset", + handler=my_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, +) +``` + +`LD_API_TOKEN` is required. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). + --- All exports, types, and behaviors are identical to `launchdarkly-ai-server`. See the [core client README](../client/README.md) for the full API reference. diff --git a/packages/client/README.md b/packages/client/README.md index 8691bab..0aecb10 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -39,6 +39,42 @@ No code changes are required — `init_client()` detects the packages at runtime | `LD_SERVICE_NAME` | No | OTel `service.name` resource attribute (default: `python-sdk`) | | `LD_ENVIRONMENT` | No | `deployment.environment` resource attribute attached to telemetry | | `OTEL_EXPORTER_OTLP_ENDPOINT` | No | OTLP endpoint override (default: LaunchDarkly Observability backend) | +| `LD_API_TOKEN` | For evaluations | API access token used by the evaluations management API | +| `LD_API_BASE_URI` | No | Evaluations management API host override; intentionally separate from `LD_BASE_URI` | + +### Run an evaluation from code + +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and client-source run, invokes your handler once per row, uploads the generations, and returns LaunchDarkly's stored verdict. Evaluation keys must be unique because every call creates a new evaluation with `POST`. + +```python +import asyncio +import sys + +from launchdarkly_ai_openai_messages import create_openai_messages_handler +from launchdarkly_ai_server import init_evaluations + + +async def main() -> int: + evals = init_evaluations() # LD_API_TOKEN required; LD_SDK_KEY optional + result = await evals.run( + project_key="my-project", + key="support-qa-2026-08-20", + dataset="support-golden", + handler=create_openai_messages_handler(), + generation={ + "provider": "OpenAI", + "model": "gpt-4o", + "instructions": "You are a support agent.", + }, + ) + print(result.url, result.summary) + return 0 if result.passed else 1 + + +sys.exit(asyncio.run(main())) +``` + +`project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/agents.md b/packages/client/agents.md index 9f143ac..7d4e2d7 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -31,6 +31,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | +| `src/launchdarkly_ai_server/evaluations/` | `init_evaluations`, the private management API operations, and generation-only `EvaluationsModule.run()` orchestration | | `src/launchdarkly_ai_server/__init__.py` | Public barrel — the only surface handler packages import from | --- @@ -68,7 +69,7 @@ from launchdarkly_ai_server import Registry, global_registry, compose, resolve_h from launchdarkly_ai_server import execute_and_track, execute_and_stream, wrap_tool_handlers # Entry points -from launchdarkly_ai_server import config, graph, resolve_graph +from launchdarkly_ai_server import config, graph, resolve_graph, init_evaluations ``` When adding a new export, add it to `__init__.py`'s imports and `__all__`. Handler packages must never import from sub-paths (e.g. `launchdarkly_ai_server.client`). @@ -125,6 +126,14 @@ Handlers may return any of these — the client normalizes them before emitting --- +## SDK-run evaluations + +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs and enables the normal handler observability path. + +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, batches generation ingest, and trusts only the server's stored verdict. + +--- + ## Conversation grouping LaunchDarkly's conversation view groups spans on `gen_ai.conversation.id`. Bind a caller-supplied id around any `invoke()` / `stream()` / `graph().invoke()` call: diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 80b959b..9a07b05 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -21,6 +21,14 @@ conversation_id, set_conversation_id_if_absent, ) +from .evaluations import ( + EvalRunResult, + EvaluationsError, + EvaluationsModule, + GenerationConfig, + RunSummary, + init_evaluations, +) from .graph import GraphInstance, graph, resolve_graph from .judges import build_judge_tasks, run_judge, run_judges from .lifecycle import ( @@ -155,6 +163,13 @@ "text_message", "to_semconv_finish_reason", "VariationMeta", + # evaluations + "EvalRunResult", + "EvaluationsError", + "EvaluationsModule", + "GenerationConfig", + "RunSummary", + "init_evaluations", # utils "create_handler", "make_track_data", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index 99340b9..6516f4a 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -10,13 +10,14 @@ urllib_transport, ) from .module import EvaluationsModule, init_evaluations -from .types import EvalRunResult, RunSummary, Usage +from .types import EvalRunResult, GenerationConfig, RunSummary, Usage __all__ = [ "DEFAULT_BASE_URI", "EvalRunResult", "EvaluationsError", "EvaluationsModule", + "GenerationConfig", "HttpResponse", "LDApiClient", "LDApiError", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/api.py b/packages/client/src/launchdarkly_ai_server/evaluations/api.py index f87f8f0..957a092 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/api.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/api.py @@ -1,10 +1,15 @@ from __future__ import annotations import json +import random +import time import urllib.error import urllib.parse import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime from typing import Any, Protocol DEFAULT_BASE_URI = "https://app.launchdarkly.com" @@ -71,11 +76,7 @@ def urllib_transport( class LDApiClient: - """ - Minimal client for the LaunchDarkly public ``/api/v2`` surface used by the - evaluations harness. Every request carries the API access token; the base - URI is overridable for non-default instances. - """ + """Minimal retrying client for the LaunchDarkly public management API.""" def __init__( self, @@ -83,11 +84,17 @@ def __init__( base_uri: str = DEFAULT_BASE_URI, transport: Transport = urllib_transport, timeout: float = 30.0, + max_retries: int = 3, + sleep: Callable[[float], None] = time.sleep, + random_value: Callable[[], float] = random.random, ) -> None: self.api_token = api_token self.base_uri = base_uri.rstrip("/") self._transport = transport self._timeout = timeout + self._max_retries = max(0, max_retries) + self._sleep = sleep + self._random_value = random_value def url_for(self, path: str, params: dict[str, Any] | None = None) -> str: url = f"{self.base_uri}/api/v2/{path.lstrip('/')}" @@ -97,6 +104,25 @@ def url_for(self, path: str, params: dict[str, Any] | None = None) -> str: url = f"{url}?{urllib.parse.urlencode(query)}" return url + def _retry_delay(self, attempt: int, response: HttpResponse | None = None) -> float: + if response is not None: + retry_after = response.headers.get("retry-after") or response.headers.get( + "Retry-After" + ) + if retry_after: + try: + return max(0.0, float(retry_after)) + except ValueError: + try: + when: datetime = parsedate_to_datetime(retry_after) + now = datetime.now(UTC) + return max(0.0, (when - now).total_seconds()) + except (TypeError, ValueError, OverflowError): + pass + exponential = float(min(30.0, 0.5 * (2**attempt))) + jitter = float(self._random_value()) * min(1.0, exponential) + return exponential + jitter + def request( self, method: str, @@ -114,14 +140,40 @@ def request( headers["Content-Type"] = "application/json" payload = json.dumps(body).encode("utf-8") - response = self._transport( - method, self.url_for(path, params), headers, payload, self._timeout - ) + response: HttpResponse | None = None + for attempt in range(self._max_retries + 1): + try: + response = self._transport( + method, self.url_for(path, params), headers, payload, self._timeout + ) + except (TimeoutError, urllib.error.URLError) as error: + if attempt >= self._max_retries: + raise EvaluationsError( + f"LaunchDarkly API {method} {path} failed after retries: {error}" + ) from error + self._sleep(self._retry_delay(attempt)) + continue + + retryable = response.status == 429 or response.status >= 500 + if retryable and attempt < self._max_retries: + self._sleep(self._retry_delay(attempt, response)) + continue + break + + if response is None: + raise EvaluationsError( + f"LaunchDarkly API {method} {path} returned no response" + ) if response.status < 200 or response.status >= 300: raise LDApiError(response.status, method, path, response.body) if not response.body: return None - return json.loads(response.body) + try: + return json.loads(response.body) + except json.JSONDecodeError as error: + raise EvaluationsError( + f"LaunchDarkly API {method} {path} returned invalid JSON" + ) from error def get(self, path: str, params: dict[str, Any] | None = None) -> Any: return self.request("GET", path, params=params) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index c73c703..b9ce4cf 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -2,7 +2,9 @@ import logging import os +from collections.abc import Mapping +from ..lifecycle import init_client from .api import ( DEFAULT_BASE_URI, EvaluationsError, @@ -10,6 +12,8 @@ Transport, urllib_transport, ) +from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment +from .types import EvalRunResult, GenerationConfig logger = logging.getLogger(__name__) @@ -21,15 +25,12 @@ def _env(name: str) -> str | None: class EvaluationsModule: - """ - Entry point for running LaunchDarkly evaluations from code. Holds the - resolved credentials and the LaunchDarkly API client; ``run()`` arrives with - the harness. - """ + """Entry point for running LaunchDarkly evaluations from customer code.""" def __init__(self, api_client: LDApiClient, sdk_key: str | None = None) -> None: self._api = api_client self._sdk_key = sdk_key + self._runner = EvaluationsRunner(api_client) @property def api(self) -> LDApiClient: @@ -40,6 +41,109 @@ def sdk_key(self) -> str | None: """SDK key used for observability traces; ``None`` disables tracing.""" return self._sdk_key + async def run( + self, + *, + project_key: str, + key: str, + dataset: str, + handler: EvalHandler, + generation: GenerationConfig, + tools: Mapping[str, ToolImplementation] | None = None, + concurrency: int = 10, + timeout: float = 300.0, + ) -> EvalRunResult: + """ + Create and run a generation-only evaluation in the caller's process. + + The returned verdict is computed by LaunchDarkly. A CI script can exit + with ``0 if result.passed else 1`` after awaiting this method. + """ + self._validate_run_args( + project_key=project_key, + key=key, + dataset=dataset, + handler=handler, + generation=generation, + concurrency=concurrency, + timeout=timeout, + ) + run_tools = dict(tools or {}) + if self._sdk_key: + await init_client({"sdkKey": self._sdk_key}) + + # Tool verification is deliberately first: a typo must not create records. + resolved_tools = self._runner._resolve_tools(project_key, run_tools) + rows = self._runner._get_dataset_rows(project_key, dataset) + evaluation = self._runner._create_evaluation( + project_key, key, generation, resolved_tools + ) + evaluation_run = self._runner._create_evaluation_run( + project_key, key, len(rows) + ) + config = self._runner._build_handler_config(generation, resolved_tools) + results = await self._runner._run_rows( + rows, + handler, + config, + run_tools, + concurrency, + ) + self._runner._ingest_results( + project_key, evaluation.id, evaluation_run.id, results + ) + completed = await self._runner._poll_run( + project_key, evaluation.id, evaluation_run.id, timeout + ) + summary = self._runner._get_summary( + project_key, evaluation.id, evaluation_run.id + ) + url = ( + f"{self._api.base_uri}/projects/{_segment(project_key)}/ai/evaluations/" + f"{_segment(evaluation.id)}/runs/{_segment(evaluation_run.id)}" + ) + return EvalRunResult( + passed=completed.verdict == "passed", + url=url, + run_id=evaluation_run.id, + summary=summary, + ) + + @staticmethod + def _validate_run_args( + *, + project_key: str, + key: str, + dataset: str, + handler: EvalHandler, + generation: GenerationConfig, + concurrency: int, + timeout: float, + ) -> None: + for name, value in ( + ("project_key", project_key), + ("key", key), + ("dataset", dataset), + ): + if not value.strip(): + raise EvaluationsError(f"{name} must not be blank") + if not callable(handler): + raise EvaluationsError("handler must be callable") + provider = generation.get("provider") + model = generation.get("model") + if not isinstance(provider, str) or not provider.strip(): + raise EvaluationsError("generation.provider is required") + if not isinstance(model, str) or not model.strip(): + raise EvaluationsError("generation.model is required") + if "instructions" in generation and "messages" in generation: + raise EvaluationsError( + "generation.instructions and generation.messages are mutually exclusive" + ) + if concurrency < 1: + raise EvaluationsError("concurrency must be at least 1") + if timeout <= 0: + raise EvaluationsError("timeout must be greater than zero") + def init_evaluations( api_token: str | None = None, @@ -47,15 +151,7 @@ def init_evaluations( base_uri: str | None = None, transport: Transport = urllib_transport, ) -> EvaluationsModule: - """ - Resolves credentials and builds the evaluations module. - - ``api_token`` (``LD_API_TOKEN``) authenticates every ``/api/v2`` call and is - required — a missing token raises before any network I/O rather than - surfacing as an opaque 401 mid-run. ``sdk_key`` (``LD_SDK_KEY``) is optional - and only makes handler calls emit observability traces. Both credentials - must point at the same project. - """ + """Resolve credentials and construct the evaluations module.""" token = api_token or _env("LD_API_TOKEN") if not token: raise EvaluationsError( @@ -71,7 +167,7 @@ def init_evaluations( api_client = LDApiClient( api_token=token, - base_uri=base_uri or _env("LD_BASE_URI") or DEFAULT_BASE_URI, + base_uri=base_uri or _env("LD_API_BASE_URI") or DEFAULT_BASE_URI, transport=transport, ) return EvaluationsModule(api_client=api_client, sdk_key=resolved_sdk_key) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py new file mode 100644 index 0000000..47375ab --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import asyncio +import json +import time +import urllib.parse +from collections.abc import Awaitable, Callable, Mapping +from datetime import UTC, datetime +from typing import Any + +from ..types import NativeTool +from ..utils import parse_template +from .api import EvaluationsError, LDApiClient, LDApiError +from .types import ( + DatasetRow, + EvaluationRef, + EvaluationRunRef, + GenerationConfig, + ResolvedTool, + RunSummary, +) + +DATASET_PAGE_SIZE = 200 +INGEST_BATCH_SIZE = 50 +MAX_INGEST_ROW_BYTES = 256 * 1024 + +EvalHandler = Callable[..., Awaitable[dict[str, Any]]] +ToolImplementation = Callable[..., Any] | NativeTool + + +def _segment(value: str) -> str: + return urllib.parse.quote(value, safe="") + + +def _mapping(value: Any, *, description: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise EvaluationsError( + f"LaunchDarkly returned an invalid {description} response" + ) + return value + + +def _required_string(data: Mapping[str, Any], key: str, description: str) -> str: + value = data.get(key) + if not isinstance(value, str) or not value: + raise EvaluationsError( + f"LaunchDarkly {description} response is missing string field {key!r}" + ) + return value + + +class ConcurrencyController: + """Owns row-worker permits; adaptive signals arrive in Phase 3.""" + + def __init__(self, limit: int = 10) -> None: + if limit < 1: + raise EvaluationsError("concurrency must be at least 1") + self._semaphore = asyncio.Semaphore(limit) + + async def acquire(self, provider: str | None = None) -> None: + del provider + await self._semaphore.acquire() + + def release(self) -> None: + self._semaphore.release() + + def record_success( + self, + provider: str | None = None, + headers: Mapping[str, str] | None = None, + ) -> None: + del provider, headers + + def record_rate_limit( + self, + provider: str | None = None, + retry_after: float | None = None, + ) -> None: + del provider, retry_after + + +class EvaluationsRunner: + """Private API operations and orchestration used by EvaluationsModule.run().""" + + def __init__(self, api: LDApiClient) -> None: + self._api = api + + def _resolve_tools( + self, + project_key: str, + tools: Mapping[str, ToolImplementation], + ) -> dict[str, ResolvedTool]: + resolved: dict[str, ResolvedTool] = {} + for key, implementation in tools.items(): + if not callable(implementation) and not isinstance( + implementation, NativeTool + ): + raise EvaluationsError( + f"Tool {key!r} must be callable or a NativeTool instance" + ) + path = f"projects/{_segment(project_key)}/ai-tools/{_segment(key)}" + try: + raw = _mapping(self._api.get(path), description=f"tool {key!r}") + except LDApiError as error: + if error.status == 404: + raise EvaluationsError( + f"LaunchDarkly AI tool {key!r} was not found in project {project_key!r}" + ) from error + raise + version = raw.get("version") + if not isinstance(version, int): + raise EvaluationsError( + f"LaunchDarkly AI tool {key!r} has no integer version" + ) + schema = raw.get("schema") + if not isinstance(schema, Mapping): + schema = {} + resolved[key] = ResolvedTool( + key=key, + version=version, + description=str(raw.get("description") or ""), + schema=dict(schema), + ) + return resolved + + def _fetch_dataset( + self, + project_key: str, + dataset_key: str, + *, + offset: int = 0, + ) -> Mapping[str, Any]: + path = ( + f"projects/{_segment(project_key)}/datasets/key/" + f"{_segment(dataset_key)}/preview" + ) + try: + return _mapping( + self._api.get( + path, params={"limit": DATASET_PAGE_SIZE, "offset": offset} + ), + description=f"dataset {dataset_key!r}", + ) + except LDApiError as error: + if error.status == 404: + raise EvaluationsError( + f"LaunchDarkly dataset {dataset_key!r} was not found in project {project_key!r}" + ) from error + raise + + def _get_dataset_rows(self, project_key: str, dataset_key: str) -> list[DatasetRow]: + rows: list[DatasetRow] = [] + offset = 0 + total: int | None = None + while total is None or len(rows) < total: + page = self._fetch_dataset(project_key, dataset_key, offset=offset) + items = page.get("items") + page_total = page.get("totalCount") + if not isinstance(items, list) or not isinstance(page_total, int): + raise EvaluationsError( + f"LaunchDarkly returned invalid rows for dataset {dataset_key!r}" + ) + total = page_total + if not items: + break + for item_value in items: + item = _mapping(item_value, description="dataset row") + row_index = item.get("rowIndex") + if not isinstance(row_index, int): + raise EvaluationsError( + "A dataset row is missing its integer rowIndex" + ) + variables_value = item.get("variables") + variables = ( + dict(variables_value) + if isinstance(variables_value, Mapping) + else {} + ) + input_value = item.get("input") + expected_value = item.get("expectedOutput") + rendered_input = ( + parse_template(input_value, variables) + if isinstance(input_value, str) + else None + ) + rendered_expected = ( + parse_template(expected_value, variables) + if isinstance(expected_value, str) + else None + ) + variables["input"] = rendered_input + variables["expected_output"] = rendered_expected + metadata_value = item.get("metadata") + rows.append( + DatasetRow( + row_index=row_index, + input=rendered_input, + expected_output=rendered_expected, + variables=variables, + metadata=( + dict(metadata_value) + if isinstance(metadata_value, Mapping) + else None + ), + ) + ) + offset += len(items) + if not rows: + raise EvaluationsError(f"Dataset {dataset_key!r} is empty") + if total is not None and len(rows) != total: + raise EvaluationsError( + f"Dataset {dataset_key!r} returned {len(rows)} of {total} rows" + ) + return rows + + def _create_evaluation( + self, + project_key: str, + key: str, + generation: GenerationConfig, + tools: Mapping[str, ResolvedTool], + ) -> EvaluationRef: + body: dict[str, Any] = { + "name": key, + "generationProvider": generation["provider"], + "generationModel": generation["model"], + } + if "parameters" in generation: + body["parameters"] = generation["parameters"] + if "instructions" in generation: + body["messages"] = [ + {"role": "system", "content": generation["instructions"]} + ] + elif "messages" in generation: + body["messages"] = generation["messages"] + else: + body["messages"] = [] + if "prompt_snippets" in generation: + body["promptSnippets"] = generation["prompt_snippets"] + if tools: + body["tools"] = [ + {"key": tool.key, "version": tool.version} for tool in tools.values() + ] + + path = f"projects/{_segment(project_key)}/evaluations" + raw = _mapping(self._api.post(path, body=body), description="evaluation") + evaluation_id = _required_string(raw, "id", "evaluation") + response_key = raw.get("name", raw.get("label", key)) + version = raw.get("version") + return EvaluationRef( + id=evaluation_id, + key=str(response_key), + version=version if isinstance(version, int) else None, + ) + + def _create_evaluation_run( + self, + project_key: str, + evaluation_key: str, + row_count: int, + ) -> EvaluationRunRef: + path = ( + f"projects/{_segment(project_key)}/evaluations/" + f"{_segment(evaluation_key)}/runs" + ) + raw = _mapping( + self._api.post(path, body={"source": "client", "rowCount": row_count}), + description="evaluation run", + ) + return self._run_ref(raw) + + def _run_ref(self, raw: Mapping[str, Any]) -> EvaluationRunRef: + return EvaluationRunRef( + id=_required_string(raw, "id", "evaluation run"), + evaluation_id=_required_string(raw, "evaluationId", "evaluation run"), + state=_required_string(raw, "state", "evaluation run"), + verdict=(str(raw["verdict"]) if raw.get("verdict") is not None else None), + status_reason=( + str(raw["statusReason"]) + if raw.get("statusReason") is not None + else None + ), + ) + + def _build_handler_config( + self, + generation: GenerationConfig, + tools: Mapping[str, ResolvedTool], + ) -> dict[str, Any]: + parameters = generation.get("parameters") + config: dict[str, Any] = { + "provider": {"name": generation["provider"]}, + "model": {"name": generation["model"], "parameters": parameters}, + "tools": { + key: { + "description": tool.description, + "parameters": tool.schema, + } + for key, tool in tools.items() + }, + } + snippet_variables = {"snippet": generation.get("prompt_snippets", {})} + if "instructions" in generation: + config["instructions"] = parse_template( + generation["instructions"], snippet_variables + ) + elif "messages" in generation: + config["messages"] = [ + { + **message, + "content": parse_template(message["content"], snippet_variables) + if isinstance(message.get("content"), str) + else message.get("content"), + } + for message in generation["messages"] + ] + if "output_format" in generation: + config["outputFormat"] = generation["output_format"] + return config + + async def _run_rows( + self, + rows: list[DatasetRow], + handler: EvalHandler, + config: dict[str, Any], + tool_handlers: dict[str, ToolImplementation], + concurrency: int, + ) -> list[dict[str, Any]]: + controller = ConcurrencyController(concurrency) + + async def invoke(row: DatasetRow) -> dict[str, Any]: + await controller.acquire(config["provider"]["name"]) + started = datetime.now(UTC) + started_clock = time.perf_counter() + try: + result = await handler( + config, row.input, tool_handlers, dict(row.variables) + ) + if not isinstance(result, Mapping): + raise TypeError("handler result must be a mapping") + completed = datetime.now(UTC) + payload: dict[str, Any] = { + "row_index": row.row_index, + "input": row.input, + "expected_output": row.expected_output, + "variables": row.variables, + "metadata": row.metadata, + "output": {"generation": result.get("output")}, + "started_at": started.isoformat().replace("+00:00", "Z"), + "generated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + "status": "COMPLETE", + } + usage = result.get("usage") + if isinstance(usage, Mapping): + payload["output"]["usage"] = dict(usage) + controller.record_success(config["provider"]["name"]) + return payload + except Exception as error: + completed = datetime.now(UTC) + return { + "row_index": row.row_index, + "input": row.input, + "expected_output": row.expected_output, + "variables": row.variables, + "metadata": row.metadata, + "started_at": started.isoformat().replace("+00:00", "Z"), + "generated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + "status": "ERROR", + "error": {"code": 5001, "message": f"handler raised: {error}"}, + } + finally: + controller.release() + + return list(await asyncio.gather(*(invoke(row) for row in rows))) + + def _ingest_results( + self, + project_key: str, + evaluation_id: str, + run_id: str, + results: list[dict[str, Any]], + ) -> None: + path = ( + f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" + f"/runs/{_segment(run_id)}/generation-results" + ) + for result in results: + size = len(json.dumps(result).encode("utf-8")) + if size > MAX_INGEST_ROW_BYTES: + raise EvaluationsError( + f"Generation result row {result['row_index']} exceeds the " + f"{MAX_INGEST_ROW_BYTES}-byte limit" + ) + for start in range(0, len(results), INGEST_BATCH_SIZE): + self._api.post( + path, body={"results": results[start : start + INGEST_BATCH_SIZE]} + ) + + async def _poll_run( + self, + project_key: str, + evaluation_id: str, + run_id: str, + timeout: float, + ) -> EvaluationRunRef: + path = ( + f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" + f"/runs/{_segment(run_id)}" + ) + deadline = time.monotonic() + timeout + delay = 0.25 + while True: + run = self._run_ref( + _mapping(self._api.get(path), description="evaluation run") + ) + if run.state == "COMPLETE": + if run.verdict not in {"passed", "failed"}: + raise EvaluationsError( + f"Evaluation run {run_id!r} completed without a verdict" + ) + return run + if run.state in {"CANCELLED", "TEMPORARY_ERROR", "PERMANENT_ERROR"}: + reason = f": {run.status_reason}" if run.status_reason else "" + raise EvaluationsError( + f"Evaluation run {run_id!r} failed in state {run.state}{reason}" + ) + if time.monotonic() >= deadline: + raise EvaluationsError( + f"Evaluation run {run_id!r} is still in progress after {timeout} seconds" + ) + await asyncio.sleep(delay) + delay = min(5.0, delay * 2) + + def _get_summary( + self, project_key: str, evaluation_id: str, run_id: str + ) -> RunSummary: + path = ( + f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" + f"/runs/{_segment(run_id)}/summary" + ) + return RunSummary.from_wire( + _mapping(self._api.get(path), description="evaluation run summary") + ) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 3b5fdb9..9717682 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -1,15 +1,13 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, TypedDict @dataclass class Usage: - """ - Token counts for a single generation, in the ingest wire shape. Handler - results carry this dict verbatim, so nothing on the eval path adapts it. - """ + """Token counts for one generation, using the ingest wire field names.""" input_tokens: int output_tokens: int @@ -21,13 +19,66 @@ def to_wire(self) -> dict[str, int]: } @classmethod - def from_wire(cls, data: dict[str, Any]) -> Usage: + def from_wire(cls, data: Mapping[str, Any]) -> Usage: return cls( input_tokens=int(data.get("input_tokens") or 0), output_tokens=int(data.get("output_tokens") or 0), ) +class GenerationConfig(TypedDict, total=False): + """Generation settings stored on the evaluation and passed to its handler.""" + + provider: str + model: str + parameters: dict[str, Any] + instructions: str + messages: list[dict[str, Any]] + prompt_snippets: dict[str, str] + output_format: dict[str, Any] + + +@dataclass +class DatasetRow: + """A rendered dataset row ready for handler invocation and ingest.""" + + row_index: int + input: str | None = None + expected_output: str | None = None + variables: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] | None = None + + +@dataclass +class ResolvedTool: + """The schema and pinned version returned by the LaunchDarkly tool API.""" + + key: str + version: int + description: str = "" + schema: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class EvaluationRef: + """Identifiers returned after creating an evaluation.""" + + id: str + key: str + version: int | None = None + + +@dataclass +class EvaluationRunRef: + """Identifiers and state returned by the evaluation-run API.""" + + id: str + evaluation_id: str + state: str + verdict: str | None = None + status_reason: str | None = None + + @dataclass class RunSummary: """Row counts for a finished evaluation run.""" @@ -38,13 +89,15 @@ class RunSummary: error_rows: int = 0 @classmethod - def from_wire(cls, data: dict[str, Any] | None) -> RunSummary: + def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: data = data or {} + counts_value = data.get("statusCounts") + counts = counts_value if isinstance(counts_value, Mapping) else data return cls( - total_rows=int(data.get("total_rows") or 0), - passed_rows=int(data.get("passed_rows") or 0), - failed_rows=int(data.get("failed_rows") or 0), - error_rows=int(data.get("error_rows") or 0), + total_rows=int(counts.get("total", counts.get("total_rows", 0)) or 0), + passed_rows=int(counts.get("passed", counts.get("passed_rows", 0)) or 0), + failed_rows=int(counts.get("failed", counts.get("failed_rows", 0)) or 0), + error_rows=int(counts.get("error", counts.get("error_rows", 0)) or 0), ) diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index fcf7dca..c950429 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -109,16 +109,19 @@ def test_missing_sdk_key_is_allowed(monkeypatch: pytest.MonkeyPatch) -> None: assert evals.sdk_key is None -def test_base_uri_override(monkeypatch: pytest.MonkeyPatch) -> None: +def test_base_uri_override_isolated_from_sdk_delivery_uri( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv("LD_API_TOKEN", "api-token") - monkeypatch.setenv("LD_BASE_URI", "https://ld.internal.example.com/") + monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com/") + monkeypatch.setenv("LD_BASE_URI", "https://relay.example.com/") from_env = init_evaluations(transport=RecordingTransport()) explicit = init_evaluations( base_uri="https://other.example.com", transport=RecordingTransport() ) - assert from_env.api.base_uri == "https://ld.internal.example.com" + assert from_env.api.base_uri == "https://api.staging.example.com" assert explicit.api.base_uri == "https://other.example.com" @@ -153,6 +156,44 @@ def test_get_encodes_query_params_and_omits_none() -> None: assert "Content-Type" not in request["headers"] +def test_rate_limit_retries_and_honors_retry_after() -> None: + transport = RecordingTransport( + [ + HttpResponse( + status=429, + body='{"message": "slow down"}', + headers={"retry-after": "2"}, + ), + HttpResponse(status=200, body='{"items": []}'), + ] + ) + sleeps: list[float] = [] + client = LDApiClient( + api_token="api-token", + transport=transport, + max_retries=1, + sleep=sleeps.append, + random_value=lambda: 0.0, + ) + + assert client.get("projects/proj/datasets") == {"items": []} + assert len(transport.requests) == 2 + assert sleeps == [2.0] + + +def test_forbidden_response_is_not_retried() -> None: + transport = RecordingTransport( + [HttpResponse(status=403, body='{"message": "forbidden"}')] + ) + client = LDApiClient(api_token="api-token", transport=transport, max_retries=3) + + with pytest.raises(LDApiError) as excinfo: + client.get("projects/proj/evaluations") + + assert excinfo.value.status == 403 + assert len(transport.requests) == 1 + + def test_error_response_raises_ld_api_error() -> None: transport = RecordingTransport( [HttpResponse(status=404, body='{"message": "nope"}')] diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py new file mode 100644 index 0000000..2254b1b --- /dev/null +++ b/packages/client/tests/test_evaluations_run.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +import pytest + +from launchdarkly_ai_server.evaluations import ( + EvaluationsError, + HttpResponse, + init_evaluations, +) + + +class SequencedTransport: + """Records requests and returns one response for each expected request.""" + + def __init__(self, responses: list[HttpResponse]) -> None: + self.responses = responses + self.requests: list[dict[str, Any]] = [] + + def __call__( + self, + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, + ) -> HttpResponse: + index = len(self.requests) + self.requests.append( + { + "method": method, + "url": url, + "headers": headers, + "body": json.loads(body) if body else None, + "timeout": timeout, + } + ) + if index >= len(self.responses): + raise AssertionError(f"unexpected request: {method} {url}") + return self.responses[index] + + +def response(status: int, body: dict[str, Any] | None = None) -> HttpResponse: + return HttpResponse( + status=status, body=json.dumps(body) if body is not None else "" + ) + + +def dataset_page( + items: list[dict[str, Any]], total: int, next_href: str | None = None +) -> dict[str, Any]: + links: dict[str, Any] = {"self": {"href": "https://api.test/current"}} + if next_href: + links["next"] = {"href": next_href} + return {"items": items, "totalCount": total, "_links": links} + + +async def successful_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], +) -> dict[str, Any]: + assert config["provider"] == {"name": "OpenAI"} + assert config["model"] == { + "name": "gpt-4o", + "parameters": {"temperature": 0.2}, + } + assert config["tools"]["lookup_order"] == { + "description": "Look up an order", + "parameters": {"type": "object"}, + } + assert "lookup_order" in tool_handlers + assert variables["input"] == user_input + return { + "output": f"generated: {user_input}", + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + + +def lookup_order(order_id: str) -> str: + return order_id + + +@pytest.mark.asyncio +async def test_run_calls_private_operations_in_order_and_returns_server_verdict() -> ( + None +): + transport = SequencedTransport( + [ + response( + 200, + { + "key": "lookup_order", + "version": 7, + "description": "Look up an order", + "schema": {"type": "object"}, + }, + ), + response( + 200, + dataset_page( + [ + { + "rowIndex": 4, + "input": "Order {{order_id}}", + "expectedOutput": "Found {{order_id}}", + "variables": {"order_id": "A19"}, + "metadata": {"suite": "orders"}, + } + ], + total=2, + next_href="https://api.test/api/v2/projects/proj/datasets/key/golden/preview?limit=1&offset=1", + ), + ), + response( + 200, + dataset_page( + [ + { + "rowIndex": 9, + "input": "Order {{order_id}}", + "expectedOutput": None, + "variables": {"order_id": "B20"}, + "metadata": None, + } + ], + total=2, + ), + ), + response( + 201, + { + "id": "11111111-1111-1111-1111-111111111111", + "name": "support-qa-unique", + "version": 1, + }, + ), + response( + 201, + { + "id": "22222222-2222-2222-2222-222222222222", + "evaluationId": "11111111-1111-1111-1111-111111111111", + "evaluationVersion": 1, + "source": "client", + "state": "PENDING", + "createdAt": 1, + }, + ), + response(202, {}), + response( + 200, + { + "id": "22222222-2222-2222-2222-222222222222", + "evaluationId": "11111111-1111-1111-1111-111111111111", + "evaluationVersion": 1, + "source": "client", + "state": "COMPLETE", + "verdict": "passed", + "createdAt": 1, + }, + ), + response( + 200, + { + "evaluationId": "11111111-1111-1111-1111-111111111111", + "evaluationVersion": 1, + "evaluationRunId": "22222222-2222-2222-2222-222222222222", + "statusCounts": { + "total": 2, + "passed": 2, + "failed": 0, + "error": 0, + "pending": 0, + }, + "createdAt": 1, + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + result = await evals.run( + project_key="proj", + key="support-qa-unique", + dataset="golden", + handler=successful_handler, + tools={"lookup_order": lookup_order}, + generation={ + "provider": "OpenAI", + "model": "gpt-4o", + "parameters": {"temperature": 0.2}, + "instructions": "Help the user.", + }, + concurrency=2, + ) + + assert result.passed is True + assert result.run_id == "22222222-2222-2222-2222-222222222222" + assert result.summary.total_rows == 2 + + assert [request["method"] for request in transport.requests] == [ + "GET", + "GET", + "GET", + "POST", + "POST", + "POST", + "GET", + "GET", + ] + assert transport.requests[0]["url"].endswith( + "/api/v2/projects/proj/ai-tools/lookup_order" + ) + assert "/projects/proj/datasets/key/golden/preview" in transport.requests[1]["url"] + assert transport.requests[3]["body"] == { + "name": "support-qa-unique", + "generationProvider": "OpenAI", + "generationModel": "gpt-4o", + "parameters": {"temperature": 0.2}, + "messages": [{"role": "system", "content": "Help the user."}], + "tools": [{"key": "lookup_order", "version": 7}], + } + assert transport.requests[4]["body"] == {"source": "client", "rowCount": 2} + + ingested = transport.requests[5]["body"]["results"] + assert [row["row_index"] for row in ingested] == [4, 9] + assert ingested[0]["input"] == "Order A19" + assert ingested[0]["expected_output"] == "Found A19" + assert ingested[0]["variables"]["input"] == "Order A19" + assert ingested[0]["variables"]["expected_output"] == "Found A19" + + +@pytest.mark.asyncio +async def test_run_rejects_instructions_and_messages_before_network_io() -> None: + transport = SequencedTransport([]) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match=r"instructions.*messages"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + generation={ + "provider": "OpenAI", + "model": "gpt-4o", + "instructions": "System prompt", + "messages": [{"role": "user", "content": "{{input}}"}], + }, + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_missing_tool_aborts_before_any_mutating_request() -> None: + transport = SequencedTransport( + [response(404, {"code": "not_found", "message": "not found"})] + ) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match="missing_tool"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + tools={"missing_tool": lookup_order}, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert [request["method"] for request in transport.requests] == ["GET"] + + +@pytest.mark.asyncio +async def test_empty_dataset_fails_before_evaluation_or_run_creation() -> None: + transport = SequencedTransport([response(200, dataset_page([], total=0))]) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match="empty"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert [request["method"] for request in transport.requests] == ["GET"] + + +@pytest.mark.asyncio +async def test_handler_error_is_ingested_and_other_rows_continue() -> None: + calls: list[str | None] = [] + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + calls.append(user_input) + if user_input == "bad": + raise RuntimeError("provider failed") + return {"output": "ok"} + + transport = SequencedTransport( + [ + response( + 200, + dataset_page( + [ + {"rowIndex": 0, "input": "bad", "variables": {}}, + {"rowIndex": 1, "input": "good", "variables": {}}, + ], + total=2, + ), + ), + response( + 201, + { + "id": "11111111-1111-1111-1111-111111111111", + "name": "eval-key", + "version": 1, + }, + ), + response( + 201, + { + "id": "22222222-2222-2222-2222-222222222222", + "evaluationId": "11111111-1111-1111-1111-111111111111", + "evaluationVersion": 1, + "source": "client", + "state": "PENDING", + "createdAt": 1, + }, + ), + response(202, {}), + response( + 200, + { + "id": "22222222-2222-2222-2222-222222222222", + "evaluationId": "11111111-1111-1111-1111-111111111111", + "evaluationVersion": 1, + "source": "client", + "state": "COMPLETE", + "verdict": "failed", + "createdAt": 1, + }, + ), + response( + 200, + { + "evaluationId": "11111111-1111-1111-1111-111111111111", + "evaluationVersion": 1, + "evaluationRunId": "22222222-2222-2222-2222-222222222222", + "statusCounts": { + "total": 2, + "passed": 1, + "failed": 0, + "error": 1, + "pending": 0, + }, + "createdAt": 1, + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert set(calls) == {"bad", "good"} + assert len(calls) == 2 + assert result.passed is False + rows = transport.requests[3]["body"]["results"] + assert {row["status"] for row in rows} == {"COMPLETE", "ERROR"} + error_row = next(row for row in rows if row["status"] == "ERROR") + assert error_row["row_index"] == 0 + assert "provider failed" in error_row["error"]["message"] From 28f8eb19770adcff0d856368f53d7b731298fc02 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 14:21:10 -0700 Subject: [PATCH 03/32] fix: align evaluations with staging dataset and run APIs --- .../evaluations/module.py | 3 +- .../evaluations/runner.py | 62 ++++++++++++------- .../evaluations/types.py | 8 +++ packages/client/tests/test_evaluations_run.py | 51 ++++++++++++--- 4 files changed, 95 insertions(+), 29 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index b9ce4cf..dba85ba 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -74,12 +74,13 @@ async def run( # Tool verification is deliberately first: a typo must not create records. resolved_tools = self._runner._resolve_tools(project_key, run_tools) + dataset_ref = self._runner._fetch_dataset(project_key, dataset) rows = self._runner._get_dataset_rows(project_key, dataset) evaluation = self._runner._create_evaluation( project_key, key, generation, resolved_tools ) evaluation_run = self._runner._create_evaluation_run( - project_key, key, len(rows) + project_key, evaluation.id, len(rows), dataset_ref.id ) config = self._runner._build_handler_config(generation, resolved_tools) results = await self._runner._run_rows( diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 47375ab..26c68e3 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -12,6 +12,7 @@ from ..utils import parse_template from .api import EvaluationsError, LDApiClient, LDApiError from .types import ( + DatasetRef, DatasetRow, EvaluationRef, EvaluationRunRef, @@ -123,37 +124,48 @@ def _resolve_tools( ) return resolved - def _fetch_dataset( - self, - project_key: str, - dataset_key: str, - *, - offset: int = 0, - ) -> Mapping[str, Any]: - path = ( - f"projects/{_segment(project_key)}/datasets/key/" - f"{_segment(dataset_key)}/preview" - ) + def _fetch_dataset(self, project_key: str, dataset_key: str) -> DatasetRef: + path = f"projects/{_segment(project_key)}/datasets/{_segment(dataset_key)}" try: - return _mapping( - self._api.get( - path, params={"limit": DATASET_PAGE_SIZE, "offset": offset} - ), - description=f"dataset {dataset_key!r}", - ) + raw = _mapping(self._api.get(path), description=f"dataset {dataset_key!r}") except LDApiError as error: if error.status == 404: raise EvaluationsError( f"LaunchDarkly dataset {dataset_key!r} was not found in project {project_key!r}" ) from error raise + dataset_id = _required_string(raw, "id", "dataset") + response_key = raw.get("key", raw.get("name", dataset_key)) + return DatasetRef(id=dataset_id, key=str(response_key)) + + def _fetch_dataset_rows_page( + self, + project_key: str, + dataset_key: str, + *, + offset: int, + ) -> Mapping[str, Any]: + path = f"projects/{_segment(project_key)}/datasets/{_segment(dataset_key)}/rows" + return _mapping( + self._api.get( + path, + params={ + "mode": "all", + "limit": DATASET_PAGE_SIZE, + "offset": offset, + }, + ), + description=f"rows for dataset {dataset_key!r}", + ) def _get_dataset_rows(self, project_key: str, dataset_key: str) -> list[DatasetRow]: rows: list[DatasetRow] = [] offset = 0 total: int | None = None while total is None or len(rows) < total: - page = self._fetch_dataset(project_key, dataset_key, offset=offset) + page = self._fetch_dataset_rows_page( + project_key, dataset_key, offset=offset + ) items = page.get("items") page_total = page.get("totalCount") if not isinstance(items, list) or not isinstance(page_total, int): @@ -256,15 +268,23 @@ def _create_evaluation( def _create_evaluation_run( self, project_key: str, - evaluation_key: str, + evaluation_id: str, row_count: int, + dataset_id: str, ) -> EvaluationRunRef: path = ( f"projects/{_segment(project_key)}/evaluations/" - f"{_segment(evaluation_key)}/runs" + f"{_segment(evaluation_id)}/runs" ) raw = _mapping( - self._api.post(path, body={"source": "client", "rowCount": row_count}), + self._api.post( + path, + body={ + "source": "client", + "rowCount": row_count, + "datasetId": dataset_id, + }, + ), description="evaluation run", ) return self._run_ref(raw) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 9717682..dda010a 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -38,6 +38,14 @@ class GenerationConfig(TypedDict, total=False): output_format: dict[str, Any] +@dataclass +class DatasetRef: + """Identifiers returned when resolving a dataset by key.""" + + id: str + key: str + + @dataclass class DatasetRow: """A rendered dataset row ready for handler invocation and ingest.""" diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 2254b1b..1d25be9 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -100,6 +100,13 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "schema": {"type": "object"}, }, ), + response( + 200, + { + "id": "33333333-3333-3333-3333-333333333333", + "name": "golden", + }, + ), response( 200, dataset_page( @@ -206,6 +213,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "GET", "GET", "GET", + "GET", "POST", "POST", "POST", @@ -215,8 +223,12 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( assert transport.requests[0]["url"].endswith( "/api/v2/projects/proj/ai-tools/lookup_order" ) - assert "/projects/proj/datasets/key/golden/preview" in transport.requests[1]["url"] - assert transport.requests[3]["body"] == { + assert transport.requests[1]["url"].endswith( + "/api/v2/projects/proj/datasets/golden" + ) + assert "/projects/proj/datasets/golden/rows" in transport.requests[2]["url"] + assert "mode=all" in transport.requests[2]["url"] + assert transport.requests[4]["body"] == { "name": "support-qa-unique", "generationProvider": "OpenAI", "generationModel": "gpt-4o", @@ -224,9 +236,16 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "messages": [{"role": "system", "content": "Help the user."}], "tools": [{"key": "lookup_order", "version": 7}], } - assert transport.requests[4]["body"] == {"source": "client", "rowCount": 2} + assert transport.requests[5]["url"].endswith( + "/api/v2/projects/proj/evaluations/11111111-1111-1111-1111-111111111111/runs" + ) + assert transport.requests[5]["body"] == { + "source": "client", + "rowCount": 2, + "datasetId": "33333333-3333-3333-3333-333333333333", + } - ingested = transport.requests[5]["body"]["results"] + ingested = transport.requests[6]["body"]["results"] assert [row["row_index"] for row in ingested] == [4, 9] assert ingested[0]["input"] == "Order A19" assert ingested[0]["expected_output"] == "Found A19" @@ -278,7 +297,18 @@ async def test_missing_tool_aborts_before_any_mutating_request() -> None: @pytest.mark.asyncio async def test_empty_dataset_fails_before_evaluation_or_run_creation() -> None: - transport = SequencedTransport([response(200, dataset_page([], total=0))]) + transport = SequencedTransport( + [ + response( + 200, + { + "id": "33333333-3333-3333-3333-333333333333", + "name": "golden", + }, + ), + response(200, dataset_page([], total=0)), + ] + ) evals = init_evaluations(api_token="token", transport=transport) with pytest.raises(EvaluationsError, match="empty"): @@ -290,7 +320,7 @@ async def test_empty_dataset_fails_before_evaluation_or_run_creation() -> None: generation={"provider": "OpenAI", "model": "gpt-4o"}, ) - assert [request["method"] for request in transport.requests] == ["GET"] + assert [request["method"] for request in transport.requests] == ["GET", "GET"] @pytest.mark.asyncio @@ -310,6 +340,13 @@ async def handler( transport = SequencedTransport( [ + response( + 200, + { + "id": "33333333-3333-3333-3333-333333333333", + "name": "golden", + }, + ), response( 200, dataset_page( @@ -383,7 +420,7 @@ async def handler( assert set(calls) == {"bad", "good"} assert len(calls) == 2 assert result.passed is False - rows = transport.requests[3]["body"]["results"] + rows = transport.requests[4]["body"]["results"] assert {row["status"] for row in rows} == {"COMPLETE", "ERROR"} error_row = next(row for row in rows if row["status"] == "ERROR") assert error_row["row_index"] == 0 From 88caec01ee7fb69c80d06d5a408609f6f09183a8 Mon Sep 17 00:00:00 2001 From: "doneill@launchdarkly.com" Date: Fri, 21 Aug 2026 21:20:16 +0000 Subject: [PATCH 04/32] chore: drop phase reference from concurrency controller docstring --- .../client/src/launchdarkly_ai_server/evaluations/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 26c68e3..19cd56a 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -51,7 +51,7 @@ def _required_string(data: Mapping[str, Any], key: str, description: str) -> str class ConcurrencyController: - """Owns row-worker permits; adaptive signals arrive in Phase 3.""" + """Owns row-worker permits.""" def __init__(self, limit: int = 10) -> None: if limit < 1: From 0ea442f96b6de6963916419bd8b4ce767f2314c8 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 21 Aug 2026 14:20:46 -0700 Subject: [PATCH 05/32] feat: gate evaluation generation result ingest --- packages/client/README.md | 2 + .../evaluations/flags.py | 44 ++++++++ .../evaluations/module.py | 13 ++- .../evaluations/runner.py | 4 + .../client/tests/test_evaluation_flags.py | 58 ++++++++++ packages/client/tests/test_evaluations_run.py | 103 +++++++++++++++++- 6 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/flags.py create mode 100644 packages/client/tests/test_evaluation_flags.py diff --git a/packages/client/README.md b/packages/client/README.md index 0aecb10..08a0917 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -76,6 +76,8 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. +When `LD_SDK_KEY` is configured, generation-result publishing is controlled by the `enable-batch-ingest-in-evals-from-code` flag evaluated for the project. Results are uploaded only when the variation is exactly `true`; false, malformed, or failed evaluations skip publishing. Without an SDK key, publishing retains its existing behavior. + The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. Call `init_client()` explicitly when you want to: diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/flags.py b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py new file mode 100644 index 0000000..ee1a138 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import inspect +import logging +from typing import Any, Final + +from ..utils import to_ld_context + +logger = logging.getLogger(__name__) + +ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY: Final[str] = ( + "enable-batch-ingest-in-evals-from-code" +) +"""Canonical rollout flag for generation-result batch ingestion.""" + + +async def is_generation_result_batch_ingest_enabled( + client: Any, + project_key: str, +) -> bool: + """Return whether the rollout flag enables generation-result batch ingest. + + Flag evaluation is fail-safe: false, malformed, or failed evaluations disable + the gated batch-ingest path. + """ + try: + context = to_ld_context( + client, + {"kind": "project", "key": project_key}, + ) + result = client.variation( + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, + context, + False, + ) + value = await result if inspect.isawaitable(result) else result + return value is True + except Exception: + logger.warning( + "Unable to evaluate %s; generation results will not be batch ingested", + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, + exc_info=True, + ) + return False diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index dba85ba..58f48f1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -12,6 +12,7 @@ Transport, urllib_transport, ) +from .flags import is_generation_result_batch_ingest_enabled from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment from .types import EvalRunResult, GenerationConfig @@ -69,8 +70,12 @@ async def run( timeout=timeout, ) run_tools = dict(tools or {}) + batch_ingest_enabled = True if self._sdk_key: - await init_client({"sdkKey": self._sdk_key}) + client = await init_client({"sdkKey": self._sdk_key}) + batch_ingest_enabled = await is_generation_result_batch_ingest_enabled( + client, project_key + ) # Tool verification is deliberately first: a typo must not create records. resolved_tools = self._runner._resolve_tools(project_key, run_tools) @@ -91,7 +96,11 @@ async def run( concurrency, ) self._runner._ingest_results( - project_key, evaluation.id, evaluation_run.id, results + project_key, + evaluation.id, + evaluation_run.id, + results, + batch_ingest_enabled=batch_ingest_enabled, ) completed = await self._runner._poll_run( project_key, evaluation.id, evaluation_run.id, timeout diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 19cd56a..0da6814 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -401,7 +401,11 @@ def _ingest_results( evaluation_id: str, run_id: str, results: list[dict[str, Any]], + *, + batch_ingest_enabled: bool = True, ) -> None: + if not batch_ingest_enabled: + return path = ( f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" f"/runs/{_segment(run_id)}/generation-results" diff --git a/packages/client/tests/test_evaluation_flags.py b/packages/client/tests/test_evaluation_flags.py new file mode 100644 index 0000000..8533cbf --- /dev/null +++ b/packages/client/tests/test_evaluation_flags.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from launchdarkly_ai_server.evaluations.flags import ( + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, + is_generation_result_batch_ingest_enabled, +) + + +@pytest.mark.asyncio +async def test_enabled_flag_enables_generation_result_batch_ingest() -> None: + client = MagicMock() + client.variation = AsyncMock(return_value=True) + + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is True + ) + client.variation.assert_awaited_once_with( + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, + {"kind": "project", "key": "project-key"}, + False, + ) + + +@pytest.mark.asyncio +async def test_disabled_flag_disables_generation_result_batch_ingest() -> None: + client = MagicMock() + client.variation = AsyncMock(return_value=False) + + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is False + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("malformed_value", [None, 1, "true", {}]) +async def test_malformed_flag_disables_generation_result_batch_ingest( + malformed_value: object, +) -> None: + client = MagicMock() + client.variation = AsyncMock(return_value=malformed_value) + + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is False + ) + + +@pytest.mark.asyncio +async def test_flag_evaluation_error_disables_generation_result_batch_ingest() -> None: + client = MagicMock() + client.variation = AsyncMock(side_effect=RuntimeError("delivery unavailable")) + + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is False + ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 1d25be9..828563e 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -3,6 +3,7 @@ import json from collections.abc import Callable from typing import Any +from unittest.mock import AsyncMock, MagicMock import pytest @@ -86,9 +87,14 @@ def lookup_order(order_id: str) -> str: @pytest.mark.asyncio -async def test_run_calls_private_operations_in_order_and_returns_server_verdict() -> ( - None -): +async def test_run_calls_private_operations_in_order_and_returns_server_verdict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LD_SDK_KEY", raising=False) + init_client = AsyncMock() + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", init_client + ) transport = SequencedTransport( [ response( @@ -189,6 +195,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( ] ) evals = init_evaluations(api_token="token", transport=transport) + assert evals.sdk_key is None result = await evals.run( project_key="proj", @@ -251,6 +258,96 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( assert ingested[0]["expected_output"] == "Found A19" assert ingested[0]["variables"]["input"] == "Order A19" assert ingested[0]["variables"]["expected_output"] == "Found A19" + init_client.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("flag_value", "expected_ingest"), + [ + pytest.param(True, True, id="enabled"), + pytest.param(False, False, id="disabled-default"), + pytest.param("true", False, id="malformed"), + pytest.param( + RuntimeError("delivery unavailable"), False, id="evaluation-error" + ), + ], +) +async def test_batch_ingest_flag_controls_generation_result_publishing( + monkeypatch: pytest.MonkeyPatch, + flag_value: object, + expected_ingest: bool, +) -> None: + responses = [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 3, "input": "hello", "variables": {}}], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "PENDING", + }, + ), + ] + if expected_ingest: + responses.append(response(202, {})) + responses.extend( + [ + response( + 200, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "COMPLETE", + "verdict": "passed", + }, + ), + response(200, {"statusCounts": {"total": 1, "passed": 1}}), + ] + ) + transport = SequencedTransport(responses) + client = MagicMock() + if isinstance(flag_value, Exception): + client.variation = AsyncMock(side_effect=flag_value) + else: + client.variation = AsyncMock(return_value=flag_value) + + async def fake_init_client(options: dict[str, Any]) -> MagicMock: + assert options == {"sdkKey": "sdk-key"} + return client + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", fake_init_client + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert result.passed is True + assert ( + any( + request["url"].endswith("/generation-results") + for request in transport.requests + ) + is expected_ingest + ) @pytest.mark.asyncio From 300c6c22e1b18a7c3b1bc4eb096f9ea7e866900e Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 21 Aug 2026 14:31:26 -0700 Subject: [PATCH 06/32] no-mistakes(document): refresh agents.md ingest gate description --- packages/client/agents.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 7d4e2d7..334c096 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs and enables the normal handler observability path. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs; when set it enables the normal handler observability path and gates generation-result ingest on the `enable-batch-ingest-in-evals-from-code` flag (only a strictly `true` variation publishes; false, default, malformed, or evaluation-error results skip publish safely). Without an SDK key the gate cannot be evaluated and ingest runs unconditionally. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, batches generation ingest, and trusts only the server's stored verdict. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, batches generation ingest when the gate permits, and trusts only the server's stored verdict. --- From 433273aa71dc5c65714228f2e4cc2fbb0e15b2cf Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 24 Aug 2026 11:52:07 -0700 Subject: [PATCH 07/32] fix(evaluations): use API run source --- packages/client/README.md | 2 +- .../src/launchdarkly_ai_server/evaluations/runner.py | 2 +- packages/client/tests/test_evaluations_run.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 08a0917..fdbfaec 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -44,7 +44,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and client-source run, invokes your handler once per row, uploads the generations, and returns LaunchDarkly's stored verdict. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and returns LaunchDarkly's stored verdict. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 0da6814..371d059 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -280,7 +280,7 @@ def _create_evaluation_run( self._api.post( path, body={ - "source": "client", + "source": "api", "rowCount": row_count, "datasetId": dataset_id, }, diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 828563e..e28c48a 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -158,7 +158,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "id": "22222222-2222-2222-2222-222222222222", "evaluationId": "11111111-1111-1111-1111-111111111111", "evaluationVersion": 1, - "source": "client", + "source": "api", "state": "PENDING", "createdAt": 1, }, @@ -170,7 +170,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "id": "22222222-2222-2222-2222-222222222222", "evaluationId": "11111111-1111-1111-1111-111111111111", "evaluationVersion": 1, - "source": "client", + "source": "api", "state": "COMPLETE", "verdict": "passed", "createdAt": 1, @@ -247,7 +247,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "/api/v2/projects/proj/evaluations/11111111-1111-1111-1111-111111111111/runs" ) assert transport.requests[5]["body"] == { - "source": "client", + "source": "api", "rowCount": 2, "datasetId": "33333333-3333-3333-3333-333333333333", } @@ -468,7 +468,7 @@ async def handler( "id": "22222222-2222-2222-2222-222222222222", "evaluationId": "11111111-1111-1111-1111-111111111111", "evaluationVersion": 1, - "source": "client", + "source": "api", "state": "PENDING", "createdAt": 1, }, @@ -480,7 +480,7 @@ async def handler( "id": "22222222-2222-2222-2222-222222222222", "evaluationId": "11111111-1111-1111-1111-111111111111", "evaluationVersion": 1, - "source": "client", + "source": "api", "state": "COMPLETE", "verdict": "failed", "createdAt": 1, From a7acc68cef558e03d580382f5fe7597e613da157 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 24 Aug 2026 12:35:19 -0700 Subject: [PATCH 08/32] fix(evaluations): derive result from summary --- packages/client/README.md | 2 +- .../evaluations/module.py | 9 ++++---- .../evaluations/runner.py | 5 ----- .../evaluations/types.py | 3 +-- packages/client/tests/test_evaluations_run.py | 22 ++++++++++++------- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index fdbfaec..867028f 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -44,7 +44,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and returns LaunchDarkly's stored verdict. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and derives pass/fail from LaunchDarkly's run summary. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 58f48f1..8b0c1df 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -57,8 +57,9 @@ async def run( """ Create and run a generation-only evaluation in the caller's process. - The returned verdict is computed by LaunchDarkly. A CI script can exit - with ``0 if result.passed else 1`` after awaiting this method. + The returned pass/fail result is derived from LaunchDarkly's run summary. + A CI script can exit with ``0 if result.passed else 1`` after awaiting + this method. """ self._validate_run_args( project_key=project_key, @@ -102,7 +103,7 @@ async def run( results, batch_ingest_enabled=batch_ingest_enabled, ) - completed = await self._runner._poll_run( + await self._runner._poll_run( project_key, evaluation.id, evaluation_run.id, timeout ) summary = self._runner._get_summary( @@ -113,7 +114,7 @@ async def run( f"{_segment(evaluation.id)}/runs/{_segment(evaluation_run.id)}" ) return EvalRunResult( - passed=completed.verdict == "passed", + passed=summary.failed_rows == 0 and summary.error_rows == 0, url=url, run_id=evaluation_run.id, summary=summary, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 371d059..306abe6 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -294,7 +294,6 @@ def _run_ref(self, raw: Mapping[str, Any]) -> EvaluationRunRef: id=_required_string(raw, "id", "evaluation run"), evaluation_id=_required_string(raw, "evaluationId", "evaluation run"), state=_required_string(raw, "state", "evaluation run"), - verdict=(str(raw["verdict"]) if raw.get("verdict") is not None else None), status_reason=( str(raw["statusReason"]) if raw.get("statusReason") is not None @@ -440,10 +439,6 @@ async def _poll_run( _mapping(self._api.get(path), description="evaluation run") ) if run.state == "COMPLETE": - if run.verdict not in {"passed", "failed"}: - raise EvaluationsError( - f"Evaluation run {run_id!r} completed without a verdict" - ) return run if run.state in {"CANCELLED", "TEMPORARY_ERROR", "PERMANENT_ERROR"}: reason = f": {run.status_reason}" if run.status_reason else "" diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index dda010a..c61fcf1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -83,7 +83,6 @@ class EvaluationRunRef: id: str evaluation_id: str state: str - verdict: str | None = None status_reason: str | None = None @@ -111,7 +110,7 @@ def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: @dataclass class EvalRunResult: - """The verdict of an evaluation run, as computed and stored by LaunchDarkly.""" + """The result of an evaluation run, derived from its row summary.""" passed: bool url: str diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index e28c48a..3ed24b8 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -87,7 +87,7 @@ def lookup_order(order_id: str) -> str: @pytest.mark.asyncio -async def test_run_calls_private_operations_in_order_and_returns_server_verdict( +async def test_complete_run_with_zero_failed_and_error_rows_passes( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.delenv("LD_SDK_KEY", raising=False) @@ -172,7 +172,6 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( "evaluationVersion": 1, "source": "api", "state": "COMPLETE", - "verdict": "passed", "createdAt": 1, }, ), @@ -307,7 +306,6 @@ async def test_batch_ingest_flag_controls_generation_result_publishing( "id": "run-id", "evaluationId": "evaluation-id", "state": "COMPLETE", - "verdict": "passed", }, ), response(200, {"statusCounts": {"total": 1, "passed": 1}}), @@ -421,7 +419,16 @@ async def test_empty_dataset_fails_before_evaluation_or_run_creation() -> None: @pytest.mark.asyncio -async def test_handler_error_is_ingested_and_other_rows_continue() -> None: +@pytest.mark.parametrize( + ("failed_rows", "error_rows"), + [ + pytest.param(1, 0, id="failed-row"), + pytest.param(0, 1, id="error-row"), + ], +) +async def test_complete_run_with_failed_or_error_rows_does_not_pass( + failed_rows: int, error_rows: int +) -> None: calls: list[str | None] = [] async def handler( @@ -482,7 +489,6 @@ async def handler( "evaluationVersion": 1, "source": "api", "state": "COMPLETE", - "verdict": "failed", "createdAt": 1, }, ), @@ -494,9 +500,9 @@ async def handler( "evaluationRunId": "22222222-2222-2222-2222-222222222222", "statusCounts": { "total": 2, - "passed": 1, - "failed": 0, - "error": 1, + "passed": 2 - failed_rows - error_rows, + "failed": failed_rows, + "error": error_rows, "pending": 0, }, "createdAt": 1, From 0b5064841d09e2e703747d5ecf880f0dce19adbf Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 24 Aug 2026 12:39:09 -0700 Subject: [PATCH 09/32] no-mistakes(review): docs: describe pass/fail derivation from run summary --- packages/client/agents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 334c096..9f8bdcd 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -130,7 +130,7 @@ Handlers may return any of these — the client normalizes them before emitting `init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs; when set it enables the normal handler observability path and gates generation-result ingest on the `enable-batch-ingest-in-evals-from-code` flag (only a strictly `true` variation publishes; false, default, malformed, or evaluation-error results skip publish safely). Without an SDK key the gate cannot be evaluated and ingest runs unconditionally. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, batches generation ingest when the gate permits, and trusts only the server's stored verdict. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, and batches generation ingest when the gate permits. It polls `GET .../runs/{id}` on lifecycle status until a terminal state, then fetches the run summary and derives `EvalRunResult.passed` from `summary.failed_rows == 0 and summary.error_rows == 0`. --- From ec3e290ad606c2a7362e6c7b1aec57f82ee9d58c Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 24 Aug 2026 15:23:56 -0700 Subject: [PATCH 10/32] fix(evaluations): return pending summaries promptly --- packages/client/README.md | 2 +- packages/client/agents.md | 2 +- .../evaluations/module.py | 13 +++-- .../evaluations/types.py | 2 + packages/client/tests/test_evaluations.py | 10 +++- packages/client/tests/test_evaluations_run.py | 55 ++++++++++++++----- 6 files changed, 62 insertions(+), 22 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 867028f..c15f047 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -44,7 +44,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and derives pass/fail from LaunchDarkly's run summary. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and derives pass/fail from LaunchDarkly's run summary. A result passes only when the summary has no failed, error, or pending rows. When generation-result ingest is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/agents.md b/packages/client/agents.md index 9f8bdcd..df52ace 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -130,7 +130,7 @@ Handlers may return any of these — the client normalizes them before emitting `init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs; when set it enables the normal handler observability path and gates generation-result ingest on the `enable-batch-ingest-in-evals-from-code` flag (only a strictly `true` variation publishes; false, default, malformed, or evaluation-error results skip publish safely). Without an SDK key the gate cannot be evaluated and ingest runs unconditionally. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, and batches generation ingest when the gate permits. It polls `GET .../runs/{id}` on lifecycle status until a terminal state, then fetches the run summary and derives `EvalRunResult.passed` from `summary.failed_rows == 0 and summary.error_rows == 0`. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, and batches generation ingest when the gate permits. When ingest is enabled, it polls `GET .../runs/{id}` on lifecycle status until a terminal state and then fetches the run summary. When ingest is disabled, it skips polling and fetches the current summary once so the call returns promptly. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 8b0c1df..943603b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -103,9 +103,10 @@ async def run( results, batch_ingest_enabled=batch_ingest_enabled, ) - await self._runner._poll_run( - project_key, evaluation.id, evaluation_run.id, timeout - ) + if batch_ingest_enabled: + await self._runner._poll_run( + project_key, evaluation.id, evaluation_run.id, timeout + ) summary = self._runner._get_summary( project_key, evaluation.id, evaluation_run.id ) @@ -114,7 +115,11 @@ async def run( f"{_segment(evaluation.id)}/runs/{_segment(evaluation_run.id)}" ) return EvalRunResult( - passed=summary.failed_rows == 0 and summary.error_rows == 0, + passed=( + summary.failed_rows == 0 + and summary.error_rows == 0 + and summary.pending_rows == 0 + ), url=url, run_id=evaluation_run.id, summary=summary, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index c61fcf1..4033a3f 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -94,6 +94,7 @@ class RunSummary: passed_rows: int = 0 failed_rows: int = 0 error_rows: int = 0 + pending_rows: int = 0 @classmethod def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: @@ -105,6 +106,7 @@ def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: passed_rows=int(counts.get("passed", counts.get("passed_rows", 0)) or 0), failed_rows=int(counts.get("failed", counts.get("failed_rows", 0)) or 0), error_rows=int(counts.get("error", counts.get("error_rows", 0)) or 0), + pending_rows=int(counts.get("pending", counts.get("pending_rows", 0)) or 0), ) diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index c950429..2447e1a 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -224,7 +224,13 @@ def test_usage_matches_ingest_wire_shape() -> None: def test_run_summary_and_result() -> None: summary = RunSummary.from_wire( - {"total_rows": 500, "passed_rows": 498, "failed_rows": 1, "error_rows": 1} + { + "total_rows": 500, + "passed_rows": 497, + "failed_rows": 1, + "error_rows": 1, + "pending_rows": 1, + } ) result = EvalRunResult( passed=False, @@ -235,6 +241,8 @@ def test_run_summary_and_result() -> None: assert summary.total_rows == 500 assert summary.error_rows == 1 + assert summary.pending_rows == 1 + assert RunSummary.from_wire({"pending": 2}).pending_rows == 2 assert RunSummary.from_wire(None) == RunSummary() assert result.passed is False assert result.run_id == "run-1" diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 3ed24b8..78a2be1 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -297,20 +297,42 @@ async def test_batch_ingest_flag_controls_generation_result_publishing( ), ] if expected_ingest: - responses.append(response(202, {})) - responses.extend( - [ + responses.extend( + [ + response(202, {}), + response( + 200, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "COMPLETE", + }, + ), + response( + 200, + { + "statusCounts": { + "total": 1, + "passed": 1, + "pending": 0, + } + }, + ), + ] + ) + else: + responses.append( response( 200, { - "id": "run-id", - "evaluationId": "evaluation-id", - "state": "COMPLETE", + "total": 1, + "passed": 0, + "failed": 0, + "error": 0, + "pending": 1, }, - ), - response(200, {"statusCounts": {"total": 1, "passed": 1}}), - ] - ) + ) + ) transport = SequencedTransport(responses) client = MagicMock() if isinstance(flag_value, Exception): @@ -338,14 +360,17 @@ async def handler(*args: object) -> dict[str, Any]: generation={"provider": "OpenAI", "model": "gpt-4o"}, ) - assert result.passed is True + assert result.passed is expected_ingest + assert result.summary.pending_rows == (0 if expected_ingest else 1) + request_urls = [request["url"] for request in transport.requests] assert ( - any( - request["url"].endswith("/generation-results") - for request in transport.requests - ) + any(url.endswith("/generation-results") for url in request_urls) is expected_ingest ) + status_url = "/evaluations/evaluation-id/runs/run-id" + assert any(url.endswith(status_url) for url in request_urls) is expected_ingest + assert request_urls[-1].endswith(f"{status_url}/summary") + assert sum(url.endswith(f"{status_url}/summary") for url in request_urls) == 1 @pytest.mark.asyncio From 4cbcbd66e2dd4eed0be931924ee7246d430474df Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 24 Aug 2026 16:11:50 -0700 Subject: [PATCH 11/32] feat(evaluations): configure run link UI base --- packages/ai/README.md | 2 +- packages/client/README.md | 3 ++- packages/client/agents.md | 2 +- .../evaluations/module.py | 24 ++++++++++++++++--- packages/client/tests/test_evaluations.py | 18 ++++++++++++++ packages/client/tests/test_evaluations_run.py | 12 +++++++++- 6 files changed, 54 insertions(+), 7 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index e1539fd..128e208 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -69,7 +69,7 @@ result = await evals.run( ) ``` -`LD_API_TOKEN` is required. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). +`LD_API_TOKEN` is required. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/client/README.md b/packages/client/README.md index c15f047..8078435 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -41,10 +41,11 @@ No code changes are required — `init_client()` detects the packages at runtime | `OTEL_EXPORTER_OTLP_ENDPOINT` | No | OTLP endpoint override (default: LaunchDarkly Observability backend) | | `LD_API_TOKEN` | For evaluations | API access token used by the evaluations management API | | `LD_API_BASE_URI` | No | Evaluations management API host override; intentionally separate from `LD_BASE_URI` | +| `LD_UI_BASE_URI` | No | LaunchDarkly application host for evaluation-run links (default: `https://app.launchdarkly.com`; staging: `https://ld-stg.launchdarkly.com`) | ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and derives pass/fail from LaunchDarkly's run summary. A result passes only when the summary has no failed, error, or pending rows. When generation-result ingest is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and derives pass/fail from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. When generation-result ingest is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/agents.md b/packages/client/agents.md index df52ace..7112637 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,7 +128,7 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs; when set it enables the normal handler observability path and gates generation-result ingest on the `enable-batch-ingest-in-evals-from-code` flag (only a strictly `true` variation publishes; false, default, malformed, or evaluation-error results skip publish safely). Without an SDK key the gate cannot be evaluated and ingest runs unconditionally. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is optional for generation-only runs; when set it enables the normal handler observability path and gates generation-result ingest on the `enable-batch-ingest-in-evals-from-code` flag (only a strictly `true` variation publishes; false, default, malformed, or evaluation-error results skip publish safely). Without an SDK key the gate cannot be evaluated and ingest runs unconditionally. `await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, and batches generation ingest when the gate permits. When ingest is enabled, it polls `GET .../runs/{id}` on lifecycle status until a terminal state and then fetches the run summary. When ingest is disabled, it skips polling and fetches the current summary once so the call returns promptly. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 943603b..36b24ad 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -18,6 +18,8 @@ logger = logging.getLogger(__name__) +DEFAULT_UI_BASE_URI = "https://app.launchdarkly.com" + def _env(name: str) -> str | None: """Read an env var, treating blank/whitespace-only values as unset.""" @@ -28,9 +30,15 @@ def _env(name: str) -> str | None: class EvaluationsModule: """Entry point for running LaunchDarkly evaluations from customer code.""" - def __init__(self, api_client: LDApiClient, sdk_key: str | None = None) -> None: + def __init__( + self, + api_client: LDApiClient, + sdk_key: str | None = None, + ui_base_uri: str = DEFAULT_UI_BASE_URI, + ) -> None: self._api = api_client self._sdk_key = sdk_key + self._ui_base_uri = ui_base_uri.rstrip("/") self._runner = EvaluationsRunner(api_client) @property @@ -42,6 +50,11 @@ def sdk_key(self) -> str | None: """SDK key used for observability traces; ``None`` disables tracing.""" return self._sdk_key + @property + def ui_base_uri(self) -> str: + """LaunchDarkly application host used for evaluation-run links.""" + return self._ui_base_uri + async def run( self, *, @@ -111,7 +124,7 @@ async def run( project_key, evaluation.id, evaluation_run.id ) url = ( - f"{self._api.base_uri}/projects/{_segment(project_key)}/ai/evaluations/" + f"{self._ui_base_uri}/projects/{_segment(project_key)}/ai/evaluations/" f"{_segment(evaluation.id)}/runs/{_segment(evaluation_run.id)}" ) return EvalRunResult( @@ -165,6 +178,7 @@ def init_evaluations( api_token: str | None = None, sdk_key: str | None = None, base_uri: str | None = None, + ui_base_uri: str | None = None, transport: Transport = urllib_transport, ) -> EvaluationsModule: """Resolve credentials and construct the evaluations module.""" @@ -186,4 +200,8 @@ def init_evaluations( base_uri=base_uri or _env("LD_API_BASE_URI") or DEFAULT_BASE_URI, transport=transport, ) - return EvaluationsModule(api_client=api_client, sdk_key=resolved_sdk_key) + return EvaluationsModule( + api_client=api_client, + sdk_key=resolved_sdk_key, + ui_base_uri=ui_base_uri or _env("LD_UI_BASE_URI") or DEFAULT_UI_BASE_URI, + ) diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index 2447e1a..e051c4c 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -65,6 +65,7 @@ def test_init_resolves_credentials_from_env(monkeypatch: pytest.MonkeyPatch) -> assert evals.api.api_token == "api-token-from-env" assert evals.sdk_key == "sdk-key-from-env" assert evals.api.base_uri == DEFAULT_BASE_URI + assert evals.ui_base_uri == "https://app.launchdarkly.com" def test_init_prefers_explicit_credentials(monkeypatch: pytest.MonkeyPatch) -> None: @@ -125,6 +126,23 @@ def test_base_uri_override_isolated_from_sdk_delivery_uri( assert explicit.api.base_uri == "https://other.example.com" +def test_ui_base_uri_precedence_and_api_base_isolation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com") + monkeypatch.setenv("LD_UI_BASE_URI", "https://ld-stg.launchdarkly.com/") + + from_env = init_evaluations(transport=RecordingTransport()) + explicit = init_evaluations( + ui_base_uri="https://ui.example.com/", transport=RecordingTransport() + ) + + assert from_env.api.base_uri == "https://api.staging.example.com" + assert from_env.ui_base_uri == "https://ld-stg.launchdarkly.com" + assert explicit.ui_base_uri == "https://ui.example.com" + + def test_requests_carry_token_auth_and_json_body() -> None: transport = RecordingTransport([HttpResponse(status=201, body='{"key": "run-1"}')]) client = LDApiClient(api_token="api-token", transport=transport) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 78a2be1..20e05c0 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -193,7 +193,12 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( ), ] ) - evals = init_evaluations(api_token="token", transport=transport) + evals = init_evaluations( + api_token="token", + base_uri="https://api.example.com", + ui_base_uri="https://ui.example.com/", + transport=transport, + ) assert evals.sdk_key is None result = await evals.run( @@ -213,6 +218,11 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert result.passed is True assert result.run_id == "22222222-2222-2222-2222-222222222222" + assert result.url == ( + "https://ui.example.com/projects/proj/ai/evaluations/" + "11111111-1111-1111-1111-111111111111/runs/" + "22222222-2222-2222-2222-222222222222" + ) assert result.summary.total_rows == 2 assert [request["method"] for request in transport.requests] == [ From 311b012937a8d4862165ee35b038597df6c7badb Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 25 Aug 2026 10:30:42 -0700 Subject: [PATCH 12/32] feat(evaluations): emit generation events --- AGENTS.md | 7 ++ CLAUDE.md | 2 + packages/ai/README.md | 2 +- packages/client/README.md | 8 +- .../evaluations/module.py | 23 +++-- .../evaluations/runner.py | 81 +++++++++++----- packages/client/tests/test_evaluations_run.py | 92 +++++++++++++------ 7 files changed, 150 insertions(+), 65 deletions(-) create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md index 6d239fc..e828b2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -703,3 +703,10 @@ response = await graph( }, ).invoke(user_input, context) ``` + +## Maintaining this file + +Keep this file for knowledge useful to almost every future agent session in this project. +Do not repeat what the codebase already shows; point to the authoritative file or command instead. +Prefer rewriting or pruning existing entries over appending new ones. +When updating this file, preserve this bar for all agents and keep entries concise. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a9d4d26 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,2 @@ + +@AGENTS.md diff --git a/packages/ai/README.md b/packages/ai/README.md index 128e208..80e04fa 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -69,7 +69,7 @@ result = await evals.run( ) ``` -`LD_API_TOKEN` is required. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). +`LD_API_TOKEN` is required. Configure `LD_SDK_KEY` to emit one `$ld:ai:offline-evals:generation` event per generated row through the standard SDK event transport. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/client/README.md b/packages/client/README.md index 8078435..6f42253 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -45,7 +45,9 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, invokes your handler once per row, uploads the generations, and derives pass/fail from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. When generation-result ingest is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Events are flushed before lifecycle polling or return; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. + +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. When generation-result processing is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio @@ -56,7 +58,7 @@ from launchdarkly_ai_server import init_evaluations async def main() -> int: - evals = init_evaluations() # LD_API_TOKEN required; LD_SDK_KEY optional + evals = init_evaluations() # LD_API_TOKEN required; LD_SDK_KEY emits generations result = await evals.run( project_key="my-project", key="support-qa-2026-08-20", @@ -77,7 +79,7 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -When `LD_SDK_KEY` is configured, generation-result publishing is controlled by the `enable-batch-ingest-in-evals-from-code` flag evaluated for the project. Results are uploaded only when the variation is exactly `true`; false, malformed, or failed evaluations skip publishing. Without an SDK key, publishing retains its existing behavior. +`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. The `enable-batch-ingest-in-evals-from-code` flag still controls whether the harness waits for terminal lifecycle processing: only an exact `true` enables polling. Events are emitted and flushed for both flag outcomes; false, malformed, or failed flag evaluations skip polling and fetch the summary once. Without an SDK key, no generation event can be emitted, so polling is skipped and the current summary is returned immediately. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 36b24ad..3c98e9c 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -1,5 +1,6 @@ from __future__ import annotations +import inspect import logging import os from collections.abc import Mapping @@ -84,7 +85,8 @@ async def run( timeout=timeout, ) run_tools = dict(tools or {}) - batch_ingest_enabled = True + client = None + batch_ingest_enabled = False if self._sdk_key: client = await init_client({"sdkKey": self._sdk_key}) batch_ingest_enabled = await is_generation_result_batch_ingest_enabled( @@ -109,13 +111,18 @@ async def run( run_tools, concurrency, ) - self._runner._ingest_results( - project_key, - evaluation.id, - evaluation_run.id, - results, - batch_ingest_enabled=batch_ingest_enabled, - ) + if client is not None: + self._runner._emit_generation_events( + client, + project_key=project_key, + evaluation=evaluation, + evaluation_run=evaluation_run, + dataset=dataset_ref, + results=results, + ) + flush_result = client.flush() + if inspect.isawaitable(flush_result): + await flush_result if batch_ingest_enabled: await self._runner._poll_run( project_key, evaluation.id, evaluation_run.id, timeout diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 306abe6..eadc6d2 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import hashlib import json import time import urllib.parse @@ -9,7 +10,7 @@ from typing import Any from ..types import NativeTool -from ..utils import parse_template +from ..utils import parse_template, to_ld_context from .api import EvaluationsError, LDApiClient, LDApiError from .types import ( DatasetRef, @@ -22,8 +23,7 @@ ) DATASET_PAGE_SIZE = 200 -INGEST_BATCH_SIZE = 50 -MAX_INGEST_ROW_BYTES = 256 * 1024 +GENERATION_EVENT_NAME = "$ld:ai:offline-evals:generation" EvalHandler = Callable[..., Awaitable[dict[str, Any]]] ToolImplementation = Callable[..., Any] | NativeTool @@ -394,32 +394,67 @@ async def invoke(row: DatasetRow) -> dict[str, Any]: return list(await asyncio.gather(*(invoke(row) for row in rows))) - def _ingest_results( + def _emit_generation_events( self, + client: Any, + *, project_key: str, - evaluation_id: str, - run_id: str, + evaluation: EvaluationRef, + evaluation_run: EvaluationRunRef, + dataset: DatasetRef, results: list[dict[str, Any]], - *, - batch_ingest_enabled: bool = True, ) -> None: - if not batch_ingest_enabled: - return - path = ( - f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" - f"/runs/{_segment(run_id)}/generation-results" + """Queue one LD custom event for each executed dataset row.""" + context = to_ld_context( + client, + { + "kind": "evaluation", + "key": evaluation_run.id, + "projectKey": project_key, + "evaluationId": evaluation.id, + }, ) for result in results: - size = len(json.dumps(result).encode("utf-8")) - if size > MAX_INGEST_ROW_BYTES: - raise EvaluationsError( - f"Generation result row {result['row_index']} exceeds the " - f"{MAX_INGEST_ROW_BYTES}-byte limit" - ) - for start in range(0, len(results), INGEST_BATCH_SIZE): - self._api.post( - path, body={"results": results[start : start + INGEST_BATCH_SIZE]} - ) + identity = { + "projectKey": project_key, + "evaluationId": evaluation.id, + "runId": evaluation_run.id, + "datasetId": dataset.id, + "rowIndex": result["row_index"], + } + event_id = hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + generated = { + "status": result["status"], + "generationOutput": result.get("output", {}).get("generation"), + "error": result.get("error"), + "usage": result.get("output", {}).get("usage"), + } + content_hash = hashlib.sha256( + json.dumps( + generated, sort_keys=True, separators=(",", ":"), default=str + ).encode() + ).hexdigest() + payload: dict[str, Any] = { + **identity, + "eventId": event_id, + "contentHash": content_hash, + "evaluationKey": evaluation.key, + "evaluationVersion": evaluation.version, + "datasetKey": dataset.key, + "status": result["status"], + "startedAt": result["started_at"], + "generatedAt": result["generated_at"], + "latencyMs": result["latency_ms"], + } + if generated["generationOutput"] is not None: + payload["generationOutput"] = generated["generationOutput"] + if generated["error"] is not None: + payload["error"] = generated["error"] + if generated["usage"] is not None: + payload["usage"] = generated["usage"] + client.track(GENERATION_EVENT_NAME, context, payload, 1) async def _poll_run( self, diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 20e05c0..dc1e014 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -163,7 +163,6 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( "createdAt": 1, }, ), - response(202, {}), response( 200, { @@ -193,13 +192,18 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( ), ] ) + client = MagicMock() + client.variation = AsyncMock(return_value=True) + client.flush = AsyncMock() + init_client.return_value = client evals = init_evaluations( api_token="token", + sdk_key="sdk-key", base_uri="https://api.example.com", ui_base_uri="https://ui.example.com/", transport=transport, ) - assert evals.sdk_key is None + assert evals.sdk_key == "sdk-key" result = await evals.run( project_key="proj", @@ -232,7 +236,6 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( "GET", "POST", "POST", - "POST", "GET", "GET", ] @@ -261,18 +264,31 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( "datasetId": "33333333-3333-3333-3333-333333333333", } - ingested = transport.requests[6]["body"]["results"] - assert [row["row_index"] for row in ingested] == [4, 9] - assert ingested[0]["input"] == "Order A19" - assert ingested[0]["expected_output"] == "Found A19" - assert ingested[0]["variables"]["input"] == "Order A19" - assert ingested[0]["variables"]["expected_output"] == "Found A19" - init_client.assert_not_awaited() + assert not any( + request["url"].endswith("/generation-results") for request in transport.requests + ) + assert client.track.call_count == 2 + event_name, context, event, metric_value = client.track.call_args_list[0].args + assert event_name == "$ld:ai:offline-evals:generation" + assert context["key"] == "22222222-2222-2222-2222-222222222222" + assert metric_value == 1 + assert event["projectKey"] == "proj" + assert event["evaluationId"] == "11111111-1111-1111-1111-111111111111" + assert event["runId"] == "22222222-2222-2222-2222-222222222222" + assert event["datasetId"] == "33333333-3333-3333-3333-333333333333" + assert event["rowIndex"] == 4 + assert event["status"] == "COMPLETE" + assert event["generationOutput"] == "generated: Order A19" + assert event["usage"] == {"input_tokens": 10, "output_tokens": 4} + assert len(event["eventId"]) == len(event["contentHash"]) == 64 + assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(event) + client.flush.assert_awaited_once_with() + init_client.assert_awaited_once_with({"sdkKey": "sdk-key"}) @pytest.mark.asyncio @pytest.mark.parametrize( - ("flag_value", "expected_ingest"), + ("flag_value", "expected_poll"), [ pytest.param(True, True, id="enabled"), pytest.param(False, False, id="disabled-default"), @@ -282,10 +298,10 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( ), ], ) -async def test_batch_ingest_flag_controls_generation_result_publishing( +async def test_batch_ingest_flag_controls_generation_result_polling( monkeypatch: pytest.MonkeyPatch, flag_value: object, - expected_ingest: bool, + expected_poll: bool, ) -> None: responses = [ response(200, {"id": "dataset-id", "name": "golden"}), @@ -306,10 +322,9 @@ async def test_batch_ingest_flag_controls_generation_result_publishing( }, ), ] - if expected_ingest: + if expected_poll: responses.extend( [ - response(202, {}), response( 200, { @@ -350,6 +365,12 @@ async def test_batch_ingest_flag_controls_generation_result_publishing( else: client.variation = AsyncMock(return_value=flag_value) + def flush_before_poll_or_summary() -> None: + assert len(transport.requests) == 4 + assert transport.requests[-1]["url"].endswith("/evaluations/evaluation-id/runs") + + client.flush.side_effect = flush_before_poll_or_summary + async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert options == {"sdkKey": "sdk-key"} return client @@ -370,15 +391,14 @@ async def handler(*args: object) -> dict[str, Any]: generation={"provider": "OpenAI", "model": "gpt-4o"}, ) - assert result.passed is expected_ingest - assert result.summary.pending_rows == (0 if expected_ingest else 1) + assert result.passed is expected_poll + assert result.summary.pending_rows == (0 if expected_poll else 1) request_urls = [request["url"] for request in transport.requests] - assert ( - any(url.endswith("/generation-results") for url in request_urls) - is expected_ingest - ) + assert not any(url.endswith("/generation-results") for url in request_urls) + client.track.assert_called_once() + client.flush.assert_called_once_with() status_url = "/evaluations/evaluation-id/runs/run-id" - assert any(url.endswith(status_url) for url in request_urls) is expected_ingest + assert any(url.endswith(status_url) for url in request_urls) is expected_poll assert request_urls[-1].endswith(f"{status_url}/summary") assert sum(url.endswith(f"{status_url}/summary") for url in request_urls) == 1 @@ -462,7 +482,7 @@ async def test_empty_dataset_fails_before_evaluation_or_run_creation() -> None: ], ) async def test_complete_run_with_failed_or_error_rows_does_not_pass( - failed_rows: int, error_rows: int + monkeypatch: pytest.MonkeyPatch, failed_rows: int, error_rows: int ) -> None: calls: list[str | None] = [] @@ -515,7 +535,6 @@ async def handler( "createdAt": 1, }, ), - response(202, {}), response( 200, { @@ -545,7 +564,17 @@ async def handler( ), ] ) - evals = init_evaluations(api_token="token", transport=transport) + client = MagicMock() + client.variation = AsyncMock(return_value=True) + + async def fake_init_client(options: dict[str, Any]) -> MagicMock: + assert options == {"sdkKey": "sdk-key"} + return client + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", fake_init_client + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) result = await evals.run( project_key="proj", @@ -558,8 +587,11 @@ async def handler( assert set(calls) == {"bad", "good"} assert len(calls) == 2 assert result.passed is False - rows = transport.requests[4]["body"]["results"] - assert {row["status"] for row in rows} == {"COMPLETE", "ERROR"} - error_row = next(row for row in rows if row["status"] == "ERROR") - assert error_row["row_index"] == 0 - assert "provider failed" in error_row["error"]["message"] + assert client.track.call_count == 2 + events = [call.args[2] for call in client.track.call_args_list] + assert {event["status"] for event in events} == {"COMPLETE", "ERROR"} + error_event = next(event for event in events if event["status"] == "ERROR") + assert error_event["rowIndex"] == 0 + assert "provider failed" in error_event["error"]["message"] + assert "generationOutput" not in error_event + assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(error_event) From 7cca8b08e9624b8b8654611c284c416e79a6ffd1 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 25 Aug 2026 10:41:49 -0700 Subject: [PATCH 13/32] no-mistakes(document): docs(evaluations): refresh agents.md for event-based ingest --- packages/client/agents.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 7112637..1b176c6 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is optional for generation-only runs; when set it enables the normal handler observability path and gates generation-result ingest on the `enable-batch-ingest-in-evals-from-code` flag (only a strictly `true` variation publishes; false, default, malformed, or evaluation-error results skip publish safely). Without an SDK key the gate cannot be evaluated and ingest runs unconditionally. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport, then flushes before any polling or return. Without an SDK key no events can be emitted, and the harness skips polling and returns the current summary immediately. The `enable-batch-ingest-in-evals-from-code` flag gates only lifecycle polling (strictly `true` enables polling; false, malformed, or failed evaluations skip polling and fetch the summary once). It never gates event emission. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, and batches generation ingest when the gate permits. When ingest is enabled, it polls `GET .../runs/{id}` on lifecycle status until a terminal state and then fetches the run summary. When ingest is disabled, it skips polling and fetches the current summary once so the call returns promptly. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. When lifecycle polling is enabled, the harness polls `GET .../runs/{id}` until a terminal state and then fetches the run summary; otherwise it fetches the current summary once so the call returns promptly. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. --- From 8cae0e4b092c475dab61481659cb21102c3e956d Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 25 Aug 2026 10:48:54 -0700 Subject: [PATCH 14/32] fix(evaluations): include evaluation run event ID --- .../client/src/launchdarkly_ai_server/evaluations/runner.py | 1 + packages/client/tests/test_evaluations_run.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index eadc6d2..e054c68 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -418,6 +418,7 @@ def _emit_generation_events( identity = { "projectKey": project_key, "evaluationId": evaluation.id, + "evaluationRunId": evaluation_run.id, "runId": evaluation_run.id, "datasetId": dataset.id, "rowIndex": result["row_index"], diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index dc1e014..38c4938 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -274,7 +274,8 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert metric_value == 1 assert event["projectKey"] == "proj" assert event["evaluationId"] == "11111111-1111-1111-1111-111111111111" - assert event["runId"] == "22222222-2222-2222-2222-222222222222" + assert event["evaluationRunId"] == "22222222-2222-2222-2222-222222222222" + assert event["runId"] == event["evaluationRunId"] assert event["datasetId"] == "33333333-3333-3333-3333-333333333333" assert event["rowIndex"] == 4 assert event["status"] == "COMPLETE" From 00792895517c7442f3c17d0591af22d49992112a Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 25 Aug 2026 12:41:46 -0700 Subject: [PATCH 15/32] feat(evaluations): print event emission timestamps --- packages/client/README.md | 2 +- .../src/launchdarkly_ai_server/evaluations/runner.py | 6 ++++++ packages/client/tests/test_evaluations_run.py | 10 ++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/client/README.md b/packages/client/README.md index 6f42253..b67ffa1 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -45,7 +45,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Events are flushed before lifecycle polling or return; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before lifecycle polling or return; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. When generation-result processing is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index e054c68..6bfc9e4 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -437,10 +437,12 @@ def _emit_generation_events( generated, sort_keys=True, separators=(",", ":"), default=str ).encode() ).hexdigest() + emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") payload: dict[str, Any] = { **identity, "eventId": event_id, "contentHash": content_hash, + "emittedAt": emitted_at, "evaluationKey": evaluation.key, "evaluationVersion": evaluation.version, "datasetKey": dataset.key, @@ -456,6 +458,10 @@ def _emit_generation_events( if generated["usage"] is not None: payload["usage"] = generated["usage"] client.track(GENERATION_EVENT_NAME, context, payload, 1) + print( + f"{GENERATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", + flush=True, + ) async def _poll_run( self, diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 38c4938..1f6ab14 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2,6 +2,7 @@ import json from collections.abc import Callable +from datetime import datetime from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -89,6 +90,7 @@ def lookup_order(order_id: str) -> str: @pytest.mark.asyncio async def test_complete_run_with_zero_failed_and_error_rows_passes( monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: monkeypatch.delenv("LD_SDK_KEY", raising=False) init_client = AsyncMock() @@ -282,7 +284,15 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert event["generationOutput"] == "generated: Order A19" assert event["usage"] == {"input_tokens": 10, "output_tokens": 4} assert len(event["eventId"]) == len(event["contentHash"]) == 64 + assert event["emittedAt"].endswith("Z") + assert datetime.fromisoformat(event["emittedAt"]).tzinfo is not None assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(event) + output_lines = capsys.readouterr().out.splitlines() + assert len(output_lines) == 2 + assert output_lines[0] == ( + "$ld:ai:offline-evals:generation " + f"emittedAt={event['emittedAt']} eventId={event['eventId']}" + ) client.flush.assert_awaited_once_with() init_client.assert_awaited_once_with({"sdkKey": "sdk-key"}) From e3d6b6c015453620135c11613444739a634ef734 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 27 Aug 2026 10:42:21 -0700 Subject: [PATCH 16/32] refactor(evaluations): remove batch ingest gate --- packages/client/README.md | 6 +- packages/client/agents.md | 4 +- .../evaluations/flags.py | 44 ------- .../evaluations/module.py | 14 -- .../evaluations/runner.py | 31 ----- .../client/tests/test_evaluation_flags.py | 58 -------- packages/client/tests/test_evaluations_run.py | 124 +++++------------- 7 files changed, 36 insertions(+), 245 deletions(-) delete mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/flags.py delete mode 100644 packages/client/tests/test_evaluation_flags.py diff --git a/packages/client/README.md b/packages/client/README.md index b67ffa1..f9a951f 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -45,9 +45,9 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before lifecycle polling or return; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. When generation-result processing is disabled by its feature gate, the harness skips polling and returns the current summary immediately, which will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. After flushing generation events, the harness does not poll or wait for backend processing; it fetches the current summary once and returns immediately, so newly created runs will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio @@ -79,7 +79,7 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. The `enable-batch-ingest-in-evals-from-code` flag still controls whether the harness waits for terminal lifecycle processing: only an exact `true` enables polling. Events are emitted and flushed for both flag outcomes; false, malformed, or failed flag evaluations skip polling and fetch the summary once. Without an SDK key, no generation event can be emitted, so polling is skipped and the current summary is returned immediately. +`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then fetches the summary once without status polling. Without an SDK key, no generation event can be emitted, but the current summary is still fetched once and returned immediately. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/agents.md b/packages/client/agents.md index 1b176c6..1cc69e9 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport, then flushes before any polling or return. Without an SDK key no events can be emitted, and the harness skips polling and returns the current summary immediately. The `enable-batch-ingest-in-evals-from-code` flag gates only lifecycle polling (strictly `true` enables polling; false, malformed, or failed evaluations skip polling and fetch the summary once). It never gates event emission. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness does not poll for backend processing; it fetches the current summary once and returns immediately. Without an SDK key no events can be emitted, but the current summary is still fetched once. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. When lifecycle polling is enabled, the harness polls `GET .../runs/{id}` until a terminal state and then fetches the run summary; otherwise it fetches the current summary once so the call returns promptly. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, fetches the current summary once, and returns promptly without lifecycle status polling. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/flags.py b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py deleted file mode 100644 index ee1a138..0000000 --- a/packages/client/src/launchdarkly_ai_server/evaluations/flags.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import inspect -import logging -from typing import Any, Final - -from ..utils import to_ld_context - -logger = logging.getLogger(__name__) - -ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY: Final[str] = ( - "enable-batch-ingest-in-evals-from-code" -) -"""Canonical rollout flag for generation-result batch ingestion.""" - - -async def is_generation_result_batch_ingest_enabled( - client: Any, - project_key: str, -) -> bool: - """Return whether the rollout flag enables generation-result batch ingest. - - Flag evaluation is fail-safe: false, malformed, or failed evaluations disable - the gated batch-ingest path. - """ - try: - context = to_ld_context( - client, - {"kind": "project", "key": project_key}, - ) - result = client.variation( - ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, - context, - False, - ) - value = await result if inspect.isawaitable(result) else result - return value is True - except Exception: - logger.warning( - "Unable to evaluate %s; generation results will not be batch ingested", - ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, - exc_info=True, - ) - return False diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 3c98e9c..e61f190 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -13,7 +13,6 @@ Transport, urllib_transport, ) -from .flags import is_generation_result_batch_ingest_enabled from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment from .types import EvalRunResult, GenerationConfig @@ -66,7 +65,6 @@ async def run( generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, concurrency: int = 10, - timeout: float = 300.0, ) -> EvalRunResult: """ Create and run a generation-only evaluation in the caller's process. @@ -82,16 +80,11 @@ async def run( handler=handler, generation=generation, concurrency=concurrency, - timeout=timeout, ) run_tools = dict(tools or {}) client = None - batch_ingest_enabled = False if self._sdk_key: client = await init_client({"sdkKey": self._sdk_key}) - batch_ingest_enabled = await is_generation_result_batch_ingest_enabled( - client, project_key - ) # Tool verification is deliberately first: a typo must not create records. resolved_tools = self._runner._resolve_tools(project_key, run_tools) @@ -123,10 +116,6 @@ async def run( flush_result = client.flush() if inspect.isawaitable(flush_result): await flush_result - if batch_ingest_enabled: - await self._runner._poll_run( - project_key, evaluation.id, evaluation_run.id, timeout - ) summary = self._runner._get_summary( project_key, evaluation.id, evaluation_run.id ) @@ -154,7 +143,6 @@ def _validate_run_args( handler: EvalHandler, generation: GenerationConfig, concurrency: int, - timeout: float, ) -> None: for name, value in ( ("project_key", project_key), @@ -177,8 +165,6 @@ def _validate_run_args( ) if concurrency < 1: raise EvaluationsError("concurrency must be at least 1") - if timeout <= 0: - raise EvaluationsError("timeout must be greater than zero") def init_evaluations( diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 6bfc9e4..d54c631 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -463,37 +463,6 @@ def _emit_generation_events( flush=True, ) - async def _poll_run( - self, - project_key: str, - evaluation_id: str, - run_id: str, - timeout: float, - ) -> EvaluationRunRef: - path = ( - f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" - f"/runs/{_segment(run_id)}" - ) - deadline = time.monotonic() + timeout - delay = 0.25 - while True: - run = self._run_ref( - _mapping(self._api.get(path), description="evaluation run") - ) - if run.state == "COMPLETE": - return run - if run.state in {"CANCELLED", "TEMPORARY_ERROR", "PERMANENT_ERROR"}: - reason = f": {run.status_reason}" if run.status_reason else "" - raise EvaluationsError( - f"Evaluation run {run_id!r} failed in state {run.state}{reason}" - ) - if time.monotonic() >= deadline: - raise EvaluationsError( - f"Evaluation run {run_id!r} is still in progress after {timeout} seconds" - ) - await asyncio.sleep(delay) - delay = min(5.0, delay * 2) - def _get_summary( self, project_key: str, evaluation_id: str, run_id: str ) -> RunSummary: diff --git a/packages/client/tests/test_evaluation_flags.py b/packages/client/tests/test_evaluation_flags.py deleted file mode 100644 index 8533cbf..0000000 --- a/packages/client/tests/test_evaluation_flags.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock - -import pytest - -from launchdarkly_ai_server.evaluations.flags import ( - ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, - is_generation_result_batch_ingest_enabled, -) - - -@pytest.mark.asyncio -async def test_enabled_flag_enables_generation_result_batch_ingest() -> None: - client = MagicMock() - client.variation = AsyncMock(return_value=True) - - assert ( - await is_generation_result_batch_ingest_enabled(client, "project-key") is True - ) - client.variation.assert_awaited_once_with( - ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, - {"kind": "project", "key": "project-key"}, - False, - ) - - -@pytest.mark.asyncio -async def test_disabled_flag_disables_generation_result_batch_ingest() -> None: - client = MagicMock() - client.variation = AsyncMock(return_value=False) - - assert ( - await is_generation_result_batch_ingest_enabled(client, "project-key") is False - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("malformed_value", [None, 1, "true", {}]) -async def test_malformed_flag_disables_generation_result_batch_ingest( - malformed_value: object, -) -> None: - client = MagicMock() - client.variation = AsyncMock(return_value=malformed_value) - - assert ( - await is_generation_result_batch_ingest_enabled(client, "project-key") is False - ) - - -@pytest.mark.asyncio -async def test_flag_evaluation_error_disables_generation_result_batch_ingest() -> None: - client = MagicMock() - client.variation = AsyncMock(side_effect=RuntimeError("delivery unavailable")) - - assert ( - await is_generation_result_batch_ingest_enabled(client, "project-key") is False - ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 1f6ab14..f986f7d 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -165,17 +165,6 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( "createdAt": 1, }, ), - response( - 200, - { - "id": "22222222-2222-2222-2222-222222222222", - "evaluationId": "11111111-1111-1111-1111-111111111111", - "evaluationVersion": 1, - "source": "api", - "state": "COMPLETE", - "createdAt": 1, - }, - ), response( 200, { @@ -239,7 +228,6 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( "POST", "POST", "GET", - "GET", ] assert transport.requests[0]["url"].endswith( "/api/v2/projects/proj/ai-tools/lookup_order" @@ -294,70 +282,33 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( f"emittedAt={event['emittedAt']} eventId={event['eventId']}" ) client.flush.assert_awaited_once_with() + client.variation.assert_not_awaited() init_client.assert_awaited_once_with({"sdkKey": "sdk-key"}) @pytest.mark.asyncio -@pytest.mark.parametrize( - ("flag_value", "expected_poll"), - [ - pytest.param(True, True, id="enabled"), - pytest.param(False, False, id="disabled-default"), - pytest.param("true", False, id="malformed"), - pytest.param( - RuntimeError("delivery unavailable"), False, id="evaluation-error" - ), - ], -) -async def test_batch_ingest_flag_controls_generation_result_polling( +async def test_generation_events_always_emit_without_flag_or_status_poll( monkeypatch: pytest.MonkeyPatch, - flag_value: object, - expected_poll: bool, ) -> None: - responses = [ - response(200, {"id": "dataset-id", "name": "golden"}), - response( - 200, - dataset_page( - [{"rowIndex": 3, "input": "hello", "variables": {}}], - total=1, - ), - ), - response(201, {"id": "evaluation-id", "name": "eval-key"}), - response( - 201, - { - "id": "run-id", - "evaluationId": "evaluation-id", - "state": "PENDING", - }, - ), - ] - if expected_poll: - responses.extend( - [ - response( - 200, - { - "id": "run-id", - "evaluationId": "evaluation-id", - "state": "COMPLETE", - }, - ), - response( - 200, - { - "statusCounts": { - "total": 1, - "passed": 1, - "pending": 0, - } - }, + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 3, "input": "hello", "variables": {}}], + total=1, ), - ] - ) - else: - responses.append( + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "PENDING", + }, + ), response( 200, { @@ -367,20 +318,17 @@ async def test_batch_ingest_flag_controls_generation_result_polling( "error": 0, "pending": 1, }, - ) - ) - transport = SequencedTransport(responses) + ), + ] + ) client = MagicMock() - if isinstance(flag_value, Exception): - client.variation = AsyncMock(side_effect=flag_value) - else: - client.variation = AsyncMock(return_value=flag_value) + client.variation = AsyncMock(side_effect=AssertionError("flag must not be read")) - def flush_before_poll_or_summary() -> None: + def flush_before_summary() -> None: assert len(transport.requests) == 4 assert transport.requests[-1]["url"].endswith("/evaluations/evaluation-id/runs") - client.flush.side_effect = flush_before_poll_or_summary + client.flush.side_effect = flush_before_summary async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert options == {"sdkKey": "sdk-key"} @@ -402,16 +350,16 @@ async def handler(*args: object) -> dict[str, Any]: generation={"provider": "OpenAI", "model": "gpt-4o"}, ) - assert result.passed is expected_poll - assert result.summary.pending_rows == (0 if expected_poll else 1) + assert result.passed is False + assert result.summary.pending_rows == 1 request_urls = [request["url"] for request in transport.requests] assert not any(url.endswith("/generation-results") for url in request_urls) + client.variation.assert_not_awaited() client.track.assert_called_once() client.flush.assert_called_once_with() status_url = "/evaluations/evaluation-id/runs/run-id" - assert any(url.endswith(status_url) for url in request_urls) is expected_poll + assert not any(url.endswith(status_url) for url in request_urls) assert request_urls[-1].endswith(f"{status_url}/summary") - assert sum(url.endswith(f"{status_url}/summary") for url in request_urls) == 1 @pytest.mark.asyncio @@ -546,17 +494,6 @@ async def handler( "createdAt": 1, }, ), - response( - 200, - { - "id": "22222222-2222-2222-2222-222222222222", - "evaluationId": "11111111-1111-1111-1111-111111111111", - "evaluationVersion": 1, - "source": "api", - "state": "COMPLETE", - "createdAt": 1, - }, - ), response( 200, { @@ -598,6 +535,7 @@ async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert set(calls) == {"bad", "good"} assert len(calls) == 2 assert result.passed is False + client.variation.assert_not_awaited() assert client.track.call_count == 2 events = [call.args[2] for call in client.track.call_args_list] assert {event["status"] for event in events} == {"COMPLETE", "ERROR"} From 485dc85ee0eb1d4a6caf9f614716824e04c680de Mon Sep 17 00:00:00 2001 From: "doneill@launchdarkly.com" Date: Thu, 27 Aug 2026 20:54:31 +0000 Subject: [PATCH 17/32] fix(evaluations): avoid replaying non-idempotent POSTs and blocking the event loop --- packages/client/agents.md | 2 +- .../launchdarkly_ai_server/evaluations/api.py | 15 +++- .../evaluations/module.py | 58 +++++++++++--- packages/client/tests/test_evaluations.py | 80 +++++++++++++++++++ 4 files changed, 141 insertions(+), 14 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 1cc69e9..c378564 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -130,7 +130,7 @@ Handlers may return any of these — the client normalizes them before emitting `init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness does not poll for backend processing; it fetches the current summary once and returns immediately. Without an SDK key no events can be emitted, but the current summary is still fetched once. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, fetches the current summary once, and returns promptly without lifecycle status polling. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, fetches the current summary once, and returns promptly without lifecycle status polling. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/api.py b/packages/client/src/launchdarkly_ai_server/evaluations/api.py index 957a092..0a6aeb4 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/api.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/api.py @@ -14,6 +14,10 @@ DEFAULT_BASE_URI = "https://app.launchdarkly.com" +# Only these methods are replayed after a 5xx or a transport failure: a POST that +# timed out may still have created a record server-side. +RETRY_SAFE_METHODS = frozenset({"GET", "HEAD"}) + class EvaluationsError(Exception): """Base error for the evaluations harness.""" @@ -147,14 +151,21 @@ def request( method, self.url_for(path, params), headers, payload, self._timeout ) except (TimeoutError, urllib.error.URLError) as error: - if attempt >= self._max_retries: + if ( + method.upper() not in RETRY_SAFE_METHODS + or attempt >= self._max_retries + ): raise EvaluationsError( f"LaunchDarkly API {method} {path} failed after retries: {error}" ) from error self._sleep(self._retry_delay(attempt)) continue - retryable = response.status == 429 or response.status >= 500 + # A 429 is rejected before the server acts on it, so it is safe to + # replay for any method. + retryable = response.status == 429 or ( + response.status >= 500 and method.upper() in RETRY_SAFE_METHODS + ) if retryable and attempt < self._max_retries: self._sleep(self._retry_delay(attempt, response)) continue diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index e61f190..1ebb075 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -1,11 +1,13 @@ from __future__ import annotations +import asyncio import inspect import logging import os from collections.abc import Mapping +from typing import Any -from ..lifecycle import init_client +from ..lifecycle import get_client, init_client from .api import ( DEFAULT_BASE_URI, EvaluationsError, @@ -84,17 +86,33 @@ async def run( run_tools = dict(tools or {}) client = None if self._sdk_key: - client = await init_client({"sdkKey": self._sdk_key}) + client = await self._resolve_client() + # The management API client is synchronous; running it in a worker thread + # keeps the caller's event loop free. # Tool verification is deliberately first: a typo must not create records. - resolved_tools = self._runner._resolve_tools(project_key, run_tools) - dataset_ref = self._runner._fetch_dataset(project_key, dataset) - rows = self._runner._get_dataset_rows(project_key, dataset) - evaluation = self._runner._create_evaluation( - project_key, key, generation, resolved_tools + resolved_tools = await asyncio.to_thread( + self._runner._resolve_tools, project_key, run_tools ) - evaluation_run = self._runner._create_evaluation_run( - project_key, evaluation.id, len(rows), dataset_ref.id + dataset_ref = await asyncio.to_thread( + self._runner._fetch_dataset, project_key, dataset + ) + rows = await asyncio.to_thread( + self._runner._get_dataset_rows, project_key, dataset + ) + evaluation = await asyncio.to_thread( + self._runner._create_evaluation, + project_key, + key, + generation, + resolved_tools, + ) + evaluation_run = await asyncio.to_thread( + self._runner._create_evaluation_run, + project_key, + evaluation.id, + len(rows), + dataset_ref.id, ) config = self._runner._build_handler_config(generation, resolved_tools) results = await self._runner._run_rows( @@ -116,8 +134,8 @@ async def run( flush_result = client.flush() if inspect.isawaitable(flush_result): await flush_result - summary = self._runner._get_summary( - project_key, evaluation.id, evaluation_run.id + summary = await asyncio.to_thread( + self._runner._get_summary, project_key, evaluation.id, evaluation_run.id ) url = ( f"{self._ui_base_uri}/projects/{_segment(project_key)}/ai/evaluations/" @@ -134,6 +152,24 @@ async def run( summary=summary, ) + async def _resolve_client(self) -> Any: + """ + Return the SDK client used for generation events. + + ``init_client`` is idempotent, so an application that already holds a + client keeps it and the evaluations SDK key is not applied. + """ + try: + existing = get_client() + except RuntimeError: + return await init_client({"sdkKey": self._sdk_key}) + logger.warning( + "A LaunchDarkly client is already initialized; evaluation events are " + "sent with it and the evaluations SDK key is ignored. Both must point " + "at the project under evaluation." + ) + return existing + @staticmethod def _validate_run_args( *, diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index e051c4c..d09e74e 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -199,6 +199,86 @@ def test_rate_limit_retries_and_honors_retry_after() -> None: assert sleeps == [2.0] +def test_server_error_retries_get_but_not_post() -> None: + server_error = HttpResponse(status=503, body='{"message": "unavailable"}') + get_transport = RecordingTransport( + [server_error, HttpResponse(200, '{"ok": true}')] + ) + client = LDApiClient( + api_token="api-token", + transport=get_transport, + max_retries=2, + sleep=lambda _: None, + random_value=lambda: 0.0, + ) + + assert client.get("projects/proj/datasets") == {"ok": True} + assert len(get_transport.requests) == 2 + + post_transport = RecordingTransport([server_error]) + client = LDApiClient( + api_token="api-token", + transport=post_transport, + max_retries=2, + sleep=lambda _: None, + random_value=lambda: 0.0, + ) + + with pytest.raises(LDApiError) as excinfo: + client.post("projects/proj/evaluations", body={"name": "eval"}) + + assert excinfo.value.status == 503 + assert len(post_transport.requests) == 1 + + +def test_transport_failure_is_not_replayed_for_post() -> None: + attempts: list[str] = [] + + def timing_out_transport( + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, + ) -> HttpResponse: + attempts.append(method) + raise TimeoutError("timed out") + + client = LDApiClient( + api_token="api-token", + transport=timing_out_transport, + max_retries=2, + sleep=lambda _: None, + random_value=lambda: 0.0, + ) + + with pytest.raises(EvaluationsError): + client.post("projects/proj/evaluations", body={"name": "eval"}) + + assert attempts == ["POST"] + + +def test_rate_limited_post_is_retried() -> None: + transport = RecordingTransport( + [ + HttpResponse(status=429, body='{"message": "slow down"}'), + HttpResponse(status=201, body='{"id": "eval-id"}'), + ] + ) + client = LDApiClient( + api_token="api-token", + transport=transport, + max_retries=1, + sleep=lambda _: None, + random_value=lambda: 0.0, + ) + + assert client.post("projects/proj/evaluations", body={"name": "eval"}) == { + "id": "eval-id" + } + assert len(transport.requests) == 2 + + def test_forbidden_response_is_not_retried() -> None: transport = RecordingTransport( [HttpResponse(status=403, body='{"message": "forbidden"}')] From 75d3072b9343e87adeaf2ff861639093d579637c Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 28 Aug 2026 10:40:49 -0700 Subject: [PATCH 18/32] fix(evaluations): poll run summary to terminal state --- packages/client/README.md | 4 +- packages/client/agents.md | 4 +- .../evaluations/module.py | 51 ++++- .../evaluations/types.py | 8 +- packages/client/tests/test_evaluations_run.py | 179 ++++++++++++++++-- 5 files changed, 221 insertions(+), 25 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index f9a951f..b3aec6d 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -47,7 +47,7 @@ No code changes are required — `init_client()` detects the packages at runtime The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. A result passes only when the summary has no failed, error, or pending rows. After flushing generation events, the harness does not poll or wait for backend processing; it fetches the current summary once and returns immediately, so newly created runs will normally show pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state, with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio @@ -79,7 +79,7 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then fetches the summary once without status polling. Without an SDK key, no generation event can be emitted, but the current summary is still fetched once and returned immediately. +`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint for terminal run state. Without an SDK key, no generation event can be emitted, but the run summary is still polled. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/agents.md b/packages/client/agents.md index c378564..4453a2f 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness does not poll for backend processing; it fetches the current summary once and returns immediately. Without an SDK key no events can be emitted, but the current summary is still fetched once. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until terminal state, timing out after three minutes. Without an SDK key no events can be emitted, but the run summary is still polled. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, fetches the current summary once, and returns promptly without lifecycle status polling. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when failed, error, and pending row counts are all zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until terminal state, and raises a timeout after three minutes if the backend never reaches one. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 1ebb075..0293742 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -4,6 +4,7 @@ import inspect import logging import os +import time from collections.abc import Mapping from typing import Any @@ -16,11 +17,21 @@ urllib_transport, ) from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment -from .types import EvalRunResult, GenerationConfig +from .types import EvalRunResult, GenerationConfig, RunSummary logger = logging.getLogger(__name__) DEFAULT_UI_BASE_URI = "https://app.launchdarkly.com" +SUMMARY_POLL_INTERVAL_SECONDS = 2.0 +SUMMARY_POLL_TIMEOUT_SECONDS = 180.0 +_TERMINAL_SUMMARY_STATES = { + "CANCELED", + "CANCELLED", + "COMPLETE", + "COMPLETED", + "ERROR", + "FAILED", +} def _env(name: str) -> str | None: @@ -29,6 +40,12 @@ def _env(name: str) -> str | None: return value if value else None +def _is_terminal_summary(summary: RunSummary) -> bool: + if summary.state is None: + return summary.pending_rows == 0 + return summary.state.upper() in _TERMINAL_SUMMARY_STATES + + class EvaluationsModule: """Entry point for running LaunchDarkly evaluations from customer code.""" @@ -134,24 +151,42 @@ async def run( flush_result = client.flush() if inspect.isawaitable(flush_result): await flush_result - summary = await asyncio.to_thread( - self._runner._get_summary, project_key, evaluation.id, evaluation_run.id + summary = await self._poll_summary_until_terminal( + project_key, evaluation.id, evaluation_run.id ) url = ( f"{self._ui_base_uri}/projects/{_segment(project_key)}/ai/evaluations/" f"{_segment(evaluation.id)}/runs/{_segment(evaluation_run.id)}" ) return EvalRunResult( - passed=( - summary.failed_rows == 0 - and summary.error_rows == 0 - and summary.pending_rows == 0 - ), + passed=(summary.error_rows == 0 and summary.pending_rows == 0), url=url, run_id=evaluation_run.id, summary=summary, ) + async def _poll_summary_until_terminal( + self, project_key: str, evaluation_id: str, run_id: str + ) -> RunSummary: + deadline = time.monotonic() + SUMMARY_POLL_TIMEOUT_SECONDS + last_summary = None + while True: + last_summary = await asyncio.to_thread( + self._runner._get_summary, project_key, evaluation_id, run_id + ) + if _is_terminal_summary(last_summary): + return last_summary + remaining = deadline - time.monotonic() + if remaining <= 0: + state = last_summary.state or "unknown" + raise EvaluationsError( + "Timed out after " + f"{SUMMARY_POLL_TIMEOUT_SECONDS:g} seconds waiting for evaluation " + f"run {run_id} summary to reach a terminal state " + f"(last state={state}, pending_rows={last_summary.pending_rows})" + ) + await asyncio.sleep(min(SUMMARY_POLL_INTERVAL_SECONDS, remaining)) + async def _resolve_client(self) -> Any: """ Return the SDK client used for generation events. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 4033a3f..6ef7aeb 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -26,6 +26,10 @@ def from_wire(cls, data: Mapping[str, Any]) -> Usage: ) +def _optional_string(value: object) -> str | None: + return value if isinstance(value, str) else None + + class GenerationConfig(TypedDict, total=False): """Generation settings stored on the evaluation and passed to its handler.""" @@ -88,13 +92,14 @@ class EvaluationRunRef: @dataclass class RunSummary: - """Row counts for a finished evaluation run.""" + """State and row counts for an evaluation run.""" total_rows: int = 0 passed_rows: int = 0 failed_rows: int = 0 error_rows: int = 0 pending_rows: int = 0 + state: str | None = None @classmethod def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: @@ -107,6 +112,7 @@ def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: failed_rows=int(counts.get("failed", counts.get("failed_rows", 0)) or 0), error_rows=int(counts.get("error", counts.get("error_rows", 0)) or 0), pending_rows=int(counts.get("pending", counts.get("pending_rows", 0)) or 0), + state=_optional_string(data.get("state") or data.get("evaluationRunState")), ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index f986f7d..f48ac49 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -171,6 +171,7 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( "evaluationId": "11111111-1111-1111-1111-111111111111", "evaluationVersion": 1, "evaluationRunId": "22222222-2222-2222-2222-222222222222", + "state": "COMPLETE", "statusCounts": { "total": 2, "passed": 2, @@ -287,7 +288,7 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( @pytest.mark.asyncio -async def test_generation_events_always_emit_without_flag_or_status_poll( +async def test_generation_events_always_emit_without_flag_or_run_status_poll( monkeypatch: pytest.MonkeyPatch, ) -> None: transport = SequencedTransport( @@ -312,6 +313,7 @@ async def test_generation_events_always_emit_without_flag_or_status_poll( response( 200, { + "state": "COMPLETE", "total": 1, "passed": 0, "failed": 0, @@ -362,6 +364,165 @@ async def handler(*args: object) -> dict[str, Any]: assert request_urls[-1].endswith(f"{status_url}/summary") +@pytest.mark.asyncio +async def test_summary_is_polled_until_terminal_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.SUMMARY_POLL_INTERVAL_SECONDS", 0 + ) + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + { + "state": "PENDING", + "statusCounts": {"total": 1, "passed": 0, "error": 0, "pending": 1}, + }, + ), + response( + 200, + { + "state": "COMPLETE", + "statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}, + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + summary_requests = [ + request for request in transport.requests if request["url"].endswith("/summary") + ] + assert len(summary_requests) == 2 + assert result.passed is True + assert result.summary.state == "COMPLETE" + + +@pytest.mark.asyncio +async def test_summary_polling_times_out_waiting_for_terminal_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.SUMMARY_POLL_TIMEOUT_SECONDS", 0 + ) + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + { + "state": "PENDING", + "statusCounts": {"total": 1, "passed": 0, "error": 0, "pending": 1}, + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + with pytest.raises( + EvaluationsError, + match=r"Timed out after 0 seconds.*terminal state.*pending_rows=1", + ): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + summary_requests = [ + request for request in transport.requests if request["url"].endswith("/summary") + ] + assert len(summary_requests) == 1 + + +@pytest.mark.asyncio +async def test_generation_failed_rows_do_not_fail_the_result() -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + { + "state": "COMPLETE", + "statusCounts": { + "total": 1, + "passed": 0, + "failed": 1, + "error": 0, + "pending": 0, + }, + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert result.summary.failed_rows == 1 + assert result.passed is True + + @pytest.mark.asyncio async def test_run_rejects_instructions_and_messages_before_network_io() -> None: transport = SequencedTransport([]) @@ -433,16 +594,10 @@ async def test_empty_dataset_fails_before_evaluation_or_run_creation() -> None: @pytest.mark.asyncio -@pytest.mark.parametrize( - ("failed_rows", "error_rows"), - [ - pytest.param(1, 0, id="failed-row"), - pytest.param(0, 1, id="error-row"), - ], -) -async def test_complete_run_with_failed_or_error_rows_does_not_pass( - monkeypatch: pytest.MonkeyPatch, failed_rows: int, error_rows: int +async def test_complete_run_with_error_rows_does_not_pass( + monkeypatch: pytest.MonkeyPatch, ) -> None: + error_rows = 1 calls: list[str | None] = [] async def handler( @@ -500,10 +655,10 @@ async def handler( "evaluationId": "11111111-1111-1111-1111-111111111111", "evaluationVersion": 1, "evaluationRunId": "22222222-2222-2222-2222-222222222222", + "state": "COMPLETE", "statusCounts": { "total": 2, - "passed": 2 - failed_rows - error_rows, - "failed": failed_rows, + "passed": 2 - error_rows, "error": error_rows, "pending": 0, }, From a0254d0a9c8d0ffe378c4eb042397965c2b7f8e1 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 28 Aug 2026 13:38:01 -0700 Subject: [PATCH 19/32] fix(evaluations): flatten generation event payload --- packages/client/README.md | 2 +- .../evaluations/runner.py | 19 ++++++++++++------- packages/client/tests/test_evaluations_run.py | 11 +++++++++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index b3aec6d..689041c 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -45,7 +45,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, usage, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, top-level `inputTokens`/`outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state, with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index d54c631..0404e3f 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -10,7 +10,7 @@ from typing import Any from ..types import NativeTool -from ..utils import parse_template, to_ld_context +from ..utils import parse_template, parse_usage, to_ld_context from .api import EvaluationsError, LDApiClient, LDApiError from .types import ( DatasetRef, @@ -428,10 +428,14 @@ def _emit_generation_events( ).hexdigest() generated = { "status": result["status"], - "generationOutput": result.get("output", {}).get("generation"), + "output": result.get("output", {}).get("generation"), "error": result.get("error"), - "usage": result.get("output", {}).get("usage"), } + usage = result.get("output", {}).get("usage") + if isinstance(usage, Mapping): + normalized_usage = parse_usage(dict(usage)) + generated["inputTokens"] = normalized_usage["input"] + generated["outputTokens"] = normalized_usage["output"] content_hash = hashlib.sha256( json.dumps( generated, sort_keys=True, separators=(",", ":"), default=str @@ -451,12 +455,13 @@ def _emit_generation_events( "generatedAt": result["generated_at"], "latencyMs": result["latency_ms"], } - if generated["generationOutput"] is not None: - payload["generationOutput"] = generated["generationOutput"] + if generated["output"] is not None: + payload["output"] = generated["output"] if generated["error"] is not None: payload["error"] = generated["error"] - if generated["usage"] is not None: - payload["usage"] = generated["usage"] + if "inputTokens" in generated: + payload["inputTokens"] = generated["inputTokens"] + payload["outputTokens"] = generated["outputTokens"] client.track(GENERATION_EVENT_NAME, context, payload, 1) print( f"{GENERATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index f48ac49..643aeff 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -270,8 +270,11 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert event["datasetId"] == "33333333-3333-3333-3333-333333333333" assert event["rowIndex"] == 4 assert event["status"] == "COMPLETE" - assert event["generationOutput"] == "generated: Order A19" - assert event["usage"] == {"input_tokens": 10, "output_tokens": 4} + assert event["output"] == "generated: Order A19" + assert event["inputTokens"] == 10 + assert event["outputTokens"] == 4 + assert "generationOutput" not in event + assert "usage" not in event assert len(event["eventId"]) == len(event["contentHash"]) == 64 assert event["emittedAt"].endswith("Z") assert datetime.fromisoformat(event["emittedAt"]).tzinfo is not None @@ -698,4 +701,8 @@ async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert error_event["rowIndex"] == 0 assert "provider failed" in error_event["error"]["message"] assert "generationOutput" not in error_event + assert "output" not in error_event + assert "usage" not in error_event + assert "inputTokens" not in error_event + assert "outputTokens" not in error_event assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(error_event) From c88e9aee533caeb36caeca1a4ea5c2adce9bfa72 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 28 Aug 2026 13:44:00 -0700 Subject: [PATCH 20/32] fix(evaluations): nest generation usage tokens --- packages/client/README.md | 2 +- .../src/launchdarkly_ai_server/evaluations/runner.py | 11 ++++++----- packages/client/tests/test_evaluations_run.py | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 689041c..74dcb1f 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -45,7 +45,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, top-level `inputTokens`/`outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state, with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 0404e3f..bea50fa 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -434,8 +434,10 @@ def _emit_generation_events( usage = result.get("output", {}).get("usage") if isinstance(usage, Mapping): normalized_usage = parse_usage(dict(usage)) - generated["inputTokens"] = normalized_usage["input"] - generated["outputTokens"] = normalized_usage["output"] + generated["usage"] = { + "inputTokens": normalized_usage["input"], + "outputTokens": normalized_usage["output"], + } content_hash = hashlib.sha256( json.dumps( generated, sort_keys=True, separators=(",", ":"), default=str @@ -459,9 +461,8 @@ def _emit_generation_events( payload["output"] = generated["output"] if generated["error"] is not None: payload["error"] = generated["error"] - if "inputTokens" in generated: - payload["inputTokens"] = generated["inputTokens"] - payload["outputTokens"] = generated["outputTokens"] + if "usage" in generated: + payload["usage"] = generated["usage"] client.track(GENERATION_EVENT_NAME, context, payload, 1) print( f"{GENERATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 643aeff..8e92e7d 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -271,10 +271,10 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert event["rowIndex"] == 4 assert event["status"] == "COMPLETE" assert event["output"] == "generated: Order A19" - assert event["inputTokens"] == 10 - assert event["outputTokens"] == 4 + assert event["usage"] == {"inputTokens": 10, "outputTokens": 4} assert "generationOutput" not in event - assert "usage" not in event + assert "inputTokens" not in event + assert "outputTokens" not in event assert len(event["eventId"]) == len(event["contentHash"]) == 64 assert event["emittedAt"].endswith("Z") assert datetime.fromisoformat(event["emittedAt"]).tzinfo is not None From b98b6434f41cbfb4d9d2b4a6d6d78cf099ec4a01 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 28 Aug 2026 13:49:05 -0700 Subject: [PATCH 21/32] fix(evaluations): emit direct generation output --- .../src/launchdarkly_ai_server/evaluations/runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index bea50fa..ec0ac01 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -364,7 +364,7 @@ async def invoke(row: DatasetRow) -> dict[str, Any]: "expected_output": row.expected_output, "variables": row.variables, "metadata": row.metadata, - "output": {"generation": result.get("output")}, + "output": result.get("output"), "started_at": started.isoformat().replace("+00:00", "Z"), "generated_at": completed.isoformat().replace("+00:00", "Z"), "latency_ms": round((time.perf_counter() - started_clock) * 1000), @@ -372,7 +372,7 @@ async def invoke(row: DatasetRow) -> dict[str, Any]: } usage = result.get("usage") if isinstance(usage, Mapping): - payload["output"]["usage"] = dict(usage) + payload["usage"] = dict(usage) controller.record_success(config["provider"]["name"]) return payload except Exception as error: @@ -428,10 +428,10 @@ def _emit_generation_events( ).hexdigest() generated = { "status": result["status"], - "output": result.get("output", {}).get("generation"), + "output": result.get("output"), "error": result.get("error"), } - usage = result.get("output", {}).get("usage") + usage = result.get("usage") if isinstance(usage, Mapping): normalized_usage = parse_usage(dict(usage)) generated["usage"] = { From 3a8e10f570297a2cb81b434e425f4ddc7ceb3053 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 14:50:01 -0700 Subject: [PATCH 22/32] Fix generation error payloads --- packages/client/README.md | 2 +- .../src/launchdarkly_ai_server/evaluations/runner.py | 11 ++++++++++- packages/client/tests/test_evaluations_run.py | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 74dcb1f..ae47816 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -45,7 +45,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error, nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state, with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index ec0ac01..bb18e07 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -426,11 +426,18 @@ def _emit_generation_events( event_id = hashlib.sha256( json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() + error = result.get("error") generated = { "status": result["status"], "output": result.get("output"), - "error": result.get("error"), + "error": error, } + if result["status"] == "ERROR": + if isinstance(error, Mapping): + message = error.get("message") + generated["errorMessage"] = str(message) if message else "Unknown error" + else: + generated["errorMessage"] = str(error) if error else "Unknown error" usage = result.get("usage") if isinstance(usage, Mapping): normalized_usage = parse_usage(dict(usage)) @@ -461,6 +468,8 @@ def _emit_generation_events( payload["output"] = generated["output"] if generated["error"] is not None: payload["error"] = generated["error"] + if generated.get("errorMessage") is not None: + payload["errorMessage"] = generated["errorMessage"] if "usage" in generated: payload["usage"] = generated["usage"] client.track(GENERATION_EVENT_NAME, context, payload, 1) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 8e92e7d..c1c008e 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -700,6 +700,7 @@ async def fake_init_client(options: dict[str, Any]) -> MagicMock: error_event = next(event for event in events if event["status"] == "ERROR") assert error_event["rowIndex"] == 0 assert "provider failed" in error_event["error"]["message"] + assert "provider failed" in error_event["errorMessage"] assert "generationOutput" not in error_event assert "output" not in error_event assert "usage" not in error_event From a60a0141eb519bdbe80f808c6db58398b6b478c7 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 15:07:32 -0700 Subject: [PATCH 23/32] chore: format evaluation runner --- .../client/src/launchdarkly_ai_server/evaluations/runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index bb18e07..951213b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -435,7 +435,9 @@ def _emit_generation_events( if result["status"] == "ERROR": if isinstance(error, Mapping): message = error.get("message") - generated["errorMessage"] = str(message) if message else "Unknown error" + generated["errorMessage"] = ( + str(message) if message else "Unknown error" + ) else: generated["errorMessage"] = str(error) if error else "Unknown error" usage = result.get("usage") From dcb08f66a57299ed13968a7dd733cc9bfe3d8f4a Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 15:26:08 -0700 Subject: [PATCH 24/32] no-mistakes(review): fix(evaluations): require terminal state to end summary polling --- .../evaluations/module.py | 2 +- packages/client/tests/test_evaluations_run.py | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 0293742..1854647 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -42,7 +42,7 @@ def _env(name: str) -> str | None: def _is_terminal_summary(summary: RunSummary) -> bool: if summary.state is None: - return summary.pending_rows == 0 + return False return summary.state.upper() in _TERMINAL_SUMMARY_STATES diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index c1c008e..3295006 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -425,6 +425,62 @@ async def handler(*args: object) -> dict[str, Any]: assert result.summary.state == "COMPLETE" +@pytest.mark.asyncio +async def test_summary_polling_ignores_missing_state_even_when_pending_is_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.SUMMARY_POLL_INTERVAL_SECONDS", 0 + ) + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response(200, {}), + response( + 200, + {"statusCounts": {"total": 1, "passed": 0, "error": 0, "pending": 0}}, + ), + response( + 200, + { + "state": "COMPLETE", + "statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}, + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + summary_requests = [ + request for request in transport.requests if request["url"].endswith("/summary") + ] + assert len(summary_requests) == 3 + assert result.summary.state == "COMPLETE" + assert result.passed is True + + @pytest.mark.asyncio async def test_summary_polling_times_out_waiting_for_terminal_state( monkeypatch: pytest.MonkeyPatch, From c84894cff495b41a0f2ba4a6ca88a04faf06ebff Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 16:04:02 -0700 Subject: [PATCH 25/32] fix evaluation summary polling without state --- .../evaluations/module.py | 12 +++- packages/client/tests/test_evaluations_run.py | 63 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 1854647..ee974e0 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -41,9 +41,15 @@ def _env(name: str) -> str | None: def _is_terminal_summary(summary: RunSummary) -> bool: - if summary.state is None: - return False - return summary.state.upper() in _TERMINAL_SUMMARY_STATES + if summary.state is not None: + return summary.state.upper() in _TERMINAL_SUMMARY_STATES + + accounted_rows = summary.passed_rows + summary.failed_rows + summary.error_rows + return ( + summary.total_rows > 0 + and summary.pending_rows == 0 + and accounted_rows == summary.total_rows + ) class EvaluationsModule: diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 3295006..64c5c74 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -425,6 +425,69 @@ async def handler(*args: object) -> dict[str, Any]: assert result.summary.state == "COMPLETE" +@pytest.mark.asyncio +async def test_summary_polling_completes_without_state_when_all_rows_are_accounted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.SUMMARY_POLL_INTERVAL_SECONDS", 0 + ) + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + { + "statusCounts": { + "total": 10, + "passed": 7, + "failed": 2, + "error": 1, + "pending": 0, + } + }, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + summary_requests = [ + request for request in transport.requests if request["url"].endswith("/summary") + ] + assert len(summary_requests) == 1 + assert result.summary.state is None + assert result.summary.total_rows == 10 + assert result.summary.pending_rows == 0 + assert ( + result.summary.passed_rows + + result.summary.failed_rows + + result.summary.error_rows + == 10 + ) + + @pytest.mark.asyncio async def test_summary_polling_ignores_missing_state_even_when_pending_is_zero( monkeypatch: pytest.MonkeyPatch, From d99dc37d78a2ce03268e7d0a0d209c9d8696d015 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 16:10:34 -0700 Subject: [PATCH 26/32] no-mistakes(document): docs(evaluations): note state-omission terminal condition --- packages/client/README.md | 2 +- packages/client/agents.md | 4 +-- packages/client/tests/test_evaluations_run.py | 28 +++++++++++-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index ae47816..65b018b 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -47,7 +47,7 @@ No code changes are required — `init_client()` detects the packages at runtime The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state, with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state — or, when the backend omits state, until passed + failed + error rows fully account for a nonzero total with no pending rows — with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/agents.md b/packages/client/agents.md index 4453a2f..e385fe8 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until terminal state, timing out after three minutes. Without an SDK key no events can be emitted, but the run summary is still polled. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until it is terminal — either a terminal `state` value or, when the backend omits `state`, a nonzero `total_rows` with `pending_rows == 0` and `passed + failed + error` rows accounting for the total — timing out after three minutes. Without an SDK key no events can be emitted, but the run summary is still polled. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until terminal state, and raises a timeout after three minutes if the backend never reaches one. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until it is terminal (a terminal `state` value, or state omitted with `total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout after three minutes if the backend never reaches one. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 64c5c74..bb57705 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -426,7 +426,7 @@ async def handler(*args: object) -> dict[str, Any]: @pytest.mark.asyncio -async def test_summary_polling_completes_without_state_when_all_rows_are_accounted( +async def test_summary_polling_completes_for_real_backend_summary_without_state( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -444,18 +444,25 @@ async def test_summary_polling_completes_without_state_when_all_rows_are_account response(201, {"id": "evaluation-id", "name": "eval-key"}), response( 201, - {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "COMPLETE", + "rowCount": 10, + "selectedRowCount": 10, + }, ), response( 200, { "statusCounts": { "total": 10, - "passed": 7, - "failed": 2, - "error": 1, + "passed": 10, + "failed": 0, + "error": 0, "pending": 0, - } + }, + "estimatedRemainingWindowMs": 0, }, ), ] @@ -480,12 +487,9 @@ async def handler(*args: object) -> dict[str, Any]: assert result.summary.state is None assert result.summary.total_rows == 10 assert result.summary.pending_rows == 0 - assert ( - result.summary.passed_rows - + result.summary.failed_rows - + result.summary.error_rows - == 10 - ) + assert result.summary.passed_rows == 10 + assert result.summary.failed_rows == 0 + assert result.summary.error_rows == 0 @pytest.mark.asyncio From ffd1d638e00f151f86a8a19cb52b0e72ed6c4070 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 16:23:57 -0700 Subject: [PATCH 27/32] remove state from evaluation run summaries --- packages/client/README.md | 4 ++-- packages/client/agents.md | 4 ++-- .../evaluations/module.py | 23 ++++++++----------- .../evaluations/types.py | 2 -- packages/client/tests/test_evaluations_run.py | 9 +++----- 5 files changed, 16 insertions(+), 26 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 65b018b..d32f423 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -47,7 +47,7 @@ No code changes are required — `init_client()` detects the packages at runtime The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until the run reaches a terminal state — or, when the backend omits state, until passed + failed + error rows fully account for a nonzero total with no pending rows — with a three-minute timeout. A generation result passes only when the terminal summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, with a three-minute timeout. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A generation result passes only when the completed summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio @@ -79,7 +79,7 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint for terminal run state. Without an SDK key, no generation event can be emitted, but the run summary is still polled. +`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. Without an SDK key, no generation event can be emitted, but the run summary is still polled. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/agents.md b/packages/client/agents.md index e385fe8..da489f6 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until it is terminal — either a terminal `state` value or, when the backend omits `state`, a nonzero `total_rows` with `pending_rows == 0` and `passed + failed + error` rows accounting for the total — timing out after three minutes. Without an SDK key no events can be emitted, but the run summary is still polled. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, timing out after three minutes. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. Without an SDK key no events can be emitted, but the run summary is still polled. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until it is terminal (a terminal `state` value, or state omitted with `total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout after three minutes if the backend never reaches one. `RunSummary` includes pending rows, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout after three minutes if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index ee974e0..fba044a 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -24,14 +24,6 @@ DEFAULT_UI_BASE_URI = "https://app.launchdarkly.com" SUMMARY_POLL_INTERVAL_SECONDS = 2.0 SUMMARY_POLL_TIMEOUT_SECONDS = 180.0 -_TERMINAL_SUMMARY_STATES = { - "CANCELED", - "CANCELLED", - "COMPLETE", - "COMPLETED", - "ERROR", - "FAILED", -} def _env(name: str) -> str | None: @@ -41,9 +33,6 @@ def _env(name: str) -> str | None: def _is_terminal_summary(summary: RunSummary) -> bool: - if summary.state is not None: - return summary.state.upper() in _TERMINAL_SUMMARY_STATES - accounted_rows = summary.passed_rows + summary.failed_rows + summary.error_rows return ( summary.total_rows > 0 @@ -184,12 +173,18 @@ async def _poll_summary_until_terminal( return last_summary remaining = deadline - time.monotonic() if remaining <= 0: - state = last_summary.state or "unknown" + accounted_rows = ( + last_summary.passed_rows + + last_summary.failed_rows + + last_summary.error_rows + ) raise EvaluationsError( "Timed out after " f"{SUMMARY_POLL_TIMEOUT_SECONDS:g} seconds waiting for evaluation " - f"run {run_id} summary to reach a terminal state " - f"(last state={state}, pending_rows={last_summary.pending_rows})" + f"run {run_id} summary rows to be fully accounted " + f"(total_rows={last_summary.total_rows}, " + f"accounted_rows={accounted_rows}, " + f"pending_rows={last_summary.pending_rows})" ) await asyncio.sleep(min(SUMMARY_POLL_INTERVAL_SECONDS, remaining)) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 6ef7aeb..e5f9413 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -99,7 +99,6 @@ class RunSummary: failed_rows: int = 0 error_rows: int = 0 pending_rows: int = 0 - state: str | None = None @classmethod def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: @@ -112,7 +111,6 @@ def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: failed_rows=int(counts.get("failed", counts.get("failed_rows", 0)) or 0), error_rows=int(counts.get("error", counts.get("error_rows", 0)) or 0), pending_rows=int(counts.get("pending", counts.get("pending_rows", 0)) or 0), - state=_optional_string(data.get("state") or data.get("evaluationRunState")), ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index bb57705..115cebf 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -368,7 +368,7 @@ async def handler(*args: object) -> dict[str, Any]: @pytest.mark.asyncio -async def test_summary_is_polled_until_terminal_state( +async def test_summary_is_polled_until_rows_are_accounted( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -422,7 +422,6 @@ async def handler(*args: object) -> dict[str, Any]: ] assert len(summary_requests) == 2 assert result.passed is True - assert result.summary.state == "COMPLETE" @pytest.mark.asyncio @@ -484,7 +483,6 @@ async def handler(*args: object) -> dict[str, Any]: request for request in transport.requests if request["url"].endswith("/summary") ] assert len(summary_requests) == 1 - assert result.summary.state is None assert result.summary.total_rows == 10 assert result.summary.pending_rows == 0 assert result.summary.passed_rows == 10 @@ -544,12 +542,11 @@ async def handler(*args: object) -> dict[str, Any]: request for request in transport.requests if request["url"].endswith("/summary") ] assert len(summary_requests) == 3 - assert result.summary.state == "COMPLETE" assert result.passed is True @pytest.mark.asyncio -async def test_summary_polling_times_out_waiting_for_terminal_state( +async def test_summary_polling_times_out_waiting_for_rows_to_be_accounted( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( @@ -585,7 +582,7 @@ async def handler(*args: object) -> dict[str, Any]: with pytest.raises( EvaluationsError, - match=r"Timed out after 0 seconds.*terminal state.*pending_rows=1", + match=r"Timed out after 0 seconds.*rows to be fully accounted.*pending_rows=1", ): await evals.run( project_key="proj", From 0e5743e9b2580d99ce493a620bc3731de9d5b8d0 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 16:30:57 -0700 Subject: [PATCH 28/32] no-mistakes(review): fix summary polling test and drop dead string helper --- .../launchdarkly_ai_server/evaluations/types.py | 4 ---- packages/client/tests/test_evaluations_run.py | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index e5f9413..6456157 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -26,10 +26,6 @@ def from_wire(cls, data: Mapping[str, Any]) -> Usage: ) -def _optional_string(value: object) -> str | None: - return value if isinstance(value, str) else None - - class GenerationConfig(TypedDict, total=False): """Generation settings stored on the evaluation and passed to its handler.""" diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 115cebf..e3339e3 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -294,6 +294,9 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( async def test_generation_events_always_emit_without_flag_or_run_status_poll( monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.SUMMARY_POLL_INTERVAL_SECONDS", 0 + ) transport = SequencedTransport( [ response(200, {"id": "dataset-id", "name": "golden"}), @@ -316,7 +319,6 @@ async def test_generation_events_always_emit_without_flag_or_run_status_poll( response( 200, { - "state": "COMPLETE", "total": 1, "passed": 0, "failed": 0, @@ -324,6 +326,16 @@ async def test_generation_events_always_emit_without_flag_or_run_status_poll( "pending": 1, }, ), + response( + 200, + { + "total": 1, + "passed": 0, + "failed": 0, + "error": 1, + "pending": 0, + }, + ), ] ) client = MagicMock() @@ -356,7 +368,8 @@ async def handler(*args: object) -> dict[str, Any]: ) assert result.passed is False - assert result.summary.pending_rows == 1 + assert result.summary.error_rows == 1 + assert result.summary.pending_rows == 0 request_urls = [request["url"] for request in transport.requests] assert not any(url.endswith("/generation-results") for url in request_urls) client.variation.assert_not_awaited() From acc196084306cb304244e58bbe472d4dd547f42f Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Mon, 31 Aug 2026 16:45:56 -0700 Subject: [PATCH 29/32] no-mistakes(document): docs(evaluations): drop stale RunSummary state docstring --- .../client/src/launchdarkly_ai_server/evaluations/types.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 6456157..5f4f4d5 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -88,7 +88,11 @@ class EvaluationRunRef: @dataclass class RunSummary: - """State and row counts for an evaluation run.""" + """Row counts for an evaluation run. + + The summary endpoint does not return run state, so terminal completion + is derived from row accounting instead. + """ total_rows: int = 0 passed_rows: int = 0 From cd843522fcf0e233e0b9cb548cb06e5c880ade57 Mon Sep 17 00:00:00 2001 From: "doneill@launchdarkly.com" Date: Tue, 1 Sep 2026 18:20:00 +0000 Subject: [PATCH 30/32] feat(evaluations): make summary poll interval and timeout configurable per run --- packages/client/README.md | 2 +- packages/client/agents.md | 4 +- .../evaluations/module.py | 37 ++++++++++-- packages/client/tests/test_evaluations_run.py | 58 +++++++++++++++++++ 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index d32f423..c5b4ad6 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -47,7 +47,7 @@ No code changes are required — `init_client()` detects the packages at runtime The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, with a three-minute timeout. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A generation result passes only when the completed summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A generation result passes only when the completed summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/agents.md b/packages/client/agents.md index da489f6..3a8f6d2 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,9 +128,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, timing out after three minutes. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. Without an SDK key no events can be emitted, but the run summary is still polled. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. Without an SDK key no events can be emitted, but the run summary is still polled. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout after three minutes if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index fba044a..dece319 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -79,14 +79,22 @@ async def run( generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, concurrency: int = 10, + poll_interval_seconds: float | None = None, + poll_timeout_seconds: float | None = None, ) -> EvalRunResult: """ Create and run a generation-only evaluation in the caller's process. The returned pass/fail result is derived from LaunchDarkly's run summary. A CI script can exit with ``0 if result.passed else 1`` after awaiting - this method. + this method. Large datasets may need a longer ``poll_timeout_seconds`` + and a wider ``poll_interval_seconds``; both default to + ``SUMMARY_POLL_TIMEOUT_SECONDS`` / ``SUMMARY_POLL_INTERVAL_SECONDS``. """ + if poll_interval_seconds is None: + poll_interval_seconds = SUMMARY_POLL_INTERVAL_SECONDS + if poll_timeout_seconds is None: + poll_timeout_seconds = SUMMARY_POLL_TIMEOUT_SECONDS self._validate_run_args( project_key=project_key, key=key, @@ -94,6 +102,8 @@ async def run( handler=handler, generation=generation, concurrency=concurrency, + poll_interval_seconds=poll_interval_seconds, + poll_timeout_seconds=poll_timeout_seconds, ) run_tools = dict(tools or {}) client = None @@ -147,7 +157,11 @@ async def run( if inspect.isawaitable(flush_result): await flush_result summary = await self._poll_summary_until_terminal( - project_key, evaluation.id, evaluation_run.id + project_key, + evaluation.id, + evaluation_run.id, + poll_interval_seconds, + poll_timeout_seconds, ) url = ( f"{self._ui_base_uri}/projects/{_segment(project_key)}/ai/evaluations/" @@ -161,9 +175,14 @@ async def run( ) async def _poll_summary_until_terminal( - self, project_key: str, evaluation_id: str, run_id: str + self, + project_key: str, + evaluation_id: str, + run_id: str, + poll_interval_seconds: float, + poll_timeout_seconds: float, ) -> RunSummary: - deadline = time.monotonic() + SUMMARY_POLL_TIMEOUT_SECONDS + deadline = time.monotonic() + poll_timeout_seconds last_summary = None while True: last_summary = await asyncio.to_thread( @@ -180,13 +199,13 @@ async def _poll_summary_until_terminal( ) raise EvaluationsError( "Timed out after " - f"{SUMMARY_POLL_TIMEOUT_SECONDS:g} seconds waiting for evaluation " + f"{poll_timeout_seconds:g} seconds waiting for evaluation " f"run {run_id} summary rows to be fully accounted " f"(total_rows={last_summary.total_rows}, " f"accounted_rows={accounted_rows}, " f"pending_rows={last_summary.pending_rows})" ) - await asyncio.sleep(min(SUMMARY_POLL_INTERVAL_SECONDS, remaining)) + await asyncio.sleep(min(poll_interval_seconds, remaining)) async def _resolve_client(self) -> Any: """ @@ -215,6 +234,8 @@ def _validate_run_args( handler: EvalHandler, generation: GenerationConfig, concurrency: int, + poll_interval_seconds: float, + poll_timeout_seconds: float, ) -> None: for name, value in ( ("project_key", project_key), @@ -237,6 +258,10 @@ def _validate_run_args( ) if concurrency < 1: raise EvaluationsError("concurrency must be at least 1") + if poll_interval_seconds < 0: + raise EvaluationsError("poll_interval_seconds must not be negative") + if poll_timeout_seconds < 0: + raise EvaluationsError("poll_timeout_seconds must not be negative") def init_evaluations( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index e3339e3..fb2c925 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -611,6 +611,64 @@ async def handler(*args: object) -> dict[str, Any]: assert len(summary_requests) == 1 +@pytest.mark.asyncio +async def test_poll_timeout_and_interval_are_configurable_per_run() -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 0, "error": 0, "pending": 1}}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + poll_interval_seconds=0, + poll_timeout_seconds=600, + ) + + assert result.passed is True + summary_requests = [ + request for request in transport.requests if request["url"].endswith("/summary") + ] + assert len(summary_requests) == 2 + + with pytest.raises(EvaluationsError, match="poll_timeout_seconds"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + poll_timeout_seconds=-1, + ) + + @pytest.mark.asyncio async def test_generation_failed_rows_do_not_fail_the_result() -> None: transport = SequencedTransport( From a1ba71edb5c8c027cfe7d978b100c4cd848a60c9 Mon Sep 17 00:00:00 2001 From: "doneill@launchdarkly.com" Date: Tue, 1 Sep 2026 19:11:04 +0000 Subject: [PATCH 31/32] feat(evaluations): require an SDK key and fail fast when it is missing --- packages/client/README.md | 7 ++-- packages/client/agents.md | 2 +- .../evaluations/module.py | 40 +++++++++---------- packages/client/tests/test_evaluations.py | 19 +++++++-- packages/client/tests/test_evaluations_run.py | 17 ++++++++ 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index c5b4ad6..6b11276 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -40,12 +40,13 @@ No code changes are required — `init_client()` detects the packages at runtime | `LD_ENVIRONMENT` | No | `deployment.environment` resource attribute attached to telemetry | | `OTEL_EXPORTER_OTLP_ENDPOINT` | No | OTLP endpoint override (default: LaunchDarkly Observability backend) | | `LD_API_TOKEN` | For evaluations | API access token used by the evaluations management API | +| `LD_SDK_KEY` | For evaluations | SDK key whose event transport carries generation results to LaunchDarkly | | `LD_API_BASE_URI` | No | Evaluations management API host override; intentionally separate from `LD_BASE_URI` | | `LD_UI_BASE_URI` | No | LaunchDarkly application host for evaluation-run links (default: `https://app.launchdarkly.com`; staging: `https://ld-stg.launchdarkly.com`) | ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. With `LD_SDK_KEY` configured, each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A generation result passes only when the completed summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. @@ -58,7 +59,7 @@ from launchdarkly_ai_server import init_evaluations async def main() -> int: - evals = init_evaluations() # LD_API_TOKEN required; LD_SDK_KEY emits generations + evals = init_evaluations() # LD_API_TOKEN and LD_SDK_KEY both required result = await evals.run( project_key="my-project", key="support-qa-2026-08-20", @@ -79,7 +80,7 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -`LD_SDK_KEY` is required to emit generation events through the standard LaunchDarkly SDK event transport. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. Without an SDK key, no generation event can be emitted, but the run summary is still polled. +`LD_SDK_KEY` is required: generation events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises when no SDK key is resolved rather than creating a run that can never complete. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/agents.md b/packages/client/agents.md index 3a8f6d2..59931b8 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,7 +128,7 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required to emit generation events: when set the harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. Without an SDK key no events can be emitted, but the run summary is still polled. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required and resolved in `init_evaluations()`, which raises before any network I/O when it is missing: generation events are the only ingest path for row results, so a run without an SDK key could never complete. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. `await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index dece319..261f1a2 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -47,7 +47,7 @@ class EvaluationsModule: def __init__( self, api_client: LDApiClient, - sdk_key: str | None = None, + sdk_key: str, ui_base_uri: str = DEFAULT_UI_BASE_URI, ) -> None: self._api = api_client @@ -60,8 +60,8 @@ def api(self) -> LDApiClient: return self._api @property - def sdk_key(self) -> str | None: - """SDK key used for observability traces; ``None`` disables tracing.""" + def sdk_key(self) -> str: + """SDK key whose event transport carries generation results to LaunchDarkly.""" return self._sdk_key @property @@ -106,9 +106,7 @@ async def run( poll_timeout_seconds=poll_timeout_seconds, ) run_tools = dict(tools or {}) - client = None - if self._sdk_key: - client = await self._resolve_client() + client = await self._resolve_client() # The management API client is synchronous; running it in a worker thread # keeps the caller's event loop free. @@ -144,18 +142,17 @@ async def run( run_tools, concurrency, ) - if client is not None: - self._runner._emit_generation_events( - client, - project_key=project_key, - evaluation=evaluation, - evaluation_run=evaluation_run, - dataset=dataset_ref, - results=results, - ) - flush_result = client.flush() - if inspect.isawaitable(flush_result): - await flush_result + self._runner._emit_generation_events( + client, + project_key=project_key, + evaluation=evaluation, + evaluation_run=evaluation_run, + dataset=dataset_ref, + results=results, + ) + flush_result = client.flush() + if inspect.isawaitable(flush_result): + await flush_result summary = await self._poll_summary_until_terminal( project_key, evaluation.id, @@ -281,8 +278,11 @@ def init_evaluations( resolved_sdk_key = sdk_key or _env("LD_SDK_KEY") if not resolved_sdk_key: - logger.info( - "No LaunchDarkly SDK key provided; evaluation runs will not emit traces." + raise EvaluationsError( + "No LaunchDarkly SDK key provided. Generation results reach " + "LaunchDarkly through the SDK event transport, so a run cannot " + "complete without one: set the LD_SDK_KEY environment variable or " + "pass sdk_key to init_evaluations()." ) api_client = LDApiClient( diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index d09e74e..5634dbf 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -101,19 +101,31 @@ def test_blank_api_token_env_is_treated_as_unset( init_evaluations(transport=failing_transport) -def test_missing_sdk_key_is_allowed(monkeypatch: pytest.MonkeyPatch) -> None: +def test_missing_sdk_key_raises_before_network_io( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv("LD_API_TOKEN", "api-token") monkeypatch.delenv("LD_SDK_KEY", raising=False) - evals = init_evaluations(transport=RecordingTransport()) + with pytest.raises(EvaluationsError, match="LD_SDK_KEY"): + init_evaluations(transport=failing_transport) + + +def test_blank_sdk_key_env_is_treated_as_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.setenv("LD_SDK_KEY", " ") - assert evals.sdk_key is None + with pytest.raises(EvaluationsError, match="LD_SDK_KEY"): + init_evaluations(transport=failing_transport) def test_base_uri_override_isolated_from_sdk_delivery_uri( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.setenv("LD_SDK_KEY", "sdk-key") monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com/") monkeypatch.setenv("LD_BASE_URI", "https://relay.example.com/") @@ -130,6 +142,7 @@ def test_ui_base_uri_precedence_and_api_base_isolation( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.setenv("LD_SDK_KEY", "sdk-key") monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com") monkeypatch.setenv("LD_UI_BASE_URI", "https://ld-stg.launchdarkly.com/") diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index fb2c925..42eebff 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -15,6 +15,23 @@ ) +@pytest.fixture(autouse=True) +def stub_sdk_client(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Give every run a resolvable SDK client, since one is now required.""" + monkeypatch.setenv("LD_SDK_KEY", "sdk-key") + client = MagicMock() + client.flush = AsyncMock() + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.get_client", + MagicMock(side_effect=RuntimeError("client not initialized")), + ) + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", + AsyncMock(return_value=client), + ) + return client + + class SequencedTransport: """Records requests and returns one response for each expected request.""" From ab5a4e5d9cb6d29b189f3efde6cea1f0336ba0b4 Mon Sep 17 00:00:00 2001 From: "doneill@launchdarkly.com" Date: Tue, 1 Sep 2026 23:33:29 +0000 Subject: [PATCH 32/32] fix(evaluations): accept a BYOC client without an SDK key and reject NaN poll values --- packages/ai/README.md | 2 +- packages/client/README.md | 4 +- packages/client/agents.md | 2 +- .../evaluations/module.py | 73 +++++++---- packages/client/tests/test_evaluations.py | 36 ++++++ packages/client/tests/test_evaluations_run.py | 118 ++++++++++++++++++ 6 files changed, 209 insertions(+), 26 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index 80e04fa..7bbf073 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -69,7 +69,7 @@ result = await evals.run( ) ``` -`LD_API_TOKEN` is required. Configure `LD_SDK_KEY` to emit one `$ld:ai:offline-evals:generation` event per generated row through the standard SDK event transport. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). +`LD_API_TOKEN` is required. Configure `LD_SDK_KEY` — or initialize your own client with `init_client(client=...)` — to emit one `$ld:ai:offline-evals:generation` event per generated row through the standard SDK event transport. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/client/README.md b/packages/client/README.md index 6b11276..852f5ef 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -59,7 +59,7 @@ from launchdarkly_ai_server import init_evaluations async def main() -> int: - evals = init_evaluations() # LD_API_TOKEN and LD_SDK_KEY both required + evals = init_evaluations() # LD_API_TOKEN required; LD_SDK_KEY unless a client is already initialized result = await evals.run( project_key="my-project", key="support-qa-2026-08-20", @@ -80,7 +80,7 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -`LD_SDK_KEY` is required: generation events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises when no SDK key is resolved rather than creating a run that can never complete. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. +Generation events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises rather than creating a run that can never complete unless it can resolve an event transport: either an SDK key (`sdk_key` or `LD_SDK_KEY`) or a client already initialized through `init_client(client=...)`. Bringing your own client lets a process emit evaluation events without an SDK key in scope. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/agents.md b/packages/client/agents.md index 59931b8..3ef0c16 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -128,7 +128,7 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. `LD_SDK_KEY` is required and resolved in `init_evaluations()`, which raises before any network I/O when it is missing: generation events are the only ingest path for row results, so a run without an SDK key could never complete. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. An event transport is resolved in `init_evaluations()`, which raises before any network I/O when it finds neither an SDK key (`sdk_key` or `LD_SDK_KEY`) nor an already-initialized event-capable client: generation events are the only ingest path for row results, so a run without a transport could never complete. The lifecycle module's bring-your-own-client path (`init_client(client=...)`) therefore satisfies the check on its own, and `run()` reuses that singleton through `_resolve_client`; `run()` raises if the client disappears before it emits. Both polling arguments reject NaN, which would otherwise never compare past a deadline and hang the run. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. `await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 261f1a2..4a3ffad 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -3,6 +3,7 @@ import asyncio import inspect import logging +import math import os import time from collections.abc import Mapping @@ -32,6 +33,20 @@ def _env(name: str) -> str | None: return value if value else None +def _initialized_client() -> Any | None: + """Return the SDK singleton when one is initialized, otherwise ``None``.""" + try: + return get_client() + except RuntimeError: + return None + + +def _can_emit_events(client: Any) -> bool: + return callable(getattr(client, "track", None)) and callable( + getattr(client, "flush", None) + ) + + def _is_terminal_summary(summary: RunSummary) -> bool: accounted_rows = summary.passed_rows + summary.failed_rows + summary.error_rows return ( @@ -47,7 +62,7 @@ class EvaluationsModule: def __init__( self, api_client: LDApiClient, - sdk_key: str, + sdk_key: str | None, ui_base_uri: str = DEFAULT_UI_BASE_URI, ) -> None: self._api = api_client @@ -60,7 +75,7 @@ def api(self) -> LDApiClient: return self._api @property - def sdk_key(self) -> str: + def sdk_key(self) -> str | None: """SDK key whose event transport carries generation results to LaunchDarkly.""" return self._sdk_key @@ -211,16 +226,21 @@ async def _resolve_client(self) -> Any: ``init_client`` is idempotent, so an application that already holds a client keeps it and the evaluations SDK key is not applied. """ - try: - existing = get_client() - except RuntimeError: - return await init_client({"sdkKey": self._sdk_key}) - logger.warning( - "A LaunchDarkly client is already initialized; evaluation events are " - "sent with it and the evaluations SDK key is ignored. Both must point " - "at the project under evaluation." - ) - return existing + existing = _initialized_client() + if existing is not None: + if self._sdk_key: + logger.warning( + "A LaunchDarkly client is already initialized; evaluation " + "events are sent with it and the evaluations SDK key is " + "ignored. Both must point at the project under evaluation." + ) + return existing + if not self._sdk_key: + raise EvaluationsError( + "No LaunchDarkly SDK key provided and no initialized " + "LaunchDarkly client is available to deliver generation events." + ) + return await init_client({"sdkKey": self._sdk_key}) @staticmethod def _validate_run_args( @@ -255,10 +275,15 @@ def _validate_run_args( ) if concurrency < 1: raise EvaluationsError("concurrency must be at least 1") - if poll_interval_seconds < 0: - raise EvaluationsError("poll_interval_seconds must not be negative") - if poll_timeout_seconds < 0: - raise EvaluationsError("poll_timeout_seconds must not be negative") + for name, seconds in ( + ("poll_interval_seconds", poll_interval_seconds), + ("poll_timeout_seconds", poll_timeout_seconds), + ): + # NaN comparisons are always false, so a NaN would poll forever. + if math.isnan(seconds): + raise EvaluationsError(f"{name} must be a number") + if seconds < 0: + raise EvaluationsError(f"{name} must not be negative") def init_evaluations( @@ -278,12 +303,16 @@ def init_evaluations( resolved_sdk_key = sdk_key or _env("LD_SDK_KEY") if not resolved_sdk_key: - raise EvaluationsError( - "No LaunchDarkly SDK key provided. Generation results reach " - "LaunchDarkly through the SDK event transport, so a run cannot " - "complete without one: set the LD_SDK_KEY environment variable or " - "pass sdk_key to init_evaluations()." - ) + byoc_client = _initialized_client() + if byoc_client is None or not _can_emit_events(byoc_client): + raise EvaluationsError( + "No LaunchDarkly SDK key provided and no initialized " + "LaunchDarkly client to emit events with. Generation results " + "reach LaunchDarkly through the SDK event transport, so a run " + "cannot complete without one: set the LD_SDK_KEY environment " + "variable, pass sdk_key to init_evaluations(), or initialize a " + "client first with init_client(client=...)." + ) api_client = LDApiClient( api_token=token, diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index 5634dbf..57edf18 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -1,10 +1,13 @@ from __future__ import annotations import json +from collections.abc import Iterator from typing import Any +from unittest.mock import AsyncMock, MagicMock import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module from launchdarkly_ai_server.evaluations import ( DEFAULT_BASE_URI, EvalRunResult, @@ -18,6 +21,13 @@ ) +@pytest.fixture(autouse=True) +def reset_sdk_singleton() -> Iterator[None]: + lifecycle_module._reset_for_testing() + yield + lifecycle_module._reset_for_testing() + + class RecordingTransport: """Mocked LD API — records requests and replays canned responses.""" @@ -121,6 +131,32 @@ def test_blank_sdk_key_env_is_treated_as_unset( init_evaluations(transport=failing_transport) +def test_missing_sdk_key_is_allowed_with_a_byoc_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.delenv("LD_SDK_KEY", raising=False) + byoc_client = MagicMock() + byoc_client.track = MagicMock() + byoc_client.flush = AsyncMock() + lifecycle_module._set_client_for_testing(byoc_client) + + evals = init_evaluations(transport=failing_transport) + + assert evals.sdk_key is None + + +def test_missing_sdk_key_raises_when_the_byoc_client_cannot_emit_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LD_API_TOKEN", "api-token") + monkeypatch.delenv("LD_SDK_KEY", raising=False) + lifecycle_module._set_client_for_testing(object()) + + with pytest.raises(EvaluationsError, match="LD_SDK_KEY"): + init_evaluations(transport=failing_transport) + + def test_base_uri_override_isolated_from_sdk_delivery_uri( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 42eebff..5784559 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -68,6 +68,16 @@ def response(status: int, body: dict[str, Any] | None = None) -> HttpResponse: ) +def failing_transport( + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + timeout: float, +) -> HttpResponse: + raise AssertionError("no network I/O expected") + + def dataset_page( items: list[dict[str, Any]], total: int, next_href: str | None = None ) -> dict[str, Any]: @@ -686,6 +696,114 @@ async def handler(*args: object) -> dict[str, Any]: ) +@pytest.mark.parametrize( + ("poll_interval_seconds", "poll_timeout_seconds"), + [(float("nan"), 1.0), (1.0, float("nan"))], + ids=["interval", "timeout"], +) +@pytest.mark.asyncio +async def test_nan_poll_values_are_rejected( + poll_interval_seconds: float, poll_timeout_seconds: float +) -> None: + evals = init_evaluations(api_token="token", transport=failing_transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + with pytest.raises(EvaluationsError, match="must be a number"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + poll_interval_seconds=poll_interval_seconds, + poll_timeout_seconds=poll_timeout_seconds, + ) + + +@pytest.mark.asyncio +async def test_run_uses_a_byoc_client_when_no_sdk_key_is_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LD_SDK_KEY", raising=False) + byoc_client = MagicMock() + byoc_client.flush = AsyncMock() + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.get_client", + MagicMock(return_value=byoc_client), + ) + init_client = AsyncMock(side_effect=AssertionError("must reuse the BYOC client")) + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", init_client + ) + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 0, "input": "hello", "variables": {}}], total=1 + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + assert evals.sdk_key is None + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert result.passed is True + init_client.assert_not_awaited() + byoc_client.track.assert_called_once() + byoc_client.flush.assert_awaited_once_with() + + +@pytest.mark.asyncio +async def test_run_raises_when_no_sdk_key_and_no_initialized_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LD_SDK_KEY", raising=False) + byoc_client = MagicMock() + byoc_client.flush = AsyncMock() + get_client = MagicMock(return_value=byoc_client) + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.get_client", get_client + ) + evals = init_evaluations(api_token="token", transport=failing_transport) + get_client.side_effect = RuntimeError("client not initialized") + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + with pytest.raises(EvaluationsError, match="no initialized LaunchDarkly client"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + @pytest.mark.asyncio async def test_generation_failed_rows_do_not_fail_the_result() -> None: transport = SequencedTransport(