diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index 65a8364..f312f26 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -97,6 +97,7 @@ able to tell from the trace which path ran. | `launchdarkly.stream.abandoned` | `True`, only when abandoned | `end_span_once` | | `gen_ai.evaluation.name` | judge config key, judge roots only | `with_judge_evaluation`, see section 4a | | `gen_ai.evaluation.score.value` | numeric score, judge roots only | `with_judge_evaluation`, see section 4a | +| `context.contextKeys.` | the context's key for that kind, one row per kind | `set_ld_span_attributes` | The root also carries one span event, `feature_flag`, with these event attributes: @@ -105,6 +106,23 @@ The root also carries one span event, `feature_flag`, with these event attribute | `feature_flag.key` | config key | | `feature_flag.provider.name` | `LaunchDarkly` | | `feature_flag.set.id` | environment id, only when present | +| `feature_flag.context.id` | canonical key of the evaluation context, only when present | +| `feature_flag.contextKeys` | JSON object of the context's per-kind keys, only when present | + +The context identity appears in two shapes on purpose, and neither is new to LaunchDarkly. +`feature_flag.context.id` is the canonical key, matching what the Go server SDK's `ldotel` hook +emits, and it is a composite of every kind for a multi-kind context. That makes it useless for +"filter this config's traces to one user", which is the question AI Config Monitoring's group-by +asks. So the per-kind keys are also written as `context.contextKeys.` span attributes — the +spelling the observability browser SDK and the product-analytics pipeline already use — where each +kind is an exact match on its own. `feature_flag.contextKeys` carries the same map as JSON on the +event, matching the browser SDK and filling the column observability's materialized view already +lifts from that attribute. + +Only context *keys* are emitted. Context attribute values are not, and there is no option to turn +them on: keys are identifiers already exposed by LaunchDarkly's other OTel integrations, whereas +attribute values are where the personal data lives. See section 7 for the same reasoning applied to +content. The root is the only span that carries the config-association attributes and the `feature_flag` event. Child spans carry neither. A test asserts this, so do not add them to children out of diff --git a/packages/client/src/launchdarkly_ai_server/ld_context.py b/packages/client/src/launchdarkly_ai_server/ld_context.py new file mode 100644 index 0000000..cae34f2 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/ld_context.py @@ -0,0 +1,102 @@ +"""Derives span-safe identity from an ``LDContext`` dict. + +Ported from the observability browser SDK's LaunchDarkly integration +(``sdk/highlight-run/src/integrations/launchdarkly/index.ts``), and mirrored by +``js-ai-sdk``'s ``packages/client/src/context.ts``, so every LaunchDarkly +emitter produces byte-identical canonical keys. + +Its own module because it is pure: no OTel, no LD client, no I/O. That is also +why it does not go through ``ldclient.Context`` — ``ldclient`` is an optional +import here (see ``utils.to_ld_context``), so relying on it would make the +attribute silently absent for anyone using a custom client. +""" + +from __future__ import annotations + +from typing import Any + + +def _encode_key(key: str) -> str: + """Escapes the two characters ambiguous inside a canonical key: ``%`` and ``:``. + + ``%`` is replaced first so an escape sequence is never double-escaped. + """ + if "%" in key or ":" in key: + return key.replace("%", "%25").replace(":", "%3A") + return key + + +def _multi_kind_pairs(context: dict[str, Any]) -> list[tuple[str, str]]: + """``(kind, key)`` pairs of a multi-kind context, sorted by kind. + + Skips any kind whose sub-context has no usable string key. Both public + functions go through this, so the canonical key and the per-kind map can + never disagree about which kinds are present. + """ + pairs: list[tuple[str, str]] = [] + for kind in sorted(context): + if kind == "kind": + continue + sub = context.get(kind) + key = sub.get("key") if isinstance(sub, dict) else None + if isinstance(key, str) and key: + pairs.append((kind, key)) + return pairs + + +def get_context_keys(context: dict[str, Any]) -> dict[str, str]: + """The per-kind keys of *context*, as ``{: }``. + + Keys are raw — only the canonical key is escaped. A legacy user (no + ``kind``) reports as kind ``user``, matching every other LaunchDarkly + integration. + """ + if context.get("kind") == "multi": + return dict(_multi_kind_pairs(context)) + key = context.get("key") + if not isinstance(key, str) or not key: + return {} + kind = context.get("kind") + return {kind if isinstance(kind, str) and kind else "user": key} + + +def get_canonical_key(context: dict[str, Any]) -> str: + """The canonical key of *context*. + + The same value the Go SDK's ``ldotel`` hook puts on + ``feature_flag.context.id`` via ``Context().FullyQualifiedKey()``. Stable + and consistent, not for presentation: it is what links a span to a context + instance. + """ + if context.get("kind") == "multi": + return ":".join( + f"{kind}:{_encode_key(key)}" for kind, key in _multi_kind_pairs(context) + ) + key = context.get("key") + if not isinstance(key, str) or not key: + return "" + kind = context.get("kind") + # A legacy user (no kind) and an explicit `user` kind both canonicalise to + # the bare key, with no `user:` prefix. + if not isinstance(kind, str) or not kind or kind == "user": + return key + return f"{kind}:{_encode_key(key)}" + + +def context_identity(context: Any) -> tuple[str, dict[str, str]] | None: + """The canonical key and per-kind keys of *context*, or ``None``. + + ``None`` whenever there is no usable identity. Never raises: this runs on + the emit path of every run, and a malformed context must degrade to + emitting nothing rather than break the caller's AI call. + """ + if not isinstance(context, dict): + return None + try: + context_keys = get_context_keys(context) + canonical_key = get_canonical_key(context) + except Exception: + return None + if not canonical_key or not context_keys: + return None + return canonical_key, context_keys diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 676934d..7a65d6c 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from typing import Any, Literal +from .ld_context import context_identity from .types import ( AiConfigRep, GraphNode, @@ -573,11 +574,18 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: * ``launchdarkly.variation.key`` = variationKey * ``launchdarkly.run.id`` = runId * ``launchdarkly.graph.key`` = graphKey (only when present) + * ``context.contextKeys.`` = one attribute per context kind, only when + ``variables['ldContext']`` yields a usable identity (AIC-3230). Per-kind + span attributes are what make a single kind filterable; the composite + canonical key on the ``feature_flag`` event cannot be. Span event (required for AI Config Monitoring Traces tab correlation): ``name='feature_flag'`` with ``feature_flag.key``, - ``feature_flag.provider.name``, and ``feature_flag.set.id`` (when - ``LD_ENVIRONMENT_ID`` is set or the TS SDK auto-resolved it). + ``feature_flag.provider.name``, ``feature_flag.set.id`` (when + ``LD_ENVIRONMENT_ID`` is set or the TS SDK auto-resolved it), + ``feature_flag.context.id`` (canonical key of the evaluation context, only + when present) and ``feature_flag.contextKeys`` (JSON object of the + context's per-kind keys, only when present). """ span.set_attribute("launchdarkly.operation.type", "gen_ai") if not variables: @@ -597,6 +605,30 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: } if ld.get("environmentId"): feature_flag_attrs["feature_flag.set.id"] = ld["environmentId"] + + # `execute_and_track` / `execute_and_stream` merge `ldContext` into + # variables after the caller's own variables, so it is always the + # evaluation context and cannot be clobbered by a same-named variable. + identity = context_identity(variables.get("ldContext")) + if identity is not None: + canonical_key, context_keys = identity + # Matches the Go SDK's ldotel hook (`feature_flag.context.id`) and the + # observability browser SDK (both, plus the per-kind span attributes). + feature_flag_attrs["feature_flag.context.id"] = canonical_key + # Compact separators on purpose. `json.dumps` defaults to `", "` and + # `": "`, which would make this string differ from what `JSON.stringify` + # produces in js-ai-sdk and in the observability browser SDK — and this + # value lands verbatim in the ClickHouse `ContextKeys` column, where a + # consumer may match on it as text. + feature_flag_attrs["feature_flag.contextKeys"] = json.dumps( + context_keys, separators=(",", ":") + ) + # The canonical key is a composite for a multi-kind context, so it + # cannot answer "filter to this one user". These per-kind attributes + # can, and they use the spelling AI Config Monitoring already speaks. + for kind, key in context_keys.items(): + span.set_attribute(f"context.contextKeys.{kind}", key) + span.add_event("feature_flag", feature_flag_attrs) diff --git a/packages/client/tests/test_ld_context.py b/packages/client/tests/test_ld_context.py new file mode 100644 index 0000000..f792023 --- /dev/null +++ b/packages/client/tests/test_ld_context.py @@ -0,0 +1,124 @@ +"""The Python port must agree with the TypeScript port, key for key. + +The fixtures here are the same ones in js-ai-sdk's +`packages/client/src/__tests__/context.test.ts`, which are in turn the +observability browser SDK's. A canonical key that differs between emitters +breaks context-instance linking, and nothing else would catch it. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from launchdarkly_ai_server.ld_context import ( + context_identity, + get_canonical_key, + get_context_keys, +) + + +@pytest.mark.parametrize( + ("context", "expected"), + [ + ({"key": "bob"}, {"user": "bob"}), + ({"kind": "user", "key": "bob"}, {"user": "bob"}), + ({"kind": "org", "key": "org123"}, {"org": "org123"}), + ({"kind": "device", "key": "device456"}, {"device": "device456"}), + ( + { + "kind": "multi", + "user": {"kind": "user", "key": "user-key", "name": "Test User"}, + "org": {"kind": "org", "key": "org-key"}, + }, + {"org": "org-key", "user": "user-key"}, + ), + ( + { + "kind": "multi", + "device": {"kind": "device", "key": "device-key"}, + "user": {"kind": "user", "key": "user-key"}, + }, + {"device": "device-key", "user": "user-key"}, + ), + ], +) +def test_get_context_keys(context: dict[str, Any], expected: dict[str, str]) -> None: + assert get_context_keys(context) == expected + + +def test_get_context_keys_does_not_escape_the_key() -> None: + # Only the canonical key is escaped. The map holds the key the customer + # actually sent, because that is what a filter compares against. + assert get_context_keys({"kind": "org", "key": "a:b%c"}) == {"org": "a:b%c"} + + +def test_get_context_keys_skips_a_multi_kind_entry_with_no_usable_key() -> None: + context = {"kind": "multi", "user": {"kind": "user", "key": "bob"}, "org": {}} + assert get_context_keys(context) == {"user": "bob"} + + +def test_get_context_keys_is_empty_without_a_key() -> None: + assert get_context_keys({}) == {} + + +@pytest.mark.parametrize( + ("context", "expected"), + [ + ({"key": "bob"}, "bob"), + ({"kind": "user", "key": "bob"}, "bob"), + ({"kind": "org", "key": "org123"}, "org:org123"), + ( + { + "kind": "multi", + "user": {"kind": "user", "key": "user-key"}, + "org": {"kind": "org", "key": "org-key"}, + }, + "org:org-key:user:user-key", + ), + ( + { + "kind": "multi", + "device": {"kind": "device", "key": "device-key"}, + "user": {"kind": "user", "key": "user-key"}, + }, + "device:device-key:user:user-key", + ), + ], +) +def test_get_canonical_key(context: dict[str, Any], expected: str) -> None: + assert get_canonical_key(context) == expected + + +def test_get_canonical_key_escapes_percent_before_colon() -> None: + # `%` first, then `:`, so an escape sequence is never double-escaped. + assert get_canonical_key({"kind": "org", "key": "a:b%c"}) == "org:a%3Ab%25c" + + +def test_get_canonical_key_is_empty_without_a_key() -> None: + assert get_canonical_key({}) == "" + + +def test_context_identity_returns_the_canonical_key_and_the_per_kind_keys() -> None: + context = { + "kind": "multi", + "user": {"kind": "user", "key": "u1"}, + "org": {"kind": "org", "key": "o1"}, + } + assert context_identity(context) == ("org:o1:user:u1", {"org": "o1", "user": "u1"}) + + +@pytest.mark.parametrize( + "context", + [ + None, + "user-key", + 42, + {}, + {"kind": "user", "key": 42}, + {"kind": "multi", "user": {}}, + ], +) +def test_context_identity_is_none_for_anything_unusable(context: Any) -> None: + assert context_identity(context) is None diff --git a/packages/client/tests/test_span_attributes.py b/packages/client/tests/test_span_attributes.py new file mode 100644 index 0000000..fbfe0d7 --- /dev/null +++ b/packages/client/tests/test_span_attributes.py @@ -0,0 +1,130 @@ +"""Emission tests for `set_ld_span_attributes`. + +Nothing covered this function before. The cross-handler parity suite in +`tests/test_cross_handler_parity.py` asserts every handler reaches it; these +tests assert what it writes once reached. +""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from launchdarkly_ai_server.utils import set_ld_span_attributes + +LD = { + "runId": "run-123", + "configKey": "test-config", + "variationKey": "variation-a", + "version": 1, + "modelName": "test-model", + "providerName": "TestProvider", +} + +MULTI = { + "kind": "multi", + "user": {"kind": "user", "key": "u1"}, + "org": {"kind": "org", "key": "o1"}, +} + + +class RecordingSpan: + """Records what was written, so a test can assert on keys and on absence.""" + + def __init__(self) -> None: + self.attributes: dict[str, Any] = {} + self.events: list[tuple[str, dict[str, Any]]] = [] + + def set_attribute(self, key: str, value: Any) -> None: + self.attributes[key] = value + + def add_event(self, name: str, attributes: dict[str, Any]) -> None: + self.events.append((name, attributes)) + + @property + def feature_flag_event(self) -> dict[str, Any]: + return next(attrs for name, attrs in self.events if name == "feature_flag") + + +def test_the_canonical_context_key_lands_on_the_feature_flag_event() -> None: + span = RecordingSpan() + set_ld_span_attributes( + span, {"__ld": LD, "ldContext": {"kind": "user", "key": "bob"}} + ) + assert span.feature_flag_event["feature_flag.context.id"] == "bob" + + +def test_a_multi_kind_context_uses_its_composite_canonical_key() -> None: + span = RecordingSpan() + set_ld_span_attributes(span, {"__ld": LD, "ldContext": MULTI}) + assert span.feature_flag_event["feature_flag.context.id"] == "org:o1:user:u1" + + +def test_the_per_kind_keys_land_on_the_event_as_json() -> None: + span = RecordingSpan() + set_ld_span_attributes(span, {"__ld": LD, "ldContext": MULTI}) + assert json.loads(span.feature_flag_event["feature_flag.contextKeys"]) == { + "org": "o1", + "user": "u1", + } + + +def test_the_json_is_byte_identical_to_what_json_stringify_produces() -> None: + # `json.dumps` defaults to `", "` / `": "` separators; JSON.stringify uses + # none. This value lands verbatim in ClickHouse's ContextKeys column, and + # the js-ai-sdk and browser SDK both write the compact form, so Python must + # too or the same context yields two different strings by language. + span = RecordingSpan() + set_ld_span_attributes(span, {"__ld": LD, "ldContext": MULTI}) + assert ( + span.feature_flag_event["feature_flag.contextKeys"] + == '{"org":"o1","user":"u1"}' + ) + + +def test_each_kind_gets_its_own_span_attribute() -> None: + # The composite canonical key cannot answer "filter to this one user" for a + # multi-kind context. These can. + span = RecordingSpan() + set_ld_span_attributes(span, {"__ld": LD, "ldContext": MULTI}) + assert span.attributes["context.contextKeys.user"] == "u1" + assert span.attributes["context.contextKeys.org"] == "o1" + + +def test_no_context_attributes_without_an_ld_context() -> None: + span = RecordingSpan() + set_ld_span_attributes(span, {"__ld": LD}) + assert "feature_flag.context.id" not in span.feature_flag_event + assert "feature_flag.contextKeys" not in span.feature_flag_event + assert [k for k in span.attributes if k.startswith("context.")] == [] + + +@pytest.mark.parametrize("bad", ["bob", 42, None, {}, {"kind": "multi", "user": {}}]) +def test_a_malformed_ld_context_emits_nothing_and_does_not_raise(bad: Any) -> None: + span = RecordingSpan() + set_ld_span_attributes(span, {"__ld": LD, "ldContext": bad}) + assert "feature_flag.context.id" not in span.feature_flag_event + assert [k for k in span.attributes if k.startswith("context.")] == [] + + +def test_only_keys_are_emitted_never_attribute_values() -> None: + # AC 5: attribute values are where the PII lives. Nothing but keys leaves + # the SDK, and no option exists to change that. + span = RecordingSpan() + set_ld_span_attributes( + span, + { + "__ld": LD, + "ldContext": { + "kind": "user", + "key": "bob", + "email": "bob@example.com", + "name": "Bob", + }, + }, + ) + emitted = json.dumps([span.attributes, span.events]) + assert "bob@example.com" not in emitted + assert "Bob" not in emitted diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index ff47d17..9786398 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -25,6 +25,7 @@ import ast import importlib import inspect +import json import re from pathlib import Path from typing import Any @@ -70,13 +71,16 @@ def __init__(self, name: str, context: Any = None) -> None: self.name = name self.context = context self.attributes: dict[str, Any] = {} - self.events: list[str] = [] + # Keyed by event name, holding the event's attributes. A list of names was + # enough while nothing asserted on an event's payload; the context identity + # rides on the feature_flag event, so the payload has to survive. + self.events: dict[str, dict[str, Any]] = {} def set_attribute(self, key: str, value: Any) -> None: self.attributes[key] = value def add_event(self, name: str, attributes: dict[str, Any] | None = None) -> None: - self.events.append(name) + self.events[name] = dict(attributes or {}) def set_status(self, code: Any, description: str | None = None) -> None: pass @@ -240,6 +244,44 @@ def test_a_model_span_carries_no_launchdarkly_identity( assert [k for k in span.attributes if k.startswith("launchdarkly.")] == [] assert "feature_flag" not in span.events + def test_the_root_carries_the_context_identity(self, handler_spans: Any) -> None: + # AC 2: the composite canonical key cannot express "filter to this one + # user" for a multi-kind context, so every handler must also write the + # per-kind attributes. Interpolated keys, so the vocabulary lock cannot + # see them — this is the only check that can. + _, module, tracer = handler_spans + module.start_root_span( + CONFIG, + { + **LD_VARIABLES, + "ldContext": { + "kind": "multi", + "user": {"kind": "user", "key": "u1"}, + "org": {"kind": "org", "key": "o1"}, + }, + }, + ) + span = tracer.spans[0] + assert span.attributes["context.contextKeys.user"] == "u1" + assert span.attributes["context.contextKeys.org"] == "o1" + event = span.events["feature_flag"] + assert event["feature_flag.context.id"] == "org:o1:user:u1" + assert json.loads(event["feature_flag.contextKeys"]) == { + "org": "o1", + "user": "u1", + } + + def test_a_tool_span_carries_no_context_identity(self, handler_spans: Any) -> None: + # Same rule as the LaunchDarkly identity above: one span per run must + # answer a context-scoped query, or one run looks like several. + _, module, tracer = handler_spans + module.start_tool_span("get_weather", "call-1", None) + assert [ + k + for k in tracer.spans[0].attributes + if k.startswith("context.contextKeys.") + ] == [] + # ─── Model identity ────────────────────────────────────────────────────────── @@ -358,6 +400,16 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "feature_flag.key", "feature_flag.provider.name", "feature_flag.set.id", + # Context identity, AIC-3230. `feature_flag.context.id` is the canonical key, matching the Go + # SDK's ldotel hook. `feature_flag.contextKeys` is a JSON object of per-kind keys, matching the + # observability browser SDK and filling the ContextKeys column the ClickHouse MV already lifts. + "feature_flag.context.id", + "feature_flag.contextKeys", + # `context.contextKeys` is a prefix, not a key: the emitted keys are + # `context.contextKeys.`, built by f-string and invisible to the scan. They are span + # attributes rather than event attributes because that is what makes a single kind an exact + # match in trace search — the canonical key above is a composite for a multi-kind context. + "context.contextKeys", # Graph spans, unchanged from before the span work "ld.ai.graph", "ld.ai.graph.key", @@ -397,8 +449,14 @@ def _without_superseded(source: str) -> str: r'|"(gen_ai\.[a-z_.0-9]+)"' r'|f"(gen_ai\.[a-z_.]+)\.\{' # The feature_flag event's own attributes are built as a plain dict before being handed to - # add_event, so they never appear inside a set_attribute call. - r'|"(feature_flag\.[a-z_.]+)"' + # add_event, so they never appear inside a set_attribute call. `contextKeys` is camelCase + # because that is the spelling the observability browser SDK already emits and the + # ClickHouse materialized view already lifts, so the character class allows uppercase. + r'|"(feature_flag\.[a-zA-Z_.]+)"' + # Per-kind context keys are `f"context.contextKeys.{kind}"`, so only the prefix has a literal + # to scan for, exactly like the gen_ai indexed carrier above. + # TestLaunchDarklyIdentity covers the interpolated keys at runtime. + r'|f"(context\.contextKeys)\.\{' ) @@ -413,6 +471,7 @@ def _emitted_vocabulary() -> set[str]: "launchdarkly", "feature_flag", "ld", + "context", ): found.add(key) return found