diff --git a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java index 7fc9f905..368b8dcf 100644 --- a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java +++ b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java @@ -24,6 +24,9 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -260,10 +263,25 @@ private void onFirstByte() { private void onStreamClosed() { try { + // Scooped first and unconditionally: tagSpanFromBuffer bails on an empty body, + // and a failed request is both the likeliest source of one and the case where + // the vendor's request id is most worth having. + InstrumentationSemConv.tagLLMSpanIdHeaders(span, headersAsMap(delegate.headers())); + byte[] bytes; synchronized (teeBuffer) { bytes = teeBuffer.toByteArray(); } + + // Recorded before tagging: the anthropic sdk raises above this layer, so the + // error status is ours alone to set, and losing it to a body-parsing problem is + // worse than losing the parsed output. + int statusCode = delegate.statusCode(); + if (statusCode >= 400) { + InstrumentationSemConv.tagLLMSpanHttpError( + span, statusCode, new String(bytes, StandardCharsets.UTF_8)); + } + // tagLLMSpanResponse also emits child spans for any server-side tool calls (web // search, etc.) nested under the LLM span while it is still live. tagSpanFromBuffer(tracer, span, bytes, timeToFirstTokenNanos.get()); @@ -387,6 +405,29 @@ private static void tagSpanFromBuffer( } } + /** + * Adapts the anthropic sdk's {@code Headers} to the vendor-neutral shape {@link + * InstrumentationSemConv} consumes. Returns null on failure so a header-shape change can never + * take down the response tagging that follows it. + */ + @Nullable + private static Map> headersAsMap( + @Nullable com.anthropic.core.http.Headers headers) { + if (headers == null) { + return null; + } + try { + var map = new HashMap>(); + for (String name : headers.names()) { + map.put(name, headers.values(name)); + } + return map; + } catch (Exception e) { + log.debug("could not read response headers", e); + return null; + } + } + private static String firstNonEmptyLine(byte[] bytes) { int start = 0; for (int i = 0; i <= bytes.length; i++) { diff --git a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicTest.java b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicTest.java index 6778a354..64faae15 100644 --- a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicTest.java +++ b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicTest.java @@ -562,4 +562,76 @@ void testWrappedClientObjectContract() { wrapped.toString().contains("ContextCapturingProxy"), "toString should identify the proxy, got: " + wrapped); } + + /** + * Anthropic returns its correlation ID as {@code request-id} (no {@code x-} prefix, unlike + * OpenAI) and its object ID as {@code msg_*}. Both must land on the span for streaming and + * non-streaming alike — the header comes off the HTTP response, the object ID out of the + * reassembled body, so the two travel independent paths. + */ + @Test + @SneakyThrows + void testCorrelationIdsCaptured() { + AnthropicClient anthropicClient = + AnthropicOkHttpClient.builder() + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) + .build(); + + var request = + MessageCreateParams.builder() + .model(Model.of(TEST_MODEL)) + .system("You are a helpful assistant") + .addUserMessage("What is the capital of France?") + .maxTokens(50) + .temperature(0.0) + .build(); + + anthropicClient.messages().create(request); + assertAnthropicIdsCaptured(testHarness.awaitExportedSpans().get(0)); + } + + @Test + @SneakyThrows + void testCorrelationIdsCapturedStreaming() { + AnthropicClient anthropicClient = + AnthropicOkHttpClient.builder() + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) + .build(); + + var request = + MessageCreateParams.builder() + .model(Model.of(TEST_MODEL)) + .system("You are a helpful assistant") + .addUserMessage("What is the capital of France?") + .maxTokens(50) + .temperature(0.0) + .build(); + + try (var stream = anthropicClient.messages().createStreaming(request)) { + stream.stream().forEach(event -> {}); + } + assertAnthropicIdsCaptured(testHarness.awaitExportedSpans().get(0)); + } + + /** + * Asserts presence only for the header: its value is an opaque vendor string, so pinning its + * shape would encode an assumption the provider never made. + */ + private static void assertAnthropicIdsCaptured(io.opentelemetry.sdk.trace.data.SpanData span) { + var attributes = span.getAttributes(); + + String requestId = attributes.get(AttributeKey.stringKey("request-id")); + assertNotNull(requestId, "request-id header must be captured"); + assertFalse(requestId.isBlank(), "request-id must not be blank"); + + String responseId = attributes.get(AttributeKey.stringKey("response_id")); + assertNotNull(responseId, "response_id must be captured from the response body"); + assertTrue(responseId.startsWith("msg_"), "unexpected response_id: " + responseId); + + assertNull( + attributes.get(AttributeKey.stringKey("x-request-id")), + "OpenAI's header name must not appear on an Anthropic span"); + } } diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java index 2d4d958b..5b32ad2a 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java @@ -20,6 +20,9 @@ import io.opentelemetry.context.Context; import java.io.*; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -68,7 +71,7 @@ private static ExtractedRequest extractCallerContext(HttpRequest request) { Context context = contextFromTraceparent(values.get(0)); HttpRequest stripped = request.toBuilder() - .replaceHeaders(ContextCapturingProxy.CONTEXT_HEADER, java.util.List.of()) + .replaceHeaders(ContextCapturingProxy.CONTEXT_HEADER, List.of()) .build(); return new ExtractedRequest(stripped, context); } @@ -259,6 +262,28 @@ private static void tagSpanFromBuffer( } } + /** + * Adapts openai-java's {@link Headers} to the vendor-neutral shape {@link + * InstrumentationSemConv} consumes. Returns null on failure so a header-shape change can never + * take down the response tagging that follows it. + */ + @Nullable + private static Map> headersAsMap(@Nullable Headers headers) { + if (headers == null) { + return null; + } + try { + var map = new HashMap>(); + for (String name : headers.names()) { + map.put(name, headers.values(name)); + } + return map; + } catch (Exception e) { + log.debug("could not read response headers", e); + return null; + } + } + private static String firstNonEmptyLine(byte[] bytes) { int start = 0; for (int i = 0; i <= bytes.length; i++) { @@ -368,12 +393,27 @@ private void onFirstByte() { /** Called back by {@link TeeInputStream} when the stream is fully drained or closed. */ private void onStreamClosed() { try { + // Scooped first and unconditionally: tagSpanFromBuffer bails on an empty body, + // and a failed request is both the likeliest source of one and the case where + // the vendor's request id is most worth having. + InstrumentationSemConv.tagLLMSpanIdHeaders(span, headersAsMap(delegate.headers())); + // Synchronize on teeBuffer to ensure any write() that was in-flight on a // concurrent read thread has fully completed before we snapshot the bytes. byte[] bytes; synchronized (teeBuffer) { bytes = teeBuffer.toByteArray(); } + + // Recorded before tagging: openai-java raises above this layer, so the error + // status is ours alone to set, and losing it to a body-parsing problem is worse + // than losing the parsed output. + int statusCode = delegate.statusCode(); + if (statusCode >= 400) { + InstrumentationSemConv.tagLLMSpanHttpError( + span, statusCode, new String(bytes, StandardCharsets.UTF_8)); + } + // tagLLMSpanResponse also emits child spans for any server-side tool calls (web // search, etc.) nested under the LLM span while it is still live. No-op for Chat // Completions responses (no `output` array). diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java index 218a4175..7f4ae514 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java @@ -24,6 +24,7 @@ import dev.braintrust.TestHarness; import dev.braintrust.instrumentation.Instrumenter; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.sdk.trace.data.SpanData; import java.util.List; import java.util.Map; @@ -649,5 +650,65 @@ private static void assertValidOpenAISpan(SpanData span, boolean isStreaming) { assertNotNull( attributes.get(AttributeKey.stringKey("braintrust.output_json")), "output must be set"); + assertOpenAIIdsCaptured(span); + } + + /** + * Both correlation IDs OpenAI hands back. Deliberately asserts only presence and the object-ID + * prefix: {@code x-request-id} is opaque, and OpenAI returns both {@code req_*} and bare UUIDs + * for it, so pinning its shape would be wrong. + */ + private static void assertOpenAIIdsCaptured(SpanData span) { + var attributes = span.getAttributes(); + + String requestId = attributes.get(AttributeKey.stringKey("x-request-id")); + assertNotNull(requestId, "x-request-id header must be captured"); + assertFalse(requestId.isBlank(), "x-request-id must not be blank"); + + String responseId = attributes.get(AttributeKey.stringKey("response_id")); + assertNotNull(responseId, "response_id must be captured from the response body"); + assertTrue( + responseId.startsWith("resp_") || responseId.startsWith("chatcmpl-"), + "unexpected response_id: " + responseId); + } + + /** + * A failed call is the case where the vendor's request id matters most, and the only one where + * it is the *sole* ID available: an error body carries no object id of its own. openai-java + * raises its exception above the HTTP layer we instrument, so the error status on the span is + * ours alone to set. + */ + @Test + @SneakyThrows + void testHttpErrorTagsSpan() { + OpenAIClient openAIClient = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + var request = + ChatCompletionCreateParams.builder() + .model("gpt-4o-mini-nonexistent-model") + .addUserMessage("What is the capital of France?") + .build(); + + assertThrows(Exception.class, () -> openAIClient.chat().completions().create(request)); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + var span = spans.get(0); + + assertEquals( + StatusCode.ERROR, + span.getStatus().getStatusCode(), + "a non-2xx response must mark the span failed"); + + var attributes = span.getAttributes(); + String requestId = attributes.get(AttributeKey.stringKey("x-request-id")); + assertNotNull(requestId, "x-request-id must still be captured on a failed call"); + assertNull( + attributes.get(AttributeKey.stringKey("response_id")), + "an error body has no object id to capture"); } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java index 76ad2255..e81bcf11 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java @@ -122,6 +122,103 @@ public static void tagLLMSpanResponse(Span span, @Nonnull Throwable responseErro span.recordException(responseError); } + /** + * Provider correlation-ID response headers. Captured onto the span under the header's own name: + * attributes no collector handler claims fall through into the Braintrust span's metadata + * verbatim, so the header name doubles as the metadata key and there is no naming convention to + * keep in sync across SDKs. + * + *

A single flat list rather than a per-provider map — OpenAI never sends {@code request-id} + * and Anthropic never sends {@code x-request-id}, so looking for both everywhere costs nothing + * and captures both if some proxy in between happens to send them. + */ + private static final List ID_HEADERS = List.of("x-request-id", "request-id"); + + /** + * Tags any {@link #ID_HEADERS} present onto the span. Best-effort: a header the vendor didn't + * send is simply not tagged. + * + *

Matching is case-insensitive here rather than at the call site because callers disagree — + * openai-java's {@code Headers} is backed by a case-insensitive {@code TreeMap}, while + * langchain4j hands over a plain {@code HashMap}. + */ + public static void tagLLMSpanIdHeaders( + @Nonnull Span span, @Nullable Map> headers) { + if (headers == null || headers.isEmpty()) { + return; + } + for (Map.Entry> entry : headers.entrySet()) { + String name = entry.getKey(); + if (name == null) { + continue; + } + for (String idHeader : ID_HEADERS) { + if (!idHeader.equalsIgnoreCase(name)) { + continue; + } + List values = entry.getValue(); + if (values == null || values.isEmpty()) { + break; + } + String value = values.get(0); + // Values are opaque — OpenAI returns both `req_*` and bare UUIDs for + // x-request-id — so never parse or validate the shape, only the emptiness. + if (value != null && !value.isBlank()) { + span.setAttribute(idHeader, value); + } + break; + } + } + } + + /** + * Marks the span failed for a non-2xx HTTP response. The vendor SDKs raise their exception + * above the HTTP client layer we instrument, so without this an errored call would otherwise + * end with an unset status and no indication anything went wrong. + */ + public static void tagLLMSpanHttpError( + @Nonnull Span span, int statusCode, @Nullable String responseBody) { + span.setStatus(StatusCode.ERROR, httpErrorMessage(statusCode, responseBody)); + } + + /** Prefers the provider's own error message, falling back to the bare status code. */ + private static String httpErrorMessage(int statusCode, @Nullable String responseBody) { + String fallback = "HTTP " + statusCode; + if (responseBody == null || responseBody.isBlank()) { + return fallback; + } + try { + JsonNode error = BraintrustJsonMapper.get().readTree(responseBody).get("error"); + if (error != null) { + // OpenAI nests the text under `error.message`; Anthropic uses the same shape. + JsonNode message = error.isTextual() ? error : error.get("message"); + if (message != null && message.isTextual() && !message.asText().isBlank()) { + return fallback + ": " + message.asText(); + } + } + } catch (Exception e) { + log.debug("could not parse error message out of response body", e); + } + return fallback; + } + + /** + * Tags the provider's object ID for the response ({@code resp_*}, {@code chatcmpl-*}, {@code + * msg_*}) onto the span. Like {@link #tagLLMSpanIdHeaders} this rides the collector's + * fall-through into metadata, but the raw field name {@code id} would be uselessly ambiguous + * there, so it is qualified — same reasoning as {@code tool_id} in {@link + * #openAIToolSpanMetadata}. + * + *

Absent on responses that carry no ID of their own (Bedrock, and any error body), which is + * exactly when the ID headers matter instead. + */ + private static void tagResponseId(Span span, JsonNode responseJson) { + JsonNode id = responseJson.get("id"); + if (id != null && id.isTextual() && !id.asText().isBlank()) { + span.setAttribute("response_id", id.asText()); + } + } + /** * Emit child {@code type:"tool"} spans for built-in tool calls the vendor executed server * side (web search, file search, code interpreter, image generation, remote MCP) that the @@ -200,6 +297,8 @@ private static void tagOpenAIRequest( @SneakyThrows private static void tagOpenAIResponse( Span span, JsonNode responseJson, @Nullable Long timeToFirstTokenNanoseconds) { + tagResponseId(span, responseJson); + // Output — chat completions API uses "choices"; Responses API uses "output"; audio // transcriptions and translations return a text-keyed object. if (responseJson.has("choices")) { @@ -493,6 +592,8 @@ private static void tagAnthropicResponse( String responseBody, JsonNode responseJson, @Nullable Long timeToFirstTokenNanoseconds) { + tagResponseId(span, responseJson); + // Anthropic response is the full Message object — output it whole span.setAttribute("braintrust.output_json", responseBody); diff --git a/test-harness/src/testFixtures/java/dev/braintrust/VCR.java b/test-harness/src/testFixtures/java/dev/braintrust/VCR.java index 5b519f60..073baa6b 100644 --- a/test-harness/src/testFixtures/java/dev/braintrust/VCR.java +++ b/test-harness/src/testFixtures/java/dev/braintrust/VCR.java @@ -20,6 +20,7 @@ import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -474,6 +475,7 @@ private void createProgrammaticStubFromMapping( com.github.tomakehurst.wiremock.client.WireMock.aResponse() .withStatus(status) .withHeader("Content-Type", responseContentType); + copyRecordedResponseHeaders(mapping, response); // Binary event-stream bodies must be served as raw bytes to avoid UTF-8 corruption if (isEventStream) { @@ -499,6 +501,45 @@ private void createProgrammaticStubFromMapping( wireMock.stubFor(stub.willReturn(response)); } + /** + * Replays the recorded response headers onto a programmatically-built stub. + * + *

Mappings that WireMock loads natively serve their recorded headers for free, but the + * programmatic path above rebuilds the response from scratch and would otherwise serve only + * {@code Content-Type} — silently hiding provider headers (rate limits, {@code x-request-id}) + * from any instrumentation under test on exactly the SSE responses these stubs exist for. + */ + private static void copyRecordedResponseHeaders( + JsonNode mapping, + com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder response) { + JsonNode headers = mapping.at("/response/headers"); + if (!headers.isObject()) { + return; + } + Iterator> fields = headers.fields(); + while (fields.hasNext()) { + Map.Entry header = fields.next(); + String name = header.getKey(); + // Content-Type is already set from the cassette; the framing headers belong to + // Jetty, and replaying a recorded Content-Length would contradict the body we serve. + if (name.equalsIgnoreCase("Content-Type") + || name.equalsIgnoreCase("Content-Length") + || name.equalsIgnoreCase("Transfer-Encoding")) { + continue; + } + JsonNode value = header.getValue(); + if (value.isArray()) { + List values = new ArrayList<>(); + value.forEach(v -> values.add(v.asText())); + if (!values.isEmpty()) { + response.withHeader(name, values.toArray(new String[0])); + } + } else if (value.isTextual()) { + response.withHeader(name, value.asText()); + } + } + } + /** * Remove dynamic fields from JSON that change between test runs. Specifically removes * parent.row_ids.span_id and parent.row_ids.root_span_id which are generated by OTEL. diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-6f74c5653e73.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-6f74c5653e73.json new file mode 100644 index 00000000..b37e9a10 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-6f74c5653e73.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "The model `gpt-4o-mini-nonexistent-model` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + } +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-6f74c5653e73.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-6f74c5653e73.json new file mode 100644 index 00000000..688a7045 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-6f74c5653e73.json @@ -0,0 +1,40 @@ +{ + "id" : "8644d738-98a1-3e0d-a6a1-a4be31ef694b", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini-nonexistent-model\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 404, + "bodyFileName" : "chat_completions-6f74c5653e73.json", + "headers" : { + "x-request-id" : "req_0b1561af84e7450b9cd1d6ef27b7d390", + "Server" : "cloudflare", + "CF-Ray" : "a30f1ec42fb7cc88-SEA", + "X-Content-Type-Options" : "nosniff", + "x-openai-proxy-wasm" : "v0.1", + "Date" : "Wed, 26 Aug 2026 01:39:01 GMT", + "set-cookie" : "__cf_bm=mHSfCIGgGG8_gp599rnyPmGl0VaCpsimYqNgbF_d9ag-1787708339.8657904-1.0.1.1-iF3F7PUuAiv8CJm_Yz75CJx8Op47zSr3OLJjs8nmGQHQkgE7ntZg9tOhLcCOVpvr01ltGLo3ka01qoycRLso4skn.L6rCAvhAVxxiV2TB.U6H1wkOhLgKmKKsFx.bX2P; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 26 Aug 2026 02:09:01 GMT", + "Access-Control-Expose-Headers" : [ "CF-Ray", "CF-Ray" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "Vary" : "Origin", + "alt-svc" : "h3=\":443\"; ma=86400", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8644d738-98a1-3e0d-a6a1-a4be31ef694b", + "persistent" : true, + "insertionIndex" : 63 +} \ No newline at end of file