From 0940a6af6366c17d70b83192a4ce498ef6871fc4 Mon Sep 17 00:00:00 2001 From: Aditya Gupta <84338325+MaiAditya@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:15:07 +0530 Subject: [PATCH] fix(ai): give Claude Agent SDK generations their tool results and every output block The Claude Code CLI sends each content block of a response as its own AssistantMessage, and each tool result as its own UserMessage after the generation's message_stop. The processor and PostHogClaudeSDKClient replaced the pending output on every AssistantMessage and promoted queued input at message_stop, so a generation kept only its last output block and the generation after a tool call had no $ai_input. Append output blocks per generation, append tool results to the queued input, and promote queued input when the next main-agent generation starts. A generation with nothing queued still has no input, as before. A request that is restarted before its message_stop drops its partial output. Without stream events, the result fallback reports every block of the final assistant turn. Messages a subagent sends (parent_tool_use_id set) stay out of the main generations' content. --- .../claude-agent-sdk-tool-results.md | 5 + posthog/ai/claude_agent_sdk/client.py | 32 ++- posthog/ai/claude_agent_sdk/formatting.py | 38 ++- posthog/ai/claude_agent_sdk/processor.py | 66 +++-- .../test/ai/claude_agent_sdk/test_client.py | 225 ++++++++++++++++++ 5 files changed, 341 insertions(+), 25 deletions(-) create mode 100644 .sampo/changesets/claude-agent-sdk-tool-results.md diff --git a/.sampo/changesets/claude-agent-sdk-tool-results.md b/.sampo/changesets/claude-agent-sdk-tool-results.md new file mode 100644 index 000000000..c09442ff8 --- /dev/null +++ b/.sampo/changesets/claude-agent-sdk-tool-results.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Claude Agent SDK: give each generation the tool results it was called with, and keep every text and tool-use block of a response in its `$ai_output_choices`. diff --git a/posthog/ai/claude_agent_sdk/client.py b/posthog/ai/claude_agent_sdk/client.py index 5d3d3d458..df1491b0f 100644 --- a/posthog/ai/claude_agent_sdk/client.py +++ b/posthog/ai/claude_agent_sdk/client.py @@ -24,13 +24,17 @@ ) from .formatting import ( + _append_assistant_output, _format_user_input, - format_assistant_blocks, + format_assistant_blocks as format_assistant_blocks, format_tool_result_content as format_tool_result_content, ) from posthog.ai.claude_agent_sdk.processor import ( PostHogClaudeAgentProcessor, _GenerationTracker, + _from_subagent, + _promote_input, + _starts_generation, ) from posthog.client import Client @@ -130,6 +134,8 @@ async def query(self, prompt: str, session_id: str = "default") -> None: """ # Track the prompt as input for the next generation self._current_input = [{"role": "user", "content": prompt}] + self._next_input = None + self._pending_output = [] await self._client.query(prompt, session_id) async def receive_response(self): @@ -143,6 +149,14 @@ async def receive_response(self): def _process_message(self, message: Any) -> None: if isinstance(message, StreamEvent): + if _starts_generation(message): + if self._tracker.current_span_id is not None: + # The previous request never finished (it was retried), + # so its partial output is dropped. + self._pending_output = [] + self._current_input, self._next_input = _promote_input( + self._current_input, self._next_input + ) self._tracker.process_stream_event(message) if self._tracker.has_completed_generation(): @@ -160,14 +174,17 @@ def _process_message(self, message: Any) -> None: self._privacy, self._groups, ) - self._current_input = self._next_input - self._next_input = None + self._current_input = None self._pending_output = [] elif isinstance(message, AssistantMessage): self._process_assistant_message(message) - elif isinstance(message, UserMessage): + elif isinstance(message, UserMessage) and not _from_subagent(message): + if not self._tracker.had_any_stream_events: + # Without stream events, a tool result ends the assistant turn; + # the result fallback reports the final turn only. + self._pending_output = [] self._next_input = _format_user_input( message.content, self._processor._client, self._next_input ) @@ -203,9 +220,10 @@ def _process_assistant_message(self, message: "AssistantMessage") -> None: self._privacy, self._groups, ) - output_content = format_assistant_blocks(message.content) - if output_content: - self._pending_output = [{"role": "assistant", "content": output_content}] + if not _from_subagent(message): + self._pending_output = _append_assistant_output( + self._pending_output, message.content + ) async def disconnect(self) -> None: """ diff --git a/posthog/ai/claude_agent_sdk/formatting.py b/posthog/ai/claude_agent_sdk/formatting.py index a99c8cbd3..bf8cbcbc6 100644 --- a/posthog/ai/claude_agent_sdk/formatting.py +++ b/posthog/ai/claude_agent_sdk/formatting.py @@ -55,11 +55,45 @@ def format_assistant_blocks(blocks: Any) -> List[Dict[str, Any]]: return out +def _append_assistant_output( + pending: List[Dict[str, Any]], blocks: Any +) -> List[Dict[str, Any]]: + """Add one AssistantMessage's blocks to the output of the current generation. + + The CLI sends each content block of a response as its own AssistantMessage. + """ + output_content = format_assistant_blocks(blocks) + if not output_content: + return pending + previous = pending[0]["content"] if pending else [] + return [{"role": "assistant", "content": previous + output_content}] + + +def _append_user_input( + previous: Optional[List[Dict[str, Any]]], content: Any +) -> List[Dict[str, Any]]: + queued = previous or [] + last = queued[-1] if queued else None + if ( + isinstance(content, list) + and last is not None + and last.get("role") == "user" + and isinstance(last.get("content"), list) + ): + return queued[:-1] + [{"role": "user", "content": last["content"] + content}] + return queued + [{"role": "user", "content": content}] + + def _format_user_input( content: Any, client: Any, previous: Optional[List[Dict[str, Any]]] ) -> Optional[List[Dict[str, Any]]]: + """Queue a UserMessage as input for the next generation. + + The CLI sends each tool result as its own UserMessage, so results are + appended to what is already queued instead of replacing it. + """ if isinstance(content, str): - return [{"role": "user", "content": content}] + return _append_user_input(previous, content) if isinstance(content, list): formatted: List[Dict[str, Any]] = [] for block in content: @@ -74,5 +108,5 @@ def _format_user_input( elif hasattr(block, "text"): formatted.append({"type": "text", "text": block.text}) if formatted: - return [{"role": "user", "content": formatted}] + return _append_user_input(previous, formatted) return previous diff --git a/posthog/ai/claude_agent_sdk/processor.py b/posthog/ai/claude_agent_sdk/processor.py index 022ad8cc5..0f86feeb0 100644 --- a/posthog/ai/claude_agent_sdk/processor.py +++ b/posthog/ai/claude_agent_sdk/processor.py @@ -26,8 +26,9 @@ from posthog import setup from .formatting import ( + _append_assistant_output, _format_user_input, - format_assistant_blocks, + format_assistant_blocks as format_assistant_blocks, format_tool_result_content as format_tool_result_content, ) from posthog.ai.media import ensure_serializable as _ensure_serializable @@ -307,12 +308,14 @@ async def _instrument_query( # Two-slot input tracking: # - current_input: input for the generation currently in progress - # - next_input: tool results that arrive mid-turn, queued for the next generation + # - next_input: tool results queued for the next generation # - # Message ordering from the SDK is: - # message_start → content_blocks → AssistantMessage → UserMessage(tool result) → message_stop - # So UserMessage arrives *before* message_stop. When message_stop fires we emit - # with current_input, then promote next_input → current_input for the next turn. + # The CLI sends each content block as its own AssistantMessage before + # message_stop, and each tool result as its own UserMessage after it. + # Queued input is promoted when the next generation starts, so it + # does not matter which side of message_stop the tool results land on. + # Input is used by one generation only: a generation with nothing + # queued has no input. next_input: Optional[List[Dict[str, Any]]] = None # Accumulate assistant output per generation @@ -325,6 +328,14 @@ async def _instrument_query( # never interrupt the underlying Claude Agent SDK query. try: if isinstance(message, StreamEvent): + if _starts_generation(message): + if tracker.current_span_id is not None: + # The previous request never finished (it was + # retried), so its partial output is dropped. + pending_output = [] + current_input, next_input = _promote_input( + current_input, next_input + ) tracker.process_stream_event(message) # Emit $ai_generation when a turn completes @@ -343,9 +354,7 @@ async def _instrument_query( privacy, groups, ) - # Promote: tool results from this turn become input for next turn - current_input = next_input - next_input = None + current_input = None pending_output = [] elif isinstance(message, AssistantMessage): @@ -361,9 +370,13 @@ async def _instrument_query( pending_output, ) - elif isinstance(message, UserMessage): - # UserMessages carry tool results. They arrive *before* message_stop - # for the current turn, so queue them as input for the *next* generation. + elif isinstance(message, UserMessage) and not _from_subagent(message): + # UserMessages carry tool results; queue them as input for the + # *next* generation. + if not tracker.had_any_stream_events: + # Without stream events, a tool result ends the assistant + # turn; the result fallback reports the final turn only. + pending_output = [] next_input = _format_user_input( message.content, self._client, next_input ) @@ -455,10 +468,10 @@ def _process_assistant_message( privacy, groups, ) - output_content = format_assistant_blocks(message.content) - if output_content: - pending_output = [{"role": "assistant", "content": output_content}] - return pending_output + if _from_subagent(message): + # A subagent's own API calls are not this query's generations. + return pending_output + return _append_assistant_output(pending_output, message.content) def _emit_generation( self, @@ -678,6 +691,27 @@ def _resolve_distinct_id( return self._get_distinct_id(result) +def _from_subagent(message: Any) -> bool: + return getattr(message, "parent_tool_use_id", None) is not None + + +def _starts_generation(message: "StreamEvent") -> bool: + """Whether a main-agent API call is starting (subagent calls are skipped).""" + return ( + not _from_subagent(message) + and isinstance(message.event, dict) + and message.event.get("type") == "message_start" + ) + + +def _promote_input( + current_input: Optional[List[Dict[str, Any]]], + next_input: Optional[List[Dict[str, Any]]], +) -> tuple[Optional[List[Dict[str, Any]]], None]: + """Make queued input the input of the generation that is starting.""" + return (current_input if next_input is None else next_input), None + + def _query_inputs( prompt: Any, options: ClaudeAgentOptions ) -> tuple[List[Dict[str, Any]], Optional[List[Dict[str, Any]]]]: diff --git a/posthog/test/ai/claude_agent_sdk/test_client.py b/posthog/test/ai/claude_agent_sdk/test_client.py index acdd38ac3..6ceb3452a 100644 --- a/posthog/test/ai/claude_agent_sdk/test_client.py +++ b/posthog/test/ai/claude_agent_sdk/test_client.py @@ -1,5 +1,6 @@ """Stateful Claude client delivery, turn boundaries, and teardown contracts.""" +from dataclasses import replace from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -370,3 +371,227 @@ async def consume(): assert finalized.is_set() ph.capture.assert_not_called() ph.flush.assert_not_called() + + +def cli_tool_turn(): + """Message order the Claude Code CLI (2.1.212) emits for a turn with two tool calls. + + Each content block arrives as its own AssistantMessage, and each tool result + arrives as its own UserMessage after the generation's message_stop. + """ + return [ + stream( + "message_start", message={"model": "claude", "usage": {"input_tokens": 12}} + ), + AssistantMessage(model="claude", content=[TextBlock(text="Reading.")]), + AssistantMessage( + model="claude", + content=[ToolUseBlock(id="tool-1", name="Read", input={"path": "a"})], + ), + AssistantMessage( + model="claude", + content=[ToolUseBlock(id="tool-2", name="Read", input={"path": "b"})], + ), + stream("message_delta", usage={"output_tokens": 4}), + stream("message_stop"), + UserMessage(content=[ToolResultBlock(tool_use_id="tool-2", content="b text")]), + UserMessage(content=[ToolResultBlock(tool_use_id="tool-1", content="a text")]), + stream( + "message_start", message={"model": "claude", "usage": {"input_tokens": 20}} + ), + AssistantMessage(model="claude", content=[TextBlock(text="Done.")]), + stream("message_delta", usage={"output_tokens": 2}), + stream("message_stop"), + result(), + ] + + +async def run_turn(sdk, monkeypatch, use_client, messages): + ph = analytics() + if use_client: + client = PostHogClaudeSDKClient(posthog_client=ph) + await client.query("question") + respond(sdk, messages) + assert [msg async for msg in client.receive_response()] == messages + else: + + async def query(**kwargs): + for message in messages: + yield message + + monkeypatch.setattr( + "posthog.ai.claude_agent_sdk.processor.original_query", query + ) + processor = PostHogClaudeAgentProcessor(client=ph) + assert [msg async for msg in processor.query(prompt="question")] == messages + return [event["properties"] for event in events(ph, "$ai_generation")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_cli_order_keeps_every_output_block_in_its_generation( + sdk, monkeypatch, use_client +): + one, two = await run_turn(sdk, monkeypatch, use_client, cli_tool_turn()) + assert one["$ai_output_choices"] == [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Reading."}, + { + "type": "function", + "function": {"name": "Read", "arguments": {"path": "a"}}, + }, + { + "type": "function", + "function": {"name": "Read", "arguments": {"path": "b"}}, + }, + ], + } + ] + assert two["$ai_output_choices"] == [ + {"role": "assistant", "content": [{"type": "text", "text": "Done."}]} + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_cli_order_gives_tool_results_to_the_next_generation( + sdk, monkeypatch, use_client +): + one, two = await run_turn(sdk, monkeypatch, use_client, cli_tool_turn()) + assert one["$ai_input"] == [{"role": "user", "content": "question"}] + assert two["$ai_input"] == [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "tool-2", "content": "b text"}, + {"type": "tool_result", "tool_use_id": "tool-1", "content": "a text"}, + ], + } + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_subagent_messages_stay_out_of_main_generation_content( + sdk, monkeypatch, use_client +): + turn = cli_tool_turn() + # The CLI forwards a subagent's messages with parent_tool_use_id set while + # the tool that started it runs, between the two main generations. + turn[7:7] = [ + AssistantMessage( + model="claude", + content=[TextBlock(text="subagent says")], + parent_tool_use_id="tool-1", + ), + UserMessage( + content=[ToolResultBlock(tool_use_id="sub-tool", content="sub result")], + parent_tool_use_id="tool-1", + ), + ] + one, two = await run_turn(sdk, monkeypatch, use_client, turn) + assert [b["tool_use_id"] for b in two["$ai_input"][0]["content"]] == [ + "tool-2", + "tool-1", + ] + assert two["$ai_output_choices"] == [ + {"role": "assistant", "content": [{"type": "text", "text": "Done."}]} + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_subagent_stream_events_do_not_take_queued_tool_results( + sdk, monkeypatch, use_client +): + turn = cli_tool_turn() + # The subagent started by tool-1 makes its own API call after tool-2's + # result is queued and before tool-1's result arrives. + turn[7:7] = [ + replace(stream(kind), parent_tool_use_id="tool-1") + for kind in ("message_start", "message_stop") + ] + generations = await run_turn(sdk, monkeypatch, use_client, turn) + assert [b["tool_use_id"] for b in generations[-1]["$ai_input"][0]["content"]] == [ + "tool-2", + "tool-1", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_retried_request_reports_only_the_finished_attempt( + sdk, monkeypatch, use_client +): + messages = [ + stream("message_start", message={"model": "claude", "usage": {}}), + AssistantMessage(model="claude", content=[TextBlock(text="partial")]), + stream("message_start", message={"model": "claude", "usage": {}}), + AssistantMessage(model="claude", content=[TextBlock(text="full")]), + stream("message_stop"), + result(), + ] + (generation,) = await run_turn(sdk, monkeypatch, use_client, messages) + assert generation["$ai_input"] == [{"role": "user", "content": "question"}] + assert generation["$ai_output_choices"] == [ + {"role": "assistant", "content": [{"type": "text", "text": "full"}]} + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_generation_without_queued_input_has_no_input( + sdk, monkeypatch, use_client +): + messages = [ + stream("message_start", message={"model": "claude", "usage": {}}), + AssistantMessage(model="claude", content=[TextBlock(text="one")]), + stream("message_stop"), + stream("message_start", message={"model": "claude", "usage": {}}), + AssistantMessage(model="claude", content=[TextBlock(text="two")]), + stream("message_stop"), + result(), + ] + one, two = await run_turn(sdk, monkeypatch, use_client, messages) + assert one["$ai_input"] == [{"role": "user", "content": "question"}] + assert "$ai_input" not in two + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_result_fallback_reports_every_block_of_the_final_turn_only( + sdk, monkeypatch, use_client +): + turn = [m for m in cli_tool_turn() if not isinstance(m, StreamEvent)] + turn[-1:-1] = [AssistantMessage(model="claude", content=[TextBlock(text="Bye.")])] + (generation,) = await run_turn(sdk, monkeypatch, use_client, turn) + assert generation["$ai_input"] == [{"role": "user", "content": "question"}] + assert generation["$ai_output_choices"] == [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Done."}, + {"type": "text", "text": "Bye."}, + ], + } + ] + + +@pytest.mark.asyncio +async def test_result_fallback_does_not_carry_output_into_the_next_query(sdk): + ph = analytics() + client = PostHogClaudeSDKClient(posthog_client=ph) + for prompt in ("first", "second"): + await client.query(prompt) + messages = [ + AssistantMessage(model="claude", content=[TextBlock(text=prompt)]), + result(), + ] + respond(sdk, messages) + assert [msg async for msg in client.receive_response()] == messages + second = events(ph, "$ai_generation")[1]["properties"] + assert second["$ai_output_choices"] == [ + {"role": "assistant", "content": [{"type": "text", "text": "second"}]} + ]