Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/claude-agent-sdk-tool-results.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

Claude Agent SDK: give each generation the tool results it was called with, and keep every text and tool-use block of a response in its `$ai_output_choices`.
32 changes: 25 additions & 7 deletions posthog/ai/claude_agent_sdk/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,17 @@
)

from .formatting import (
_append_assistant_output,
_format_user_input,
format_assistant_blocks,
format_assistant_blocks as format_assistant_blocks,
format_tool_result_content as format_tool_result_content,
)
from posthog.ai.claude_agent_sdk.processor import (
PostHogClaudeAgentProcessor,
_GenerationTracker,
_from_subagent,
_promote_input,
_starts_generation,
)
from posthog.client import Client

Expand Down Expand Up @@ -130,6 +134,8 @@ async def query(self, prompt: str, session_id: str = "default") -> None:
"""
# Track the prompt as input for the next generation
self._current_input = [{"role": "user", "content": prompt}]
self._next_input = None
self._pending_output = []
await self._client.query(prompt, session_id)

async def receive_response(self):
Expand All @@ -143,6 +149,14 @@ async def receive_response(self):

def _process_message(self, message: Any) -> None:
if isinstance(message, StreamEvent):
if _starts_generation(message):
if self._tracker.current_span_id is not None:
# The previous request never finished (it was retried),
# so its partial output is dropped.
self._pending_output = []
self._current_input, self._next_input = _promote_input(
self._current_input, self._next_input
)
self._tracker.process_stream_event(message)

if self._tracker.has_completed_generation():
Expand All @@ -160,14 +174,17 @@ def _process_message(self, message: Any) -> None:
self._privacy,
self._groups,
)
self._current_input = self._next_input
self._next_input = None
self._current_input = None
self._pending_output = []

elif isinstance(message, AssistantMessage):
self._process_assistant_message(message)

elif isinstance(message, UserMessage):
elif isinstance(message, UserMessage) and not _from_subagent(message):
if not self._tracker.had_any_stream_events:
# Without stream events, a tool result ends the assistant turn;
# the result fallback reports the final turn only.
self._pending_output = []
self._next_input = _format_user_input(
message.content, self._processor._client, self._next_input
)
Expand Down Expand Up @@ -203,9 +220,10 @@ def _process_assistant_message(self, message: "AssistantMessage") -> None:
self._privacy,
self._groups,
)
output_content = format_assistant_blocks(message.content)
if output_content:
self._pending_output = [{"role": "assistant", "content": output_content}]
if not _from_subagent(message):
self._pending_output = _append_assistant_output(
self._pending_output, message.content
)

async def disconnect(self) -> None:
"""
Expand Down
38 changes: 36 additions & 2 deletions posthog/ai/claude_agent_sdk/formatting.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,45 @@ def format_assistant_blocks(blocks: Any) -> List[Dict[str, Any]]:
return out


def _append_assistant_output(
pending: List[Dict[str, Any]], blocks: Any
) -> List[Dict[str, Any]]:
"""Add one AssistantMessage's blocks to the output of the current generation.

The CLI sends each content block of a response as its own AssistantMessage.
"""
output_content = format_assistant_blocks(blocks)
if not output_content:
return pending
previous = pending[0]["content"] if pending else []
return [{"role": "assistant", "content": previous + output_content}]


def _append_user_input(
previous: Optional[List[Dict[str, Any]]], content: Any
) -> List[Dict[str, Any]]:
queued = previous or []
last = queued[-1] if queued else None
if (
isinstance(content, list)
and last is not None
and last.get("role") == "user"
and isinstance(last.get("content"), list)
):
return queued[:-1] + [{"role": "user", "content": last["content"] + content}]
return queued + [{"role": "user", "content": content}]


def _format_user_input(
content: Any, client: Any, previous: Optional[List[Dict[str, Any]]]
) -> Optional[List[Dict[str, Any]]]:
"""Queue a UserMessage as input for the next generation.

The CLI sends each tool result as its own UserMessage, so results are
appended to what is already queued instead of replacing it.
"""
if isinstance(content, str):
return [{"role": "user", "content": content}]
return _append_user_input(previous, content)
if isinstance(content, list):
formatted: List[Dict[str, Any]] = []
for block in content:
Expand All @@ -74,5 +108,5 @@ def _format_user_input(
elif hasattr(block, "text"):
formatted.append({"type": "text", "text": block.text})
if formatted:
return [{"role": "user", "content": formatted}]
return _append_user_input(previous, formatted)
return previous
66 changes: 50 additions & 16 deletions posthog/ai/claude_agent_sdk/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@

from posthog import setup
from .formatting import (
_append_assistant_output,
_format_user_input,
format_assistant_blocks,
format_assistant_blocks as format_assistant_blocks,
format_tool_result_content as format_tool_result_content,
)
from posthog.ai.media import ensure_serializable as _ensure_serializable
Expand Down Expand Up @@ -307,12 +308,14 @@ async def _instrument_query(

# Two-slot input tracking:
# - current_input: input for the generation currently in progress
# - next_input: tool results that arrive mid-turn, queued for the next generation
# - next_input: tool results queued for the next generation
#
# Message ordering from the SDK is:
# message_start → content_blocks → AssistantMessage → UserMessage(tool result) → message_stop
# So UserMessage arrives *before* message_stop. When message_stop fires we emit
# with current_input, then promote next_input → current_input for the next turn.
# The CLI sends each content block as its own AssistantMessage before
# message_stop, and each tool result as its own UserMessage after it.
# Queued input is promoted when the next generation starts, so it
# does not matter which side of message_stop the tool results land on.
# Input is used by one generation only: a generation with nothing
# queued has no input.
next_input: Optional[List[Dict[str, Any]]] = None

# Accumulate assistant output per generation
Expand All @@ -325,6 +328,14 @@ async def _instrument_query(
# never interrupt the underlying Claude Agent SDK query.
try:
if isinstance(message, StreamEvent):
if _starts_generation(message):
if tracker.current_span_id is not None:
# The previous request never finished (it was
# retried), so its partial output is dropped.
pending_output = []
current_input, next_input = _promote_input(
current_input, next_input
)
tracker.process_stream_event(message)

# Emit $ai_generation when a turn completes
Expand All @@ -343,9 +354,7 @@ async def _instrument_query(
privacy,
groups,
)
# Promote: tool results from this turn become input for next turn
current_input = next_input
next_input = None
current_input = None
pending_output = []

elif isinstance(message, AssistantMessage):
Expand All @@ -361,9 +370,13 @@ async def _instrument_query(
pending_output,
)

elif isinstance(message, UserMessage):
# UserMessages carry tool results. They arrive *before* message_stop
# for the current turn, so queue them as input for the *next* generation.
elif isinstance(message, UserMessage) and not _from_subagent(message):
# UserMessages carry tool results; queue them as input for the
# *next* generation.
if not tracker.had_any_stream_events:
# Without stream events, a tool result ends the assistant
# turn; the result fallback reports the final turn only.
pending_output = []
next_input = _format_user_input(
message.content, self._client, next_input
)
Expand Down Expand Up @@ -455,10 +468,10 @@ def _process_assistant_message(
privacy,
groups,
)
output_content = format_assistant_blocks(message.content)
if output_content:
pending_output = [{"role": "assistant", "content": output_content}]
return pending_output
if _from_subagent(message):
# A subagent's own API calls are not this query's generations.
return pending_output
return _append_assistant_output(pending_output, message.content)

def _emit_generation(
self,
Expand Down Expand Up @@ -678,6 +691,27 @@ def _resolve_distinct_id(
return self._get_distinct_id(result)


def _from_subagent(message: Any) -> bool:
return getattr(message, "parent_tool_use_id", None) is not None


def _starts_generation(message: "StreamEvent") -> bool:
"""Whether a main-agent API call is starting (subagent calls are skipped)."""
return (
not _from_subagent(message)
and isinstance(message.event, dict)
and message.event.get("type") == "message_start"
)


def _promote_input(
current_input: Optional[List[Dict[str, Any]]],
next_input: Optional[List[Dict[str, Any]]],
) -> tuple[Optional[List[Dict[str, Any]]], None]:
"""Make queued input the input of the generation that is starting."""
return (current_input if next_input is None else next_input), None


def _query_inputs(
prompt: Any, options: ClaudeAgentOptions
) -> tuple[List[Dict[str, Any]], Optional[List[Dict[str, Any]]]]:
Expand Down
Loading
Loading