diff --git a/CHANGELOG.md b/CHANGELOG.md index fc4f8a8..1008a81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [0.2.0] - 2026-08-14 + +### Added + +- The RubyLLM adapter captures conversation content when the configuration's + `capture_bodies` is enabled: `llm.prompt`, `llm.instructions` and + `llm.completion` on the root span, and `tool.arguments` / `tool.result` on + each tool span. Each value is truncated to 4,000 characters + (`RubyLLM::CONTENT_LIMIT`). `llm.instructions` joins every system message, + since RubyLLM's `with_instructions` appends by default and reporting only + the last one would hide a layered base prompt. A tool span records a result + only when the call succeeded; a raised tool reports its truncated error + message instead. + + Capture stays **off** by default. Prompts and tool results carry whatever + the application sends the model, so enabling this is a data-handling + decision — the truncation is a cap on trace size, not a redaction boundary. + ## [Unreleased] ### Added diff --git a/README.md b/README.md index 9e66e00..ff9c873 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ root SupportBot.respond 1,240ms OK └─ tool tool.search_docs 310ms OK ``` -Tool arguments and results are never sent. Error messages are truncated and +Prompts, completions, and tool arguments/results are sent only when +`capture_bodies` is enabled (off by default). Error messages are truncated and backtraces are never transmitted. ## Self-hosting diff --git a/adapters/ruby_llm/README.md b/adapters/ruby_llm/README.md index d6db536..311b22e 100644 --- a/adapters/ruby_llm/README.md +++ b/adapters/ruby_llm/README.md @@ -48,7 +48,9 @@ enclosing event, 2.x drives a flat `step until complete?` loop whose rounds are siblings with tool calls between them. Rounds are accumulated and flushed on the round that ends the turn, so both produce the same trace. -Tool arguments and results are never sent; error messages are truncated. +Prompts, completions, and tool arguments/results are sent only when the +configuration's `capture_bodies` is enabled (off by default, truncated to +4,000 characters); error messages are truncated. ## Naming the traffic diff --git a/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm.rb b/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm.rb index 18e575f..f8687ca 100644 --- a/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm.rb +++ b/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm.rb @@ -27,7 +27,9 @@ module Telemetry # Tokens are summed per round from the assistant messages that round added, # so a repeated event-level count is never double counted. # - # Tool arguments and results are never sent; error messages are truncated. + # Prompts, completions, and tool arguments/results are not sent unless the + # configuration's capture_bodies is enabled (they may contain sensitive + # data); error messages are truncated. module RubyLLM AGENT_KEY = :activeagents_telemetry_ruby_llm_agent STATE_KEY = :activeagents_telemetry_ruby_llm_state @@ -36,6 +38,8 @@ module RubyLLM # A turn that never reaches a final round (a halted tool call, or an app # driving RubyLLM 2.x's `step` by hand) would otherwise accumulate forever. MAX_TURN_SECONDS = 600 + # Captured prompt/completion/tool content is truncated to this many characters. + CONTENT_LIMIT = 4_000 DEFAULT_AGENT = { name: "RubyLLM::Chat", action: "chat" }.freeze @@ -139,11 +143,16 @@ def finish_round(payload) def build_tool_span(payload, started_at, finished_at) error = payload[:exception_object] + attributes = { "tool.name" => payload[:tool_name].to_s, "tool.call_id" => payload[:tool_call_id].to_s } + if configuration.capture_bodies? + attributes["tool.arguments"] = tool_io_json(payload[:tool_arguments]) + attributes["tool.result"] = tool_io_json(payload[:result_content]) unless error + end span = Span.new( "tool.#{payload[:tool_name]}", type: "tool", start_time: started_at, - attributes: { "tool.name" => payload[:tool_name].to_s, "tool.call_id" => payload[:tool_call_id].to_s } + attributes: attributes ) span.record_error(error, message_limit: configuration.error_message_limit) if error span.finish(at: finished_at) @@ -163,14 +172,17 @@ def report_turn(payload, turn) resource_attributes: configuration.resource_attributes ) + root_attributes = { + "agent.class" => agent[:name], + "agent.action" => agent[:action], + "agent.provider" => payload[:provider].to_s, + "agent.model" => payload[:model].to_s + } + root_attributes.merge!(conversation_attributes(payload)) if configuration.capture_bodies? + root = trace.span( "#{agent[:name]}.#{agent[:action]}", type: "root", start_time: started_at, - attributes: { - "agent.class" => agent[:name], - "agent.action" => agent[:action], - "agent.provider" => payload[:provider].to_s, - "agent.model" => payload[:model].to_s - } + attributes: root_attributes ) llm = trace.span( @@ -215,6 +227,59 @@ def resolve_agent(payload) nil end + # The prompt that opened the turn and the answer that closed it — the two + # ends a trace is otherwise missing. System instructions are included: + # they are the most common cause of a surprising answer. + def conversation_attributes(payload) + input = Array(payload[:input_messages]) + new_messages = Array(payload[:messages_after])[input.size..] || [] + + attributes = {} + if (prompt = last_message_text(input, "user")) + attributes["llm.prompt"] = truncate_captured(prompt) + end + if (instructions = system_instructions(input)) + attributes["llm.instructions"] = truncate_captured(instructions) + end + if (completion = last_message_text(new_messages, "assistant")) + attributes["llm.completion"] = truncate_captured(completion) + end + attributes + end + + # RubyLLM's `with_instructions` appends by default, so a chat can carry + # several system messages and the model sees all of them. Join rather + # than taking the last, or an app that layers a base prompt with a + # per-request one would report only the fragment. + def system_instructions(messages) + texts = messages.select do |candidate| + candidate.respond_to?(:role) && candidate.role.to_s == "system" && + candidate.respond_to?(:content) && !candidate.content.to_s.empty? + end.map { |message| message.content.to_s } + + texts.empty? ? nil : texts.join("\n\n") + end + + def last_message_text(messages, role) + message = messages.reverse.find do |candidate| + candidate.respond_to?(:role) && candidate.role.to_s == role && + candidate.respond_to?(:content) && !candidate.content.to_s.empty? + end + text = message&.content.to_s + text.empty? ? nil : text + end + + def tool_io_json(value) + json = value.is_a?(String) ? value : JSON.generate(value) + truncate_captured(json) + rescue StandardError + value.inspect[0, CONTENT_LIMIT] + end + + def truncate_captured(text) + text.to_s[0, CONTENT_LIMIT] + end + def token_totals(payload) initial_count = Array(payload[:input_messages]).size new_messages = Array(payload[:messages_after])[initial_count..] || [] diff --git a/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm/version.rb b/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm/version.rb index 7e7125e..220f1f1 100644 --- a/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm/version.rb +++ b/adapters/ruby_llm/lib/activeagents/telemetry/ruby_llm/version.rb @@ -3,7 +3,7 @@ module ActiveAgents module Telemetry module RubyLLM - VERSION = "0.1.0" + VERSION = "0.2.0" end end end diff --git a/adapters/ruby_llm/test/test_ruby_llm.rb b/adapters/ruby_llm/test/test_ruby_llm.rb index 3dd3972..469cbb7 100644 --- a/adapters/ruby_llm/test/test_ruby_llm.rb +++ b/adapters/ruby_llm/test/test_ruby_llm.rb @@ -144,6 +144,50 @@ def test_inherits_destination_from_the_shared_configuration ActiveAgents::Telemetry.reset! end + ContentMsg = Struct.new(:role, :content, :input_tokens, :output_tokens, :thinking_tokens) + + def test_does_not_capture_content_by_default + payload = chat_payload(input_messages: [ ContentMsg.new("user", "what is up?") ]) + instrument("tool_call.ruby_llm", tool_name: "search", tool_call_id: "t1", tool_arguments: { q: "x" }, result_content: "y") { nil } + instrument("chat.ruby_llm", payload) do + payload[:messages_after] = payload[:input_messages] + [ ContentMsg.new("assistant", "not much", 1, 1, 0) ] + end + + trace = traces.first + root_attributes = spans_of(trace, "root").first["attributes"] + tool_attributes = spans_of(trace, "tool").first["attributes"] + refute_includes root_attributes.keys, "llm.prompt" + refute_includes root_attributes.keys, "llm.completion" + refute_includes tool_attributes.keys, "tool.arguments" + refute_includes tool_attributes.keys, "tool.result" + end + + def test_captures_conversation_and_tool_io_when_capture_bodies_enabled + ActiveAgents::Telemetry.configure { |config| config.capture_bodies = true } + subscribe + + payload = chat_payload(input_messages: [ + ContentMsg.new("system", "Be terse."), + ContentMsg.new("system", "Answer in English."), + ContentMsg.new("user", "what is up?" * 1_000) + ]) + instrument("tool_call.ruby_llm", tool_name: "search", tool_call_id: "t1", tool_arguments: { q: "x" }, result_content: "y") { nil } + instrument("chat.ruby_llm", payload) do + payload[:messages_after] = payload[:input_messages] + [ ContentMsg.new("assistant", "not much", 1, 1, 0) ] + end + + trace = traces.first + root_attributes = spans_of(trace, "root").first["attributes"] + tool_attributes = spans_of(trace, "tool").first["attributes"] + assert_equal ActiveAgents::Telemetry::RubyLLM::CONTENT_LIMIT, root_attributes["llm.prompt"].length + assert_equal "Be terse.\n\nAnswer in English.", root_attributes["llm.instructions"] + assert_equal "not much", root_attributes["llm.completion"] + assert_equal '{"q":"x"}', tool_attributes["tool.arguments"] + assert_equal "y", tool_attributes["tool.result"] + ensure + ActiveAgents::Telemetry.reset! + end + private def subscribe_inheriting diff --git a/lib/activeagents/telemetry/version.rb b/lib/activeagents/telemetry/version.rb index f4ead21..b5747b4 100644 --- a/lib/activeagents/telemetry/version.rb +++ b/lib/activeagents/telemetry/version.rb @@ -2,6 +2,6 @@ module ActiveAgents module Telemetry - VERSION = "0.1.0" + VERSION = "0.2.0" end end