diff --git a/docs/examples/components/README.md b/docs/examples/components/README.md new file mode 100644 index 0000000000..30ccacda4c --- /dev/null +++ b/docs/examples/components/README.md @@ -0,0 +1,113 @@ +# Component Examples + +This directory contains examples demonstrating Mellea's component system, particularly focusing on component ID-based tool prefixing for collision-free tool composition. + +## Files + +### `duplicate_tool_names_public_api.py` (Public API - Recommended) +**Modern example** - Demonstrates component ID-based tool prefixing using public APIs only. + +**What it shows:** +- Two components with identical tool names handled via automatic prefixing +- Public `MelleaSession.act()` API with `tool_calls=True` +- Backend automatically extracts and prefixes tools (no manual extraction needed) +- Tool execution via public `call_tools()` API with telemetry support +- All public APIs, no private imports + +**Important:** Uses `ChatContext` instead of `SimpleContext` (required for tool extraction from session context). + +**Run it:** +```bash +uv run python docs/examples/components/duplicate_tool_names_public_api.py +uv run pytest docs/examples/components/duplicate_tool_names_public_api.py -v +``` + +**View telemetry metrics:** +```bash +export MELLEA_METRICS_ENABLED=true +export MELLEA_METRICS_CONSOLE=true +uv run python docs/examples/components/duplicate_tool_names_public_api.py +``` + +**Key points:** +- Uses public `act()` with `tool_calls=True` instead of `instruct()` + explicit tool passing +- **Requires `ChatContext`** — `SimpleContext` returns empty history and tools won't be extracted +- Backend auto-extracts tools from context components (no `ModelOption.TOOLS` needed) +- **Important:** `act()` does NOT automatically execute tools; call `call_tools()` explicitly to execute them and record telemetry +- Cleaner, more declarative API for handling multiple components + +--- + +### `pattern2_public_api.py` (Public API - Recommended) +**Modern example** - Shows Pattern 2 (components in context) using public APIs only. + +**What it shows:** +- Components with templates live in session context +- Public `MelleaSession.act()` API with `tool_calls=True` +- Backend automatically extracts tools from all context components +- Tool execution via public `call_tools()` API with telemetry support +- Multi-component composition with stable component IDs +- All public APIs, no private imports + +**Important:** Uses `ChatContext` instead of `SimpleContext` (required for tool extraction from session context). + +**Run it:** +```bash +uv run python docs/examples/components/pattern2_public_api.py +uv run pytest docs/examples/components/pattern2_public_api.py -v +``` + +**View telemetry metrics:** +```bash +export MELLEA_METRICS_ENABLED=true +export MELLEA_METRICS_CONSOLE=true +uv run python docs/examples/components/pattern2_public_api.py +``` + +**Key concepts:** +- Pattern 1: Extract tools only (simple tool calling) → use `duplicate_tool_names_public_api.py` +- Pattern 2: Components in context with auto-extraction → use `pattern2_public_api.py` +- Both patterns use component ID-based prefixing +- **Requires `ChatContext`** — `SimpleContext` is stateless and won't extract tools from context +- Backend auto-extracts tools—NO explicit `ModelOption.TOOLS` needed +- **Important:** `act()` only extracts and passes tools to LLM; doesn't execute them. Call `call_tools(response, backend)` to execute tool calls and trigger telemetry +- Components must have valid templates for rendering + +--- + +## Concepts + +### Component ID-Based Prefixing + +When multiple components define tools with the same name, Mellea prevents collisions by prefixing each tool name with its component ID: + +``` +Original: query, query +Prefixed: component_1adeba40__query, component_1c611a00__query +``` + +**How it works:** +1. Each component instance gets a unique ID: `hex(id(object))[-8:]` +2. Tools from each component are extracted and prefixed +3. Prefixed names are collision-free +4. Same component instances always produce same IDs (stable for multi-turn) + +--- + +## Key Takeaways + +1. **Composability**: Multiple components can safely define tools with identical names +2. **Determinism**: Component IDs are stable within a session for the same instance +3. **Flexibility**: You can control which tools reach the LLM via filtering +4. **Scalability**: Works smoothly with 2, 3, or more components +5. **Observability**: Prefixed names and component IDs enable tracing and debugging + +--- + +## Related Source Files + +- `mellea/backends/tools.py` - `add_tools_from_context_actions()` implementation +- `mellea/stdlib/functional.py` - `call_tools()` implementation (executes tools via pipeline) +- `mellea/telemetry/metrics_plugins.py` - `ToolMetricsPlugin` (records tool metrics) +- `mellea/telemetry/metrics.py` - `record_tool_call()` function (telemetry recording) +- Tests: `test/backends/test_tool_helpers.py` - Unit tests for tool prefixing diff --git a/docs/examples/components/duplicate_tool_names_public_api.py b/docs/examples/components/duplicate_tool_names_public_api.py new file mode 100644 index 0000000000..acbddcaab5 --- /dev/null +++ b/docs/examples/components/duplicate_tool_names_public_api.py @@ -0,0 +1,251 @@ +# pytest: ollama, e2e +"""Example demonstrating component ID-based tool prefixing using public APIs. + +This is the recommended approach for handling multiple components with identical +tool names. It uses the public MelleaSession.act() API with tool_calls=True, +which automatically extracts and prefixes tools from context. + +When multiple components define tools with identical names, Mellea automatically +prefixes each tool name with its component ID (component_{ID}__tool_name) to prevent +naming collisions. + +In this example: +- DatabaseComponent has a "query" tool for querying data +- SearchComponent also has a "query" tool for searching +- Both tools are available to the LLM with prefixed names to avoid conflicts +- The LLM is prompted to use both tools and demonstrate collision handling +- Tool calls are executed via call_tools() to enable telemetry recording + +To view tool calling telemetry metrics: + export MELLEA_METRICS_ENABLED=true + export MELLEA_METRICS_CONSOLE=true + uv run python this_script.py +""" + +import os +from typing import Any + +from mellea.backends import ModelOption +from mellea.backends.model_ids import IBM_GRANITE_4_HYBRID_MICRO +from mellea.backends.openai import OpenAIBackend +from mellea.backends.tools import MelleaTool +from mellea.core import CBlock, Component, ModelOutputThunk, TemplateRepresentation +from mellea.core.base import AbstractMelleaTool +from mellea.formatters import TemplateFormatter +from mellea.stdlib.context import ChatContext +from mellea.stdlib.functional import call_tools +from mellea.stdlib.session import MelleaSession + + +class QueryDatabaseTool(AbstractMelleaTool): + """Tool for querying a database.""" + + name = "query" + + def run(self, sql: str) -> str: + """Execute a SQL query on the database. + + Args: + sql: The SQL query to execute + + Returns: + Mock query results as a string + """ + return f"Database query result: [{sql}] returned 42 rows" + + @property + def as_json_tool(self) -> dict[str, Any]: + """Return JSON schema for this tool.""" + return { + "type": "function", + "function": { + "name": self.name, + "description": "Query a database with SQL", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string", "description": "SQL query to execute"} + }, + "required": ["sql"], + }, + }, + } + + +class SearchIndexTool(AbstractMelleaTool): + """Tool for searching an index.""" + + name = "query" + + def run(self, text: str) -> str: + """Search the index for matching documents. + + Args: + text: The search query text + + Returns: + Mock search results as a string + """ + return f"Search results for '{text}': found 5 documents" + + @property + def as_json_tool(self) -> dict[str, Any]: + """Return JSON schema for this tool.""" + return { + "type": "function", + "function": { + "name": self.name, + "description": "Search an index for documents", + "parameters": { + "type": "object", + "properties": { + "text": {"type": "string", "description": "Search query text"} + }, + "required": ["text"], + }, + }, + } + + +class DatabaseComponent(Component): + """Component that provides database querying capabilities.""" + + description = "Database query interface" + _tool = QueryDatabaseTool() + + def parts(self) -> list[Component | CBlock | ModelOutputThunk]: + """Return parts of this component.""" + return [] + + def format_for_llm(self) -> TemplateRepresentation | str: + """Format component for LLM with database query tool.""" + return TemplateRepresentation( + obj=self, + args={"description": "Database query interface"}, + tools={"query": MelleaTool.from_callable(self._tool.run)}, + template="🗄️ **Database Interface**: {{description}}\nAvailable: SQL query tool", + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + """Parse the LLM response.""" + return str(computed.value) + + +class SearchComponent(Component): + """Component that provides search capabilities.""" + + description = "Search interface" + _tool = SearchIndexTool() + + def parts(self) -> list[Component | CBlock | ModelOutputThunk]: + """Return parts of this component.""" + return [] + + def format_for_llm(self) -> TemplateRepresentation | str: + """Format component for LLM with search tool.""" + return TemplateRepresentation( + obj=self, + args={"description": "Search interface"}, + tools={"query": MelleaTool.from_callable(self._tool.run)}, + template="🔍 **Search Interface**: {{description}}\nAvailable: Document search tool", + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + """Parse the LLM response.""" + return str(computed.value) + + +class TaskComponent(Component): + """Component that prompts the LLM to use available tools.""" + + def parts(self) -> list[Component | CBlock | ModelOutputThunk]: + """Return parts of this component.""" + return [] + + def format_for_llm(self) -> TemplateRepresentation | str: + """Format component for LLM with explicit tool use instructions.""" + return ( + "Please complete these tasks using the available tools:\n" + "1. Query the database: SELECT * FROM users\n" + "2. Search documentation: best practices\n" + "Use both tools to demonstrate collision handling." + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + """Parse the LLM response.""" + return str(computed.value) + + +def main() -> None: + """Main function demonstrating component ID-based tool prefixing.""" + + print("\n" + "=" * 70) + print("Component ID-Based Tool Prefixing (Public API Example)") + print("=" * 70) + + backend = OpenAIBackend( + model_id=IBM_GRANITE_4_HYBRID_MICRO.ollama_name, # type: ignore[arg-type] + formatter=TemplateFormatter( + model_id=IBM_GRANITE_4_HYBRID_MICRO.hf_model_name # type: ignore[arg-type] + ), + base_url=os.getenv("OLLAMA_BASE_URL", "http://localhost:11434/v1"), + api_key="ollama", + ) + + # Create a session with ChatContext (required for context-based tool extraction) + session = MelleaSession(backend=backend, ctx=ChatContext()) + + # Add both components to the session context + db_component = DatabaseComponent() + search_component = SearchComponent() + + session.ctx = session.ctx.add(db_component).add(search_component) + + print("\nStep 1: Add components to session context") + print(" ✓ Added DatabaseComponent (has 'query' tool)") + print(" ✓ Added SearchComponent (also has 'query' tool)") + + print("\nStep 2: Use act() with tool_calls=True") + print(" This automatically:") + print(" - Extracts tools from all components in context") + print(" - Prefixes duplicate tool names with component IDs") + print(" - Enables tool calling") + print(" Then execute tools via call_tools() to record telemetry") + + # Create a task component that will request tool use + action = TaskComponent() + + # Use the public API: act() with tool_calls=True + # This handles tool extraction, execution, and telemetry automatically + response = session.act( + action, + model_options={ + ModelOption.MAX_NEW_TOKENS: 500, + ModelOption.TOOL_CHOICE: "auto", + }, + tool_calls=True, + ) + + print(f"\nLLM Response:\n{response.value}\n") + + # Check if tool calls were made and execute them + if hasattr(response, "tool_calls") and response.tool_calls: + print(f"Tool calls requested: {[tc.name for tc in response.tool_calls]}") + print( + "\nExecuting tool calls via Mellea's pipeline (enables telemetry recording):" + ) + tool_messages = call_tools(response, backend) + for msg in tool_messages: + print(f" {msg.name}() → {msg.content}") + print() + else: + print("(No tool calls in this response)\n") + + print("\n" + "=" * 70) + print("✓ Component ID-based tool prefixing successfully demonstrated") + print("✓ Tools extracted from context and executed via call_tools()") + print("=" * 70) + + +if __name__ == "__main__": + main() diff --git a/docs/examples/components/pattern2_public_api.py b/docs/examples/components/pattern2_public_api.py new file mode 100644 index 0000000000..3e68f62a23 --- /dev/null +++ b/docs/examples/components/pattern2_public_api.py @@ -0,0 +1,260 @@ +# pytest: ollama, e2e +"""Example demonstrating Pattern 2 (components in context) using public APIs. + +PATTERN 2: Components in Context + Auto Tool Extraction + +This example shows the recommended approach for tool calling: add components +to the session context, then use act() with tool_calls=True. The backend +automatically extracts tools via add_tools_from_context_actions(). + +Key features: +1. Components live in session context with templates +2. Backend auto-extracts tools when tool_calls=True (NO ModelOption.TOOLS needed) +3. Component ID-based prefixing prevents name collisions +4. Tool calls executed via call_tools() to enable telemetry +5. Multi-turn stability: same components always get same IDs + +To view tool calling telemetry metrics: + export MELLEA_METRICS_ENABLED=true + export MELLEA_METRICS_CONSOLE=true + uv run python this_script.py +""" + +import os +from typing import Any + +from mellea.backends import ModelOption +from mellea.backends.model_ids import IBM_GRANITE_4_HYBRID_MICRO +from mellea.backends.openai import OpenAIBackend +from mellea.backends.tools import MelleaTool +from mellea.core import CBlock, Component, ModelOutputThunk, TemplateRepresentation +from mellea.core.base import AbstractMelleaTool +from mellea.formatters import TemplateFormatter +from mellea.stdlib.context import ChatContext +from mellea.stdlib.functional import call_tools +from mellea.stdlib.session import MelleaSession + + +class QueryDatabaseTool(AbstractMelleaTool): + """Tool for querying a database.""" + + name = "query" + + def run(self, sql: str) -> str: + """Execute a SQL query on the database. + + Args: + sql: The SQL query to execute + + Returns: + Mock query results as a string + """ + return f"Database: [{sql}] returned 42 rows" + + @property + def as_json_tool(self) -> dict[str, Any]: + """Return JSON schema for this tool.""" + return { + "type": "function", + "function": { + "name": self.name, + "description": "Query a database with SQL", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string", "description": "SQL query to execute"} + }, + "required": ["sql"], + }, + }, + } + + +class SearchIndexTool(AbstractMelleaTool): + """Tool for searching an index.""" + + name = "query" + + def run(self, text: str) -> str: + """Search the index for matching documents. + + Args: + text: The search query text + + Returns: + Mock search results as a string + """ + return f"Search: '{text}' found 5 documents" + + @property + def as_json_tool(self) -> dict[str, Any]: + """Return JSON schema for this tool.""" + return { + "type": "function", + "function": { + "name": self.name, + "description": "Search an index for documents", + "parameters": { + "type": "object", + "properties": { + "text": {"type": "string", "description": "Search query text"} + }, + "required": ["text"], + }, + }, + } + + +class DatabaseComponent(Component): + """Component that provides database querying capabilities.""" + + description = "Database query interface" + + def parts(self) -> list[Component | CBlock | ModelOutputThunk]: + """Return parts of this component.""" + return [] + + def format_for_llm(self) -> TemplateRepresentation | str: + """Format component for LLM with database query tool.""" + return TemplateRepresentation( + obj=self, + args={"description": "Database query interface"}, + tools={"query": MelleaTool.from_callable(QueryDatabaseTool().run)}, + template="🗄️ **Database**: {{description}}\nAvailable: SQL query tool", + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + """Parse the LLM response.""" + return str(computed.value) + + +class SearchComponent(Component): + """Component that provides search capabilities.""" + + description = "Search interface" + + def parts(self) -> list[Component | CBlock | ModelOutputThunk]: + """Return parts of this component.""" + return [] + + def format_for_llm(self) -> TemplateRepresentation | str: + """Format component for LLM with search tool.""" + return TemplateRepresentation( + obj=self, + args={"description": "Search interface"}, + tools={"query": MelleaTool.from_callable(SearchIndexTool().run)}, + template="🔍 **Search**: {{description}}\nAvailable: Document search tool", + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + """Parse the LLM response.""" + return str(computed.value) + + +class QueryComponent(Component): + """Component that prompts the LLM to use available tools.""" + + def parts(self) -> list[Component | CBlock | ModelOutputThunk]: + """Return parts of this component.""" + return [] + + def format_for_llm(self) -> TemplateRepresentation | str: + """Format component for LLM with explicit tool use instructions.""" + return ( + "Please complete these tasks using the available tools:\n" + "1. Query the database: SELECT * FROM users WHERE country = 'USA'\n" + "2. Search documentation: user management best practices\n" + "Use both the database query tool and the search tool." + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + """Parse the LLM response.""" + return str(computed.value) + + +def main() -> None: + """Demonstrate Pattern 2: Components in context with auto tool extraction.""" + + print("\n" + "=" * 70) + print("PATTERN 2: Components in Context + Auto Tool Extraction") + print("=" * 70) + + backend = OpenAIBackend( + model_id=IBM_GRANITE_4_HYBRID_MICRO.ollama_name, # type: ignore[arg-type] + formatter=TemplateFormatter( + model_id=IBM_GRANITE_4_HYBRID_MICRO.hf_model_name # type: ignore[arg-type] + ), + base_url=os.getenv("OLLAMA_BASE_URL", "http://localhost:11434/v1"), + api_key="ollama", + ) + # Use ChatContext (required for context-based tool extraction) + session = MelleaSession(backend=backend, ctx=ChatContext()) + + print("\nStep 1: Add components to session context") + db_component = DatabaseComponent() + search_component = SearchComponent() + query_component = QueryComponent() + + session.ctx = ( + session.ctx.add(db_component).add(search_component).add(query_component) + ) + print(" ✓ Added DatabaseComponent (has 'query' tool)") + print(" ✓ Added SearchComponent (also has 'query' tool)") + print(" ✓ Added QueryComponent (provides context)") + + print("\nStep 2: Use act() with tool_calls=True") + print(" Configuration:") + print(" - ModelOption.TOOLS: NO (not needed!)") + print(" - tool_calls=True: YES (enables auto-extraction)") + print(" Backend will:") + print(" - Auto-extract tools from context components") + print(" - Prefix duplicate names with component IDs") + print(" - Generate tool calls if LLM requests them") + print(" Then execute tools via call_tools() to record telemetry") + + # Use the public API: act() with tool_calls=True + # NO ModelOption.TOOLS - backend auto-extracts from context! + # strategy=None to avoid sampling (which may suppress tool calls) + response = session.act( + query_component, + strategy=None, + model_options={ + ModelOption.MAX_NEW_TOKENS: 500, + ModelOption.TOOL_CHOICE: "auto", + }, + tool_calls=True, + ) + + print(f"\nLLM Response:\n{response.value}\n") + + # Check if tool calls were made and execute them + if hasattr(response, "tool_calls") and response.tool_calls: + print(f"Tool calls requested: {[tc.name for tc in response.tool_calls]}") + print( + "\nExecuting tool calls via Mellea's pipeline (enables telemetry recording):" + ) + tool_messages = call_tools(response, backend) + for msg in tool_messages: + print(f" {msg.name}() → {msg.content}") + print() + else: + print("(No tool calls in this response)\n") + + print("=" * 70) + print("✓ Pattern 2 (components in context) successfully demonstrated") + print("✓ Tools automatically extracted from context") + print("✓ Tool calls executed via call_tools() for telemetry") + print("=" * 70) + + print("\nKey Concepts:") + print(" - Pattern 1: Extract tools only (simple tool calling)") + print( + " - Pattern 2: Components in context with auto-extraction (implicit tool passing)" + ) + print(" - Both patterns use component ID-based prefixing") + print(" - Use act() with tool_calls=True (recommended public API)") + print(" - Tool execution and telemetry handled automatically") + + +if __name__ == "__main__": + main() diff --git a/mellea/backends/ollama.py b/mellea/backends/ollama.py index 2babc396ce..40fb45020e 100644 --- a/mellea/backends/ollama.py +++ b/mellea/backends/ollama.py @@ -43,7 +43,11 @@ from ..telemetry.context import generate_request_id, with_context from .backend import FormatterBackend from .model_options import ModelOption -from .tools import add_tools_from_context_actions, add_tools_from_model_options +from .tools import ( + add_tools_from_context_actions, + add_tools_from_model_options, + convert_tools_to_json, +) format: None = None # typing this variable in order to shadow the global format function and ensure mypy checks for errors @@ -575,7 +579,7 @@ async def generate_from_chat_context( ] = self._async_client.chat( model=self._model_id, messages=conversation, - tools=[t.as_json_tool for t in tools.values()], + tools=convert_tools_to_json(tools), think=model_opts.get(ModelOption.THINKING, None), stream=model_opts.get(ModelOption.STREAM, False), options=self._make_backend_specific_and_remove(model_opts), diff --git a/mellea/backends/tools.py b/mellea/backends/tools.py index 0bc3d04d2c..20495ace0a 100644 --- a/mellea/backends/tools.py +++ b/mellea/backends/tools.py @@ -367,12 +367,16 @@ def add_tools_from_model_options( def add_tools_from_context_actions( tools_dict: dict[str, AbstractMelleaTool], ctx_actions: list[Span] | None ): - """If any of the actions in ctx_actions have tools in their template_representation, add those to the tools_dict. + """Extract and merge tools from component actions, with auto-prefixing to avoid collisions. + + Tools from each component are prefixed with "component_{ID}__" to prevent naming collisions when + multiple components define tools with identical names. The component ID is derived from the + component object's identity for multi-turn stability. This allows safe composition of + multiple agents or tool-bearing components. Args: - tools_dict: Mutable mapping of tool name to tool instance; modified in-place. Dict keys are unique, - so if multiple components define tools with the same name, the last one wins (earlier definitions - are silently overwritten). + tools_dict: Mutable mapping of tool name to tool instance; modified in-place. Prefixed names + ensure tools from different components coexist without overwriting each other. ctx_actions: List of `Component`, `CBlock`, or `ModelOutputThunk` objects whose template representations may declare tools, or `None` to skip. """ @@ -387,13 +391,48 @@ def add_tools_from_context_actions( if not isinstance(tr, TemplateRepresentation) or tr.tools is None: continue - for tool_name, func in tr.tools.items(): - tools_dict[tool_name] = func + component_id = hex(id(action))[-8:] + component_type = type(action).__name__ + + for original_tool_name, tool_instance in tr.tools.items(): + # Auto-prefix tool name using component ID to avoid collisions + prefixed_name = f"component_{component_id}__{original_tool_name}" + + # Validate function name length (OpenAI and most providers enforce 64-char limit) + if len(prefixed_name) > 64: + MelleaLogger.get_logger().warning( + f"Tool name exceeds 64-character limit (OpenAI/Ollama/HF constraint): " + f"'{prefixed_name}' ({len(prefixed_name)} chars). " + f"Original: '{original_tool_name}' ({len(original_tool_name)} chars). " + f"Providers may reject this tool call. Consider using shorter tool names." + ) + + # Validate function name pattern (OpenAI constraint: characters must be [a-zA-Z0-9_-]) + # Length is checked separately above; this regex only validates character set + if not re.match(r"^[a-zA-Z0-9_-]+$", prefixed_name): + MelleaLogger.get_logger().warning( + f"Tool name contains invalid characters (allowed: [a-zA-Z0-9_-]): " + f"'{prefixed_name}'. Providers may reject this tool call." + ) + + # Detect collision and warn if it still occurs (defensive) + if prefixed_name in tools_dict: + MelleaLogger.get_logger().warning( + f"Tool name collision even after prefixing: '{prefixed_name}' " + f"already exists (component {component_type} {component_id}); skipping tool '{original_tool_name}'" + ) + continue + + # Add tool with prefixed name + tools_dict[prefixed_name] = tool_instance def convert_tools_to_json(tools: dict[str, AbstractMelleaTool]) -> list[dict]: """Convert tools to json dict representation. + Ensures that tool names in JSON schemas match the keys in the tools dict, + which is necessary when tools have been renamed (e.g., prefixed for conflict avoidance). + Args: tools: Mapping of tool name to `AbstractMelleaTool` instance. @@ -405,7 +444,20 @@ def convert_tools_to_json(tools: dict[str, AbstractMelleaTool]) -> list[dict]: - WatsonxAI uses `from langchain_ibm.chat_models import convert_to_openai_tool` in their demos, but it gives the same values. - OpenAI uses the same format / schema. """ - return [t.as_json_tool for t in tools.values()] + result = [] + for tool_name, tool_instance in tools.items(): + tool_json = tool_instance.as_json_tool.copy() + + # Update the function name in JSON to match the dict key (for prefixed names). + # This ensures the model sees and requests the prefixed name. + if tool_json.get("function", {}).get("name") != tool_name: + if "function" in tool_json: + tool_json["function"] = tool_json["function"].copy() + tool_json["function"]["name"] = tool_name + + result.append(tool_json) + + return result def json_extraction(text: str) -> Generator[dict, None, None]: diff --git a/mellea/telemetry/metrics_plugins.py b/mellea/telemetry/metrics_plugins.py index bca95ec6e0..5a9540a343 100644 --- a/mellea/telemetry/metrics_plugins.py +++ b/mellea/telemetry/metrics_plugins.py @@ -531,13 +531,13 @@ async def record_tool_call( """Record one tool invocation after it completes. Args: - payload: Contains model_tool_call (with name) and success flag. + payload: Contains model_tool_call (with func.name) and success flag. context: Plugin context (unused). """ from mellea.telemetry.metrics import record_tool_call tool_name = ( - payload.model_tool_call.name + payload.model_tool_call.func.name if payload.model_tool_call is not None else "unknown" ) diff --git a/test/backends/test_tool_calls.py b/test/backends/test_tool_calls.py index 05aef52a01..5974a5fa72 100644 --- a/test/backends/test_tool_calls.py +++ b/test/backends/test_tool_calls.py @@ -41,6 +41,8 @@ def table() -> Table: def test_tool_called_from_context_action(m: MelleaSession, table: Table): """Make sure tools can be called from actions in the context.""" + import re + m.reset() # Insert a component with tools into the context. @@ -61,7 +63,15 @@ def test2(): ... assert "test2" in tools add_tools_from_context_actions(tools, m.ctx.actions_for_available_tools()) - assert "to_markdown" in tools + # Component tools are now auto-prefixed using component ID to avoid collisions + # Pattern: component_{ID}__tool_name where ID is hex-encoded object identity + table_tools = [ + k for k in tools if k.startswith("component_") and k.endswith("__to_markdown") + ] + assert len(table_tools) == 1, ( + f"Expected 1 table tool with ID-based prefix, found {table_tools}" + ) + assert re.match(r"component_[0-9a-f]{8}__to_markdown", table_tools[0]) @pytest.mark.xfail( diff --git a/test/backends/test_tool_helpers.py b/test/backends/test_tool_helpers.py index a6c4bc611c..ab4d05331a 100644 --- a/test/backends/test_tool_helpers.py +++ b/test/backends/test_tool_helpers.py @@ -93,21 +93,101 @@ def get_weather(location: str) -> int: def test_add_tools_from_context_actions(): + import re + ftc1 = FakeToolComponentWithExtraTool() ftc2 = FakeToolComponent() + # Extract component IDs before adding tools (ID is based on Python object identity) + ftc1_id = hex(id(ftc1))[-8:] + ftc2_id = hex(id(ftc2))[-8:] + ctx_actions = [CBlock("Hello"), ftc1, ftc2] tools = {} add_tools_from_context_actions(tools, ctx_actions) - # Check that tools with the same name get properly overwritten in order of ctx. - tool1 = tools["tool1"]._call_tool - assert tool1 == ftc2.tool1, f"{tool1} should == {ftc2.tool1}" + # With auto-prefixing using component IDs, tools with the same name no longer collide. + # Both are preserved with prefixed names: component_{ID}__tool1 + tool1_key_ftc1 = f"component_{ftc1_id}__tool1" + tool1_key_ftc2 = f"component_{ftc2_id}__tool1" + + assert tool1_key_ftc1 in tools, f"Expected {tool1_key_ftc1} in tools" + assert tool1_key_ftc2 in tools, f"Expected {tool1_key_ftc2} in tools" + + tool1_from_ftc1 = tools[tool1_key_ftc1]._call_tool + assert tool1_from_ftc1 == ftc1.tool1, f"{tool1_from_ftc1} should == {ftc1.tool1}" + + tool1_from_ftc2 = tools[tool1_key_ftc2]._call_tool + assert tool1_from_ftc2 == ftc2.tool1, f"{tool1_from_ftc2} should == {ftc2.tool1}" - # Check that tools that aren't overwritten are still there. - tool2 = tools["tool2"]._call_tool + # Check that tools that aren't duplicated are still there with prefixed names. + tool2_key = f"component_{ftc1_id}__tool2" + assert tool2_key in tools, f"Expected {tool2_key} in tools" + + tool2 = tools[tool2_key]._call_tool assert tool2 == ftc1.tool2, f"{tool2} should == {ftc1.tool2}" + # Verify that all tool prefixes match the expected ID pattern + for tool_name in tools: + if tool_name.startswith("component_"): + assert re.match(r"component_[0-9a-f]{8}__", tool_name), ( + f"Tool name {tool_name} does not match ID-based prefix pattern" + ) + + +def test_add_tools_from_context_actions_exceeds_length_limit(caplog): + """Verify warning when tool name exceeds 64-character provider limit.""" + import logging + + # Create a custom component with a very long tool name + # The tool name (key in tools dict) is what gets prefixed, so we need a 45+ char name + # to exceed the 64 char limit (20 char prefix + 45+ char name = 65+ chars) + class ComponentWithLongToolName(FakeToolComponent): + def format_for_llm(self) -> TemplateRepresentation: + long_tool_key = ( + "x" * 45 + ) # Will exceed 64 chars after prefixing (20 + 45 = 65) + long_tool = MelleaTool.from_callable(lambda: None, name="tool") + return TemplateRepresentation( + obj=self, args={"arg": None}, tools={long_tool_key: long_tool} + ) + + component = ComponentWithLongToolName() + tools = {} + with caplog.at_level(logging.WARNING, logger="mellea"): + add_tools_from_context_actions(tools, [component]) + + # Verify warning was logged for length constraint + assert any( + "exceeds 64-character limit" in record.message for record in caplog.records + ), f"Expected length warning in logs; got: {[r.message for r in caplog.records]}" + + +def test_add_tools_from_context_actions_invalid_characters(caplog): + """Verify warning when tool name contains invalid characters.""" + import logging + + # Create a custom component with a tool name containing invalid characters + class ComponentWithInvalidToolName(FakeToolComponent): + def format_for_llm(self) -> TemplateRepresentation: + # Invalid character: @ (not in [a-zA-Z0-9_-]) + # The key in tools dict is what gets prefixed, so use @ in the key + invalid_tool_key = "my@tool" + invalid_tool = MelleaTool.from_callable(lambda: None, name="tool") + return TemplateRepresentation( + obj=self, args={"arg": None}, tools={invalid_tool_key: invalid_tool} + ) + + component = ComponentWithInvalidToolName() + tools = {} + with caplog.at_level(logging.WARNING, logger="mellea"): + add_tools_from_context_actions(tools, [component]) + + # Verify warning was logged for invalid characters + assert any("invalid characters" in record.message for record in caplog.records), ( + f"Expected invalid character warning in logs; got: {[r.message for r in caplog.records]}" + ) + if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/telemetry/test_metrics_plugins.py b/test/telemetry/test_metrics_plugins.py index a952c22070..d4b555207c 100644 --- a/test/telemetry/test_metrics_plugins.py +++ b/test/telemetry/test_metrics_plugins.py @@ -1158,11 +1158,17 @@ async def test_requirement_metrics_ignores_non_quick_check_events(requirement_pl # ToolMetricsPlugin tests -class _MockToolCall: +class _MockToolFunc: def __init__(self, name: str) -> None: self.name = name +class _MockToolCall: + def __init__(self, prefixed_name: str, original_name: str) -> None: + self.name = prefixed_name + self.func = _MockToolFunc(original_name) + + @pytest.fixture def tool_plugin(): return ToolMetricsPlugin() @@ -1170,9 +1176,10 @@ def tool_plugin(): @pytest.mark.asyncio async def test_tool_plugin_records_success(tool_plugin): - """Successful tool calls are recorded with status='success'.""" + """Successful tool calls are recorded with status='success' using original tool name.""" payload = ToolPostInvokePayload( - model_tool_call=_MockToolCall("search"), success=True + model_tool_call=_MockToolCall("component_abc123__search", "search"), + success=True, ) with patch("mellea.telemetry.metrics.record_tool_call") as mock_record: @@ -1183,9 +1190,10 @@ async def test_tool_plugin_records_success(tool_plugin): @pytest.mark.asyncio async def test_tool_plugin_records_failure(tool_plugin): - """Failed tool calls are recorded with status='failure'.""" + """Failed tool calls are recorded with status='failure' using original tool name.""" payload = ToolPostInvokePayload( - model_tool_call=_MockToolCall("calculator"), success=False + model_tool_call=_MockToolCall("component_def456__calculator", "calculator"), + success=False, ) with patch("mellea.telemetry.metrics.record_tool_call") as mock_record: