From ddd57aaf5df0e7dc99c1351890ceef6683dcef0d Mon Sep 17 00:00:00 2001 From: dvlpjrs Date: Fri, 4 Sep 2026 01:18:22 +0000 Subject: [PATCH 1/3] [GMLP-9378] Add evaluation metrics, options, run and list filters to the agents SDK and CLI --- src/gumloop/cli/commands/agents.py | 115 +++++++++++++++++++++++++++++ src/gumloop/resources/agents.py | 62 +++++++++++++++- src/gumloop/types.py | 37 ++++++++++ 3 files changed, 212 insertions(+), 2 deletions(-) diff --git a/src/gumloop/cli/commands/agents.py b/src/gumloop/cli/commands/agents.py index dce14fb..df87526 100644 --- a/src/gumloop/cli/commands/agents.py +++ b/src/gumloop/cli/commands/agents.py @@ -550,3 +550,118 @@ def detach_mcp_server( console.print(f"[green]Detached[/green] MCP server {escape_markup(server_id)} from {agent_id}") console.print(f" detached: {'true' if response.detached else 'false'}", markup=False, highlight=False) + + +@agents_app.command("eval-options", epilog="Example:\n gumloop agents eval-options --json") +def get_evaluation_options( + ctx: typer.Context, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the raw SDK response as JSON."), + ] = False, +) -> None: + """Show the available agent evaluation settings.""" + cli: CliContext = ctx.obj + try: + response = cli.call_with_refresh(lambda client: client.agents.get_evaluation_options()) + except GumloopError as error: + exit_with_error(error, json_output=json_output) + + if json_output: + print_json(response) + return + + for field in ( + "interaction_types", + "criterion_types", + "criterion_priorities", + "data_point_types", + "frequencies", + "default_interaction_types", + ): + console.print(f"{field}: {', '.join(getattr(response, field))}", markup=False, highlight=False) + for name, value in response.limits.model_dump().items(): + console.print(f"limits.{name}: {value}", markup=False, highlight=False) + + +@agents_app.command( + "eval-metrics", + epilog="Example:\n gumloop agents eval-metrics agent_abc --days 30 --json", +) +def get_evaluation_metrics( + ctx: typer.Context, + agent_id: Annotated[str, typer.Argument(help="ID of the agent whose evaluation metrics should be shown.")], + days: Annotated[ + int, + typer.Option("--days", help="Number of days to include."), + ] = 30, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the raw SDK response as JSON."), + ] = False, +) -> None: + """Show an agent's evaluation grade and tag counts.""" + cli: CliContext = ctx.obj + try: + response = cli.call_with_refresh(lambda client: client.agents.get_evaluation_metrics(agent_id, days=days)) + except GumloopError as error: + exit_with_error(error, json_output=json_output) + + if json_output: + print_json(response) + return + + console.print("GRADE", "COUNT", sep="\t", soft_wrap=True) + for grade, count in response.grades.items(): + console.print(grade, str(count), sep="\t", soft_wrap=True) + console.print("TAG", "COUNT", sep="\t", soft_wrap=True) + for tag, count in response.tags.items(): + console.print(tag, str(count), sep="\t", soft_wrap=True) + + +@agents_app.command( + "eval-run", + epilog=( + "Examples:\n" + " gumloop agents eval-run agent_abc --session-id session_1\n" + " gumloop agents eval-run agent_abc --session-id session_1 --session-id session_2 --json" + ), +) +def run_evaluations( + ctx: typer.Context, + agent_id: Annotated[str, typer.Argument(help="ID of the agent that owns the sessions.")], + session_ids: Annotated[ + list[str] | None, + typer.Option("--session-id", help="Session ID to evaluate. Repeat for multiple sessions."), + ] = None, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the raw SDK response as JSON."), + ] = False, +) -> None: + """Queue evaluations for one or more agent sessions.""" + cli: CliContext = ctx.obj + try: + if not session_ids: + raise GumloopError("Pass at least one --session-id.") + response = cli.call_with_refresh(lambda client: client.agents.run_evaluations(agent_id, session_ids)) + except GumloopError as error: + exit_with_error(error, json_output=json_output) + + if json_output: + print_json(response) + return + + console.print(f"[green]Queued[/green] {response.queued} evaluation(s)") + if response.skipped.ineligible: + console.print( + f" Ineligible: {', '.join(response.skipped.ineligible)}", + markup=False, + highlight=False, + ) + if response.skipped.in_flight: + console.print( + f" In flight: {', '.join(response.skipped.in_flight)}", + markup=False, + highlight=False, + ) diff --git a/src/gumloop/resources/agents.py b/src/gumloop/resources/agents.py index 90b13b3..93333ee 100644 --- a/src/gumloop/resources/agents.py +++ b/src/gumloop/resources/agents.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from collections.abc import Sequence +from datetime import datetime from typing import Any from gumloop._http import AsyncHttpClient @@ -18,8 +19,11 @@ from gumloop.types import AgentVersionsResponse from gumloop.types import EvaluationConfigResponse from gumloop.types import EvaluationConfigUpdateRequest +from gumloop.types import EvaluationMetrics +from gumloop.types import EvaluationOptions from gumloop.types import EvaluationResultListResponse from gumloop.types import EvaluationResultResponse +from gumloop.types import EvaluationRunResult from gumloop.types import ModelListResponse from gumloop.types import SkillListResponse @@ -132,6 +136,9 @@ def detach_mcp_server(self, agent_id: str, server_id: str) -> AgentMcpServerDeta def list_mcp_servers(self, agent_id: str) -> AgentMcpServersResponse: return AgentMcpServersResponse.model_validate(self._client.get(f"agents/{agent_id}/mcp-servers")) + def get_evaluation_options(self) -> EvaluationOptions: + return EvaluationOptions.model_validate(self._client.get("agents/evaluation-options")) + def get_evaluation_config(self, agent_id: str) -> EvaluationConfigResponse: return EvaluationConfigResponse.model_validate(self._client.get(f"agents/{agent_id}/evaluation-config")) @@ -156,6 +163,10 @@ def list_evaluations( agent_id: str, *, grade: str | None = None, + status: str | None = None, + created_after: datetime | None = None, + created_before: datetime | None = None, + session_id: str | None = None, page_size: int | None = None, cursor: str | None = None, **kwargs: Any, @@ -163,10 +174,29 @@ def list_evaluations( return EvaluationResultListResponse.model_validate( self._client.get( f"agents/{agent_id}/evaluations", - params={"grade": grade, "page_size": page_size, "cursor": cursor, **kwargs}, + params={ + "grade": grade, + "status": status, + "created_after": created_after.isoformat() if created_after is not None else None, + "created_before": created_before.isoformat() if created_before is not None else None, + "session_id": session_id, + "page_size": page_size, + "cursor": cursor, + **kwargs, + }, ) ) + def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetrics: + return EvaluationMetrics.model_validate( + self._client.get(f"agents/{agent_id}/evaluations/metrics", params={"days": days}) + ) + + def run_evaluations(self, agent_id: str, session_ids: list[str]) -> EvaluationRunResult: + return EvaluationRunResult.model_validate( + self._client.post(f"agents/{agent_id}/evaluations/run", json={"session_ids": session_ids}) + ) + def get_evaluation(self, agent_id: str, evaluation_id: str) -> EvaluationResultResponse: return EvaluationResultResponse.model_validate( self._client.get(f"agents/{agent_id}/evaluations/{evaluation_id}") @@ -280,6 +310,10 @@ async def list_mcp_servers(self, agent_id: str) -> AgentMcpServersResponse: data = await self._client.get(f"agents/{agent_id}/mcp-servers") return AgentMcpServersResponse.model_validate(data) + async def get_evaluation_options(self) -> EvaluationOptions: + data = await self._client.get("agents/evaluation-options") + return EvaluationOptions.model_validate(data) + async def get_evaluation_config(self, agent_id: str) -> EvaluationConfigResponse: data = await self._client.get(f"agents/{agent_id}/evaluation-config") return EvaluationConfigResponse.model_validate(data) @@ -304,16 +338,40 @@ async def list_evaluations( agent_id: str, *, grade: str | None = None, + status: str | None = None, + created_after: datetime | None = None, + created_before: datetime | None = None, + session_id: str | None = None, page_size: int | None = None, cursor: str | None = None, **kwargs: Any, ) -> EvaluationResultListResponse: data = await self._client.get( f"agents/{agent_id}/evaluations", - params={"grade": grade, "page_size": page_size, "cursor": cursor, **kwargs}, + params={ + "grade": grade, + "status": status, + "created_after": created_after.isoformat() if created_after is not None else None, + "created_before": created_before.isoformat() if created_before is not None else None, + "session_id": session_id, + "page_size": page_size, + "cursor": cursor, + **kwargs, + }, ) return EvaluationResultListResponse.model_validate(data) + async def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetrics: + data = await self._client.get(f"agents/{agent_id}/evaluations/metrics", params={"days": days}) + return EvaluationMetrics.model_validate(data) + + async def run_evaluations(self, agent_id: str, session_ids: list[str]) -> EvaluationRunResult: + data = await self._client.post( + f"agents/{agent_id}/evaluations/run", + json={"session_ids": session_ids}, + ) + return EvaluationRunResult.model_validate(data) + async def get_evaluation(self, agent_id: str, evaluation_id: str) -> EvaluationResultResponse: data = await self._client.get(f"agents/{agent_id}/evaluations/{evaluation_id}") return EvaluationResultResponse.model_validate(data) diff --git a/src/gumloop/types.py b/src/gumloop/types.py index 902f552..ce614f1 100644 --- a/src/gumloop/types.py +++ b/src/gumloop/types.py @@ -689,6 +689,7 @@ class EvaluationConfig(_Model): tags: list[Any] = Field(default_factory=list) data_points: list[Any] = Field(default_factory=list) sentiment: dict[str, Any] | None = None + notifications: dict[str, Any] | None = None updated_ts: str | None = None @@ -707,6 +708,42 @@ class EvaluationConfigUpdateRequest(_Model): tags: list[Any] | None = None data_points: list[Any] | None = None sentiment: dict[str, Any] | None = None + notifications: dict[str, Any] | None = None + + +class EvaluationOptionLimits(_Model): + criteria: int + tags: int + data_points: int + tag_name_max_len: int + tag_description_max_len: int + notification_recipients: int + + +class EvaluationOptions(_Model): + interaction_types: list[str] = Field(default_factory=list) + criterion_types: list[str] = Field(default_factory=list) + criterion_priorities: list[str] = Field(default_factory=list) + data_point_types: list[str] = Field(default_factory=list) + frequencies: list[str] = Field(default_factory=list) + default_interaction_types: list[str] = Field(default_factory=list) + limits: EvaluationOptionLimits + + +class EvaluationMetrics(_Model): + grades: dict[str, int] = Field(default_factory=dict) + tags: dict[str, int] = Field(default_factory=dict) + + +class EvaluationRunSkipped(_Model): + ineligible: list[str] = Field(default_factory=list) + in_flight: list[str] = Field(default_factory=list) + + +class EvaluationRunResult(_Model): + status: Literal["queued"] + queued: int + skipped: EvaluationRunSkipped class EvaluationResult(_Model): From bbf5a7d4c3d3c8152d664c5a660038386f45c62a Mon Sep 17 00:00:00 2001 From: dvlpjrs Date: Fri, 4 Sep 2026 05:32:24 +0000 Subject: [PATCH 2/3] [GMLP-9378] Align evaluation run with the request-model pattern and Response type naming --- src/gumloop/cli/commands/agents.py | 4 ++- src/gumloop/resources/agents.py | 46 ++++++++++++++++++------------ src/gumloop/types.py | 10 +++++-- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/gumloop/cli/commands/agents.py b/src/gumloop/cli/commands/agents.py index df87526..adee812 100644 --- a/src/gumloop/cli/commands/agents.py +++ b/src/gumloop/cli/commands/agents.py @@ -644,7 +644,9 @@ def run_evaluations( try: if not session_ids: raise GumloopError("Pass at least one --session-id.") - response = cli.call_with_refresh(lambda client: client.agents.run_evaluations(agent_id, session_ids)) + response = cli.call_with_refresh( + lambda client: client.agents.run_evaluations(agent_id, session_ids=session_ids) + ) except GumloopError as error: exit_with_error(error, json_output=json_output) diff --git a/src/gumloop/resources/agents.py b/src/gumloop/resources/agents.py index 93333ee..eb83ba0 100644 --- a/src/gumloop/resources/agents.py +++ b/src/gumloop/resources/agents.py @@ -19,11 +19,12 @@ from gumloop.types import AgentVersionsResponse from gumloop.types import EvaluationConfigResponse from gumloop.types import EvaluationConfigUpdateRequest -from gumloop.types import EvaluationMetrics -from gumloop.types import EvaluationOptions +from gumloop.types import EvaluationMetricsResponse +from gumloop.types import EvaluationOptionsResponse from gumloop.types import EvaluationResultListResponse from gumloop.types import EvaluationResultResponse -from gumloop.types import EvaluationRunResult +from gumloop.types import EvaluationRunRequest +from gumloop.types import EvaluationRunResponse from gumloop.types import ModelListResponse from gumloop.types import SkillListResponse @@ -136,8 +137,8 @@ def detach_mcp_server(self, agent_id: str, server_id: str) -> AgentMcpServerDeta def list_mcp_servers(self, agent_id: str) -> AgentMcpServersResponse: return AgentMcpServersResponse.model_validate(self._client.get(f"agents/{agent_id}/mcp-servers")) - def get_evaluation_options(self) -> EvaluationOptions: - return EvaluationOptions.model_validate(self._client.get("agents/evaluation-options")) + def get_evaluation_options(self) -> EvaluationOptionsResponse: + return EvaluationOptionsResponse.model_validate(self._client.get("agents/evaluation-options")) def get_evaluation_config(self, agent_id: str) -> EvaluationConfigResponse: return EvaluationConfigResponse.model_validate(self._client.get(f"agents/{agent_id}/evaluation-config")) @@ -187,14 +188,19 @@ def list_evaluations( ) ) - def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetrics: - return EvaluationMetrics.model_validate( + def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetricsResponse: + return EvaluationMetricsResponse.model_validate( self._client.get(f"agents/{agent_id}/evaluations/metrics", params={"days": days}) ) - def run_evaluations(self, agent_id: str, session_ids: list[str]) -> EvaluationRunResult: - return EvaluationRunResult.model_validate( - self._client.post(f"agents/{agent_id}/evaluations/run", json={"session_ids": session_ids}) + def run_evaluations( + self, + agent_id: str, + request: EvaluationRunRequest | Mapping[str, Any] | None = None, + **kwargs: Any, + ) -> EvaluationRunResponse: + return EvaluationRunResponse.model_validate( + self._client.post(f"agents/{agent_id}/evaluations/run", json=EvaluationRunRequest.build(request, **kwargs)) ) def get_evaluation(self, agent_id: str, evaluation_id: str) -> EvaluationResultResponse: @@ -310,9 +316,9 @@ async def list_mcp_servers(self, agent_id: str) -> AgentMcpServersResponse: data = await self._client.get(f"agents/{agent_id}/mcp-servers") return AgentMcpServersResponse.model_validate(data) - async def get_evaluation_options(self) -> EvaluationOptions: + async def get_evaluation_options(self) -> EvaluationOptionsResponse: data = await self._client.get("agents/evaluation-options") - return EvaluationOptions.model_validate(data) + return EvaluationOptionsResponse.model_validate(data) async def get_evaluation_config(self, agent_id: str) -> EvaluationConfigResponse: data = await self._client.get(f"agents/{agent_id}/evaluation-config") @@ -361,16 +367,20 @@ async def list_evaluations( ) return EvaluationResultListResponse.model_validate(data) - async def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetrics: + async def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetricsResponse: data = await self._client.get(f"agents/{agent_id}/evaluations/metrics", params={"days": days}) - return EvaluationMetrics.model_validate(data) + return EvaluationMetricsResponse.model_validate(data) - async def run_evaluations(self, agent_id: str, session_ids: list[str]) -> EvaluationRunResult: + async def run_evaluations( + self, + agent_id: str, + request: EvaluationRunRequest | Mapping[str, Any] | None = None, + **kwargs: Any, + ) -> EvaluationRunResponse: data = await self._client.post( - f"agents/{agent_id}/evaluations/run", - json={"session_ids": session_ids}, + f"agents/{agent_id}/evaluations/run", json=EvaluationRunRequest.build(request, **kwargs) ) - return EvaluationRunResult.model_validate(data) + return EvaluationRunResponse.model_validate(data) async def get_evaluation(self, agent_id: str, evaluation_id: str) -> EvaluationResultResponse: data = await self._client.get(f"agents/{agent_id}/evaluations/{evaluation_id}") diff --git a/src/gumloop/types.py b/src/gumloop/types.py index ce614f1..f2b3628 100644 --- a/src/gumloop/types.py +++ b/src/gumloop/types.py @@ -720,7 +720,7 @@ class EvaluationOptionLimits(_Model): notification_recipients: int -class EvaluationOptions(_Model): +class EvaluationOptionsResponse(_Model): interaction_types: list[str] = Field(default_factory=list) criterion_types: list[str] = Field(default_factory=list) criterion_priorities: list[str] = Field(default_factory=list) @@ -730,17 +730,21 @@ class EvaluationOptions(_Model): limits: EvaluationOptionLimits -class EvaluationMetrics(_Model): +class EvaluationMetricsResponse(_Model): grades: dict[str, int] = Field(default_factory=dict) tags: dict[str, int] = Field(default_factory=dict) +class EvaluationRunRequest(_Model): + session_ids: list[str] + + class EvaluationRunSkipped(_Model): ineligible: list[str] = Field(default_factory=list) in_flight: list[str] = Field(default_factory=list) -class EvaluationRunResult(_Model): +class EvaluationRunResponse(_Model): status: Literal["queued"] queued: int skipped: EvaluationRunSkipped From a9fa4b3e8eb95c93675f6c528e0b1c7ea879dab0 Mon Sep 17 00:00:00 2001 From: dvlpjrs Date: Fri, 4 Sep 2026 05:38:53 +0000 Subject: [PATCH 3/3] [GMLP-9378] Bump version to 0.4.10 --- src/gumloop/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gumloop/_version.py b/src/gumloop/_version.py index 2a716d4..0266b88 100644 --- a/src/gumloop/_version.py +++ b/src/gumloop/_version.py @@ -1,3 +1,3 @@ from __future__ import annotations -__version__ = "0.4.9" +__version__ = "0.4.10"