diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ce05be..e6391b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `COMPATIBILITY_MANIFEST` records each built-in adapter's install extra, import module, accepted SDK range, tested lockfile version, and runtime binary versions such as `openai-codex-cli-bin`. +- `RuntimeReadiness`, `ReadinessStatus`, the optional + `RuntimeReadinessProvider`, and bounded async `check_readiness()` probes + distinguish `READY_TO_ATTEMPT`, `NOT_READY`, and `INDETERMINATE` without + running an agent task. Registry, `AgentKit`, and provider-diagnostics helpers + expose the same async checks. ### Changed @@ -37,6 +42,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 tool-filter, and per-MCP-server environment support. Built-in `run()` methods consume the same support report used by preflight, eliminating divergent rejection paths. +- `availability()` is now strictly package-only and side-effect-free. Credential + and setup checks moved to async readiness probes: Codex uses the supported + account API with guaranteed cleanup, and Antigravity runs Google ADC discovery + off the event loop. Probe failures/timeouts are secret-safe and indeterminate. ### Fixed diff --git a/README.md b/README.md index a0e1960..66cf597 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ capabilities visible. `agent-runtime-kit` is for Python developers who want to run coding-agent tasks through vendor runtimes without rewriting their application around each SDK. It normalizes the runtime boundary: task inputs, capability checks, event streams, -tool audits, availability diagnostics, and typed results. +tool audits, package/readiness diagnostics, and typed results. The library keeps vendor differences visible. Claude, Codex, and Antigravity still expose different capabilities, permission models, setup requirements, and @@ -23,7 +23,7 @@ shape instead of hiding them behind a lowest-common-denominator wrapper. The package is intentionally not a router, benchmark harness, queue, hosted service, or full agent framework. It is the reusable layer underneath those -systems: task models, runtime capabilities, event sinks, availability +systems: task models, runtime capabilities, event sinks, package/readiness diagnostics, and adapters. ## Install @@ -83,7 +83,7 @@ caches them per kind, and turns Python types into structured output. import asyncio from dataclasses import dataclass -from agent_runtime_kit import AgentKit +from agent_runtime_kit import AgentKit, ReadinessStatus @dataclass @@ -94,9 +94,9 @@ class RepoSummary: async def main() -> None: async with AgentKit() as kit: - diagnostic = kit.availability_for("claude") - if not diagnostic.available: - raise RuntimeError(diagnostic.message) + readiness = await kit.readiness_for("claude") + if readiness.status is ReadinessStatus.NOT_READY: + raise RuntimeError(readiness.message) result = await kit.run( "claude", goal="Summarize this repository", @@ -114,15 +114,15 @@ Adapters also work standalone when you need vendor-specific configuration: ```python import asyncio -from agent_runtime_kit import AgentTask +from agent_runtime_kit import AgentTask, ReadinessStatus, check_readiness from agent_runtime_kit.adapters import ClaudeAgentRuntime async def main() -> None: runtime = ClaudeAgentRuntime(default_model="claude-sonnet-4-6") - diagnostic = runtime.availability() - if not diagnostic.available: - raise RuntimeError(diagnostic.message) + readiness = await check_readiness(runtime) + if readiness.status is ReadinessStatus.NOT_READY: + raise RuntimeError(readiness.message) result = await runtime.run(AgentTask(goal="Summarize this repository")) print(result.output) @@ -149,6 +149,13 @@ issue. Third-party runtimes can opt into provider-specific checks with the `TaskSupportProvider` protocol, while older `AgentRuntime` implementations continue to work through capability-based fallback checks. +`availability()` is deliberately synchronous, side-effect-free, and package-only. +Use `await check_readiness(runtime)` (or `kit.readiness_for(...)`) for an explicit, +bounded credential/setup probe. `READY_TO_ATTEMPT` means setup was positively +detected, not that a future provider call is guaranteed to succeed; +`INDETERMINATE` lets callers decide whether to attempt provider-chain or local +login authentication. Probe failures and timeouts never include credential values. + `AgentResult` returns output, finish reason (see `FinishReason`), locally validated structured output, usage, cost, session id, tool-call audits, and provider metadata. `is_success` is true only for a natural, error-free diff --git a/docs/api-stability.md b/docs/api-stability.md index e302f29..6b54726 100644 --- a/docs/api-stability.md +++ b/docs/api-stability.md @@ -50,6 +50,13 @@ import the names from the top-level package instead. `TaskSupportProvider` is an optional extension, not a new requirement on the `AgentRuntime` protocol, so existing third-party runtimes retain structural compatibility and use capability-based fallback checks. +- **Package availability and execution readiness are separate.** Synchronous + `availability()` is package-only and side-effect-free. The bounded async + `check_readiness()` helper uses the optional `RuntimeReadinessProvider` + extension without changing the `AgentRuntime` protocol. A third-party runtime + without the extension maps a missing package to `NOT_READY` and present + package to `INDETERMINATE`. `READY_TO_ATTEMPT` establishes only that known + setup signals are present; it is not a guarantee of future execution. - **`AgentKit` is sugar, not a second API.** The hub assembles the same frozen `AgentTask` and returns the same `AgentResult` the runtimes produce (`ParsedResult` is a runtime-identical subclass adding only the typed diff --git a/docs/capability-matrix.md b/docs/capability-matrix.md index 98bf1bc..8db793e 100644 --- a/docs/capability-matrix.md +++ b/docs/capability-matrix.md @@ -12,8 +12,9 @@ | Streaming output events | Yes — incremental `output.delta` while the SDK runs | No — a single `output.delta` at the end (non-streaming SDK) | Yes — from response chunks | | Tool audit events | Yes — streamed from message blocks | Yes — emitted from parsed `TurnResult` items after the turn | Yes — from tool chunks | | `vendor.turn` events | No | No | Yes — from thought/unknown chunks | -| Missing package diagnostics | Yes (`AgentRuntimeUnavailableError`) | Yes (`AgentRuntimeUnavailableError`) | Yes (`AgentRuntimeUnavailableError`) | -| Missing credential diagnostics | No — `availability()` reports available with an `auth_source` label; auth failures surface at `run()` | No — same as Claude (`auth_source` label; deferred) | Yes — `availability()` returns `MISSING_CREDENTIALS` when no API key / ADC-Vertex project is configured | +| Package availability | Sync, side-effect-free, package-only | Sync, side-effect-free, package-only | Sync, side-effect-free, package-only; never calls ADC | +| Async readiness | Direct API key/OAuth/Bedrock bearer signal → `READY_TO_ATTEMPT`; provider/local chains → `INDETERMINATE` | Supported `AsyncCodex.account(refresh_token=False)` probe; absent account → `NOT_READY` | API key or off-loop ADC/project probe; missing → `NOT_READY` | +| Probe failure/timeout | `INDETERMINATE` | `INDETERMINATE`, app-server always closed | `INDETERMINATE` | | Live smoke test | Opt-in | Opt-in | Opt-in | The matrix is intentionally not a lowest-common-denominator contract. Adapters diff --git a/docs/providers.md b/docs/providers.md index df6793f..5a49457 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -1,7 +1,8 @@ # Provider Diagnostics -`agent-runtime-kit` keeps provider setup checks explicit. Each runtime exposes -`availability()` and returns a `RuntimeAvailability` value with: +`agent-runtime-kit` separates cheap package discovery from provider setup +probes. Each runtime's synchronous `availability()` is side-effect-free and +package-only. It returns a `RuntimeAvailability` value with: - runtime kind - availability flag @@ -10,6 +11,64 @@ - package name - installed version when discoverable +Availability never imports a vendor SDK, starts a subprocess, reads a +credential store, calls Google ADC, or contacts a provider. Package presence is +not a claim that execution will work. + +Use the public `await check_readiness(runtime, timeout=5.0)` for an explicit +credential/setup probe. Registry and hub forms are also available: + +```python +readiness = await registry.readiness_for("codex") +readiness = await kit.readiness_for("codex") +all_readiness = await kit.readiness() +``` + +`RuntimeReadiness.status` has three outcomes: + +- `READY_TO_ATTEMPT`: a supported setup or credential signal was positively + established. It does not promise that a later model/network request succeeds. +- `NOT_READY`: a concrete problem such as a missing package, account, API key, + or ADC project was established. +- `INDETERMINATE`: the runtime uses a provider/local credential chain that + cannot be verified safely, lacks the optional readiness extension, or the + bounded probe failed/timed out. The caller chooses whether to attempt work. + +The built-in probes never return credential values or raw exception messages. +They report only safe categories such as auth source and exception type. The +optional `RuntimeReadinessProvider` protocol does not change the required +`AgentRuntime` protocol. For older third-party runtimes, +`check_readiness()` maps negative availability to `NOT_READY` and positive +package presence conservatively to `INDETERMINATE`. + +Provider behavior is deliberately specific: + +- Claude treats direct API key, OAuth-token, and Bedrock bearer-token signals + as `READY_TO_ATTEMPT`. Provider chains and provider-owned local login remain + `INDETERMINATE`; probing them would require starting a real task or scraping + unsupported credential stores. +- Codex starts the supported `AsyncCodex` client, calls + `account(refresh_token=False)`, and always closes the app-server context. An + account is `READY_TO_ATTEMPT`, no account is `NOT_READY`, and startup/API/ + cleanup failures are `INDETERMINATE`. +- Antigravity accepts an explicit or ambient Gemini API key, otherwise probes + Google Application Default Credentials and project setup in a worker thread + so synchronous Google discovery cannot block the event loop. Missing setup + is `NOT_READY`; errors and timeouts are `INDETERMINATE`. + +For all built-ins at once, the existing sync collector stays package-only while +the async collector performs bounded readiness checks: + +```python +from agent_runtime_kit.adapters.diagnostics import ( + collect_provider_diagnostics, + collect_provider_readiness, +) + +packages = collect_provider_diagnostics() +readiness = await collect_provider_readiness(timeout=5.0) +``` + ## Install Model The plain `agent-runtime-kit` package is the vendor-SDK-free core and includes diff --git a/docs/quickstart.md b/docs/quickstart.md index ad6cb43..7f57c09 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -122,5 +122,10 @@ runtime = ClaudeAgentRuntime(default_model="claude-sonnet-4-6", reuse_process=Tr result = await runtime.run(AgentTask(goal="Summarize this repository")) ``` -Use `kit.availability()` (or `runtime.availability()`) before dispatching work -in applications that need clear setup diagnostics. +Use synchronous `kit.availability()` (or `runtime.availability()`) for a +side-effect-free package check. Use `await kit.readiness()` or +`await check_readiness(runtime)` when an application also needs a bounded +credential/setup probe. Treat `ReadinessStatus.READY_TO_ATTEMPT` as permission +to try — not a guarantee that the provider, model, or network will accept the +next request. `NOT_READY` is a confirmed setup problem; `INDETERMINATE` leaves +the attempt policy to the caller. diff --git a/src/agent_runtime_kit/__init__.py b/src/agent_runtime_kit/__init__.py index bd7648d..416207c 100644 --- a/src/agent_runtime_kit/__init__.py +++ b/src/agent_runtime_kit/__init__.py @@ -25,7 +25,10 @@ ParsedResult, PermissionMode, PermissionProfile, + ReadinessStatus, RuntimeAvailability, + RuntimeReadiness, + RuntimeReadinessProvider, SessionResumeState, TaskSupportIssue, TaskSupportProvider, @@ -50,6 +53,7 @@ tool_requested_event, vendor_turn_event, ) +from agent_runtime_kit.readiness import DEFAULT_READINESS_TIMEOUT, check_readiness from agent_runtime_kit.registry import RuntimeRegistry, create_default_registry from agent_runtime_kit.support import validate_task @@ -64,12 +68,13 @@ "AgentTask", "ArtifactRef", "AvailabilityReason", + "COMPATIBILITY_MANIFEST", + "DEFAULT_READINESS_TIMEOUT", "EventSink", "FakeAgentRuntime", "FilesystemAccess", "FinishReason", "KIND_ALIASES", - "COMPATIBILITY_MANIFEST", "McpServerConfig", "OutputSchemaError", "OutputTypeError", @@ -77,9 +82,12 @@ "ParsedResult", "PermissionMode", "PermissionProfile", + "ReadinessStatus", "RuntimeAvailability", "RuntimeCompatibility", "RuntimeNotRegisteredError", + "RuntimeReadiness", + "RuntimeReadinessProvider", "RuntimeRegistry", "SessionResumeState", "TaskSupportIssue", @@ -90,6 +98,7 @@ "Usage", "create_default_registry", "compatibility_for", + "check_readiness", "runtime_kind_value", "output_delta_event", "safe_emit", diff --git a/src/agent_runtime_kit/_kit.py b/src/agent_runtime_kit/_kit.py index 2268e7e..53f070f 100644 --- a/src/agent_runtime_kit/_kit.py +++ b/src/agent_runtime_kit/_kit.py @@ -25,9 +25,11 @@ PermissionMode, PermissionProfile, RuntimeAvailability, + RuntimeReadiness, SessionResumeState, TaskSupportReport, ) +from agent_runtime_kit.readiness import DEFAULT_READINESS_TIMEOUT, check_readiness from agent_runtime_kit.registry import RuntimeFactory, RuntimeRegistry, create_default_registry from agent_runtime_kit.support import validate_task as validate_runtime_task @@ -64,8 +66,8 @@ class AgentKit: ``AgentKit()`` builds a registry with the fake runtime and the vendor adapters registered (adapters resolve their SDKs lazily, so this works - without any extra installed — ``availability_for`` reports what is - missing). Pass ``registry=`` to bring your own; the other flags then do + without any extra installed — ``availability_for`` reports missing + packages). Pass ``registry=`` to bring your own; the other flags then do not apply. Runtimes resolved through the hub are constructed zero-arg, cached per @@ -101,13 +103,37 @@ def kinds(self) -> tuple[AgentRuntimeKind | str, ...]: return self._registry.kinds() def availability(self) -> tuple[RuntimeAvailability, ...]: - """Availability diagnostics for every registered kind.""" + """Side-effect-free package diagnostics for every registered kind.""" return tuple(self._registry.availability_for(kind) for kind in self._registry.kinds()) def availability_for(self, kind: AgentRuntimeKind | str) -> RuntimeAvailability: return self._registry.availability_for(self._normalize_kind(kind)) + async def readiness( + self, + *, + timeout: float = DEFAULT_READINESS_TIMEOUT, + ) -> tuple[RuntimeReadiness, ...]: + """Probe every registered runtime concurrently.""" + + return tuple( + await asyncio.gather( + *(self.readiness_for(kind, timeout=timeout) for kind in self.kinds()) + ) + ) + + async def readiness_for( + self, + kind: AgentRuntimeKind | str, + *, + timeout: float = DEFAULT_READINESS_TIMEOUT, + ) -> RuntimeReadiness: + """Probe one cached runtime without executing an agent task.""" + + runtime = await self._runtime_for(self._normalize_kind(kind)) + return await check_readiness(runtime, timeout=timeout) + def capabilities_for(self, kind: AgentRuntimeKind | str) -> AgentCapabilities: return self._registry.capabilities_for(self._normalize_kind(kind)) diff --git a/src/agent_runtime_kit/_runtime.py b/src/agent_runtime_kit/_runtime.py index fde1ccd..b03d9ce 100644 --- a/src/agent_runtime_kit/_runtime.py +++ b/src/agent_runtime_kit/_runtime.py @@ -12,6 +12,7 @@ AgentRuntimeKind, AgentTask, RuntimeAvailability, + RuntimeReadiness, TaskSupportReport, ToolCallAudit, ) @@ -64,6 +65,17 @@ def validate_task(self, task: AgentTask) -> TaskSupportReport: return _validate_declared_task_support(self.kind, self.capabilities, task) + async def check_readiness(self) -> RuntimeReadiness: + """Fake runtime is deterministic and always ready to attempt.""" + + availability = self.availability() + return RuntimeReadiness.ready_to_attempt( + self.kind, + package=availability.package, + version=availability.version, + metadata=availability.metadata, + ) + async def run(self, task: AgentTask) -> AgentResult: """Return a deterministic result after validating capabilities.""" diff --git a/src/agent_runtime_kit/_types.py b/src/agent_runtime_kit/_types.py index ea37c8a..1b69af2 100644 --- a/src/agent_runtime_kit/_types.py +++ b/src/agent_runtime_kit/_types.py @@ -166,6 +166,14 @@ class AvailabilityReason(str, Enum): UNKNOWN = "unknown" +class ReadinessStatus(str, Enum): + """Whether a runtime can start work in the current environment.""" + + READY_TO_ATTEMPT = "ready-to-attempt" + NOT_READY = "not-ready" + INDETERMINATE = "indeterminate" + + class PermissionMode(str, Enum): """High-level permission intent for vendor runtimes.""" @@ -280,7 +288,7 @@ def validate_task(self, task: AgentTask) -> TaskSupportReport: @dataclass(frozen=True) class RuntimeAvailability: - """Availability diagnostic for a runtime in the current environment.""" + """Side-effect-free package/loadability diagnostic for a runtime.""" kind: AgentRuntimeKind | str available: bool @@ -336,6 +344,168 @@ def unavailable( ) +@dataclass(frozen=True) +class RuntimeReadiness: + """Execution-readiness diagnostic produced by an explicit async probe. + + ``status`` is the readiness conclusion. ``reason`` retains the related + package/setup category, so an installed package can legitimately produce + ``INDETERMINATE`` with ``AvailabilityReason.AVAILABLE`` when credentials or + provider connectivity cannot be verified safely. + """ + + kind: AgentRuntimeKind | str + status: ReadinessStatus + message: str + reason: AvailabilityReason = AvailabilityReason.UNKNOWN + package: str | None = None + version: str | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "kind", AgentRuntimeKind.coerce(self.kind)) + object.__setattr__( + self, + "status", + _coerce_enum(ReadinessStatus, self.status, "RuntimeReadiness.status"), + ) + object.__setattr__( + self, + "reason", + _coerce_enum(AvailabilityReason, self.reason, "RuntimeReadiness.reason"), + ) + _require_nonblank(self.message, "RuntimeReadiness.message") + if self.package is not None: + _require_nonblank(self.package, "RuntimeReadiness.package") + if self.version is not None: + _require_nonblank(self.version, "RuntimeReadiness.version") + if not isinstance(self.metadata, Mapping): + raise ValueError("RuntimeReadiness.metadata must be a mapping") + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + @property + def is_ready_to_attempt(self) -> bool: + """Return whether setup is sufficient to attempt execution.""" + + return self.status is ReadinessStatus.READY_TO_ATTEMPT + + @classmethod + def ready_to_attempt( + cls, + kind: AgentRuntimeKind | str, + *, + message: str = "ready to attempt", + package: str | None = None, + version: str | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> RuntimeReadiness: + """Build a positive setup diagnostic without promising execution.""" + + return cls( + kind=kind, + status=ReadinessStatus.READY_TO_ATTEMPT, + reason=AvailabilityReason.AVAILABLE, + message=message, + package=package, + version=version, + metadata=dict(metadata or {}), + ) + + @classmethod + def not_ready( + cls, + kind: AgentRuntimeKind | str, + *, + reason: AvailabilityReason, + message: str, + package: str | None = None, + version: str | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> RuntimeReadiness: + """Build a negative readiness diagnostic.""" + + return cls( + kind=kind, + status=ReadinessStatus.NOT_READY, + reason=reason, + message=message, + package=package, + version=version, + metadata=dict(metadata or {}), + ) + + @classmethod + def indeterminate( + cls, + kind: AgentRuntimeKind | str, + *, + message: str, + reason: AvailabilityReason = AvailabilityReason.UNKNOWN, + package: str | None = None, + version: str | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> RuntimeReadiness: + """Build a diagnostic for a probe that could not reach a conclusion.""" + + return cls( + kind=kind, + status=ReadinessStatus.INDETERMINATE, + reason=reason, + message=message, + package=package, + version=version, + metadata=dict(metadata or {}), + ) + + @classmethod + def from_availability( + cls, + availability: RuntimeAvailability, + *, + status: ReadinessStatus, + message: str | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> RuntimeReadiness: + """Build readiness while preserving non-metadata availability fields. + + Availability metadata is not copied implicitly because third-party + implementations may have placed credential material there. Callers may + pass explicitly curated readiness metadata instead. + """ + + combined_metadata = dict(metadata or {}) + fallback_messages = { + ReadinessStatus.READY_TO_ATTEMPT: "ready to attempt", + ReadinessStatus.NOT_READY: "not ready", + ReadinessStatus.INDETERMINATE: "readiness could not be determined", + } + normalized_status = _coerce_enum( + ReadinessStatus, status, "RuntimeReadiness.status" + ) + if ( + normalized_status is ReadinessStatus.READY_TO_ATTEMPT + and not availability.available + ): + raise ValueError("an unavailable package cannot be ready to attempt") + return cls( + kind=availability.kind, + status=normalized_status, + reason=availability.reason, + message=message or availability.message or fallback_messages[normalized_status], + package=availability.package, + version=availability.version, + metadata=combined_metadata, + ) + + +@runtime_checkable +class RuntimeReadinessProvider(Protocol): + """Optional extension for runtimes that can probe execution readiness.""" + + async def check_readiness(self) -> RuntimeReadiness: + """Probe credentials/setup without executing an agent task.""" + + @dataclass(frozen=True) class McpServerConfig: """Configuration for a stdio MCP server owned by a vendor runtime.""" @@ -638,7 +808,7 @@ def kind(self) -> AgentRuntimeKind | str: ... capabilities: AgentCapabilities def availability(self) -> RuntimeAvailability: - """Report whether this runtime can execute in the current environment.""" + """Report package/loadability without credential or network probes.""" async def run(self, task: AgentTask) -> AgentResult: """Execute one task.""" diff --git a/src/agent_runtime_kit/adapters/_common.py b/src/agent_runtime_kit/adapters/_common.py index d3cf550..e3c6240 100644 --- a/src/agent_runtime_kit/adapters/_common.py +++ b/src/agent_runtime_kit/adapters/_common.py @@ -5,7 +5,7 @@ import inspect import os from collections.abc import Iterable, Mapping -from importlib import metadata, util +from importlib import metadata from math import isfinite from typing import Any @@ -28,14 +28,11 @@ def package_availability(kind: AgentRuntimeKind) -> RuntimeAvailability: - """Return import/package availability without importing the package.""" + """Return installed-distribution availability without resolving a module.""" compatibility = compatibility_for(kind) - try: - module_spec = util.find_spec(compatibility.module) - except ModuleNotFoundError: - module_spec = None - if module_spec is None: + version = package_version(compatibility.package) + if version is None: return RuntimeAvailability.unavailable( kind, reason=AvailabilityReason.MISSING_PACKAGE, @@ -48,7 +45,7 @@ def package_availability(kind: AgentRuntimeKind) -> RuntimeAvailability: return RuntimeAvailability.ok( kind, package=compatibility.package, - version=package_version(compatibility.package), + version=version, ) diff --git a/src/agent_runtime_kit/adapters/antigravity.py b/src/agent_runtime_kit/adapters/antigravity.py index 898b4cb..07c7399 100644 --- a/src/agent_runtime_kit/adapters/antigravity.py +++ b/src/agent_runtime_kit/adapters/antigravity.py @@ -22,7 +22,9 @@ AvailabilityReason, FilesystemAccess, PermissionMode, + ReadinessStatus, RuntimeAvailability, + RuntimeReadiness, TaskSupportIssue, TaskSupportReport, ToolCallAudit, @@ -116,35 +118,78 @@ async def __aexit__(self, _exc_type: object, _exc: object, _tb: object) -> None: await self.aclose() def availability(self) -> RuntimeAvailability: - """Report Antigravity package and credential availability. - - Synchronous by design (a setup diagnostic), but ADC discovery may read - files or hit the GCE metadata server — from async code, call it via - ``asyncio.to_thread`` like ``run()`` does internally. - """ + """Report Antigravity package presence without probing Google ADC.""" if self._agent_cls is not None: return RuntimeAvailability.ok(self.kind, package="google-antigravity") - package = package_availability(self.kind) - if not package.available: - return package - auth = self._auth_config() - if not auth.available: - return RuntimeAvailability.unavailable( + return package_availability(self.kind) + + async def check_readiness(self) -> RuntimeReadiness: + """Check API-key or Google ADC setup without starting an agent task.""" + + availability = self.availability() + if not availability.available: + return RuntimeReadiness.from_availability( + availability, + status=ReadinessStatus.NOT_READY, + ) + ambient_key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") + if self._api_key or (ambient_key and self._vertex is not True): + return RuntimeReadiness.ready_to_attempt( + self.kind, + message="An Antigravity API-key credential signal is configured.", + package=availability.package, + version=availability.version, + metadata={"auth_source": "api-key"}, + ) + if self._vertex is False: + return self._missing_credentials_readiness(availability) + try: + adc = await asyncio.to_thread(_probe_google_adc) + except Exception as exc: + if _is_missing_google_credentials(exc): + return self._missing_credentials_readiness(availability) + return RuntimeReadiness.indeterminate( self.kind, - reason=AvailabilityReason.MISSING_CREDENTIALS, + reason=AvailabilityReason.SETUP_FAILED, message=( - "Set GEMINI_API_KEY or GOOGLE_API_KEY, or configure Google " - "Application Default Credentials with a Vertex AI project/location." + "The Google Application Default Credentials probe failed before " + "readiness was known." ), - package="google-antigravity", - metadata=package.metadata, + package=availability.package, + version=availability.version, + metadata={"failure": "adc-probe", "error_type": type(exc).__name__}, ) - return RuntimeAvailability.ok( + project = ( + self._project + or _env_first("GOOGLE_CLOUD_PROJECT", "GCLOUD_PROJECT") + or adc.project + ) + if not adc.credentials_available or not project: + return self._missing_credentials_readiness(availability) + return RuntimeReadiness.ready_to_attempt( + self.kind, + message="Google Application Default Credentials and a Vertex project are configured.", + package=availability.package, + version=availability.version, + metadata={ + "auth_source": "application-default-credentials", + "project_configured": True, + }, + ) + + def _missing_credentials_readiness( + self, availability: RuntimeAvailability + ) -> RuntimeReadiness: + return RuntimeReadiness.not_ready( self.kind, - package="google-antigravity", - version=package.version, - metadata={**dict(package.metadata), "auth_source": auth.source}, + reason=AvailabilityReason.MISSING_CREDENTIALS, + message=( + "Set GEMINI_API_KEY or GOOGLE_API_KEY, or configure Google Application " + "Default Credentials with a Vertex AI project." + ), + package=availability.package, + version=availability.version, ) def validate_task(self, task: AgentTask) -> TaskSupportReport: @@ -659,6 +704,12 @@ def available(self) -> bool: ) +@dataclass(frozen=True) +class _GoogleADCProbe: + credentials_available: bool + project: str | None + + def _capability_policy( kind: AgentRuntimeKind, task: AgentTask, sdk: _AntigravitySDK ) -> tuple[Any, list[Any]]: @@ -764,13 +815,28 @@ def _env_first(*names: str) -> str | None: return None +def _probe_google_adc() -> _GoogleADCProbe: + google_auth = importlib.import_module("google.auth") + credentials, project = google_auth.default() + return _GoogleADCProbe( + credentials_available=credentials is not None, + project=str(project) if project else None, + ) + + def _google_adc_project() -> str | None: try: - google_auth = importlib.import_module("google.auth") - _, project = google_auth.default() + probe = _probe_google_adc() except Exception: return None - return str(project) if project else None + return probe.project if probe.credentials_available else None + + +def _is_missing_google_credentials(exc: Exception) -> bool: + return ( + type(exc).__name__ == "DefaultCredentialsError" + and type(exc).__module__.startswith("google.auth") + ) def _default_tools(task: AgentTask, builtin: Any) -> list[Any]: diff --git a/src/agent_runtime_kit/adapters/claude.py b/src/agent_runtime_kit/adapters/claude.py index 1f236ae..b5812d4 100644 --- a/src/agent_runtime_kit/adapters/claude.py +++ b/src/agent_runtime_kit/adapters/claude.py @@ -16,10 +16,13 @@ AgentResult, AgentRuntimeKind, AgentTask, + AvailabilityReason, FilesystemAccess, PermissionMode, PermissionProfile, + ReadinessStatus, RuntimeAvailability, + RuntimeReadiness, TaskSupportReport, ToolCallAudit, Usage, @@ -103,23 +106,43 @@ async def __aexit__(self, _exc_type: object, _exc: object, _tb: object) -> None: await self.aclose() def availability(self) -> RuntimeAvailability: - """Report Claude Agent SDK package availability.""" + """Report Claude package presence without probing credentials.""" - auth_metadata = _claude_auth_metadata(self._env) if self._query_func is not None or self._client_cls is not None: - return RuntimeAvailability.ok( + return RuntimeAvailability.ok(self.kind, package="claude-agent-sdk") + return package_availability(self.kind) + + async def check_readiness(self) -> RuntimeReadiness: + """Inspect supported direct credential signals without starting Claude.""" + + availability = self.availability() + if not availability.available: + return RuntimeReadiness.from_availability( + availability, + status=ReadinessStatus.NOT_READY, + ) + auth_metadata = _claude_auth_metadata(self._env) + if auth_metadata["auth_source"] in { + "anthropic-api-key", + "claude-code-oauth-token", + } or auth_metadata.get("bedrock_api_key_configured") is True: + return RuntimeReadiness.ready_to_attempt( self.kind, - package="claude-agent-sdk", + message="A direct Claude credential signal is configured.", + package=availability.package, + version=availability.version, metadata=auth_metadata, ) - package = package_availability(self.kind) - if not package.available: - return package - return RuntimeAvailability.ok( + return RuntimeReadiness.indeterminate( self.kind, - package="claude-agent-sdk", - version=package.version, - metadata={**dict(package.metadata), **auth_metadata}, + reason=AvailabilityReason.AVAILABLE, + message=( + "The Claude package is installed, but provider-chain or local-login " + "credentials cannot be verified without starting a task." + ), + package=availability.package, + version=availability.version, + metadata=auth_metadata, ) def validate_task(self, task: AgentTask) -> TaskSupportReport: @@ -757,6 +780,8 @@ def _claude_auth_metadata(runtime_env: Mapping[str, str] | None) -> dict[str, An return {"auth_source": "azure-ai-foundry"} if _env_first(env, "ANTHROPIC_API_KEY"): return {"auth_source": "anthropic-api-key"} + if _env_first(env, "CLAUDE_CODE_OAUTH_TOKEN"): + return {"auth_source": "claude-code-oauth-token"} return {"auth_source": "provider-owned-local"} diff --git a/src/agent_runtime_kit/adapters/codex.py b/src/agent_runtime_kit/adapters/codex.py index 1772e9b..caf78d7 100644 --- a/src/agent_runtime_kit/adapters/codex.py +++ b/src/agent_runtime_kit/adapters/codex.py @@ -14,9 +14,12 @@ AgentResult, AgentRuntimeKind, AgentTask, + AvailabilityReason, FilesystemAccess, PermissionMode, + ReadinessStatus, RuntimeAvailability, + RuntimeReadiness, TaskSupportReport, ToolCallAudit, Usage, @@ -112,23 +115,62 @@ async def __aexit__(self, _exc_type: object, _exc: object, _tb: object) -> None: await self.aclose() def availability(self) -> RuntimeAvailability: - """Report OpenAI Codex SDK package availability.""" + """Report Codex package presence without starting app-server.""" - auth_metadata = _codex_auth_metadata(self._config_overrides, self._env) if self._codex_cls is not None: - return RuntimeAvailability.ok( + return RuntimeAvailability.ok(self.kind, package="openai-codex") + return package_availability(self.kind) + + async def check_readiness(self) -> RuntimeReadiness: + """Ask the supported SDK account API whether Codex is authenticated.""" + + availability = self.availability() + if not availability.available: + return RuntimeReadiness.from_availability( + availability, + status=ReadinessStatus.NOT_READY, + ) + auth_metadata = _codex_auth_metadata(self._config_overrides, self._env) + try: + account_response = await self._read_account() + except Exception as exc: + return RuntimeReadiness.indeterminate( self.kind, - package="openai-codex", + reason=AvailabilityReason.SETUP_FAILED, + message="The Codex account probe failed before authentication was known.", + package=availability.package, + version=availability.version, + metadata={ + **auth_metadata, + "failure": "account-probe", + "error_type": type(exc).__name__, + }, + ) + account = field_value(account_response, "account") + if account is None: + return RuntimeReadiness.not_ready( + self.kind, + reason=AvailabilityReason.MISSING_CREDENTIALS, + message="The Codex SDK reported no authenticated account.", + package=availability.package, + version=availability.version, metadata=auth_metadata, ) - package = package_availability(self.kind) - if not package.available: - return package - return RuntimeAvailability.ok( + account_root = field_value(account, "root", account) + account_type = field_value(account_root, "type") + safe_account_types = { + "apiKey": "api-key", + "chatgpt": "chatgpt", + "amazonBedrock": "amazon-bedrock", + } + if account_type in safe_account_types: + auth_metadata["account_type"] = safe_account_types[account_type] + return RuntimeReadiness.ready_to_attempt( self.kind, - package="openai-codex", - version=package.version, - metadata={**dict(package.metadata), **auth_metadata}, + message="The Codex SDK reported an authenticated account.", + package=availability.package, + version=availability.version, + metadata=auth_metadata, ) def validate_task(self, task: AgentTask) -> TaskSupportReport: @@ -239,6 +281,25 @@ def _load_sdk(self) -> tuple[Any, Any, Any, Any]: self._approval_mode_cls or ApprovalMode, ) + async def _read_account(self) -> Any: + codex_cls, config_cls, _, _ = self._load_sdk() + config_kwargs: dict[str, Any] = {"config_overrides": self._config_overrides} + if self._env is not None: + config_kwargs["env"] = dict(self._env) + supported, _ = filter_supported_kwargs(config_cls, config_kwargs) + context = codex_cls(config=config_cls(**supported)) + client = context + try: + enter = getattr(context, "__aenter__", None) + if callable(enter): + client = await enter() + account = getattr(client, "account", None) + if not callable(account): + raise TypeError("installed Codex SDK does not expose AsyncCodex.account") + return await account(refresh_token=False) + finally: + await close_vendor_resource(context, fallback=client) + async def _run_codex( self, task: AgentTask, diff --git a/src/agent_runtime_kit/adapters/diagnostics.py b/src/agent_runtime_kit/adapters/diagnostics.py index c045ca8..99a0712 100644 --- a/src/agent_runtime_kit/adapters/diagnostics.py +++ b/src/agent_runtime_kit/adapters/diagnostics.py @@ -2,17 +2,41 @@ from __future__ import annotations -from agent_runtime_kit._types import RuntimeAvailability +import asyncio + +from agent_runtime_kit._types import RuntimeAvailability, RuntimeReadiness from agent_runtime_kit.adapters.antigravity import AntigravityAgentRuntime from agent_runtime_kit.adapters.claude import ClaudeAgentRuntime from agent_runtime_kit.adapters.codex import CodexAgentRuntime +from agent_runtime_kit.readiness import DEFAULT_READINESS_TIMEOUT, check_readiness def collect_provider_diagnostics() -> tuple[RuntimeAvailability, ...]: - """Return availability diagnostics for installed provider adapters.""" + """Return synchronous, package-only diagnostics for provider adapters.""" return ( ClaudeAgentRuntime().availability(), CodexAgentRuntime().availability(), AntigravityAgentRuntime().availability(), ) + + +async def collect_provider_readiness( + *, + timeout: float = DEFAULT_READINESS_TIMEOUT, +) -> tuple[RuntimeReadiness, ...]: + """Probe provider credentials/setup concurrently with a bound per runtime.""" + + runtimes = ( + ClaudeAgentRuntime(), + CodexAgentRuntime(), + AntigravityAgentRuntime(), + ) + return tuple( + await asyncio.gather( + *(check_readiness(runtime, timeout=timeout) for runtime in runtimes) + ) + ) + + +__all__ = ["collect_provider_diagnostics", "collect_provider_readiness"] diff --git a/src/agent_runtime_kit/readiness.py b/src/agent_runtime_kit/readiness.py new file mode 100644 index 0000000..b19a9ca --- /dev/null +++ b/src/agent_runtime_kit/readiness.py @@ -0,0 +1,128 @@ +"""Bounded, explicit runtime-readiness probes.""" + +from __future__ import annotations + +import asyncio +from math import isfinite + +from agent_runtime_kit._types import ( + AgentRuntime, + AvailabilityReason, + ReadinessStatus, + RuntimeAvailability, + RuntimeReadiness, + runtime_kind_value, +) + +DEFAULT_READINESS_TIMEOUT = 5.0 + + +async def check_readiness( + runtime: AgentRuntime, + *, + timeout: float = DEFAULT_READINESS_TIMEOUT, +) -> RuntimeReadiness: + """Return bounded execution readiness without running an agent task. + + Runtimes may implement the optional :class:`RuntimeReadinessProvider` + extension. Older third-party runtimes retain compatibility: a negative + package availability result maps to ``NOT_READY`` while a positive one maps + conservatively to ``INDETERMINATE`` because package presence alone does not + prove credentials or provider setup. + + Probe exceptions and timeouts are converted to secret-safe indeterminate + diagnostics. Task cancellation still propagates normally. + """ + + _validate_timeout(timeout) + probe = getattr(runtime, "check_readiness", None) + if not callable(probe): + return _legacy_readiness(runtime) + + try: + readiness = await asyncio.wait_for(probe(), timeout=float(timeout)) + if not isinstance(readiness, RuntimeReadiness): + raise TypeError("readiness provider returned an invalid result") + if runtime_kind_value(readiness.kind) != runtime_kind_value(runtime.kind): + raise ValueError("readiness provider returned a different runtime kind") + return readiness + except asyncio.TimeoutError: + return _probe_failure( + runtime, + message=f"Readiness probe timed out after {float(timeout):g} seconds.", + failure="timeout", + ) + except Exception as exc: + return _probe_failure( + runtime, + message="Readiness probe failed before it could reach a conclusion.", + failure="error", + error_type=type(exc).__name__, + ) + + +def _validate_timeout(timeout: float) -> None: + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not isfinite(float(timeout)) + or timeout <= 0 + ): + raise ValueError("timeout must be a positive finite number") + + +def _legacy_readiness(runtime: AgentRuntime) -> RuntimeReadiness: + try: + availability = runtime.availability() + except Exception as exc: + return RuntimeReadiness.indeterminate( + runtime.kind, + reason=AvailabilityReason.SETUP_FAILED, + message="Package availability failed before readiness could be determined.", + metadata={"failure": "availability", "error_type": type(exc).__name__}, + ) + if not availability.available: + return RuntimeReadiness.from_availability( + availability, + status=ReadinessStatus.NOT_READY, + ) + return RuntimeReadiness.from_availability( + availability, + status=ReadinessStatus.INDETERMINATE, + message=( + "Package is available, but this runtime does not implement an " + "execution-readiness probe." + ), + metadata={"probe": "unsupported"}, + ) + + +def _probe_failure( + runtime: AgentRuntime, + *, + message: str, + failure: str, + error_type: str | None = None, +) -> RuntimeReadiness: + availability = _safe_availability(runtime) + metadata = {"failure": failure} + if error_type is not None: + metadata["error_type"] = error_type + return RuntimeReadiness.indeterminate( + runtime.kind, + reason=AvailabilityReason.SETUP_FAILED, + message=message, + package=availability.package if availability is not None else None, + version=availability.version if availability is not None else None, + metadata=metadata, + ) + + +def _safe_availability(runtime: AgentRuntime) -> RuntimeAvailability | None: + try: + return runtime.availability() + except Exception: + return None + + +__all__ = ["DEFAULT_READINESS_TIMEOUT", "check_readiness"] diff --git a/src/agent_runtime_kit/registry.py b/src/agent_runtime_kit/registry.py index 66d1a99..dff9567 100644 --- a/src/agent_runtime_kit/registry.py +++ b/src/agent_runtime_kit/registry.py @@ -12,10 +12,13 @@ AgentRuntime, AgentRuntimeKind, AgentTask, + AvailabilityReason, RuntimeAvailability, + RuntimeReadiness, TaskSupportReport, runtime_kind_value, ) +from agent_runtime_kit.readiness import DEFAULT_READINESS_TIMEOUT, check_readiness from agent_runtime_kit.support import validate_task RuntimeFactory = Callable[..., AgentRuntime] @@ -80,7 +83,7 @@ def validate_task_for( return validate_task(self.resolve(kind), task) def availability_for(self, kind: AgentRuntimeKind | str) -> RuntimeAvailability: - """Construct a runtime and return its availability diagnostic. + """Construct a runtime and return its package diagnostic. This constructs the runtime with no arguments, so any registered factory must be callable with zero args (the built-in adapters are). @@ -88,6 +91,36 @@ def availability_for(self, kind: AgentRuntimeKind | str) -> RuntimeAvailability: return self.resolve(kind).availability() + async def readiness_for( + self, + kind: AgentRuntimeKind | str, + *, + timeout: float = DEFAULT_READINESS_TIMEOUT, + ) -> RuntimeReadiness: + """Construct, probe, and close a runtime for one registered kind.""" + + runtime = self.resolve(kind) + try: + readiness = await check_readiness(runtime, timeout=timeout) + except BaseException: + try: + await runtime.aclose() + except Exception: + pass + raise + try: + await runtime.aclose() + except Exception as exc: + return RuntimeReadiness.indeterminate( + runtime.kind, + reason=AvailabilityReason.SETUP_FAILED, + message="Runtime cleanup failed after the readiness probe.", + package=readiness.package, + version=readiness.version, + metadata={"failure": "cleanup", "error_type": type(exc).__name__}, + ) + return readiness + def create_default_registry( *, diff --git a/src/agent_runtime_kit/testing/fakes.py b/src/agent_runtime_kit/testing/fakes.py index 8444fc2..0bd37b8 100644 --- a/src/agent_runtime_kit/testing/fakes.py +++ b/src/agent_runtime_kit/testing/fakes.py @@ -14,7 +14,9 @@ AgentRuntimeKind, AgentTask, AvailabilityReason, + ReadinessStatus, RuntimeAvailability, + RuntimeReadiness, TaskSupportReport, ToolCallAudit, ) @@ -149,6 +151,22 @@ def validate_task(self, task: AgentTask) -> TaskSupportReport: return _validate_declared_task_support(self.kind, self.capabilities, task) + async def check_readiness(self) -> RuntimeReadiness: + """Return deterministic readiness without invoking the fake harness.""" + + availability = self.availability() + if not availability.available: + return RuntimeReadiness.from_availability( + availability, + status=ReadinessStatus.NOT_READY, + ) + return RuntimeReadiness.ready_to_attempt( + self.kind, + package=availability.package, + version=availability.version, + metadata=availability.metadata, + ) + async def run(self, task: AgentTask) -> AgentResult: """Invoke the fake SDK and emit normalized events.""" diff --git a/tests/test_adapter_common.py b/tests/test_adapter_common.py index f32020f..0b41df9 100644 --- a/tests/test_adapter_common.py +++ b/tests/test_adapter_common.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib.util import inspect import pytest @@ -14,9 +15,31 @@ filter_supported_kwargs, optional_int, output_schema_from, + package_availability, ) +def test_package_availability_never_resolves_or_imports_vendor_module( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "agent_runtime_kit.adapters._common.metadata.version", + lambda _package: "1.2.3", + ) + monkeypatch.setattr( + importlib.util, + "find_spec", + lambda _module: (_ for _ in ()).throw( + AssertionError("package availability must not resolve imports") + ), + ) + + availability = package_availability(AgentRuntimeKind.ANTIGRAVITY_AGENT_SDK) + + assert availability.available is True + assert availability.version == "1.2.3" + + def test_filter_supported_kwargs_rejects_required_key_hidden_by_var_kwargs() -> None: def opaque(**_kwargs: object) -> None: return None diff --git a/tests/test_adapter_protocol.py b/tests/test_adapter_protocol.py index 9b68525..a65b7b5 100644 --- a/tests/test_adapter_protocol.py +++ b/tests/test_adapter_protocol.py @@ -21,6 +21,7 @@ AgentRuntimeKind, AgentTask, RuntimeAvailability, + RuntimeReadinessProvider, ) from agent_runtime_kit._types import AgentRuntime from agent_runtime_kit.adapters import ( @@ -76,6 +77,18 @@ def test_adapter_exposes_async_lifecycle( assert inspect.iscoroutinefunction(method), f"{kind.value}.{name} must be async" +@pytest.mark.parametrize("kind,builder", BUILDERS, ids=lambda v: getattr(v, "value", "")) +def test_adapter_exposes_optional_async_readiness_probe( + kind: AgentRuntimeKind, builder: Callable[[Path], AgentRuntime], tmp_path: Path +) -> None: + runtime = builder(tmp_path) + + assert isinstance(runtime, RuntimeReadinessProvider) + assert inspect.iscoroutinefunction(runtime.check_readiness), ( + f"{kind.value}.check_readiness must be async" + ) + + @pytest.mark.parametrize("kind,builder", BUILDERS, ids=lambda v: getattr(v, "value", "")) def test_adapter_availability_returns_diagnostic( kind: AgentRuntimeKind, builder: Callable[[Path], AgentRuntime], tmp_path: Path diff --git a/tests/test_antigravity_adapter.py b/tests/test_antigravity_adapter.py index c74cec2..b7d4590 100644 --- a/tests/test_antigravity_adapter.py +++ b/tests/test_antigravity_adapter.py @@ -1345,7 +1345,7 @@ def test_antigravity_availability_uses_injected_sdk(tmp_path: Path) -> None: assert diagnostic.available is True -def test_antigravity_availability_accepts_application_default_credentials( +def test_antigravity_availability_does_not_probe_application_default_credentials( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.delenv("GEMINI_API_KEY", raising=False) @@ -1360,18 +1360,18 @@ def test_antigravity_availability_accepts_application_default_credentials( ), ) monkeypatch.setattr( - "agent_runtime_kit.adapters.antigravity._google_adc_project", - lambda: "adc-project", + "agent_runtime_kit.adapters.antigravity._probe_google_adc", + lambda: (_ for _ in ()).throw(AssertionError("availability must not probe ADC")), ) runtime = AntigravityAgentRuntime(api_key=None) diagnostic = runtime.availability() assert diagnostic.available is True - assert diagnostic.metadata["auth_source"] == "application-default-credentials" + assert diagnostic.metadata == {} -def test_antigravity_availability_rejects_missing_credentials( +def test_antigravity_availability_does_not_classify_missing_credentials( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.delenv("GEMINI_API_KEY", raising=False) @@ -1387,10 +1387,13 @@ def test_antigravity_availability_rejects_missing_credentials( version="0.1.2", ), ) - monkeypatch.setattr("agent_runtime_kit.adapters.antigravity._google_adc_project", lambda: None) + monkeypatch.setattr( + "agent_runtime_kit.adapters.antigravity._probe_google_adc", + lambda: (_ for _ in ()).throw(AssertionError("availability must not probe ADC")), + ) runtime = AntigravityAgentRuntime(api_key=None) diagnostic = runtime.availability() - assert diagnostic.available is False - assert diagnostic.reason.value == "missing-credentials" + assert diagnostic.available is True + assert diagnostic.metadata == {} diff --git a/tests/test_claude_adapter.py b/tests/test_claude_adapter.py index 5800075..e0ea7a3 100644 --- a/tests/test_claude_adapter.py +++ b/tests/test_claude_adapter.py @@ -932,7 +932,7 @@ async def test_claude_passes_runtime_env_for_bedrock() -> None: } -def test_claude_availability_reports_bedrock_auth_source() -> None: +def test_claude_availability_is_package_only_for_bedrock() -> None: runtime = ClaudeAgentRuntime( env={ "CLAUDE_CODE_USE_BEDROCK": "1", @@ -946,6 +946,4 @@ def test_claude_availability_reports_bedrock_auth_source() -> None: diagnostic = runtime.availability() assert diagnostic.available is True - assert diagnostic.metadata["auth_source"] == "amazon-bedrock" - assert diagnostic.metadata["credential_chain"] == "aws-sdk" - assert diagnostic.metadata["aws_profile_configured"] is True + assert diagnostic.metadata == {} diff --git a/tests/test_codex_adapter.py b/tests/test_codex_adapter.py index 7055201..c8ce20a 100644 --- a/tests/test_codex_adapter.py +++ b/tests/test_codex_adapter.py @@ -1083,7 +1083,7 @@ def codex_factory(*, config: FakeCodexConfig) -> FakeCodex: assert seen["config"].env == {"AWS_PROFILE": "agent-runtime-kit"} -def test_codex_availability_reports_bedrock_provider() -> None: +def test_codex_availability_is_package_only_for_bedrock() -> None: runtime = CodexAgentRuntime( config_overrides=( "features.plugins=false", @@ -1098,6 +1098,4 @@ def test_codex_availability_reports_bedrock_provider() -> None: diagnostic = runtime.availability() assert diagnostic.available is True - assert diagnostic.metadata["auth_source"] == "amazon-bedrock" - assert diagnostic.metadata["credential_chain"] == "aws-sdk" - assert diagnostic.metadata["aws_profile_configured"] is True + assert diagnostic.metadata == {} diff --git a/tests/test_live_smoke.py b/tests/test_live_smoke.py index 88ef461..d60bcab 100644 --- a/tests/test_live_smoke.py +++ b/tests/test_live_smoke.py @@ -4,7 +4,7 @@ import pytest -from agent_runtime_kit import AgentTask +from agent_runtime_kit import AgentRuntime, AgentTask, ReadinessStatus, check_readiness from agent_runtime_kit.adapters import ( AntigravityAgentRuntime, ClaudeAgentRuntime, @@ -19,10 +19,10 @@ def _enabled(provider: str) -> bool: return selected in {provider, "all"} -async def _run_smoke(runtime: object) -> None: - availability = runtime.availability() - if not availability.available: - pytest.skip(availability.message) +async def _run_smoke(runtime: AgentRuntime) -> None: + readiness = await check_readiness(runtime) + if readiness.status is ReadinessStatus.NOT_READY: + pytest.skip(readiness.message) result = await runtime.run(AgentTask(goal="Reply with exactly: agent-runtime-kit smoke")) assert result.output diff --git a/tests/test_readiness.py b/tests/test_readiness.py new file mode 100644 index 0000000..7fe72df --- /dev/null +++ b/tests/test_readiness.py @@ -0,0 +1,662 @@ +from __future__ import annotations + +import asyncio +import threading +from dataclasses import dataclass +from typing import Any + +import pytest + +from agent_runtime_kit import ( + AgentCapabilities, + AgentKit, + AgentResult, + AgentRuntime, + AgentRuntimeKind, + AgentTask, + AvailabilityReason, + FakeAgentRuntime, + ReadinessStatus, + RuntimeAvailability, + RuntimeReadiness, + RuntimeReadinessProvider, + RuntimeRegistry, + check_readiness, +) +from agent_runtime_kit.adapters import ( + AntigravityAgentRuntime, + ClaudeAgentRuntime, + CodexAgentRuntime, +) +from agent_runtime_kit.adapters import antigravity as antigravity_adapter +from agent_runtime_kit.adapters import claude as claude_adapter +from agent_runtime_kit.adapters import codex as codex_adapter +from agent_runtime_kit.adapters import diagnostics as provider_diagnostics +from agent_runtime_kit.adapters.antigravity import _GoogleADCProbe + + +class LegacyRuntime: + """Protocol-complete third-party runtime predating readiness probes.""" + + kind = "x-third-party" + capabilities = AgentCapabilities() + + def __init__(self, availability: RuntimeAvailability) -> None: + self._availability = availability + self.closed = False + + def availability(self) -> RuntimeAvailability: + return self._availability + + async def run(self, task: AgentTask) -> AgentResult: + return AgentResult(output=task.goal) + + async def cancel(self, task_id: str) -> None: + del task_id + + async def aclose(self) -> None: + self.closed = True + + async def __aenter__(self) -> LegacyRuntime: + return self + + async def __aexit__(self, _exc_type: object, _exc: object, _tb: object) -> None: + await self.aclose() + + +class FailingReadinessRuntime(LegacyRuntime): + async def check_readiness(self) -> RuntimeReadiness: + raise RuntimeError("secret-token-value") + + +class SlowReadinessRuntime(LegacyRuntime): + async def check_readiness(self) -> RuntimeReadiness: + await asyncio.sleep(60) + return RuntimeReadiness.ready_to_attempt(self.kind) + + +class InvalidReadinessRuntime(LegacyRuntime): + async def check_readiness(self) -> Any: + return {"status": "ready-to-attempt"} + + +class MismatchedReadinessRuntime(LegacyRuntime): + async def check_readiness(self) -> RuntimeReadiness: + return RuntimeReadiness.ready_to_attempt("x-different-runtime") + + +class BrokenAvailabilityRuntime(LegacyRuntime): + def availability(self) -> RuntimeAvailability: + raise RuntimeError("secret-availability-detail") + + +class BrokenProbeAndAvailabilityRuntime(FailingReadinessRuntime): + def availability(self) -> RuntimeAvailability: + raise RuntimeError("secret-availability-detail") + + +class CleanupFailingRuntime(LegacyRuntime): + async def aclose(self) -> None: + raise RuntimeError("secret-cleanup-detail") + + +def _available_third_party() -> RuntimeAvailability: + return RuntimeAvailability.ok( + "x-third-party", + package="third-party-sdk", + version="1.2.3", + metadata={"source": "package"}, + ) + + +def test_readiness_value_coerces_enums_and_freezes_metadata() -> None: + readiness = RuntimeReadiness( + kind="fake", + status=ReadinessStatus.READY_TO_ATTEMPT, + reason=AvailabilityReason.AVAILABLE, + message="ready", + metadata={"source": "test"}, + ) + + assert readiness.kind is AgentRuntimeKind.FAKE + assert readiness.is_ready_to_attempt is True + with pytest.raises(TypeError, match="read-only"): + readiness.metadata["source"] = "changed" # type: ignore[index] + with pytest.raises(ValueError, match="message"): + RuntimeReadiness.ready_to_attempt("fake", message=" ") + with pytest.raises(ValueError, match="unavailable package"): + RuntimeReadiness.from_availability( + RuntimeAvailability.unavailable( + "fake", + reason=AvailabilityReason.MISSING_PACKAGE, + message="missing", + ), + status=ReadinessStatus.READY_TO_ATTEMPT, + ) + + +@pytest.mark.asyncio +async def test_fake_runtime_is_ready_to_attempt() -> None: + readiness = await check_readiness(FakeAgentRuntime()) + + assert readiness.status is ReadinessStatus.READY_TO_ATTEMPT + assert readiness.is_ready_to_attempt is True + assert readiness.package == "agent-runtime-kit" + + +@pytest.mark.asyncio +async def test_legacy_runtime_with_package_is_conservatively_indeterminate() -> None: + runtime = LegacyRuntime(_available_third_party()) + + readiness = await check_readiness(runtime) + + assert isinstance(runtime, AgentRuntime) + assert not isinstance(runtime, RuntimeReadinessProvider) + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.reason is AvailabilityReason.AVAILABLE + assert readiness.package == "third-party-sdk" + assert readiness.version == "1.2.3" + assert readiness.metadata == {"probe": "unsupported"} + + +@pytest.mark.asyncio +async def test_legacy_runtime_preserves_negative_availability() -> None: + availability = RuntimeAvailability( + kind="x-third-party", + available=False, + reason=AvailabilityReason.MISSING_PACKAGE, + message="install x-third-party", + package="third-party-sdk", + version="1.0.0", + metadata={"extra": "third-party"}, + ) + + readiness = await check_readiness(LegacyRuntime(availability)) + + assert readiness.status is ReadinessStatus.NOT_READY + assert readiness.reason is AvailabilityReason.MISSING_PACKAGE + assert readiness.message == "install x-third-party" + assert readiness.package == "third-party-sdk" + assert readiness.version == "1.0.0" + assert readiness.metadata == {} + + +@pytest.mark.asyncio +async def test_readiness_failure_is_indeterminate_and_secret_safe() -> None: + readiness = await check_readiness(FailingReadinessRuntime(_available_third_party())) + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.reason is AvailabilityReason.SETUP_FAILED + assert readiness.metadata["failure"] == "error" + assert readiness.metadata["error_type"] == "RuntimeError" + assert "secret-token-value" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_invalid_readiness_result_is_indeterminate() -> None: + readiness = await check_readiness(InvalidReadinessRuntime(_available_third_party())) + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["error_type"] == "TypeError" + + +@pytest.mark.asyncio +async def test_mismatched_readiness_kind_is_indeterminate() -> None: + readiness = await check_readiness( + MismatchedReadinessRuntime(_available_third_party()) + ) + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["error_type"] == "ValueError" + + +@pytest.mark.asyncio +async def test_legacy_availability_failure_is_secret_safe() -> None: + readiness = await check_readiness(BrokenAvailabilityRuntime(_available_third_party())) + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata == { + "failure": "availability", + "error_type": "RuntimeError", + } + assert "secret-availability-detail" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_probe_and_availability_failure_stays_secret_safe() -> None: + readiness = await check_readiness( + BrokenProbeAndAvailabilityRuntime(_available_third_party()) + ) + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.package is None + assert readiness.metadata["error_type"] == "RuntimeError" + assert "secret" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_readiness_timeout_is_indeterminate() -> None: + readiness = await check_readiness( + SlowReadinessRuntime(_available_third_party()), timeout=0.01 + ) + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["failure"] == "timeout" + assert "0.01 seconds" in readiness.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan"), True]) +async def test_readiness_rejects_invalid_timeout(timeout: float) -> None: + with pytest.raises(ValueError, match="positive finite"): + await check_readiness(FakeAgentRuntime(), timeout=timeout) + + +@pytest.mark.asyncio +async def test_registry_readiness_closes_ephemeral_runtime() -> None: + registry = RuntimeRegistry() + runtimes: list[LegacyRuntime] = [] + + def factory() -> LegacyRuntime: + runtime = LegacyRuntime(_available_third_party()) + runtimes.append(runtime) + return runtime + + registry.register("x-third-party", factory) + + readiness = await registry.readiness_for("x-third-party") + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert runtimes[0].closed is True + + +@pytest.mark.asyncio +async def test_registry_cleanup_failure_is_secret_safe_and_indeterminate() -> None: + registry = RuntimeRegistry() + registry.register( + "x-third-party", + lambda: CleanupFailingRuntime(_available_third_party()), + ) + + readiness = await registry.readiness_for("x-third-party") + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata == {"failure": "cleanup", "error_type": "RuntimeError"} + assert "secret-cleanup-detail" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_agent_kit_readiness_helpers_use_owned_cached_runtimes() -> None: + kit = AgentKit(register_default_adapters=False) + + one = await kit.readiness_for("fake") + all_results = await kit.readiness() + + assert one.status is ReadinessStatus.READY_TO_ATTEMPT + assert all_results == (one,) + await kit.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("env", "source"), + [ + ({"ANTHROPIC_API_KEY": "secret-anthropic-key"}, "anthropic-api-key"), + ({"CLAUDE_CODE_OAUTH_TOKEN": "secret-oauth-token"}, "claude-code-oauth-token"), + ( + { + "CLAUDE_CODE_USE_BEDROCK": "1", + "AWS_BEARER_TOKEN_BEDROCK": "secret-bedrock-token", + }, + "amazon-bedrock", + ), + ], +) +async def test_claude_direct_credential_signal_is_ready_without_exposing_it( + env: dict[str, str], source: str +) -> None: + runtime = ClaudeAgentRuntime( + env=env, + query_func=object(), + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.READY_TO_ATTEMPT + assert readiness.metadata["auth_source"] == source + assert not any(secret in repr(readiness) for secret in env.values()) + + +@pytest.mark.asyncio +async def test_claude_provider_chain_is_indeterminate() -> None: + runtime = ClaudeAgentRuntime( + env={ + "CLAUDE_CODE_USE_BEDROCK": "1", + "AWS_PROFILE": "agent-runtime-kit", + "AWS_REGION": "us-east-1", + }, + query_func=object(), + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["auth_source"] == "amazon-bedrock" + assert readiness.metadata["credential_chain"] == "aws-sdk" + assert readiness.metadata["aws_profile_configured"] is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("module", "factory", "package"), + [ + (claude_adapter, ClaudeAgentRuntime, "claude-agent-sdk"), + (codex_adapter, CodexAgentRuntime, "openai-codex"), + (antigravity_adapter, AntigravityAgentRuntime, "google-antigravity"), + ], +) +async def test_provider_missing_package_is_not_ready_without_probe( + monkeypatch: pytest.MonkeyPatch, + module: Any, + factory: Any, + package: str, +) -> None: + def missing(kind: AgentRuntimeKind) -> RuntimeAvailability: + return RuntimeAvailability.unavailable( + kind, + reason=AvailabilityReason.MISSING_PACKAGE, + message=f"install {package}", + package=package, + ) + + monkeypatch.setattr(module, "package_availability", missing) + + readiness = await factory().check_readiness() + + assert readiness.status is ReadinessStatus.NOT_READY + assert readiness.reason is AvailabilityReason.MISSING_PACKAGE + assert readiness.package == package + + +@dataclass +class ProbeCodexConfig: + config_overrides: tuple[str, ...] = () + env: dict[str, str] | None = None + + +def _codex_probe_runtime( + *, + response: Any = None, + error: Exception | None = None, + close_error: Exception | None = None, + env: dict[str, str] | None = None, +) -> tuple[CodexAgentRuntime, list[Any]]: + instances: list[Any] = [] + + class ProbeCodex: + def __init__(self, config: ProbeCodexConfig) -> None: + self.config = config + self.closed = False + self.refresh_token: bool | None = None + instances.append(self) + + async def __aenter__(self) -> ProbeCodex: + return self + + async def __aexit__(self, *_args: object) -> None: + self.closed = True + if close_error is not None: + raise close_error + + async def account(self, *, refresh_token: bool = True) -> Any: + self.refresh_token = refresh_token + if error is not None: + raise error + return response + + return ( + CodexAgentRuntime( + env=env, + codex_cls=ProbeCodex, + config_cls=ProbeCodexConfig, + ), + instances, + ) + + +@pytest.mark.asyncio +async def test_codex_account_probe_is_ready_and_always_cleans_up() -> None: + runtime, instances = _codex_probe_runtime( + response={ + "account": { + "root": { + "type": "chatgpt", + "email": "private@example.invalid", + } + }, + "requires_openai_auth": True, + }, + env={"OPENAI_API_KEY": "secret-openai-key"}, + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.READY_TO_ATTEMPT + assert readiness.metadata["auth_source"] == "openai-api-key" + assert readiness.metadata["account_type"] == "chatgpt" + assert "private@example.invalid" not in repr(readiness) + assert "secret-openai-key" not in repr(readiness) + assert instances[0].refresh_token is False + assert instances[0].closed is True + + +@pytest.mark.asyncio +async def test_codex_absent_account_is_not_ready_and_cleans_up() -> None: + runtime, instances = _codex_probe_runtime( + response={"account": None, "requires_openai_auth": True} + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.NOT_READY + assert readiness.reason is AvailabilityReason.MISSING_CREDENTIALS + assert instances[0].closed is True + + +@pytest.mark.asyncio +async def test_codex_probe_failure_is_indeterminate_and_cleans_up() -> None: + runtime, instances = _codex_probe_runtime( + error=RuntimeError("secret-codex-token") + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["error_type"] == "RuntimeError" + assert "secret-codex-token" not in repr(readiness) + assert instances[0].closed is True + + +@pytest.mark.asyncio +async def test_codex_cleanup_failure_makes_probe_indeterminate() -> None: + runtime, instances = _codex_probe_runtime( + response={"account": {"root": {"type": "apiKey"}}}, + close_error=RuntimeError("secret-cleanup-detail"), + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["error_type"] == "RuntimeError" + assert "secret-cleanup-detail" not in repr(readiness) + assert instances[0].closed is True + + +@pytest.mark.asyncio +async def test_antigravity_api_key_is_ready_without_adc( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "agent_runtime_kit.adapters.antigravity._probe_google_adc", + lambda: (_ for _ in ()).throw(AssertionError("API key must bypass ADC")), + ) + runtime = AntigravityAgentRuntime(api_key="secret-google-key", agent_cls=object()) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.READY_TO_ATTEMPT + assert readiness.metadata == {"auth_source": "api-key"} + assert "secret-google-key" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_antigravity_missing_credentials_is_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + runtime = AntigravityAgentRuntime( + api_key=None, + vertex=False, + agent_cls=object(), + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.NOT_READY + assert readiness.reason is AvailabilityReason.MISSING_CREDENTIALS + + +@pytest.mark.asyncio +async def test_antigravity_adc_probe_runs_off_event_loop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + release_probe = threading.Event() + probe_started = threading.Event() + heartbeat_ran = asyncio.Event() + + def probe() -> _GoogleADCProbe: + probe_started.set() + assert release_probe.wait(timeout=1) + return _GoogleADCProbe(credentials_available=True, project="private-project-id") + + async def heartbeat() -> None: + while not probe_started.is_set(): + await asyncio.sleep(0) + heartbeat_ran.set() + release_probe.set() + + monkeypatch.setattr("agent_runtime_kit.adapters.antigravity._probe_google_adc", probe) + runtime = AntigravityAgentRuntime( + api_key=None, + vertex=True, + agent_cls=object(), + ) + + readiness, _ = await asyncio.gather(runtime.check_readiness(), heartbeat()) + + assert heartbeat_ran.is_set() + assert readiness.status is ReadinessStatus.READY_TO_ATTEMPT + assert readiness.metadata["project_configured"] is True + assert "private-project-id" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_antigravity_adc_error_is_indeterminate_and_secret_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + + def fail_probe() -> _GoogleADCProbe: + raise RuntimeError("secret-google-error") + + monkeypatch.setattr( + "agent_runtime_kit.adapters.antigravity._probe_google_adc", fail_probe + ) + runtime = AntigravityAgentRuntime( + api_key=None, + vertex=True, + agent_cls=object(), + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["error_type"] == "RuntimeError" + assert "secret-google-error" not in repr(readiness) + + +@pytest.mark.asyncio +async def test_antigravity_missing_adc_is_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + + class DefaultCredentialsError(Exception): + pass + + DefaultCredentialsError.__module__ = "google.auth.exceptions" + + def missing_probe() -> _GoogleADCProbe: + raise DefaultCredentialsError("credentials were not found") + + monkeypatch.setattr( + "agent_runtime_kit.adapters.antigravity._probe_google_adc", missing_probe + ) + runtime = AntigravityAgentRuntime( + api_key=None, + vertex=True, + agent_cls=object(), + ) + + readiness = await runtime.check_readiness() + + assert readiness.status is ReadinessStatus.NOT_READY + assert readiness.reason is AvailabilityReason.MISSING_CREDENTIALS + + +@pytest.mark.asyncio +async def test_antigravity_adc_timeout_is_indeterminate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + release_probe = threading.Event() + + def blocked_probe() -> _GoogleADCProbe: + release_probe.wait(timeout=1) + return _GoogleADCProbe(credentials_available=True, project="project") + + monkeypatch.setattr( + "agent_runtime_kit.adapters.antigravity._probe_google_adc", blocked_probe + ) + runtime = AntigravityAgentRuntime( + api_key=None, + vertex=True, + agent_cls=object(), + ) + try: + readiness = await check_readiness(runtime, timeout=0.01) + finally: + release_probe.set() + + assert readiness.status is ReadinessStatus.INDETERMINATE + assert readiness.metadata["failure"] == "timeout" + + +@pytest.mark.asyncio +async def test_async_provider_diagnostics_are_separate_from_package_checks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(provider_diagnostics, "ClaudeAgentRuntime", FakeAgentRuntime) + monkeypatch.setattr(provider_diagnostics, "CodexAgentRuntime", FakeAgentRuntime) + monkeypatch.setattr(provider_diagnostics, "AntigravityAgentRuntime", FakeAgentRuntime) + + readiness = await provider_diagnostics.collect_provider_readiness(timeout=0.1) + + assert len(readiness) == 3 + assert all(item.status is ReadinessStatus.READY_TO_ATTEMPT for item in readiness)