diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index be3ce5f..ea0a5b9 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -1,10 +1,10 @@ name: Desktop native builds on: pull_request: - paths: ['umbod/api/**', 'packages/umbod-sdk/**', 'desktop/**', 'website/**', '.github/workflows/desktop-*.yml', '.dockerignore'] + paths: ['umbod/api/**', 'packages/umbod-sdk/**', 'desktop/**', 'website/**', '.github/workflows/desktop-*.yml', '.dockerignore', 'flake.nix', 'flake.lock'] push: branches: [main, feat/desktop-monorepo-releases] - paths: ['umbod/api/**', 'packages/umbod-sdk/**', 'desktop/**', 'website/**', '.github/workflows/desktop-*.yml', '.dockerignore'] + paths: ['umbod/api/**', 'packages/umbod-sdk/**', 'desktop/**', 'website/**', '.github/workflows/desktop-*.yml', '.dockerignore', 'flake.nix', 'flake.lock'] workflow_dispatch: workflow_call: permissions: @@ -48,10 +48,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Verify native Apple Silicon runner and Swift 6 - run: | - test "$(uname -m)" = arm64 - swift --version | grep -E "Swift version 6\." + - name: Verify full Xcode and SwiftUI toolchain + run: bash scripts/check-macos-toolchain.sh - uses: actions/setup-python@v5 with: python-version: '3.12' diff --git a/desktop/README.md b/desktop/README.md index 2f2a807..9b70a5a 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -6,7 +6,7 @@ Verified desktop releases are available from https://umbod.ai/download/. Local b ## Build and test -Run from the monorepo root unless a command explicitly changes directory. Build prerequisites: uv, Python 3.11+, Swift 6 and Apple developer tools on macOS. Packaging downloads a pinned CPython 3.14.3 runtime and verifies its SHA-256. No command below installs or launches the production app. +Run from the monorepo root unless a command explicitly changes directory. Build prerequisites: uv, Python 3.11+, and full Xcode with Swift 6 or newer on Apple Silicon macOS; Command Line Tools alone are insufficient. Install Xcode and open it to finish first-launch setup and accept its license. Select the installation with `DEVELOPER_DIR` or `sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer`. Run `bash desktop/scripts/check-macos-toolchain.sh` to verify the selection and SwiftUI macros before building. On Apple Silicon macOS, `nix develop .#desktop` supplies the normal repository tools and pinned Python/uv environment and checks the external Apple compiler and SDK; it does not install Swift, Xcode, or an SDK. The general `nix develop` shell has no Xcode requirement. Packaging downloads a pinned CPython 3.14.3 runtime and verifies its SHA-256. No command below installs or launches the production app. ```sh cd desktop @@ -55,11 +55,15 @@ OAuth supports native public clients with loopback callbacks, discovery, PKCE an Older installed versions that require a Windows asset in update metadata cannot read new macOS-only manifests. Download the first macOS-only release manually from the website; its updater accepts both historical and macOS-only manifests. Existing published releases are not changed. +## Shared composition + +The Python host uses the backend-owned [`umbod.hosting`](../umbod/api/src/umbod/hosting/README.md) API also used by the separate Kubernetes REST/MCP services. Native lifecycle and credential ownership remain desktop-specific. Desktop migration and tool-name repair live exclusively in the removable [`backward_compatibility`](python/umbod_desktop/backward_compatibility/README.md) package, alongside old-host retirement, historical profile fallback, OAuth conversion and migrated token matching. Compatibility orchestration and permanent usage recording depend on desktop-owned typed ports; concrete shared HTTP, hosting, transport and OAuth adapters live in `python/umbod_desktop/backend_integration/`. The Python index documents the exact remaining naming, configuration and OAuth persistence coupling; the compatibility index owns historical formats, lifecycle entrypoints and concrete removal steps. + ## Code and release index - [`python/`](python/README.md): macOS host, transport adapters, Keychain, migration and tests; reuses `umbod/api` and `packages/umbod-sdk`. - `native/`: SwiftUI app and Swift contract/lifecycle tests; `UmbodCore/AdvancedSettings.swift` owns backend-reported pending/effective settings, confirmation and verified restart/recovery, while `Umbod/AdvancedSettingsView.swift` presents Settings → Advanced; Settings provides compact update controls and generic stdio/HTTP connection details, with the bearer token masked and available only through an explicit copy action. Connector authentication stays optional and credentials remain in Keychain; `UmbodCore/ConnectorEditor.swift` owns the actual editor submission and retry state; `UmbodCore/ToolPermissionQueue.swift` owns optimistic permission edits, ordered save state, and stale-refresh protection; `UmbodCore/DesktopUpdate.swift` validates update metadata and `Umbod/UpdatesView.swift` owns update preferences and the install guidance. -- `scripts/`: local builds, packaging, isolated fixtures, and release policy/metadata. +- `scripts/`: local builds, packaging, isolated fixtures, and release policy/metadata; `check-macos-toolchain.sh` owns the shared full-Xcode, Swift 6, SDK and SwiftUI macro preflight used by builds, verification, acceptance preparation and the desktop Nix shell. - `VERSION`: independent desktop version; the single source for bundle and package versions. - [Release procedure](docs/RELEASING.md), [manifest application](docs/APPLY-MANIFEST.md), and [import provenance and audit](docs/PROVENANCE.md). - [0.2.4 release candidate](docs/releases/0.2.4.md): announcement draft and release-owner acceptance checklist; not a publication record. diff --git a/desktop/python/README.md b/desktop/python/README.md index ccec75f..012cf54 100644 --- a/desktop/python/README.md +++ b/desktop/python/README.md @@ -5,21 +5,27 @@ The SwiftUI app keeps its native screens. `AdministrationClient.swift` maps the ## Entry points and ownership - `main.py`: packaged `serve` and `bridge` commands, loopback socket, stable port, process lock, readiness pipe, owner shutdown and dev/release isolation. Its `credential` command lets Swift reattach through a private pipe using the same installed backend identity; Swift does not access Keychain independently. Legacy positional bridge identities remain accepted. `build.sh` bundles a relocatable interpreter and locked dependencies, then signs a fresh bundle. -- `runtime.py`, `configuration.py`: shared application composition and authenticated `/desktop` routes. SQL event polling synchronizes API writes with MCP. Native single-tool and bulk enablement publish the connector before applying activation and batched permissions; rename-only saves preserve the live transport. Native removal revokes the selected connector’s desktop-owner grants, including stale tool grants, before deleting its registry entry and transport. Dedicated metrics listeners are disabled for desktop. MCP defaults to gateway exposure (`search_tools` and `execute_tool`); saved backend settings select gateway, flat or codemode before constructing the application. Settings never hot-swap registered tools. +- `runtime.py`, `backend_integration/configuration.py`: shared application composition and authenticated `/desktop` routes. SQL event polling synchronizes API writes with MCP. Native single-tool and bulk enablement publish the connector before applying activation and batched permissions; rename-only saves preserve the live transport. Native removal revokes the selected connector’s desktop-owner grants, including stale tool grants, before deleting its registry entry and transport. Dedicated metrics listeners are disabled for desktop. MCP defaults to gateway exposure (`search_tools` and `execute_tool`); saved backend settings select gateway, flat or codemode before constructing the application. Settings never hot-swap registered tools. +- `contracts.py`, `ports.py`: immutable desktop DTOs and semantic administration, naming, invocation recording, metadata and upstream preparation protocols. Use cases consume these ports; `DesktopState` satisfies the shared metadata protocol without a persistence rewrite. `CredentialVault` and `DESKTOP_GROUP` remain available through their established public security imports. +- [`backend_integration/`](umbod_desktop/backend_integration/README.md): concrete HTTP administration, hosting authentication/configuration, shared naming, Keychain OAuth, transport and usage mapping adapters. Permanent backend adapters live here; the compatibility OAuth adapter has a narrow historical conversion import, and runtime additionally uses supported hosting root composition exports. - `security.py`: separate random admin, agent, upstream and encryption credentials in macOS Keychain. No development auth or plaintext fallback. Existing agent token hashes are retained for migration; they never grant REST owner access. Agent identities receive `desktop-owner`, so opting into administrator MCP tools grants all of these clients MCP administration access. -- `upstreams.py`, `models.py`: validated stdio and Streamable HTTP transports exposed through private authenticated loopback proxies to the shared downstream connector API. A proxy owns its lifespan in one task. Offline drafts remain editable and cannot retain an obsolete live transport after an invalid edit. Upstream HTTP challenges and runtime OAuth reauthentication persist a sign-in-required status; successful requests clear it. The private loopback bearer is consumed before proxying, so only connector-owned credentials reach upstream servers. -- `oauth.py`: macOS Keychain storage and the `DesktopOAuthHost` adapter for shared OAuth connection storage, stable loopback callback addresses, and local transport reconnection. `callback_transport` forwards callback parameters to shared protocol validation. Discovery, public-client registration, scopes, status transitions, safe errors, diagnostics, cancellation, token exchange/refresh, and authorization-versus-discovery outcomes live in `umbod.core.connectors.downstream_mcp.oauth`. Native clients use the shared `/api/admin/connectors/mcp/oauth` API; private `oauth_*` host actions are no longer supported. Existing callback paths and Keychain records remain compatible. -- `migration.py`, `legacy.py`: authenticated retirement of the old process, resumable settings import, OAuth credential conversion and stable endpoint/client identity preservation. Original settings and Keychain entries are preserved. No PID from disk is signalled. -- `tool_names.py`: startup repair of desktop-generated UUID/hash tool prefixes through the shared metadata API. Readable prefixes use the shared core naming policy; duplicate imported/repaired names receive numeric suffixes. Only managed connections whose prefix matches their generated public path are repaired. Explicit prefixes, IDs, credentials, paths, activation and grants are preserved; rejected repairs keep the working prefix, log a warning, and retry on restart. New native connections omit the prefix so the shared Python backend derives it from the display name. Renaming a display label does not implicitly rename published tools. +- `backend_integration/transports.py`, `backend_integration/models.py`: validated stdio and Streamable HTTP transports exposed through private authenticated loopback proxies to the shared downstream connector API. A proxy owns its lifespan in one task. Offline drafts remain editable and cannot retain an obsolete live transport after an invalid edit. Upstream HTTP challenges and runtime OAuth reauthentication persist a sign-in-required status; successful requests clear it. The private loopback bearer is consumed before proxying, so only connector-owned credentials reach upstream servers. +- `backend_integration/oauth.py`: macOS Keychain storage and the `DesktopOAuthHost` adapter for shared OAuth connection storage, stable loopback callback addresses, and local transport reconnection. `callback_transport` forwards callback parameters to shared protocol validation. Discovery, public-client registration, scopes, status transitions, safe errors, diagnostics, cancellation, token exchange/refresh, and authorization-versus-discovery outcomes live in `umbod.core.connectors.downstream_mcp.oauth`. Native clients use the shared `/api/admin/connectors/mcp/oauth` API; private `oauth_*` host actions are no longer supported. Existing callback paths and Keychain records remain compatible. +- [`backward_compatibility/`](umbod_desktop/backward_compatibility/README.md): exclusively historical settings import, generated tool-prefix repair, authenticated old-host retirement, legacy OAuth conversion, saved-port fallback and migrated agent hash matching. Migration-only contracts and adapters live inside this removable package. Original settings and Keychain entries are preserved; no PID from disk is signalled. Its index describes lifecycle entrypoints and the deletion checklist. New connections use permanent backend naming rather than compatibility repair. +- `endpoint.py`: validates the current `python-endpoint.json` port independently of historical formats; `main.py` explicitly supplies the compatibility fallback. - `settings.py`: strict exposure-mode and boolean administrator settings, defaulting to gateway/off for existing profiles. `GET /desktop/settings` returns `pending`, immutable process `effective`, and `restart_required`; `PUT /desktop/settings` validates and atomically saves pending values without changing the running app. Both require the private owner/admin bearer; the agent token cannot access this API even when administrator MCP is enabled. Swift verifies health and reads back effective settings after restarting. Unknown modes, malformed booleans and unexpected fields are rejected. Invalid persisted settings fail closed without rewriting or resetting the profile; repair only the `settings` object (or restore its valid backup), leaving connectors, credentials, migration journals and grants intact. Operator `.env` settings remain ignored. -- `storage.py`, `usage.py`: owner-only atomic host metadata and aggregate call counts, without arguments or results. Shared connectors and encrypted configuration live in `backend.sqlite3`. +- `persistence.py`, `storage.py`, `usage.py`: owner-only atomic file helpers, host metadata and typed aggregate completion recording, without arguments or results. `backend_integration/usage.py` filters and maps hosting events before recording. Shared connectors and encrypted configuration live in `backend.sqlite3`. - `client_setup.py`: consented configuration previews, conflict detection and byte-preserving backup restoration. - `testing.py`: explicitly selected fixture credentials, restricted to owned `umbod-test-*` directories under the system temporary directory. This is never a fallback for unavailable Keychain access; normal profiles always use Keychain. - `tests/`: real HTTP/stdio MCP calls, default-deny and live revocation, resumable import, authentication isolation, OAuth refresh/callback races and safe client-file editing. Native tests additionally exercise the packaged interpreter, an explicit disposable fixture vault and persistent-process reattachment. Routine tests never open the user’s Keychain. Automated native UI and window-lifecycle tests are currently removed. ## Shared backend extension points -Shared backend host extension points include OAuth storage/callback composition, an explicitly supplied REST identity verifier, optional REST/MCP metrics listeners (enabled by default), SQL-backed group permission reloads, and an injectable MCP invocation sink. The host injects the shared `OAuthService` through `create_app(oauth_service=...)` and closes it with the bundled process lifecycle. Enterprise verifier selection and domain behavior retain their defaults. Desktop authentication is fixed to one local owner and one permission group. +`runtime.py` is the outer composition root. Desktop use cases depend on `ports.py` and `contracts.py`; `backend_integration` implements those desktop-owned ports. Supported backend assembly, verifiers, configuration and invocation events come from [`umbod.hosting`](../../umbod/api/src/umbod/hosting/README.md). Integration adapters retain exact naming, secret derivation, EndpointUrl and OAuth storage/connection/provider imports, detailed in their index; the compatibility OAuth adapter separately owns historical authorization conversion. The shared administration HTTP JSON contract remains authoritative. This is dependency inversion for desktop use cases, not a replacement backend or a claim that all desktop-local file I/O is abstracted. + +Runtime loads DesktopState before mapping its immutable settings snapshot. Administration owns the authenticated ASGI client, and runtime closes it in the former migration-client callback position. Naming repair is composed independently of migration. Existing startup/shutdown order, credentials, journal dictionaries and public JSON remain unchanged. Changes to these interfaces require explicit compatibility review and updated contract tests. `tests/test_architecture.py` and API hosting boundary tests walk nested imports, resolve relative imports, check exact residual symbols by adapter file and test their scanners against violating snippets; desktop tests additionally verify transitive use-case independence. + +Shared backend host extension points include OAuth storage/callback composition, an explicitly supplied REST identity verifier, optional REST/MCP metrics listeners (enabled by default), SQL-backed group permission reloads, and an injectable MCP invocation sink. The host injects the shared `OAuthService` through the `oauth_service` hook on `create_rest_app` and closes it with the bundled process lifecycle. Enterprise verifier selection and domain behavior retain their defaults. Desktop authentication is fixed to one local owner and one permission group. The desktop host provides authenticated stdio/HTTP transport adapters instead of adding desktop subprocess, Keychain or browser lifecycle behavior to the shared core. The REST API remains the authority for connector discovery, tool activation, publication and permissions. diff --git a/desktop/python/tests/fakes.py b/desktop/python/tests/fakes.py new file mode 100644 index 0000000..3c039c6 --- /dev/null +++ b/desktop/python/tests/fakes.py @@ -0,0 +1,118 @@ +from typing import Any + +from umbod_desktop.backward_compatibility.contracts import ( + LegacyAuthorization, + PrefixUpdate, +) +from umbod_desktop.contracts import ( + AdministrationRejected, + ConnectorBinding, + ConnectorCreation, + ConnectorDefinition, + PreparedUpstream, + ProxyConfiguration, + ToolGrant, + ToolSelection, + UpstreamInput, +) + + +class MemoryState: + def __init__(self, data: dict[str, Any]) -> None: + self.data = data + self.saves = 0 + + def save(self) -> None: + self.saves += 1 + + +class MemoryAdministration: + def __init__(self, definitions: tuple[ConnectorDefinition, ...]) -> None: + self.definitions = {item.connector_id: item for item in definitions} + self.patches: list[tuple[str, dict[str, str]]] = [] + self.rejected: set[str] = set() + self.created: list[ConnectorCreation] = [] + self.discovered: list[str] = [] + self.names: tuple[str, ...] = ("read", "write") + self.enabled: list[ToolSelection] = [] + self.grants: list[ToolGrant] = [] + self.groups: list[str] = [] + self.published: list[str] = [] + + async def inventory(self) -> tuple[ConnectorDefinition, ...]: + return tuple(self.definitions.values()) + + async def create(self, command: ConnectorCreation) -> ConnectorDefinition: + self.created.append(command) + definition = ConnectorDefinition( + connector_id=f"connector-{len(self.created)}", + display_name=command.display_name, + tool_name_prefix=command.tool_name_prefix, + public_path=command.public_path, + ) + self.definitions[definition.connector_id] = definition + return definition + + async def discover(self, connector_id: str) -> None: + self.discovered.append(connector_id) + + async def tool_names(self, connector_id: str) -> tuple[str, ...]: + return self.names + + async def enable_tools(self, selection: ToolSelection) -> None: + self.enabled.append(selection) + + async def ensure_group(self, group_id: str) -> None: + self.groups.append(group_id) + + async def grant_tools(self, grant: ToolGrant) -> None: + self.grants.append(grant) + + async def publish(self, connector_id: str) -> None: + self.published.append(connector_id) + + async def update_prefix(self, update: PrefixUpdate) -> None: + self.patches.append((update.connector_id, {"tool_name_prefix": update.prefix})) + if update.connector_id in self.rejected: + raise AdministrationRejected() + self.definitions[update.connector_id] = self.definitions[update.connector_id].model_copy( + update={"tool_name_prefix": update.prefix} + ) + + +class MemoryNamingPolicy: + def readable_prefix(self, display_name: str) -> str: + return {"Finance API": "Finance_API"}.get(display_name, display_name) + + def collision_key(self, prefix: str) -> str: + return {"Finance-API": "Finance_API"}.get(prefix, prefix) + + +class MemoryUpstreams: + def __init__(self) -> None: + self.prepared: list[UpstreamInput] = [] + self.bindings: list[ConnectorBinding] = [] + + async def prepare(self, server: UpstreamInput) -> PreparedUpstream: + self.prepared.append(server) + return PreparedUpstream( + host_id="host", connection_changed=True, + configuration=self.proxy_configuration("host"), + ) + + def proxy_configuration(self, host_id: str) -> ProxyConfiguration: + return ProxyConfiguration( + auth_mode="static_bearer", endpoint_url=f"http://127.0.0.1/{host_id}", + header_type="bearer", bearer_token="private-loopback-token", + ) + + def bind(self, binding: ConnectorBinding) -> None: + self.bindings.append(binding) + + +class MemoryAuthorizationImporter: + def __init__(self) -> None: + self.imported: list[LegacyAuthorization] = [] + + def import_authorization(self, authorization: LegacyAuthorization) -> None: + self.imported.append(authorization) diff --git a/desktop/python/tests/test_administration.py b/desktop/python/tests/test_administration.py new file mode 100644 index 0000000..9899cbf --- /dev/null +++ b/desktop/python/tests/test_administration.py @@ -0,0 +1,43 @@ +import httpx +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.routing import Route + +from umbod_desktop.backend_integration.administration import BackendAdministration +from umbod_desktop.contracts import AdministrationError, AdministrationRejected +from umbod_desktop.ports import ConnectorAdministration + + +@pytest.mark.parametrize("status,error", [(403, AdministrationRejected), (409, AdministrationRejected), (500, AdministrationRejected)]) +async def test_administration_translates_rejections_without_response_secrets(status: int, error: type[AdministrationError]) -> None: + async def reject(request: Request) -> JSONResponse: + assert request.headers["authorization"] == "Bearer private-token" + return JSONResponse({"detail": "private-token"}, status_code=status) + + adapter = BackendAdministration(Starlette(routes=[Route("/api/admin/mcp-permissions/groups/desktop-owner", reject, methods=["POST"])]), "private-token") + administration: ConnectorAdministration = adapter + try: + with pytest.raises(error) as caught: + await administration.ensure_group("desktop-owner") + assert "private-token" not in str(caught.value) + assert caught.value.__suppress_context__ + finally: + await adapter.close() + assert adapter.client.is_closed + + +async def test_administration_translates_transport_errors_without_secrets() -> None: + async def fail(request: Request) -> JSONResponse: + raise httpx.ConnectError("private-token") + + adapter = BackendAdministration(Starlette(routes=[Route("/api/admin/mcp-permissions/groups/desktop-owner", fail, methods=["POST"])]), "private-token") + administration: ConnectorAdministration = adapter + try: + with pytest.raises(AdministrationError) as caught: + await administration.ensure_group("desktop-owner") + assert not isinstance(caught.value, AdministrationRejected) + assert "private-token" not in str(caught.value) + finally: + await adapter.close() diff --git a/desktop/python/tests/test_architecture.py b/desktop/python/tests/test_architecture.py new file mode 100644 index 0000000..c73d3e9 --- /dev/null +++ b/desktop/python/tests/test_architecture.py @@ -0,0 +1,206 @@ +import ast +import subprocess +import sys +from pathlib import Path + +import pytest + +PACKAGE = Path(__file__).resolve().parents[1] / "umbod_desktop" +CORE = { + "contracts", "ports", "usage", "backward_compatibility.contracts", + "backward_compatibility.ports", "backward_compatibility.migration", + "backward_compatibility.tool_names", +} +HOSTING = { + "AccessToken", "AppConfig", "ConnectorRegistration", + "InMemoryGroupConnectorToolPermissions", "JwtClaims", "JwtVerificationError", + "JwtVerifier", "MCPAuthProvider", "McpToolInvocationLogEvent", + "McpToolInvocationLogSink", "OAuthService", "TokenVerifier", + "create_mcp_app", "create_rest_app", +} +RESIDUAL = { + "umbod_desktop.backward_compatibility.adapters.oauth": { + "umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth": {"AuthorizationDocument"}, + }, + "umbod_desktop.backend_integration.configuration": { + "umbod.config.secret_derivation": {"derive_secret"}, + }, + "umbod_desktop.backend_integration.naming": { + "umbod.core.capabilities.tools.names": {"mangle_public_tool_name"}, + "umbod.core.connectors.downstream_mcp.models": {"default_tool_name_prefix"}, + }, + "umbod_desktop.backend_integration.models": { + "umbod.core.connectors.downstream_mcp.models": {"EndpointUrl"}, + }, + "umbod_desktop.backend_integration.oauth": { + "umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth": { + "AuthorizationDocument", "ConnectorOAuthStorage", "SignInRequired", + }, + "umbod.core.connectors.downstream_mcp.oauth.ports": {"OAuthConnection"}, + }, + "umbod_desktop.backend_integration.transports": { + "umbod.core.connectors.downstream_mcp.oauth.provider": { + "RuntimeOAuth", "response_authentication_status", + }, + }, +} + + +def imports(source: str, module: str, is_package: bool) -> list[tuple[str, tuple[str, ...]]]: + found = [] + package = module.split(".") if is_package else module.split(".")[:-1] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + found.extend((alias.name, ()) for alias in node.names) + elif isinstance(node, ast.ImportFrom): + base = node.module or "" + if node.level: + prefix = package[:len(package) - node.level + 1] + base = ".".join((*prefix, *base.split("."))).rstrip(".") + found.append((base, tuple(alias.name for alias in node.names))) + return found + + +def violations(source: str, module: str, is_package: bool = False) -> list[str]: + failures = [] + adapter = module.startswith("umbod_desktop.backend_integration") + runtime = module == "umbod_desktop.runtime" + core = module.removeprefix("umbod_desktop.") in CORE + for target, names in imports(source, module, is_package): + candidates = (target, *(f"{target}.{name}" for name in names)) + if not module.startswith("umbod_desktop.backward_compatibility") and module not in { + "umbod_desktop.main", "umbod_desktop.runtime", + } and any(candidate == "umbod_desktop.backward_compatibility" or candidate.startswith("umbod_desktop.backward_compatibility.") for candidate in candidates): + failures.append(target) + if target == "umbod" or target.startswith("umbod."): + permitted = (adapter or runtime) and target == "umbod.hosting" and bool(names) and set(names) <= HOSTING + permitted = permitted or (bool(names) and target in RESIDUAL.get(module, {}) and set(names) <= RESIDUAL[module][target]) + if not permitted: + failures.append(target) + if not adapter and not runtime and any( + candidate == "umbod_desktop.backend_integration" or candidate.startswith("umbod_desktop.backend_integration.") + for candidate in candidates + ): + failures.append(target) + if core and any(candidate in { + "umbod_desktop.runtime", "umbod_desktop.storage", "umbod_desktop.security", + } for candidate in candidates): + failures.append(target) + if core and target.split(".")[0] in {"httpx", "httpx2", "fastapi", "starlette", "keyring"}: + failures.append(target) + if not adapter and module not in {"umbod_desktop.backward_compatibility.adapters.legacy_host", "umbod_desktop.backward_compatibility.adapters.administration"} and target.split(".")[0] == "httpx": + failures.append(target) + if module not in {"umbod_desktop.backend_integration.administration", "umbod_desktop.backward_compatibility.adapters.administration"} and any( + isinstance(node, ast.Constant) and isinstance(node.value, str) and "/api/admin" in node.value + for node in ast.walk(ast.parse(source)) + ): + failures.append("administration HTTP path") + return failures + + +def transitive_violations(sources: dict[str, tuple[str, bool]], root: str) -> list[str]: + pending = [root] + visited: set[str] = set() + failures = [] + while pending: + module = pending.pop() + if module in visited: + continue + visited.add(module) + source, is_package = sources[module] + for target, names in imports(source, module, is_package): + candidates = (target, *(f"{target}.{name}" for name in names)) + for candidate in candidates: + if candidate in sources: + pending.append(candidate) + if target.split(".")[0] in {"umbod", "httpx", "httpx2", "fastapi", "starlette", "keyring"}: + failures.append(target) + if any(candidate in { + "umbod_desktop.runtime", "umbod_desktop.storage", "umbod_desktop.security", + } or candidate.startswith("umbod_desktop.backend_integration") for candidate in candidates): + failures.append(target) + return failures + + +@pytest.mark.parametrize("path", sorted(PACKAGE.rglob("*.py"))) +def test_desktop_dependency_direction(path: Path) -> None: + relative = path.relative_to(PACKAGE).with_suffix("") + parts = relative.parts + is_package = parts[-1] == "__init__" + module = ".".join(("umbod_desktop", *(parts[:-1] if is_package else parts))) + assert violations(path.read_text(), module, is_package) == [] + + +@pytest.mark.parametrize("source,module,is_package", [ + ("def run():\n import umbod.rest.dependencies", "umbod_desktop.runtime", False), + ("from umbod.hosting.contracts import AppConfig", "umbod_desktop.backend_integration.configuration", False), + ("from umbod.core.connectors.downstream_mcp.models import Connector", "umbod_desktop.backend_integration.models", False), + ("from umbod.core.connectors.downstream_mcp.models import EndpointUrl", "umbod_desktop.backend_integration.naming", False), + ("from umbod.core.connectors.downstream_mcp.oauth.provider import RuntimeOAuth", "umbod_desktop.backend_integration.usage", False), + ("import umbod.core.capabilities.tools.names", "umbod_desktop.backend_integration.naming", False), + ("from ..backend_integration.naming import BackendToolNamingPolicy", "umbod_desktop.backward_compatibility.migration", False), + ("from .backward_compatibility import migration", "umbod_desktop.endpoint", False), + ("from ..backward_compatibility.authentication import LegacyAgentCredentials", "umbod_desktop.backend_integration.authentication", False), + ("from umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth import ConnectorOAuthStorage", "umbod_desktop.backward_compatibility.adapters.oauth", False), + ("import httpx", "umbod_desktop.backward_compatibility.profile", False), + ("from .. import backend_integration", "umbod_desktop.backward_compatibility.tool_names", False), + ("from umbod_desktop import backend_integration", "umbod_desktop.usage", False), + ("from ..runtime import create_desktop_app", "umbod_desktop.backward_compatibility.migration", False), + ("from . import runtime", "umbod_desktop.ports", False), + ("from .storage import DesktopState", "umbod_desktop.usage", False), + ("from .security import CredentialVault", "umbod_desktop.ports", False), + ("from starlette.types import ASGIApp", "umbod_desktop.contracts", False), + ("import httpx", "umbod_desktop.backward_compatibility.migration", False), + ('path = "/api/admin/connectors/mcp"', "umbod_desktop.legacy", False), + ("from ..backend_integration import naming", "umbod_desktop.nested", True), +]) +def test_checker_rejects_forbidden_synthetic_imports(source: str, module: str, is_package: bool) -> None: + assert violations(source, module, is_package) + + +@pytest.mark.parametrize("root", sorted(CORE)) +def test_use_case_transitive_dependencies_are_desktop_owned(root: str) -> None: + sources = {} + for path in PACKAGE.rglob("*.py"): + parts = path.relative_to(PACKAGE).with_suffix("").parts + is_package = parts[-1] == "__init__" + module = ".".join(("umbod_desktop", *(parts[:-1] if is_package else parts))) + sources[module] = (path.read_text(), is_package) + assert transitive_violations(sources, f"umbod_desktop.{root}") == [] + + +def test_transitive_checker_follows_relative_helper_imports() -> None: + sources = { + "umbod_desktop.migration": ("from .helpers import prepare", False), + "umbod_desktop.helpers": ("def prepare():\n from starlette.types import ASGIApp", False), + } + assert transitive_violations(sources, "umbod_desktop.migration") == ["starlette.types"] + + +def test_fresh_core_imports_do_not_load_backend_or_outer_adapters(tmp_path: Path) -> None: + program = ''' +import importlib.abc +import sys + +class BoundaryGuard(importlib.abc.MetaPathFinder): + def find_spec(self, fullname: str, path: object = None, target: object = None) -> None: + root = fullname.split(".")[0] + if root in {"umbod", "httpx", "httpx2", "fastapi", "starlette", "keyring"} or fullname.startswith("umbod_desktop.backend_integration") or fullname in {"umbod_desktop.runtime", "umbod_desktop.security", "umbod_desktop.storage"}: + raise AssertionError("Unexpected core dependency: " + fullname) + +sys.meta_path.insert(0, BoundaryGuard()) +from umbod_desktop import contracts, ports, usage +from umbod_desktop.backward_compatibility import contracts as legacy_contracts, ports as legacy_ports, migration, tool_names +assert all(module is not None for module in (contracts, ports, usage, legacy_contracts, legacy_ports, migration, tool_names)) +assert not any(name == "umbod" or name.startswith("umbod.") for name in sys.modules) +''' + result = subprocess.run([sys.executable, "-c", program], cwd=tmp_path, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_checker_resolves_nested_relative_imports() -> None: + assert imports("def factory():\n from ..contracts import ConnectorDefinition", "umbod_desktop.backend_integration.administration", False) == [ + ("umbod_desktop.contracts", ("ConnectorDefinition",)) + ] + assert violations("from ..contracts import ConnectorDefinition", "umbod_desktop.backend_integration.administration") == [] + assert violations("from umbod.hosting import AppConfig", "umbod_desktop.backend_integration.configuration") == [] diff --git a/desktop/python/tests/test_authentication.py b/desktop/python/tests/test_authentication.py new file mode 100644 index 0000000..85048e0 --- /dev/null +++ b/desktop/python/tests/test_authentication.py @@ -0,0 +1,29 @@ +import hashlib + +import pytest +from umbod.hosting import TokenVerifier + +from umbod_desktop.backend_integration.authentication import DesktopAgentVerifier +from umbod_desktop.backward_compatibility.authentication import LegacyAgentCredentials + + +@pytest.mark.parametrize("token", ["", "unrecognized", "historical-token"]) +async def test_agent_verifier_rejects_noncurrent_credentials_by_default(token: str) -> None: + verifier: TokenVerifier = DesktopAgentVerifier("current-token") + assert await verifier.verify_token(token) is None + + +async def test_agent_verifier_accepts_current_token_with_owner_group() -> None: + verifier: TokenVerifier = DesktopAgentVerifier("current-token") + result = await verifier.verify_token("current-token") + assert result is not None + assert result.claims == {"sub": "desktop-owner", "groups": ["desktop-owner"]} + + +async def test_agent_verifier_accepts_only_injected_historical_credentials() -> None: + historical = "historical-token" + digest = hashlib.sha256(historical.encode()).hexdigest() + verifier: TokenVerifier = DesktopAgentVerifier("current-token", LegacyAgentCredentials((digest,))) + assert await verifier.verify_token(historical) is not None + assert await verifier.verify_token("current-token") is not None + assert await verifier.verify_token("unknown") is None diff --git a/desktop/python/tests/test_composition.py b/desktop/python/tests/test_composition.py new file mode 100644 index 0000000..ef2ba77 --- /dev/null +++ b/desktop/python/tests/test_composition.py @@ -0,0 +1,110 @@ +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any, AsyncIterator + +from fastapi import FastAPI +from test_runtime import MemoryVault + +from umbod_desktop import runtime +from umbod_desktop.backend_integration.configuration import desktop_settings +from umbod_desktop.settings import DesktopSettings +from umbod_desktop.storage import atomic_json + + +def test_configuration_maps_supplied_snapshot_without_reloading_profile(tmp_path: Path) -> None: + atomic_json(tmp_path / "desktop-v2.json", { + "version": 2, "servers": {}, + "settings": {"connector_tool_exposure_mode": "flat", "mcp_administrator_enabled": True}, + }) + snapshot = DesktopSettings(connector_tool_exposure_mode="codemode", mcp_administrator_enabled=False) + config = desktop_settings(tmp_path, "http://127.0.0.1:8000", "root-secret", snapshot) + assert config.mcp.connector_tool_exposure_mode == "codemode" + assert config.feature_toggles.mcp_administrator_enabled is False + + +async def test_runtime_owns_administration_and_preserves_lifecycle_order(tmp_path: Path, monkeypatch: Any) -> None: + events: list[str] = [] + config = object() + snapshots: list[DesktopSettings] = [] + + def map_config(directory: Path, origin: str, root: str, snapshot: DesktopSettings) -> object: + snapshots.append(snapshot) + return config + + def service(name: str) -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + events.append(name + ":enter") + yield + events.append(name + ":exit") + return FastAPI(lifespan=lifespan) + + api = service("rest") + mcp = service("mcp") + + def rest_factory(settings: object, **kwargs: Any) -> FastAPI: + assert settings is config + return api + + async def mcp_factory(settings: object, **kwargs: Any) -> FastAPI: + assert settings is config + return mcp + + class Administration: + def __init__(self, application: FastAPI, token: str) -> None: + assert application is api + self.client = object() + events.append("administration:create") + + async def close(self) -> None: + events.append("administration:close") + + class Names: + def __init__(self, administration: Administration, policy: object, prefix_repair: object) -> None: + assert isinstance(administration, Administration) + + async def repair(self, state: object) -> None: + events.append("names:repair") + + class Upstreams: + def __init__(self, *args: Any) -> None: + pass + + async def restore(self) -> None: + events.append("upstreams:restore") + + async def close(self) -> None: + events.append("upstreams:close") + + class OAuth: + def __init__(self, *args: Any) -> None: + pass + + async def close(self) -> None: + events.append("oauth:close") + + monkeypatch.setattr(runtime, "desktop_settings", map_config) + monkeypatch.setattr(runtime, "create_rest_app", rest_factory) + monkeypatch.setattr(runtime, "create_mcp_app", mcp_factory) + monkeypatch.setattr(runtime, "BackendAdministration", Administration) + monkeypatch.setattr(runtime, "DesktopToolNames", Names) + monkeypatch.setattr(runtime, "DesktopUpstreams", Upstreams) + monkeypatch.setattr(runtime, "OAuthService", OAuth) + monkeypatch.setattr(runtime, "DesktopOAuthHost", lambda upstreams: upstreams) + app, token = await runtime.create_desktop_app(tmp_path, "dev", "http://127.0.0.1:8000", MemoryVault(), lambda: None) + sent: list[dict[str, Any]] = [] + messages = iter([{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]) + + async def receive() -> dict[str, Any]: + return next(messages) + + async def send(message: dict[str, Any]) -> None: + sent.append(message) + + await app({"type": "lifespan", "state": {}}, receive, send) + assert [message["type"] for message in sent] == ["lifespan.startup.complete", "lifespan.shutdown.complete"] + assert snapshots == [DesktopSettings()] + assert events == [ + "administration:create", "rest:enter", "names:repair", "mcp:enter", "upstreams:restore", + "administration:close", "oauth:close", "upstreams:close", "mcp:exit", "rest:exit", + ] diff --git a/desktop/python/tests/test_endpoint.py b/desktop/python/tests/test_endpoint.py new file mode 100644 index 0000000..ea35c47 --- /dev/null +++ b/desktop/python/tests/test_endpoint.py @@ -0,0 +1,42 @@ +import json +from pathlib import Path + +import pytest + +from umbod_desktop.backward_compatibility.profile import saved_legacy_port +from umbod_desktop.endpoint import no_saved_port, saved_port + + +@pytest.mark.parametrize("current", [{}, {"port": None}]) +def test_missing_current_port_uses_composed_legacy_fallback(tmp_path: Path, current: dict[str, object]) -> None: + (tmp_path / "python-endpoint.json").write_text(json.dumps(current)) + (tmp_path / "config.json").write_text('{"gateway_port":19432}') + assert saved_port(tmp_path, saved_legacy_port) == 19432 + + +def test_current_port_precedes_legacy_without_reading_it(tmp_path: Path) -> None: + (tmp_path / "python-endpoint.json").write_text('{"port":19433}') + (tmp_path / "config.json").write_text("invalid-json") + assert saved_port(tmp_path, saved_legacy_port) == 19433 + + +def test_permanent_port_reader_does_not_read_legacy_profile(tmp_path: Path) -> None: + (tmp_path / "config.json").write_text('{"gateway_port":19432}') + assert saved_port(tmp_path, no_saved_port) == 0 + + +def test_absent_current_file_uses_composed_legacy_fallback(tmp_path: Path) -> None: + (tmp_path / "config.json").write_text('{"gateway_port":19432}') + assert saved_port(tmp_path, saved_legacy_port) == 19432 + + +@pytest.mark.parametrize("port", [True, False, 0, -1, 65536, "19432", 19432.0]) +@pytest.mark.parametrize("filename,field", [("python-endpoint.json", "port"), ("config.json", "gateway_port")]) +def test_invalid_saved_port_is_rejected(tmp_path: Path, port: object, filename: str, field: str) -> None: + (tmp_path / filename).write_text(json.dumps({field: port})) + with pytest.raises(ValueError, match="Invalid saved local port"): + saved_port(tmp_path, saved_legacy_port) + + +def test_no_saved_port_returns_ephemeral_port(tmp_path: Path) -> None: + assert saved_port(tmp_path, saved_legacy_port) == 0 diff --git a/desktop/python/tests/test_legacy_authorization.py b/desktop/python/tests/test_legacy_authorization.py new file mode 100644 index 0000000..b142c08 --- /dev/null +++ b/desktop/python/tests/test_legacy_authorization.py @@ -0,0 +1,51 @@ +import json +from typing import Any + +import pytest +from test_runtime import MemoryVault + +from umbod_desktop.backward_compatibility.adapters.oauth import ( + KeychainLegacyAuthorizationImporter, +) +from umbod_desktop.backward_compatibility.contracts import LegacyAuthorization +from umbod_desktop.backward_compatibility.ports import LegacyAuthorizationImporter +from umbod_desktop.ports import CredentialVault + + +def test_legacy_authorization_preserves_serialization_and_refresh_metadata(monkeypatch: Any) -> None: + monkeypatch.setattr("umbod_desktop.backward_compatibility.adapters.oauth.time.time", lambda: 1000) + vault: CredentialVault = MemoryVault() + original = json.dumps({ + "resource": "https://tools.example/mcp/", "client_id": "client", + "access": "access-secret", "refresh": "refresh-secret", "expires": 1060, + }) + vault.set("upstream:oauth:legacy", original) + importer: LegacyAuthorizationImporter = KeychainLegacyAuthorizationImporter(vault) + command = LegacyAuthorization(legacy_id="legacy", host_id="host", endpoint_url="https://tools.example/mcp") + importer.import_authorization(command) + imported = vault.get("python:oauth:host") + assert imported is not None + document = json.loads(imported) + assert document["endpoint_url"] == command.endpoint_url + assert document["client_info"]["client_id"] == "client" + assert document["tokens"]["access_token"] == "access-secret" + assert document["tokens"]["refresh_token"] == "refresh-secret" + assert document["tokens"]["expires_in"] == 60 + assert document["expires_at"] == 1060 + assert vault.get("upstream:oauth:legacy") == original + vault.set("python:oauth:host", "new-authorization") + importer.import_authorization(command) + assert vault.get("python:oauth:host") == "new-authorization" + + +@pytest.mark.parametrize("saved", [None, json.dumps({"resource": "https://other.example/mcp"})]) +def test_legacy_authorization_rejects_missing_or_mismatched_resource(saved: str | None) -> None: + vault: CredentialVault = MemoryVault() + if saved is not None: + vault.set("upstream:oauth:legacy", saved) + importer: LegacyAuthorizationImporter = KeychainLegacyAuthorizationImporter(vault) + with pytest.raises(ValueError): + importer.import_authorization(LegacyAuthorization( + legacy_id="legacy", host_id="host", endpoint_url="https://tools.example/mcp", + )) + assert vault.get("python:oauth:host") is None diff --git a/desktop/python/tests/test_migration.py b/desktop/python/tests/test_migration.py new file mode 100644 index 0000000..1c67548 --- /dev/null +++ b/desktop/python/tests/test_migration.py @@ -0,0 +1,140 @@ +import asyncio +import json +from pathlib import Path +from typing import Any + +import pytest +from fakes import ( + MemoryAdministration, + MemoryAuthorizationImporter, + MemoryNamingPolicy, + MemoryState, + MemoryUpstreams, +) + +from umbod_desktop.backward_compatibility.migration import DesktopMigration +from umbod_desktop.backward_compatibility.tool_names import DesktopToolNames +from umbod_desktop.contracts import ConnectorCreation, ConnectorDefinition +from umbod_desktop.ports import ConnectorAdministration, MetadataState + + +def migration( + directory: Path, state: MetadataState, administration: ConnectorAdministration, + upstreams: MemoryUpstreams, importer: MemoryAuthorizationImporter, +) -> DesktopMigration: + return DesktopMigration( + directory, state, upstreams, administration, + DesktopToolNames(administration, MemoryNamingPolicy(), administration), importer, + ) + + +def legacy_source(directory: Path, extra: dict[str, Any]) -> bytes: + source = json.dumps({ + "servers": [{"id": "legacy", "name": "Finance API", "transport": "http", "url": "https://tools.example/mcp", "oauth": True}], + **extra, + }).encode() + (directory / "config.json").write_bytes(source) + return source + + +async def test_resume_after_discovery_failure_preserves_journal_and_credentials(tmp_path: Path) -> None: + class InterruptedAdministration(MemoryAdministration): + async def discover(self, connector_id: str) -> None: + if not self.discovered: + self.discovered.append(connector_id) + raise OSError("private-secret") + await super().discover(connector_id) + + source = legacy_source(tmp_path, {"allowed_tools": ["legacy.read"]}) + state = MemoryState({"servers": {}}) + admin = InterruptedAdministration(()) + upstreams = MemoryUpstreams() + importer = MemoryAuthorizationImporter() + use_case = migration(tmp_path, state, admin, upstreams, importer) + await use_case.run() + entry = state.data["migration"]["connectors"]["legacy"] + assert entry["id"] == "connector-1" + assert state.data["migration"]["status"] == "needs_attention" + assert "private-secret" not in json.dumps(state.data) + await use_case.run() + assert state.data["migration"]["status"] == "complete" + assert len(admin.created) == len(upstreams.prepared) == 1 + assert len(importer.imported) == 2 + assert importer.imported[0].endpoint_url == "https://tools.example/mcp" + assert admin.enabled[0].names == ("read",) + assert admin.grants[0].selection == admin.enabled[0] + assert upstreams.bindings[0].connector_id == "connector-1" + assert (tmp_path / "config.json").read_bytes() == source + assert json.loads((tmp_path / "config.before-python.json").read_bytes()) == json.loads(source) + await use_case.run() + assert len(admin.published) == 1 + + +async def test_recovers_creation_committed_before_journal_by_stable_path(tmp_path: Path) -> None: + class CommittedAdministration(MemoryAdministration): + async def create(self, command: ConnectorCreation) -> ConnectorDefinition: + await super().create(command) + raise OSError("reply lost") + + legacy_source(tmp_path, {"allowed_tools": []}) + state = MemoryState({"servers": {}}) + admin = CommittedAdministration(()) + upstreams = MemoryUpstreams() + use_case = migration(tmp_path, state, admin, upstreams, MemoryAuthorizationImporter()) + await use_case.run() + assert "id" not in state.data["migration"]["connectors"]["legacy"] + await use_case.run() + assert state.data["migration"]["status"] == "complete" + assert len(admin.created) == len(upstreams.prepared) == 1 + assert upstreams.bindings[0].connector_id == "connector-1" + assert not admin.enabled + assert not admin.grants + + +@pytest.mark.parametrize("clients,grants,selected", [ + ([{"id": "a"}, {"id": "b"}], {"a": ["legacy.read", "legacy.write"], "b": ["legacy.read"]}, ("read",)), + ([{"id": "a"}, {"id": "b"}], {"a": ["legacy.read"]}, ()), + ([], {}, ()), +]) +async def test_migration_grants_only_intersection( + tmp_path: Path, clients: list[dict[str, str]], grants: dict[str, list[str]], selected: tuple[str, ...], +) -> None: + legacy_source(tmp_path, {"clients": clients, "grants": grants}) + state = MemoryState({"servers": {}}) + admin = MemoryAdministration(()) + await migration(tmp_path, state, admin, MemoryUpstreams(), MemoryAuthorizationImporter()).run() + assert state.data["migration"]["status"] == "complete" + assert [selection.names for selection in admin.enabled] == ([selected] if selected else []) + assert [grant.selection.names for grant in admin.grants] == ([selected] if selected else []) + assert admin.published == ["connector-1"] + + +async def test_cancellation_propagates_and_saves_retryable_journal(tmp_path: Path) -> None: + class CancelledAdministration(MemoryAdministration): + async def discover(self, connector_id: str) -> None: + raise asyncio.CancelledError() + + legacy_source(tmp_path, {}) + state = MemoryState({"servers": {}}) + use_case = migration(tmp_path, state, CancelledAdministration(()), MemoryUpstreams(), MemoryAuthorizationImporter()) + with pytest.raises(asyncio.CancelledError): + await use_case.run() + assert state.data["migration"]["status"] == "importing" + assert "error" not in state.data["migration"] + assert state.saves > 0 + + +async def test_changed_source_is_rejected_without_overwriting_backup(tmp_path: Path) -> None: + class FailingAdministration(MemoryAdministration): + async def discover(self, connector_id: str) -> None: + raise OSError("private-access-token") + + source = legacy_source(tmp_path, {}) + state = MemoryState({"servers": {}}) + use_case = migration(tmp_path, state, FailingAdministration(()), MemoryUpstreams(), MemoryAuthorizationImporter()) + await use_case.run() + legacy_source(tmp_path, {"allowed_tools": []}) + with pytest.raises(ValueError, match="Legacy settings changed"): + await use_case.run() + assert json.loads((tmp_path / "config.before-python.json").read_bytes()) == json.loads(source) + assert "private-access-token" not in json.dumps(state.data) diff --git a/desktop/python/tests/test_oauth.py b/desktop/python/tests/test_oauth.py index cbc9b6c..03c6e0b 100644 --- a/desktop/python/tests/test_oauth.py +++ b/desktop/python/tests/test_oauth.py @@ -1,6 +1,8 @@ import asyncio import time +from pathlib import Path from types import SimpleNamespace +from typing import Any from urllib.parse import parse_qs import httpx2 @@ -20,14 +22,15 @@ ) from umbod.core.connectors.downstream_mcp.oauth.service import OAuthService -from umbod_desktop.oauth import ( +from umbod_desktop.backend_integration.oauth import ( DesktopOAuthHost, KeychainOAuthStorage, callback_transport, ) +from umbod_desktop.contracts import UpstreamInput -def desktop_oauth(upstreams): +def desktop_oauth(upstreams: Any) -> OAuthService: return OAuthService(DesktopOAuthHost(upstreams)) @@ -56,12 +59,12 @@ async def test_fresh_authorization_keeps_saved_tokens_and_hosted_identity(): assert auth.token_received -async def test_explicit_scopes_limit_browser_request_after_discovery(): +async def test_explicit_scopes_limit_browser_request_after_discovery() -> None: from urllib.parse import urlsplit from mcp.client.auth import AuthorizationCodeResult - from umbod_desktop.models import DesktopServer + from umbod_desktop.backend_integration.models import DesktopServer query = {} @@ -349,11 +352,11 @@ async def test_declined_browser_consent_finishes_pending_callback(): await future -async def test_runtime_reauthentication_is_reported_and_persisted(tmp_path): +async def test_runtime_reauthentication_is_reported_and_persisted(tmp_path: Path) -> None: from umbod.core.connectors.downstream_mcp.oauth.provider import RuntimeOAuth + from umbod_desktop.backend_integration.transports import DesktopUpstreams from umbod_desktop.storage import DesktopState - from umbod_desktop.upstreams import DesktopUpstreams vault = MemoryVault() upstreams = DesktopUpstreams( @@ -386,7 +389,7 @@ async def handle(request): assert restored.authentication_status["fixture"] == "sign_in_required" -async def test_http_challenge_and_success_update_connection_status(tmp_path): +async def test_http_challenge_and_success_update_connection_status(tmp_path: Path) -> None: import socket import uvicorn @@ -394,8 +397,8 @@ async def test_http_challenge_and_success_update_connection_status(tmp_path): from fastmcp.client.transports import StreamableHttpTransport from fastmcp.server.auth import AccessToken, TokenVerifier + from umbod_desktop.backend_integration.transports import DesktopUpstreams from umbod_desktop.storage import DesktopState - from umbod_desktop.upstreams import DesktopUpstreams class FixtureVerifier(TokenVerifier): async def verify_token(self, token): @@ -424,10 +427,8 @@ def echo() -> str: async with asyncio.timeout(15): while not server.started: await asyncio.sleep(0.01) - prepared = await upstreams.prepare( - {"name": "Protected", "transport": "http", "url": url} - ) - key = prepared["host_id"] + prepared = await upstreams.prepare(UpstreamInput({"name": "Protected", "transport": "http", "url": url})) + key = prepared.host_id proxy = upstreams.apps[key] def transport(): @@ -444,15 +445,13 @@ def transport(): await client.list_tools() assert upstreams.authentication_status[key] == "sign_in_required" upstreams.vault.set("upstream:fixture", "Bearer fixture-token") - await upstreams.prepare( - { + await upstreams.prepare(UpstreamInput({ "host_id": key, "name": "Protected", "transport": "http", "url": url, "headers": {"Authorization": "upstream:fixture"}, - } - ) + })) proxy = upstreams.apps[key] async with Client(transport()) as client: assert [tool.name for tool in await client.list_tools()] == ["echo"] diff --git a/desktop/python/tests/test_prefix_repair.py b/desktop/python/tests/test_prefix_repair.py new file mode 100644 index 0000000..bb495b1 --- /dev/null +++ b/desktop/python/tests/test_prefix_repair.py @@ -0,0 +1,47 @@ +import httpx +import pytest + +from umbod_desktop.backward_compatibility.adapters.administration import ( + BackendPrefixRepair, +) +from umbod_desktop.backward_compatibility.contracts import PrefixUpdate +from umbod_desktop.backward_compatibility.ports import PrefixRepair +from umbod_desktop.contracts import AdministrationError, AdministrationRejected + + +async def test_prefix_repair_updates_metadata_without_owning_client() -> None: + def handle(request: httpx.Request) -> httpx.Response: + assert request.method == "PATCH" + assert request.url.path == "/api/admin/connectors/mcp/connector" + assert request.content == b'{"tool_name_prefix":"Readable"}' + return httpx.Response(200, json={}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handle), base_url="http://127.0.0.1") as client: + repair: PrefixRepair = BackendPrefixRepair(client) + await repair.update_prefix(PrefixUpdate(connector_id="connector", prefix="Readable")) + assert not client.is_closed + + +@pytest.mark.parametrize("status", [403, 409, 500]) +async def test_prefix_repair_maps_rejection_without_secrets(status: int) -> None: + def handle(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, json={"detail": "private-token"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handle), base_url="http://127.0.0.1") as client: + repair: PrefixRepair = BackendPrefixRepair(client) + with pytest.raises(AdministrationRejected) as caught: + await repair.update_prefix(PrefixUpdate(connector_id="connector", prefix="Readable")) + assert "private-token" not in str(caught.value) + assert caught.value.__suppress_context__ + + +async def test_prefix_repair_maps_transport_failure_without_secrets() -> None: + def handle(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("private-token") + + async with httpx.AsyncClient(transport=httpx.MockTransport(handle), base_url="http://127.0.0.1") as client: + repair: PrefixRepair = BackendPrefixRepair(client) + with pytest.raises(AdministrationError) as caught: + await repair.update_prefix(PrefixUpdate(connector_id="connector", prefix="Readable")) + assert not isinstance(caught.value, AdministrationRejected) + assert "private-token" not in str(caught.value) diff --git a/desktop/python/tests/test_recovery.py b/desktop/python/tests/test_recovery.py index 2f13141..06a562a 100644 --- a/desktop/python/tests/test_recovery.py +++ b/desktop/python/tests/test_recovery.py @@ -1,16 +1,22 @@ import json +from pathlib import Path +from typing import Any import pytest from umbod.rest.authentication.authorization import JwtVerificationError +from umbod_desktop.backend_integration.authentication import ( + DesktopAdminVerifier, + DesktopAgentVerifier, +) +from umbod_desktop.backend_integration.configuration import desktop_settings from umbod_desktop.client_setup import ClientSetup -from umbod_desktop.configuration import desktop_settings -from umbod_desktop.security import DesktopAdminVerifier, DesktopAgentVerifier +from umbod_desktop.contracts import UpstreamInput from umbod_desktop.storage import DesktopState -def test_desktop_configuration_never_uses_development_authentication(tmp_path): - settings = desktop_settings(tmp_path, "http://127.0.0.1:12345", "unique-secret" * 5) +def test_desktop_configuration_never_uses_development_authentication(tmp_path: Path) -> None: + settings = desktop_settings(tmp_path, "http://127.0.0.1:12345", "unique-secret" * 5, DesktopState(tmp_path).settings) assert settings.runtime.auth == "desktop" assert settings.admin_authentication.mode == "jwt" assert settings.admin_authentication.environment == "production" @@ -70,10 +76,17 @@ def test_client_setup_rejects_symlinks(tmp_path): ) -async def test_legacy_agent_credentials_keep_only_agent_access(tmp_path): +async def test_legacy_agent_credentials_keep_only_agent_access(tmp_path: Path) -> None: import hashlib - from umbod_desktop.legacy import saved_agent_hashes, saved_port + from umbod_desktop.backward_compatibility.authentication import ( + LegacyAgentCredentials, + ) + from umbod_desktop.backward_compatibility.profile import ( + saved_agent_hashes, + saved_legacy_port, + ) + from umbod_desktop.endpoint import saved_port old_token = "existing-private-agent-token" document = { @@ -87,14 +100,14 @@ async def test_legacy_agent_credentials_keep_only_agent_access(tmp_path): (tmp_path / "config.json").write_text(json.dumps(document)) hashes = saved_agent_hashes(tmp_path) assert len(hashes) == 1 - verifier = DesktopAgentVerifier("new-token", tuple(hashes)) + verifier = DesktopAgentVerifier("new-token", LegacyAgentCredentials(tuple(hashes))) assert await verifier.verify_token(old_token) is not None assert await verifier.verify_token("wrong-token") is None with pytest.raises(JwtVerificationError): DesktopAdminVerifier("admin-token").verify(old_token) - assert saved_port(tmp_path) == 19432 + assert saved_port(tmp_path, saved_legacy_port) == 19432 (tmp_path / "python-endpoint.json").write_text('{"port":19433}') - assert saved_port(tmp_path) == 19433 + assert saved_port(tmp_path, saved_legacy_port) == 19433 async def test_existing_encrypted_database_never_generates_replacement_key(tmp_path): @@ -198,35 +211,29 @@ async def test_moved_profile_stops_host_even_when_replacement_exists(tmp_path): assert replacement_record.read_text() == '{"pid": 123}' -async def test_renaming_upstream_preserves_live_transport(tmp_path): +async def test_renaming_upstream_preserves_live_transport(tmp_path: Path) -> None: from test_runtime import MemoryVault - from umbod_desktop.upstreams import DesktopUpstreams + from umbod_desktop.backend_integration.transports import DesktopUpstreams host = DesktopUpstreams( DesktopState(tmp_path), MemoryVault(), "fixture", "http://127.0.0.1:1" ) starts = [] - async def start(key, server): + async def start(key: str, server: dict[str, Any]) -> None: starts.append(key) host.start = start - prepared = await host.prepare( - {"name": "Original", "transport": "stdio", "command": "/usr/bin/python3"} - ) - key = prepared["host_id"] - renamed = await host.prepare({**host.state.data["servers"][key], "name": "Renamed"}) - assert renamed["connection_changed"] is False + prepared = await host.prepare(UpstreamInput({"name": "Original", "transport": "stdio", "command": "/usr/bin/python3"})) + key = prepared.host_id + renamed = await host.prepare(UpstreamInput({**host.state.data["servers"][key], "name": "Renamed"})) + assert renamed.connection_changed is False assert len(starts) == 1 - changed = await host.prepare( - {**host.state.data["servers"][key], "command": "/different/python"} - ) - assert changed["connection_changed"] is True + changed = await host.prepare(UpstreamInput({**host.state.data["servers"][key], "command": "/different/python"})) + assert changed.connection_changed is True assert len(starts) == 2 - refreshed = await host.prepare( - {**host.state.data["servers"][key], "reconnect": True} - ) - assert refreshed["connection_changed"] is True + refreshed = await host.prepare(UpstreamInput({**host.state.data["servers"][key], "reconnect": True})) + assert refreshed.connection_changed is True assert len(starts) == 3 assert "reconnect" not in host.state.data["servers"][key] diff --git a/desktop/python/tests/test_runtime.py b/desktop/python/tests/test_runtime.py index 03d6f9e..c7a94a6 100644 --- a/desktop/python/tests/test_runtime.py +++ b/desktop/python/tests/test_runtime.py @@ -12,23 +12,26 @@ from fastmcp.exceptions import ToolError from umbod.mcp.agent_instructions import AGENT_INSTRUCTIONS, AGENT_INSTRUCTIONS_URI +from umbod_desktop.backward_compatibility.adapters.administration import ( + BackendPrefixRepair, +) +from umbod_desktop.backward_compatibility.tool_names import DesktopToolNames from umbod_desktop.runtime import create_desktop_app from umbod_desktop.security import DESKTOP_GROUP, persistent_secret from umbod_desktop.storage import DesktopState -from umbod_desktop.tool_names import DesktopToolNames class MemoryVault: - def __init__(self): - self.values = {} + def __init__(self) -> None: + self.values: dict[str, str] = {} - def get(self, reference): + def get(self, reference: str) -> str | None: return self.values.get(reference) - def set(self, reference, value): + def set(self, reference: str, value: str) -> None: self.values[reference] = value - def delete(self, reference): + def delete(self, reference: str) -> None: self.values.pop(reference, None) @@ -39,7 +42,7 @@ async def discovered_names(agent): @pytest.mark.asyncio @pytest.mark.parametrize("transport", ["http", "stdio", "legacy_stdio"]) -async def test_shared_backend_connector_flow(tmp_path, transport, monkeypatch): +async def test_shared_backend_connector_flow(tmp_path: Path, transport: str, monkeypatch: pytest.MonkeyPatch) -> None: fixture = FastMCP("fixture") @fixture.tool @@ -265,14 +268,20 @@ def echo(message: str) -> str: while legacy_prefix + "_echo" not in await discovered_names(agent): await asyncio.sleep(0.1) - async def admin_request(method, path, body=None): - response = await client.request(method, "/api/admin" + path, json=body) - response.raise_for_status() - return response.json() + from umbod_desktop.backend_integration.administration import ( + BackendAdministration, + ) + from umbod_desktop.backend_integration.naming import ( + BackendToolNamingPolicy, + ) - names = DesktopToolNames(admin_request) - await names.repair(DesktopState(tmp_path)) - await names.repair(DesktopState(tmp_path)) + administration = BackendAdministration(app, token) + names = DesktopToolNames(administration, BackendToolNamingPolicy(), BackendPrefixRepair(administration.client)) + try: + await names.repair(DesktopState(tmp_path)) + await names.repair(DesktopState(tmp_path)) + finally: + await administration.close() async with asyncio.timeout(15): while "Fixture_echo" not in await discovered_names(agent): await asyncio.sleep(0.1) @@ -319,7 +328,9 @@ async def admin_request(method, path, body=None): ) ).status_code == 401 - from umbod_desktop.migration import DesktopMigration + from umbod_desktop.backend_integration.administration import ( + BackendAdministration, + ) legacy = { "servers": [{**draft, "id": "legacy"}], @@ -328,17 +339,17 @@ async def admin_request(method, path, body=None): } source = json.dumps(legacy) (tmp_path / "config.json").write_text(source) - original_request = DesktopMigration.request + original_discover = BackendAdministration.discover failed_once = False - async def fail_discovery_once(migration, method, path, body=None): + async def fail_discovery_once(administration: BackendAdministration, connector_id: str) -> None: nonlocal failed_once - if path.endswith("/discoveries") and not failed_once: + if not failed_once: failed_once = True raise OSError("fixture interrupted") - return await original_request(migration, method, path, body) + await original_discover(administration, connector_id) - monkeypatch.setattr(DesktopMigration, "request", fail_discovery_once) + monkeypatch.setattr(BackendAdministration, "discover", fail_discovery_once) interrupted = await client.post("/desktop/migration") assert interrupted.json()["status"] == "needs_attention" resumed = await client.post("/desktop/migration") diff --git a/desktop/python/tests/test_settings.py b/desktop/python/tests/test_settings.py index cabc302..9e6f102 100644 --- a/desktop/python/tests/test_settings.py +++ b/desktop/python/tests/test_settings.py @@ -2,18 +2,19 @@ import json import socket from contextlib import asynccontextmanager +from pathlib import Path import httpx import pytest import uvicorn from test_runtime import MemoryVault -from umbod_desktop.configuration import desktop_settings +from umbod_desktop.backend_integration.configuration import desktop_settings from umbod_desktop.runtime import create_desktop_app from umbod_desktop.storage import DesktopState -def test_existing_profile_defaults_ignore_operator_environment(tmp_path, monkeypatch): +def test_existing_profile_defaults_ignore_operator_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: original = { "version": 2, "servers": {}, @@ -25,13 +26,13 @@ def test_existing_profile_defaults_ignore_operator_environment(tmp_path, monkeyp path.write_text(json.dumps(original)) monkeypatch.setenv("UMBOD_MCP_TOOL_EXPOSURE", "flat") monkeypatch.setenv("UMBOD_FEATURE_MCP_ADMINISTRATOR_ENABLED", "true") - config = desktop_settings(tmp_path, "http://127.0.0.1:8000", "test-root") + config = desktop_settings(tmp_path, "http://127.0.0.1:8000", "test-root", DesktopState(tmp_path).settings) assert config.mcp.connector_tool_exposure_mode == "gateway" assert config.feature_toggles.mcp_administrator_enabled is False assert DesktopState(tmp_path).data == original -def test_saved_settings_drive_backend_composition(tmp_path): +def test_saved_settings_drive_backend_composition(tmp_path: Path) -> None: original = { "version": 2, "servers": {"saved": {"id": "keep"}}, @@ -43,7 +44,7 @@ def test_saved_settings_drive_backend_composition(tmp_path): "legacy_agent_hashes": ["keep"], } (tmp_path / "desktop-v2.json").write_text(json.dumps(original)) - config = desktop_settings(tmp_path, "http://127.0.0.1:8000", "test-root") + config = desktop_settings(tmp_path, "http://127.0.0.1:8000", "test-root", DesktopState(tmp_path).settings) assert config.mcp.connector_tool_exposure_mode == "flat" assert config.feature_toggles.mcp_administrator_enabled is True assert DesktopState(tmp_path).data == original diff --git a/desktop/python/tests/test_tool_names.py b/desktop/python/tests/test_tool_names.py index ab50de0..7d71cd9 100644 --- a/desktop/python/tests/test_tool_names.py +++ b/desktop/python/tests/test_tool_names.py @@ -1,49 +1,36 @@ -from copy import deepcopy -from types import SimpleNamespace +from typing import Any -import httpx import pytest +from fakes import MemoryAdministration, MemoryNamingPolicy, MemoryState -from umbod_desktop.tool_names import DesktopToolNames +from umbod_desktop.backward_compatibility.tool_names import DesktopToolNames +from umbod_desktop.contracts import AdministrationError, ConnectorDefinition LEGACY = "desktop_46b824c3_b082_4a1a_8177_c2c096eec4ce" -def definition(identifier, name, prefix=LEGACY, path=None): - return { - "connector_id": identifier, - "display_name": name, - "tool_name_prefix": prefix, - "public_path": path or "/mcp/proxies/" + prefix.replace("_", "-"), - } - +def definition( + identifier: str, name: str, prefix: str = LEGACY, path: str = "" +) -> ConnectorDefinition: + return ConnectorDefinition( + connector_id=identifier, display_name=name, tool_name_prefix=prefix, + public_path=path or "/mcp/proxies/" + prefix.replace("_", "-"), + ) -class Administration: - def __init__(self, *definitions, rejected=()): - self.definitions = {item["connector_id"]: item for item in definitions} - self.patches = [] - self.rejected = rejected - async def request(self, method, path, body=None): - if path == "/connectors/mcp": - return {"connectors": list(self.definitions.values())} - identifier = path.rsplit("/", 1)[1] - if method == "PATCH": - self.patches.append((identifier, body)) - if identifier in self.rejected: - response = httpx.Response(409, request=httpx.Request(method, "http://test")) - response.raise_for_status() - self.definitions[identifier].update(body) - return deepcopy(self.definitions[identifier]) +def managed(*identifiers: str) -> MemoryState: + return MemoryState({"servers": {key: {"id": key} for key in identifiers}}) -def managed(*identifiers): - return SimpleNamespace(data={"servers": {key: {"id": key} for key in identifiers}}) +def administration(*definitions: ConnectorDefinition, rejected: tuple[str, ...] = ()) -> MemoryAdministration: + admin = MemoryAdministration(definitions) + admin.rejected = set(rejected) + return admin @pytest.mark.asyncio -async def test_repairs_only_managed_generated_prefixes_and_is_idempotent(): - admin = Administration( +async def test_repairs_only_managed_generated_prefixes_and_is_idempotent() -> None: + admin = administration( definition("dinero", "Dinero"), definition("imported", "Cloudflare", "import_" + "a" * 24), definition("custom", "Dinero", "my_finance"), @@ -51,25 +38,25 @@ async def test_repairs_only_managed_generated_prefixes_and_is_idempotent(): definition("different_path", "Deliberate", path="/mcp/proxies/custom"), ) state = managed("dinero", "imported", "custom", "different_path") - names = DesktopToolNames(admin.request) + names = DesktopToolNames(admin, MemoryNamingPolicy(), admin) await names.repair(state) assert admin.patches == [ ("dinero", {"tool_name_prefix": "Dinero"}), ("imported", {"tool_name_prefix": "Cloudflare"}), ] - assert admin.definitions["dinero"]["public_path"] == "/mcp/proxies/" + LEGACY.replace("_", "-") + assert admin.definitions["dinero"].public_path == "/mcp/proxies/" + LEGACY.replace("_", "-") await names.repair(state) assert len(admin.patches) == 2 @pytest.mark.asyncio -async def test_duplicate_names_receive_readable_stable_suffixes(): - admin = Administration( +async def test_duplicate_names_receive_readable_stable_suffixes() -> None: + admin = administration( definition("a", "Finance API"), definition("b", "Finance API"), definition("custom", "Other", "Finance-API"), ) - names = DesktopToolNames(admin.request) + names = DesktopToolNames(admin, MemoryNamingPolicy(), admin) await names.repair(managed("a", "b")) assert admin.patches == [ ("a", {"tool_name_prefix": "Finance_API_2"}), @@ -80,11 +67,26 @@ async def test_duplicate_names_receive_readable_stable_suffixes(): @pytest.mark.asyncio -async def test_rejected_repair_preserves_working_prefix_and_continues(caplog): - admin = Administration( - definition("a", "Dinero"), definition("b", "Cloudflare"), rejected={"a"} +async def test_inventory_failure_does_not_block_startup_and_retries(caplog: Any) -> None: + class UnavailableAdministration(MemoryAdministration): + async def inventory(self) -> tuple[ConnectorDefinition, ...]: + raise AdministrationError() + + admin = UnavailableAdministration((definition("a", "Dinero"),)) + await DesktopToolNames(admin, MemoryNamingPolicy(), admin).repair(managed("a")) + assert not admin.patches + assert "retry on restart" in caplog.text + working = administration(definition("a", "Dinero")) + await DesktopToolNames(working, MemoryNamingPolicy(), working).repair(managed("a")) + assert working.definitions["a"].tool_name_prefix == "Dinero" + + +@pytest.mark.asyncio +async def test_rejected_repair_preserves_working_prefix_and_continues(caplog: Any) -> None: + admin = administration( + definition("a", "Dinero"), definition("b", "Cloudflare"), rejected=("a",) ) - await DesktopToolNames(admin.request).repair(managed("a", "b")) - assert admin.definitions["a"]["tool_name_prefix"] == LEGACY - assert admin.definitions["b"]["tool_name_prefix"] == "Cloudflare" + await DesktopToolNames(admin, MemoryNamingPolicy(), admin).repair(managed("a", "b")) + assert admin.definitions["a"].tool_name_prefix == LEGACY + assert admin.definitions["b"].tool_name_prefix == "Cloudflare" assert "repair deferred for connector a" in caplog.text diff --git a/desktop/python/tests/test_usage.py b/desktop/python/tests/test_usage.py new file mode 100644 index 0000000..574fb48 --- /dev/null +++ b/desktop/python/tests/test_usage.py @@ -0,0 +1,58 @@ +from types import SimpleNamespace +from typing import Any + +import pytest +from fakes import MemoryState +from umbod.hosting import McpToolInvocationLogSink + +from umbod_desktop.backend_integration.usage import DesktopUsageSink +from umbod_desktop.contracts import InvocationCompletion +from umbod_desktop.ports import InvocationRecorder +from umbod_desktop.usage import DesktopUsageRecorder + + +class MemoryRecorder: + def __init__(self) -> None: + self.completions: list[InvocationCompletion] = [] + + def record(self, completion: InvocationCompletion) -> None: + self.completions.append(completion) + + +@pytest.mark.parametrize("connector,operation,category,outcome,expected", [ + ("connector", "read", "", "success", "success"), + ("connector", "read", "connector_error", "failure", "failure"), + ("connector", "read", "authorization", "failure", ""), + ("connector", "read", "validation", "failure", ""), + (None, "search_tools", "", "success", ""), + ("connector", None, "", "success", ""), +]) +def test_invocation_sink_filters_before_mapping( + connector: str | None, operation: str | None, category: str, outcome: str, expected: str, +) -> None: + recorder = MemoryRecorder() + sink: McpToolInvocationLogSink = DesktopUsageSink(recorder) + event = SimpleNamespace( + started=SimpleNamespace(tool=SimpleNamespace(connector_id=connector, operation_name=operation)), + outcome=outcome, failure_category=category, + arguments={"secret": "not recorded"}, result="not recorded", + ) + sink.record_invocation_completed(event) + assert [completion.outcome for completion in recorder.completions] == ([expected] if expected else []) + if expected: + assert recorder.completions[0].model_dump() == { + "connector_id": "connector", "operation_name": "read", "outcome": expected, + } + + +def test_recorder_updates_existing_counts_and_daily_totals(monkeypatch: Any) -> None: + monkeypatch.setattr("umbod_desktop.usage.time.time", lambda: 86400 * 200) + state = MemoryState({"tool_usage": {"connector.read": {"succeeded": 3, "failed": 2}}}) + recorder: InvocationRecorder = DesktopUsageRecorder(state) + recorder.record(InvocationCompletion(connector_id="connector", operation_name="read", outcome="success")) + recorder.record(InvocationCompletion(connector_id="connector", operation_name="read", outcome="failure")) + assert state.data == { + "tool_usage": {"connector.read": {"succeeded": 4, "failed": 3}}, + "daily_calls": {"200": 2}, "daily_tracking_started": 200, + } + assert state.saves == 2 diff --git a/desktop/python/umbod_desktop/backend_integration/README.md b/desktop/python/umbod_desktop/backend_integration/README.md new file mode 100644 index 0000000..dc6e3d6 --- /dev/null +++ b/desktop/python/umbod_desktop/backend_integration/README.md @@ -0,0 +1,24 @@ +# Shared backend integration + +This package is the desktop-owned outer adapter layer. The dependency graph is runtime composition → these adapters → desktop contracts and ports. Migration, naming repair and aggregate recording depend on desktop ports, not this package, HTTP or shared backend types. The package initializer intentionally exports nothing; composition imports each adapter explicitly. + +## Module index + +- `administration.py`: `BackendAdministration` owns the authenticated httpx ASGI client and exact `/api/admin` connector, discovery, tool activation, publication and group-grant translations. Repair-only prefix PATCH translation is owned by the compatibility adapter, which borrows this client without owning closure. Inventory and creation become immutable desktop connector definitions; tool responses become tuples of names. Its HTTP helper is private. Status failures become desktop administration rejections; transport failures become desktop administration errors without response or credential text. Runtime owns closure, not migration. +- `authentication.py`: owner/admin and current agent token verifiers implement the supported hosting contracts. An injected credential predicate defaults to rejection; historical hash matching belongs exclusively to `backward_compatibility/authentication.py`. Desktop vault and loopback authentication remain in `security.py`; the credential port and desktop group identity belong to desktop ports/contracts. +- `configuration.py`: maps a required, already-loaded immutable `DesktopSettings` snapshot to hosting `AppConfig`. It never reloads a profile or operator environment. Existing encryption derivation labels and storage settings remain unchanged. +- `naming.py`: implements the desktop naming policy with the backend's existing readable-prefix and public-name collision functions; it does not normalize names independently. +- `oauth.py`: permanent Keychain OAuth storage and host connection/callback composition. Historical credential conversion lives only in [`../backward_compatibility/`](../backward_compatibility/README.md), with its own narrow authorization-document import. Resource comparisons, absolute expiry, refresh data, serialization and stale-write rejection retain backend OAuth persistence semantics. Existing Keychain references and callback routes are unchanged. +- `transports.py`: stdio/Streamable HTTP proxy infrastructure, single-task proxy lifespans, OAuth runtime authentication and response-status observation. It implements upstream preparation, proxy configuration and binding through desktop DTOs while keeping existing persistence dictionaries. Runtime serializes preparation DTOs to the existing `/desktop/upstreams` JSON shape. +- `models.py`: shared EndpointUrl-based validation of native transport requests. `loopback_origin` remains desktop-owned in `models.py` outside this package. +- `usage.py`: filters nonconnector invocations and authorization/validation failures, then maps hosting invocation events to desktop completion DTOs consumed by the injected recorder. + +## Supported and residual coupling + +Hosting root imports are the supported backend composition boundary. Administration deliberately uses the shared HTTP JSON API rather than backend repositories or REST internals. Residual deep imports are restricted by consumer file, module and symbol: configuration uses `derive_secret`; naming uses `default_tool_name_prefix` and `mangle_public_tool_name`; transport validation uses `EndpointUrl`; OAuth storage uses `AuthorizationDocument`, `ConnectorOAuthStorage`, `SignInRequired` and `OAuthConnection`; transports use `RuntimeOAuth` and `response_authentication_status`. OAuthService comes from hosting. The compatibility OAuth adapter separately imports only `AuthorizationDocument` for historical conversion. No adapter imports deep REST/MCP modules or hosting implementation submodules. + +Runtime retains REST construction, adapter/migration setup, MCP construction, then REST lifespan, independent naming repair, MCP lifespan and upstream restoration. Shutdown closes administration, OAuth and upstreams in that order before MCP and REST lifespans exit. This refactor does not add application-state service lookup, broaden shared lifetimes or promise additional construction rollback. + +Desktop owns the protocols in `ports.py`, including the small shared metadata `data`/`save` protocol. `DesktopState` structurally implements that protocol; use cases do not import the filesystem adapter. Migration's legacy source reads and atomic backup remain explicit desktop-local filesystem coupling through `persistence.py`, not a claim that all desktop I/O is inverted. + +Changes to supported hosting exports, HTTP JSON, desktop credentials, preparation DTO serialization, OAuth persistence or startup/shutdown semantics require compatibility review and corresponding contract tests. Private adapter paths are not compatibility shims for deleted desktop modules. Desktop and API hosting AST tests enforce direct and transitive dependency direction and exercise forbidden synthetic imports. In-memory administration, naming, metadata, authorization and upstream doubles prove the desktop ports without constructing the backend; real backend tests retain runtime/OAuth/settings coverage. diff --git a/desktop/python/umbod_desktop/backend_integration/__init__.py b/desktop/python/umbod_desktop/backend_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/desktop/python/umbod_desktop/backend_integration/administration.py b/desktop/python/umbod_desktop/backend_integration/administration.py new file mode 100644 index 0000000..a00504c --- /dev/null +++ b/desktop/python/umbod_desktop/backend_integration/administration.py @@ -0,0 +1,103 @@ +from typing import Any + +import httpx +from pydantic import BaseModel +from starlette.types import ASGIApp + +from ..contracts import ( + AdministrationError, + AdministrationRejected, + ConnectorCreation, + ConnectorDefinition, + ToolGrant, + ToolSelection, +) + + +class ConnectorReference(BaseModel): + connector_id: str + + +class ConnectorInventory(BaseModel): + connectors: tuple[ConnectorReference, ...] + + +class ToolReference(BaseModel): + name: str + + +class ToolInventory(BaseModel): + tools: tuple[ToolReference, ...] + + +def connector_definition(payload: dict[str, Any]) -> ConnectorDefinition: + return ConnectorDefinition.model_validate({ + field: payload[field] for field in ConnectorDefinition.model_fields + }) + + +class BackendAdministration: + def __init__(self, api: ASGIApp, token: str) -> None: + self.client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=api), + base_url="http://127.0.0.1", + headers={"Authorization": f"Bearer {token}"}, + ) + + async def _request( + self, method: str, path: str, body: Any = None + ) -> dict[str, Any]: + try: + response = await self.client.request(method, "/api/admin" + path, json=body) + response.raise_for_status() + except httpx.HTTPStatusError: + raise AdministrationRejected() from None + except httpx.HTTPError: + raise AdministrationError() from None + return response.json() if response.content else {} + + async def inventory(self) -> tuple[ConnectorDefinition, ...]: + listed = ConnectorInventory.model_validate(await self._request("GET", "/connectors/mcp")) + return tuple([ + connector_definition(await self._request("GET", f"/connectors/mcp/{item.connector_id}")) + for item in listed.connectors + ]) + + async def create(self, command: ConnectorCreation) -> ConnectorDefinition: + payload = command.model_dump(exclude={"configuration"}) + created = await self._request("POST", "/connectors/mcp", { + "metadata": payload, + "configuration": command.configuration.model_dump(), + }) + return connector_definition(created) + + async def discover(self, connector_id: str) -> None: + await self._request("POST", f"/connectors/mcp/{connector_id}/discoveries") + + async def tool_names(self, connector_id: str) -> tuple[str, ...]: + tools = ToolInventory.model_validate(await self._request("GET", f"/connectors/mcp/{connector_id}/tools")) + return tuple(tool.name for tool in tools.tools) + + async def enable_tools(self, selection: ToolSelection) -> None: + await self._request("PUT", f"/connectors/mcp/{selection.connector_id}/tools/activation", { + "tools": [{"tool_id": name, "activation_status": "enabled"} for name in selection.names] + }) + + async def ensure_group(self, group_id: str) -> None: + await self._request("POST", f"/mcp-permissions/groups/{group_id}") + + async def grant_tools(self, grant: ToolGrant) -> None: + await self._request("PUT", f"/mcp-permissions/groups/{grant.group_id}/permissions", { + "capabilities": [{ + "connector_id": grant.selection.connector_id, + "capability_kind": "tool", + "capability_key": name, + "permission_status": "enabled", + } for name in grant.selection.names] + }) + + async def publish(self, connector_id: str) -> None: + await self._request("PUT", f"/connectors/mcp/{connector_id}/publication") + + async def close(self) -> None: + await self.client.aclose() diff --git a/desktop/python/umbod_desktop/backend_integration/authentication.py b/desktop/python/umbod_desktop/backend_integration/authentication.py new file mode 100644 index 0000000..63695c4 --- /dev/null +++ b/desktop/python/umbod_desktop/backend_integration/authentication.py @@ -0,0 +1,45 @@ +import secrets +from collections.abc import Callable + +from umbod.hosting import AccessToken, JwtClaims, JwtVerificationError, TokenVerifier + +from ..contracts import DESKTOP_GROUP + + +class DesktopAdminVerifier: + def __init__(self, token: str) -> None: + self.token = token + + def verify(self, token: str) -> JwtClaims: + if not secrets.compare_digest(token.encode(), self.token.encode()): + raise JwtVerificationError("Invalid desktop credential") + return JwtClaims( + claims={"sub": "desktop-owner", "name": "You", "groups": [DESKTOP_GROUP]} + ) + + +def reject_additional_credential(token: str) -> bool: + return False + + +class DesktopAgentVerifier(TokenVerifier): + def __init__( + self, token: str, + additional_credential: Callable[[str], bool] = reject_additional_credential, + ) -> None: + super().__init__() + self.token = token + self.additional_credential = additional_credential + + async def verify_token(self, token: str) -> AccessToken | None: + if not secrets.compare_digest(token.encode(), self.token.encode()) and not self.additional_credential(token): + return None + return AccessToken( + token=token, + client_id="desktop-agent", + scopes=[], + subject="desktop-owner", + claims={"sub": "desktop-owner", "groups": [DESKTOP_GROUP]}, + ) + + diff --git a/desktop/python/umbod_desktop/configuration.py b/desktop/python/umbod_desktop/backend_integration/configuration.py similarity index 86% rename from desktop/python/umbod_desktop/configuration.py rename to desktop/python/umbod_desktop/backend_integration/configuration.py index 88f0e07..a6904df 100644 --- a/desktop/python/umbod_desktop/configuration.py +++ b/desktop/python/umbod_desktop/backend_integration/configuration.py @@ -1,16 +1,15 @@ -"""Explicit desktop composition: never load operator .env or development defaults.""" - from pathlib import Path -from umbod.config import AppConfig from umbod.config.secret_derivation import derive_secret +from umbod.hosting import AppConfig -from .security import DESKTOP_GROUP -from .storage import DesktopState +from ..contracts import DESKTOP_GROUP +from ..settings import DesktopSettings -def desktop_settings(directory: Path, origin: str, root_secret: str) -> AppConfig: - advanced = DesktopState(directory).settings +def desktop_settings( + directory: Path, origin: str, root_secret: str, advanced: DesktopSettings +) -> AppConfig: return AppConfig.model_validate( { "runtime": { diff --git a/desktop/python/umbod_desktop/backend_integration/models.py b/desktop/python/umbod_desktop/backend_integration/models.py new file mode 100644 index 0000000..e127349 --- /dev/null +++ b/desktop/python/umbod_desktop/backend_integration/models.py @@ -0,0 +1,35 @@ +from typing import Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator +from umbod.core.connectors.downstream_mcp.models import EndpointUrl + + +class DesktopServer(BaseModel): + model_config = ConfigDict(extra="forbid") + id: str = "" + host_id: str = "" + name: str = Field(min_length=1, max_length=200) + transport: Literal["stdio", "http"] + command: str = "" + args: list[str] = Field(default_factory=list) + url: str = "" + oauth: bool = False + oauth_scopes: str = "" + reconnect: bool = Field(default=False, exclude=True) + headers: dict[str, str] = Field(default_factory=dict) + env: dict[str, str] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_transport(self) -> Self: + if self.transport == "stdio" and not self.command.strip(): + raise ValueError("A local executable is required") + if self.transport == "http": + TypeAdapter(EndpointUrl).validate_python(self.url) + if any( + not reference.startswith("upstream:") + for reference in (*self.headers.values(), *self.env.values()) + ): + raise ValueError("Credentials must be upstream Keychain references") + return self + + diff --git a/desktop/python/umbod_desktop/backend_integration/naming.py b/desktop/python/umbod_desktop/backend_integration/naming.py new file mode 100644 index 0000000..187848d --- /dev/null +++ b/desktop/python/umbod_desktop/backend_integration/naming.py @@ -0,0 +1,10 @@ +from umbod.core.capabilities.tools.names import mangle_public_tool_name +from umbod.core.connectors.downstream_mcp.models import default_tool_name_prefix + + +class BackendToolNamingPolicy: + def readable_prefix(self, display_name: str) -> str: + return default_tool_name_prefix(display_name) + + def collision_key(self, prefix: str) -> str: + return mangle_public_tool_name(prefix, "") diff --git a/desktop/python/umbod_desktop/oauth.py b/desktop/python/umbod_desktop/backend_integration/oauth.py similarity index 75% rename from desktop/python/umbod_desktop/oauth.py rename to desktop/python/umbod_desktop/backend_integration/oauth.py index 3b63247..3cf4691 100644 --- a/desktop/python/umbod_desktop/oauth.py +++ b/desktop/python/umbod_desktop/backend_integration/oauth.py @@ -1,4 +1,7 @@ -"""macOS Keychain and loopback transport adapters for shared OAuth.""" +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING + +from starlette.requests import Request from starlette.responses import PlainTextResponse from umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth import ( AuthorizationDocument, @@ -6,9 +9,12 @@ SignInRequired, ) from umbod.core.connectors.downstream_mcp.oauth.ports import OAuthConnection -from umbod.core.connectors.downstream_mcp.oauth.service import OAuthService +from umbod.hosting import OAuthService + +from ..ports import CredentialVault -from .security import CredentialVault +if TYPE_CHECKING: + from .transports import DesktopUpstreams class KeychainOAuthStorage(ConnectorOAuthStorage): @@ -36,11 +42,11 @@ async def save(self) -> None: class DesktopOAuthHost: - def __init__(self, upstreams): + def __init__(self, upstreams: "DesktopUpstreams") -> None: self.upstreams = upstreams self.statuses = getattr(upstreams, "authentication_status", {}) - def resolve(self, connector_id): + def resolve(self, connector_id: str) -> str: servers = self.upstreams.state.data["servers"] if connector_id.startswith("pending:"): key = connector_id.removeprefix("pending:") @@ -51,35 +57,36 @@ def resolve(self, connector_id): return key raise KeyError(connector_id) - def connection(self, key): + def connection(self, key: str) -> OAuthConnection: server = self.upstreams.state.data["servers"][key] if server.get("transport", "http") != "http": raise ValueError("Browser sign-in requires an HTTPS connector") return OAuthConnection(server["url"], server.get("oauth_scopes", "")) - def storage(self, key): + def storage(self, key: str) -> KeychainOAuthStorage: return KeychainOAuthStorage(self.upstreams.vault, key, self.connection(key).endpoint_url) - def callback_url(self, key, *, shared): + def callback_url(self, key: str, *, shared: bool) -> str: return f"{self.upstreams.origin}/desktop/oauth/callback" + ("" if shared else f"/{key}") - def set_scopes(self, key, scopes): + def set_scopes(self, key: str, scopes: str) -> None: self.upstreams.state.data["servers"][key]["oauth_scopes"] = scopes self.upstreams.state.save() - def set_authorized(self, key, authorized): + def set_authorized(self, key: str, authorized: bool) -> None: self.upstreams.state.data["servers"][key]["oauth"] = authorized self.upstreams.state.save() - async def delete_authorization(self, key): + async def delete_authorization(self, key: str) -> None: self.upstreams.vault.delete(f"python:oauth:{key}") - async def reconnect(self, key): + async def reconnect(self, key: str) -> None: await self.upstreams.start(key, self.upstreams.state.data["servers"][key]) -def callback_transport(oauth: OAuthService): - async def callback(request): +def callback_transport(oauth: OAuthService) -> Callable[[Request], Awaitable[PlainTextResponse]]: + async def callback(request: Request) -> PlainTextResponse: result = await oauth.callback(request.path_params.get("nonce"), dict(request.query_params)) return PlainTextResponse(result.content, status_code=result.status_code) return callback + diff --git a/desktop/python/umbod_desktop/upstreams.py b/desktop/python/umbod_desktop/backend_integration/transports.py similarity index 84% rename from desktop/python/umbod_desktop/upstreams.py rename to desktop/python/umbod_desktop/backend_integration/transports.py index 096f8af..a9abaf2 100644 --- a/desktop/python/umbod_desktop/upstreams.py +++ b/desktop/python/umbod_desktop/backend_integration/transports.py @@ -1,7 +1,6 @@ -"""Adapt desktop-owned transports to the shared downstream HTTP connector API.""" - import asyncio import sys +from typing import Any from uuid import uuid4 import httpx2 @@ -10,21 +9,26 @@ from starlette.responses import JSONResponse from starlette.types import Receive, Scope, Send +from ..contracts import ( + ConnectorBinding, + PreparedUpstream, + ProxyConfiguration, + UpstreamInput, +) +from ..ports import CredentialVault +from ..security import LoopbackAuthentication +from ..storage import DesktopState from .models import DesktopServer -from .security import CredentialVault, LoopbackAuthentication -from .storage import DesktopState class ProxyLifetime: - """Enter and exit ASGI lifespan in one task (required by AnyIO/ContextVars).""" - - def __init__(self, app): + def __init__(self, app: Any) -> None: self.app = app self.ready = asyncio.get_running_loop().create_future() self.stopped = asyncio.Event() self.task = asyncio.create_task(self.run()) - async def run(self): + async def run(self) -> None: try: async with self.app.lifespan(self.app): self.ready.set_result(None) @@ -34,14 +38,14 @@ async def run(self): self.ready.set_exception(error) raise - async def start(self): + async def start(self) -> None: try: await self.ready except BaseException: await asyncio.gather(self.task, return_exceptions=True) raise - async def aclose(self): + async def aclose(self) -> None: self.stopped.set() await self.task @@ -94,7 +98,7 @@ async def start(self, key: str, server: dict) -> None: lambda: self.set_authentication_status(key, "sign_in_required") ) - async def observe_response(response): + async def observe_response(response: httpx2.Response) -> None: if str(response.request.url).rstrip("/") != server["url"].rstrip("/"): return from umbod.core.connectors.downstream_mcp.oauth.provider import ( @@ -105,7 +109,7 @@ async def observe_response(response): self.authentication_status[key] = status self.state.save() - def http_client(**kwargs): + def http_client(**kwargs: Any) -> httpx2.AsyncClient: kwargs.setdefault("timeout", httpx2.Timeout(30.0, read=300.0)) return httpx2.AsyncClient( **kwargs, event_hooks={"response": [observe_response]} @@ -142,8 +146,8 @@ def _credential(self, reference: str) -> str: raise ValueError("A connector credential is missing from Keychain") return value - async def prepare(self, server: dict) -> dict: - draft = DesktopServer.model_validate(server) + async def prepare(self, server: UpstreamInput) -> PreparedUpstream: + draft = DesktopServer.model_validate(server.root) server = draft.model_dump() key = server.get("host_id") or uuid4().hex if key not in self.state.data["servers"] and server.get("host_id"): @@ -170,7 +174,6 @@ async def prepare(self, server: dict) -> dict: await self.start(key, server) self.state.data.setdefault("transport_errors", {}).pop(key, None) except ValueError, OSError: - # Keep editable offline drafts, but stop the previously configured transport. old = self.lifetimes.pop(key, None) self.apps.pop(key, None) if old: @@ -180,16 +183,23 @@ async def prepare(self, server: dict) -> dict: ) self.state.data["servers"][key] = {**server, "host_id": key} self.state.save() - return { - "host_id": key, - "connection_changed": connection_changed, - "configuration": { - "auth_mode": "static_bearer", - "endpoint_url": f"{self.origin}/desktop/upstreams/{key}/mcp", - "header_type": "bearer", - "bearer_token": self.token, - }, - } + return PreparedUpstream( + host_id=key, + connection_changed=connection_changed, + configuration=self.proxy_configuration(key), + ) + + def proxy_configuration(self, host_id: str) -> ProxyConfiguration: + return ProxyConfiguration( + auth_mode="static_bearer", + endpoint_url=f"{self.origin}/desktop/upstreams/{host_id}/mcp", + header_type="bearer", + bearer_token=self.token, + ) + + def bind(self, binding: ConnectorBinding) -> None: + self.state.data["servers"][binding.host_id]["id"] = binding.connector_id + self.state.save() async def restore(self) -> None: errors = self.state.data.setdefault("transport_errors", {}) diff --git a/desktop/python/umbod_desktop/backend_integration/usage.py b/desktop/python/umbod_desktop/backend_integration/usage.py new file mode 100644 index 0000000..2b67c2f --- /dev/null +++ b/desktop/python/umbod_desktop/backend_integration/usage.py @@ -0,0 +1,21 @@ +from umbod.hosting import McpToolInvocationLogEvent + +from ..contracts import InvocationCompletion +from ..ports import InvocationRecorder + + +class DesktopUsageSink: + def __init__(self, recorder: InvocationRecorder) -> None: + self.recorder = recorder + + def record_invocation_completed(self, event: McpToolInvocationLogEvent) -> None: + tool = event.started.tool + if tool.connector_id is None or tool.operation_name is None: + return + if getattr(event, "failure_category", None) in ("authorization", "validation"): + return + self.recorder.record(InvocationCompletion( + connector_id=tool.connector_id, + operation_name=tool.operation_name, + outcome="success" if event.outcome == "success" else "failure", + )) diff --git a/desktop/python/umbod_desktop/backward_compatibility/README.md b/desktop/python/umbod_desktop/backward_compatibility/README.md new file mode 100644 index 0000000..d5779f7 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/README.md @@ -0,0 +1,28 @@ +# Desktop backward compatibility only + +This package exists exclusively to repair or import historical desktop data and retain old client access. It is not a home for new features or permanent desktop behavior. Remove it once support for these historical profiles and credentials ends. + +## Ownership and entrypoints + +- `migration.py`: `DesktopMigration.run` imports historical `config.json` servers, intersected grants, usage, activation, diagnostics and client setup. It preserves the original source, `config.before-python.json` backup and resumable `desktop-v2.json` migration journal, including deterministic import public paths. One runtime-owned instance retains its asynchronous lock so concurrent migration requests serialize safely. +- `tool_names.py`: `DesktopToolNames` repairs managed `desktop_` and `import_` prefixes matching their generated public paths, and allocates readable collision-free prefixes during import. Explicit prefixes and failed repairs remain unchanged; rejected repairs retry on restart. +- `contracts.py`, `ports.py`: compatibility-only authorization/prefix-update commands and authorization-import/prefix-repair protocols. Orchestration depends on permanent desktop administration, naming, metadata and upstream preparation ports; it never imports backend adapters. +- `profile.py`: reads historical `config.json` agent hashes and gateway-port fallback. Permanent `endpoint.py` reads the current `python-endpoint.json`; `main.py` supplies the historical fallback only when needed. +- `authentication.py`: `LegacyAgentCredentials` matches retained SHA-256 agent hashes. Runtime injects this predicate into the permanent verifier; absent injection, only the current agent credential is accepted. Historical credentials never grant owner REST access. +- `adapters/legacy_host.py`: authenticates the old `runtime.json` management endpoint, verifies profile/PID identity and requests shutdown without signalling a PID from disk. +- `adapters/administration.py`: `BackendPrefixRepair` owns only historical prefix PATCH requests and error translation. Runtime supplies the permanent administration adapter’s HTTP client; that adapter retains ownership of closure. +- `adapters/oauth.py`: converts historical `upstream:oauth:*` Keychain records into current `python:oauth:*` authorization documents, preserving source records, expiry and refresh data without overwriting an existing target. + +The initializers intentionally export nothing and import no backend code. Only the OAuth adapter imports the exact backend authorization document needed for conversion; the legacy-host adapter owns historical HTTP retirement and the prefix-repair adapter owns repair-only HTTP updates. + +## Lifecycle + +`main.py` retires the authenticated old host before binding the saved endpoint. `runtime.py` captures historical hashes only when not already persisted, then composes migration, naming repair and credential conversion. Startup enters REST, repairs names, enters MCP and restores upstreams in that order. Migration runs only through the authenticated `POST /desktop/migration` route, including native Swift callers. Shutdown ordering and permanent services are unchanged. + +## Deletion checklist + +- End support for historical config/runtime formats, generated tool prefixes, legacy OAuth Keychain records and migrated agent hashes; communicate endpoint and credential replacement requirements first. +- Remove legacy retirement and saved-port fallback imports/calls from `main.py`, retaining the current endpoint reader. +- Remove migration, repair, historical hash capture and credential-predicate injection from `runtime.py`, retaining current-token authentication and normal REST/MCP/upstream lifecycle. +- Remove `/desktop/migration` and its native Swift client/model call sites together; do not leave a broken native request. +- Remove this package and its dedicated tests, update architecture boundaries and README indexes, and retire persisted migration/hash metadata only with an explicit data-retention decision. Do not delete shared backend storage, source backups or Keychain credentials as incidental cleanup. diff --git a/desktop/python/umbod_desktop/backward_compatibility/__init__.py b/desktop/python/umbod_desktop/backward_compatibility/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/desktop/python/umbod_desktop/backward_compatibility/adapters/__init__.py b/desktop/python/umbod_desktop/backward_compatibility/adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/desktop/python/umbod_desktop/backward_compatibility/adapters/administration.py b/desktop/python/umbod_desktop/backward_compatibility/adapters/administration.py new file mode 100644 index 0000000..058637c --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/adapters/administration.py @@ -0,0 +1,21 @@ +import httpx + +from ...contracts import AdministrationError, AdministrationRejected +from ..contracts import PrefixUpdate + + +class BackendPrefixRepair: + def __init__(self, client: httpx.AsyncClient) -> None: + self.client = client + + async def update_prefix(self, update: PrefixUpdate) -> None: + try: + response = await self.client.patch( + f"/api/admin/connectors/mcp/{update.connector_id}", + json={"tool_name_prefix": update.prefix}, + ) + response.raise_for_status() + except httpx.HTTPStatusError: + raise AdministrationRejected() from None + except httpx.HTTPError: + raise AdministrationError() from None diff --git a/desktop/python/umbod_desktop/backward_compatibility/adapters/legacy_host.py b/desktop/python/umbod_desktop/backward_compatibility/adapters/legacy_host.py new file mode 100644 index 0000000..064ccf0 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/adapters/legacy_host.py @@ -0,0 +1,37 @@ +import asyncio +import json +from pathlib import Path + +import httpx + +from ...models import loopback_origin + + +async def retire_legacy_host(directory: Path, profile: str) -> None: + path = directory / "runtime.json" + if not path.exists(): + return + record = json.loads(path.read_text()) + if record.get("profile") != profile: + raise ValueError("Legacy runtime profile does not match") + origin = loopback_origin(record["management"]) + async with httpx.AsyncClient( + base_url=origin, + timeout=3, + trust_env=False, + headers={"Authorization": f"Bearer {record['admin_token']}"}, + ) as client: + try: + response = await client.post("/manage", json={"action": "health"}) + except httpx.ConnectError: + return + response.raise_for_status() + health = response.json() + if health.get("profile") != profile or health.get("pid") != record["pid"]: + raise ValueError("Legacy runtime identity does not match") + response = await client.post("/manage", json={"action": "shutdown"}) + response.raise_for_status() + async with asyncio.timeout(15): + while path.exists(): + await asyncio.sleep(0.05) + diff --git a/desktop/python/umbod_desktop/backward_compatibility/adapters/oauth.py b/desktop/python/umbod_desktop/backward_compatibility/adapters/oauth.py new file mode 100644 index 0000000..f99a72e --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/adapters/oauth.py @@ -0,0 +1,38 @@ +import json +import time + +from mcp.shared.auth import OAuthClientInformationFull, OAuthToken +from umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth import ( + AuthorizationDocument, +) + +from ...ports import CredentialVault +from ..contracts import LegacyAuthorization + + +class KeychainLegacyAuthorizationImporter: + def __init__(self, vault: CredentialVault) -> None: + self.vault = vault + + def import_authorization(self, authorization: LegacyAuthorization) -> None: + reference = f"python:oauth:{authorization.host_id}" + if self.vault.get(reference) is not None: + return + raw = self.vault.get(f"upstream:oauth:{authorization.legacy_id}") + if raw is None: + raise ValueError("Legacy OAuth credential is missing") + saved = json.loads(raw) + if saved["resource"].rstrip("/") != authorization.endpoint_url.rstrip("/"): + raise ValueError("OAuth resource does not match") + document = AuthorizationDocument( + endpoint_url=authorization.endpoint_url, + client_info=OAuthClientInformationFull(client_id=saved["client_id"]), + tokens=OAuthToken( + access_token=saved["access"], + token_type="Bearer", + refresh_token=saved.get("refresh"), + expires_in=max(0, int(saved["expires"] - time.time())), + ), + expires_at=saved["expires"], + ) + self.vault.set(reference, document.model_dump_json()) diff --git a/desktop/python/umbod_desktop/backward_compatibility/authentication.py b/desktop/python/umbod_desktop/backward_compatibility/authentication.py new file mode 100644 index 0000000..cae9c26 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/authentication.py @@ -0,0 +1,11 @@ +import hashlib +import secrets + + +class LegacyAgentCredentials: + def __init__(self, hashes: tuple[str, ...]) -> None: + self.hashes = hashes + + def __call__(self, token: str) -> bool: + digest = hashlib.sha256(token.encode()).hexdigest() + return any(secrets.compare_digest(digest, saved) for saved in self.hashes) diff --git a/desktop/python/umbod_desktop/backward_compatibility/contracts.py b/desktop/python/umbod_desktop/backward_compatibility/contracts.py new file mode 100644 index 0000000..921b389 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/contracts.py @@ -0,0 +1,12 @@ +from ..contracts import DesktopContract + + +class PrefixUpdate(DesktopContract): + connector_id: str + prefix: str + + +class LegacyAuthorization(DesktopContract): + legacy_id: str + host_id: str + endpoint_url: str diff --git a/desktop/python/umbod_desktop/backward_compatibility/migration.py b/desktop/python/umbod_desktop/backward_compatibility/migration.py new file mode 100644 index 0000000..3e06500 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/migration.py @@ -0,0 +1,148 @@ +import asyncio +import hashlib +import json +from pathlib import Path + +from ..contracts import ( + DESKTOP_GROUP, + ConnectorBinding, + ConnectorCreation, + ToolGrant, + ToolSelection, + UpstreamInput, +) +from ..persistence import atomic_json +from ..ports import ( + ConnectorAdministration, + MetadataState, + UpstreamPreparation, +) +from .contracts import LegacyAuthorization +from .ports import LegacyAuthorizationImporter +from .tool_names import DesktopToolNames + + +class DesktopMigration: + def __init__( + self, directory: Path, state: MetadataState, upstreams: UpstreamPreparation, + administration: ConnectorAdministration, tool_names: DesktopToolNames, + authorization_importer: LegacyAuthorizationImporter, + ) -> None: + self.directory, self.state, self.upstreams = directory, state, upstreams + self.administration = administration + self.tool_names = tool_names + self.authorization_importer = authorization_importer + self.lock = asyncio.Lock() + + async def run(self) -> None: + async with self.lock: + await self._run() + + async def _run(self) -> None: + source = self.directory / "config.json" + if ( + not source.exists() + or self.state.data.get("migration", {}).get("status") == "complete" + ): + return + payload = source.read_bytes() + legacy = json.loads(payload) + digest = hashlib.sha256(payload).hexdigest() + journal = self.state.data.setdefault( + "migration", + {"status": "pending", "source_sha256": digest, "connectors": {}}, + ) + if journal["source_sha256"] != digest: + raise ValueError( + "Legacy settings changed during migration; restore the saved source before retrying" + ) + backup = self.directory / "config.before-python.json" + if not backup.exists(): + atomic_json(backup, legacy) + journal["status"] = "importing" + self.state.save() + try: + await self.administration.ensure_group(DESKTOP_GROUP) + for server in legacy.get("servers", []): + old_id = server["id"] + entry = journal["connectors"].setdefault(old_id, {}) + if entry.get("complete"): + continue + if "host_id" not in entry: + prepared = await self.upstreams.prepare(UpstreamInput(server)) + entry["host_id"] = prepared.host_id + self.state.save() + host_id = entry["host_id"] + if server.get("oauth"): + self.authorization_importer.import_authorization(LegacyAuthorization( + legacy_id=old_id, host_id=host_id, endpoint_url=server["url"] + )) + config = self.upstreams.proxy_configuration(host_id) + if "id" not in entry: + slug = "import-" + hashlib.sha256(old_id.encode()).hexdigest()[:24] + definitions = await self.administration.inventory() + existing = next(( + item for item in definitions + if item.public_path == f"/mcp/proxies/{slug}" + ), None) + created = existing or await self.administration.create(ConnectorCreation( + display_name=server["name"], + tool_name_prefix=self.tool_names.available_prefix( + server["name"], await self.administration.inventory() + ), + capability_description=f"Tools from {server['name']}", + public_path=f"/mcp/proxies/{slug}", + configuration=config, + )) + entry["id"] = created.connector_id + self.upstreams.bind(ConnectorBinding(host_id=host_id, connector_id=entry["id"])) + self.state.save() + await self.administration.discover(entry["id"]) + tools = await self.administration.tool_names(entry["id"]) + allowed = legacy.get("allowed_tools") + if allowed is None: + client_ids = [client["id"] for client in legacy.get("clients", [])] + sets = [ + set(legacy.get("grants", {}).get(client, [])) + for client in client_ids + ] + allowed = set.intersection(*sets) if sets else set() + selected = ToolSelection( + connector_id=entry["id"], + names=tuple(name for name in tools if f"{old_id}.{name}" in allowed), + ) + if selected.names: + await self.administration.enable_tools(selected) + await self.administration.grant_tools(ToolGrant( + group_id=DESKTOP_GROUP, selection=selected + )) + await self.administration.publish(entry["id"]) + entry["complete"] = True + self.state.save() + for field in ( + "daily_calls", + "daily_tracking_started", + "diagnostics", + "client_setup", + ): + if field in legacy: + self.state.data.setdefault(field, legacy[field]) + if legacy.get("activation") and "activation" not in self.state.data: + journey = dict(legacy["activation"]) + entry = journal["connectors"].get(journey.get("server"), {}) + journey["server"] = entry.get("id", journey.get("server")) + self.state.data["activation"] = journey + usage = self.state.data.setdefault("tool_usage", {}) + for name, counts in legacy.get("tool_usage", {}).items(): + old_id, _, tool = name.partition(".") + new_id = journal["connectors"].get(old_id, {}).get("id", old_id) + usage.setdefault(f"{new_id}.{tool}", counts) + journal["status"] = "complete" + journal.pop("error", None) + except Exception: + journal["status"] = "needs_attention" + journal["error"] = ( + "Some connectors could not be imported. Original settings and credentials are preserved; reconnect the upstream and retry." + ) + finally: + self.state.save() diff --git a/desktop/python/umbod_desktop/backward_compatibility/ports.py b/desktop/python/umbod_desktop/backward_compatibility/ports.py new file mode 100644 index 0000000..a8da767 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/ports.py @@ -0,0 +1,11 @@ +from typing import Protocol + +from .contracts import LegacyAuthorization, PrefixUpdate + + +class PrefixRepair(Protocol): + async def update_prefix(self, update: PrefixUpdate) -> None: ... + + +class LegacyAuthorizationImporter(Protocol): + def import_authorization(self, authorization: LegacyAuthorization) -> None: ... diff --git a/desktop/python/umbod_desktop/backward_compatibility/profile.py b/desktop/python/umbod_desktop/backward_compatibility/profile.py new file mode 100644 index 0000000..6347ba9 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/profile.py @@ -0,0 +1,32 @@ +import json +from pathlib import Path + +from ..endpoint import validate_saved_port + + +def saved_agent_hashes(directory: Path) -> list[str]: + path = directory / "config.json" + if not path.exists(): + return [] + document = json.loads(path.read_text()) + clients = list(document.get("clients", [])) + if document.get("connection"): + clients.append(document["connection"]) + return sorted( + { + client["token_hash"] + for client in clients + if isinstance(client.get("token_hash"), str) + and len(client["token_hash"]) == 64 + and all(c in "0123456789abcdef" for c in client["token_hash"]) + } + ) + + +def saved_legacy_port(directory: Path) -> int: + path = directory / "config.json" + if path.exists(): + port = json.loads(path.read_text()).get("gateway_port") + if port is not None: + return validate_saved_port(port) + return 0 diff --git a/desktop/python/umbod_desktop/backward_compatibility/tool_names.py b/desktop/python/umbod_desktop/backward_compatibility/tool_names.py new file mode 100644 index 0000000..3f20943 --- /dev/null +++ b/desktop/python/umbod_desktop/backward_compatibility/tool_names.py @@ -0,0 +1,70 @@ +import logging +import re +from collections.abc import Sequence + +from ..contracts import ( + AdministrationError, + AdministrationRejected, + ConnectorDefinition, +) +from ..ports import ConnectorAdministration, MetadataState, ToolNamingPolicy +from .contracts import PrefixUpdate +from .ports import PrefixRepair + +_LOGGER = logging.getLogger(__name__) +_GENERATED_PREFIX = re.compile( + r"(?:desktop_[0-9a-f]{8}(?:_[0-9a-f]{4}){3}_[0-9a-f]{12}|import_[0-9a-f]{24})" +) + + +class DesktopToolNames: + def __init__( + self, administration: ConnectorAdministration, policy: ToolNamingPolicy, + prefix_repair: PrefixRepair, + ) -> None: + self.administration = administration + self.policy = policy + self.prefix_repair = prefix_repair + + def available_prefix(self, display_name: str, definitions: Sequence[ConnectorDefinition]) -> str: + base = self.policy.readable_prefix(display_name) + reserved = {self.policy.collision_key(item.tool_name_prefix) for item in definitions} + candidate, suffix = base, 2 + while self.policy.collision_key(candidate) in reserved: + candidate = f"{base}_{suffix}" + suffix += 1 + return candidate + + async def repair(self, state: MetadataState) -> None: + managed_ids = { + item.get("id") for item in state.data.get("servers", {}).values() + } + if not managed_ids: + return + try: + definitions = list(await self.administration.inventory()) + for definition in sorted(definitions, key=lambda item: item.connector_id): + prefix = definition.tool_name_prefix + if ( + definition.connector_id not in managed_ids + or not _GENERATED_PREFIX.fullmatch(prefix) + or definition.public_path.removeprefix("/mcp/proxies/") + .replace("-", "_") != prefix + ): + continue + candidate = self.available_prefix(definition.display_name, definitions) + try: + await self.prefix_repair.update_prefix(PrefixUpdate( + connector_id=definition.connector_id, prefix=candidate, + )) + except AdministrationRejected: + _LOGGER.warning( + "Desktop tool namespace repair deferred for connector %s", + definition.connector_id, + ) + continue + definitions[definitions.index(definition)] = definition.model_copy( + update={"tool_name_prefix": candidate} + ) + except AdministrationError: + _LOGGER.warning("Desktop tool namespace repair deferred; retry on restart") diff --git a/desktop/python/umbod_desktop/contracts.py b/desktop/python/umbod_desktop/contracts.py new file mode 100644 index 0000000..074e410 --- /dev/null +++ b/desktop/python/umbod_desktop/contracts.py @@ -0,0 +1,71 @@ +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, RootModel + +DESKTOP_GROUP = "desktop-owner" + + +class DesktopContract(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class ConnectorDefinition(DesktopContract): + connector_id: str + display_name: str + tool_name_prefix: str + public_path: str + + +class ProxyConfiguration(DesktopContract): + auth_mode: Literal["static_bearer"] + endpoint_url: str + header_type: Literal["bearer"] + bearer_token: str = Field(repr=False) + + +class ConnectorCreation(DesktopContract): + display_name: str + tool_name_prefix: str + public_path: str + capability_description: str + configuration: ProxyConfiguration + + +class ToolSelection(DesktopContract): + connector_id: str + names: tuple[str, ...] + + +class ToolGrant(DesktopContract): + group_id: str + selection: ToolSelection + + +class UpstreamInput(RootModel[dict[str, Any]]): + model_config = ConfigDict(frozen=True) + + +class PreparedUpstream(DesktopContract): + host_id: str + connection_changed: bool + configuration: ProxyConfiguration + + +class ConnectorBinding(DesktopContract): + host_id: str + connector_id: str + + +class InvocationCompletion(DesktopContract): + connector_id: str + operation_name: str + outcome: Literal["success", "failure"] + + +class AdministrationError(Exception): + def __init__(self) -> None: + super().__init__("Connector administration failed") + + +class AdministrationRejected(AdministrationError): + pass diff --git a/desktop/python/umbod_desktop/endpoint.py b/desktop/python/umbod_desktop/endpoint.py new file mode 100644 index 0000000..c0cb7e6 --- /dev/null +++ b/desktop/python/umbod_desktop/endpoint.py @@ -0,0 +1,22 @@ +import json +from collections.abc import Callable +from pathlib import Path + + +def validate_saved_port(port: object) -> int: + if type(port) is not int or not 1 <= port <= 65535: + raise ValueError("Invalid saved local port") + return port + + +def no_saved_port(directory: Path) -> int: + return 0 + + +def saved_port(directory: Path, fallback: Callable[[Path], int]) -> int: + path = directory / "python-endpoint.json" + if path.exists(): + port = json.loads(path.read_text()).get("port") + if port is not None: + return validate_saved_port(port) + return fallback(directory) diff --git a/desktop/python/umbod_desktop/legacy.py b/desktop/python/umbod_desktop/legacy.py deleted file mode 100644 index 94721f0..0000000 --- a/desktop/python/umbod_desktop/legacy.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Retire a running legacy host through its authenticated lifecycle endpoint.""" - -import asyncio -import json - -import httpx - -from .models import loopback_origin - - -async def retire_legacy_host(directory, profile): - path = directory / "runtime.json" - if not path.exists(): - return - record = json.loads(path.read_text()) - if record.get("profile") != profile: - raise ValueError("Legacy runtime profile does not match") - origin = loopback_origin(record["management"]) - async with httpx.AsyncClient( - base_url=origin, - timeout=3, - trust_env=False, - headers={"Authorization": f"Bearer {record['admin_token']}"}, - ) as client: - try: - response = await client.post("/manage", json={"action": "health"}) - except httpx.ConnectError: - return - response.raise_for_status() - health = response.json() - if health.get("profile") != profile or health.get("pid") != record["pid"]: - raise ValueError("Legacy runtime identity does not match") - response = await client.post("/manage", json={"action": "shutdown"}) - response.raise_for_status() - async with asyncio.timeout(15): - while path.exists(): - await asyncio.sleep(0.05) - - -def saved_agent_hashes(directory): - """Retain existing global-permission client identities without extracting secrets.""" - path = directory / "config.json" - if not path.exists(): - return [] - document = json.loads(path.read_text()) - clients = list(document.get("clients", [])) - if document.get("connection"): - clients.append(document["connection"]) - return sorted( - { - client["token_hash"] - for client in clients - if isinstance(client.get("token_hash"), str) - and len(client["token_hash"]) == 64 - and all(c in "0123456789abcdef" for c in client["token_hash"]) - } - ) - - -def saved_port(directory): - for filename, field in ( - ("python-endpoint.json", "port"), - ("config.json", "gateway_port"), - ): - path = directory / filename - if path.exists(): - port = json.loads(path.read_text()).get(field) - if port is not None: - if type(port) is not int or not 1 <= port <= 65535: - raise ValueError("Invalid saved local port") - return port - return 0 diff --git a/desktop/python/umbod_desktop/main.py b/desktop/python/umbod_desktop/main.py index 20b3ff7..8606199 100644 --- a/desktop/python/umbod_desktop/main.py +++ b/desktop/python/umbod_desktop/main.py @@ -12,7 +12,9 @@ import uvicorn -from .legacy import retire_legacy_host, saved_port +from .backward_compatibility.adapters.legacy_host import retire_legacy_host +from .backward_compatibility.profile import saved_legacy_port +from .endpoint import saved_port from .models import loopback_origin from .security import KeychainVault, persistent_secret from .storage import atomic_json @@ -98,7 +100,7 @@ async def serve(arguments) -> None: endpoint_path = directory / "python-endpoint.json" with socket.socket() as listener: listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", saved_port(directory))) + listener.bind(("127.0.0.1", saved_port(directory, saved_legacy_port))) listener.listen(128) port = listener.getsockname()[1] origin = f"http://127.0.0.1:{port}" diff --git a/desktop/python/umbod_desktop/migration.py b/desktop/python/umbod_desktop/migration.py deleted file mode 100644 index a3064bf..0000000 --- a/desktop/python/umbod_desktop/migration.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Resumable legacy import. Original JSON and Keychain entries remain untouched.""" - -import asyncio -import hashlib -import json -import time - -import httpx -from mcp.shared.auth import OAuthClientInformationFull, OAuthToken -from umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth import ( - AuthorizationDocument, -) - -from .security import DESKTOP_GROUP -from .storage import atomic_json -from .tool_names import DesktopToolNames - - -class DesktopMigration: - def __init__(self, directory, state, upstreams, api, token): - self.directory, self.state, self.upstreams = directory, state, upstreams - self.lock = asyncio.Lock() - self.client = httpx.AsyncClient( - transport=httpx.ASGITransport(app=api), - base_url="http://127.0.0.1", - headers={"Authorization": f"Bearer {token}"}, - ) - self.tool_names = DesktopToolNames(self.request) - - async def request(self, method, path, body=None): - response = await self.client.request(method, "/api/admin" + path, json=body) - response.raise_for_status() - return response.json() if response.content else {} - - async def run(self): - async with self.lock: - await self._run() - - async def _run(self): - source = self.directory / "config.json" - if ( - not source.exists() - or self.state.data.get("migration", {}).get("status") == "complete" - ): - return - payload = source.read_bytes() - legacy = json.loads(payload) - digest = hashlib.sha256(payload).hexdigest() - journal = self.state.data.setdefault( - "migration", - {"status": "pending", "source_sha256": digest, "connectors": {}}, - ) - if journal["source_sha256"] != digest: - raise ValueError( - "Legacy settings changed during migration; restore the saved source before retrying" - ) - backup = self.directory / "config.before-python.json" - if not backup.exists(): - atomic_json(backup, legacy) - journal["status"] = "importing" - self.state.save() - try: - await self.request("POST", f"/mcp-permissions/groups/{DESKTOP_GROUP}") - for server in legacy.get("servers", []): - old_id = server["id"] - entry = journal["connectors"].setdefault(old_id, {}) - if entry.get("complete"): - continue - if "host_id" not in entry: - prepared = await self.upstreams.prepare(server) - entry["host_id"] = prepared["host_id"] - self.state.save() - host_id = entry["host_id"] - if ( - server.get("oauth") - and self.upstreams.vault.get(f"python:oauth:{host_id}") is None - ): - raw = self.upstreams.vault.get(f"upstream:oauth:{old_id}") - if raw is None: - raise ValueError("Legacy OAuth credential is missing") - saved = json.loads(raw) - if saved["resource"].rstrip("/") != server["url"].rstrip("/"): - raise ValueError("OAuth resource does not match") - document = AuthorizationDocument( - endpoint_url=server["url"], - client_info=OAuthClientInformationFull( - client_id=saved["client_id"] - ), - tokens=OAuthToken( - access_token=saved["access"], - token_type="Bearer", - refresh_token=saved.get("refresh"), - expires_in=max(0, int(saved["expires"] - time.time())), - ), - expires_at=saved["expires"], - ) - self.upstreams.vault.set( - f"python:oauth:{host_id}", document.model_dump_json() - ) - config = { - "auth_mode": "static_bearer", - "endpoint_url": f"{self.upstreams.origin}/desktop/upstreams/{host_id}/mcp", - "header_type": "bearer", - "bearer_token": self.upstreams.token, - } - if "id" not in entry: - # Stable public path allows recovery if creation committed before journaling. - slug = "import-" + hashlib.sha256(old_id.encode()).hexdigest()[:24] - inventory = await self.request("GET", "/connectors/mcp") - existing = None - for item in inventory["connectors"]: - detail = await self.request( - "GET", f"/connectors/mcp/{item['connector_id']}" - ) - if detail["public_path"] == f"/mcp/proxies/{slug}": - existing = detail - break - created = existing or await self.request( - "POST", - "/connectors/mcp", - { - "metadata": { - "display_name": server["name"], - "tool_name_prefix": self.tool_names.available_prefix( - server["name"], await self.tool_names.inventory() - ), - "capability_description": f"Tools from {server['name']}", - "public_path": f"/mcp/proxies/{slug}", - }, - "configuration": config, - }, - ) - entry["id"] = created["connector_id"] - self.upstreams.state.data["servers"][host_id]["id"] = entry["id"] - self.state.save() - base = f"/connectors/mcp/{entry['id']}" - await self.request("POST", base + "/discoveries") - tools = (await self.request("GET", base + "/tools"))["tools"] - allowed = legacy.get("allowed_tools") - if allowed is None: - client_ids = [client["id"] for client in legacy.get("clients", [])] - sets = [ - set(legacy.get("grants", {}).get(client, [])) - for client in client_ids - ] - allowed = set.intersection(*sets) if sets else set() - selected = [ - tool["name"] - for tool in tools - if f"{old_id}.{tool['name']}" in allowed - ] - if selected: - await self.request( - "PUT", - base + "/tools/activation", - { - "tools": [ - {"tool_id": name, "activation_status": "enabled"} - for name in selected - ] - }, - ) - await self.request( - "PUT", - f"/mcp-permissions/groups/{DESKTOP_GROUP}/permissions", - { - "capabilities": [ - { - "connector_id": entry["id"], - "capability_kind": "tool", - "capability_key": name, - "permission_status": "enabled", - } - for name in selected - ] - }, - ) - await self.request("PUT", base + "/publication") - entry["complete"] = True - self.state.save() - for field in ( - "daily_calls", - "daily_tracking_started", - "diagnostics", - "client_setup", - ): - if field in legacy: - self.state.data.setdefault(field, legacy[field]) - if legacy.get("activation") and "activation" not in self.state.data: - journey = dict(legacy["activation"]) - entry = journal["connectors"].get(journey.get("server"), {}) - journey["server"] = entry.get("id", journey.get("server")) - self.state.data["activation"] = journey - usage = self.state.data.setdefault("tool_usage", {}) - for name, counts in legacy.get("tool_usage", {}).items(): - old_id, _, tool = name.partition(".") - new_id = journal["connectors"].get(old_id, {}).get("id", old_id) - usage.setdefault(f"{new_id}.{tool}", counts) - journal["status"] = "complete" - journal.pop("error", None) - except Exception: # noqa: BLE001 - boundary redacts upstream and vault secrets - journal["status"] = "needs_attention" - journal["error"] = ( - "Some connectors could not be imported. Original settings and credentials are preserved; reconnect the upstream and retry." - ) - finally: - self.state.save() - - async def close(self): - await self.client.aclose() diff --git a/desktop/python/umbod_desktop/models.py b/desktop/python/umbod_desktop/models.py index ab55c83..1be2094 100644 --- a/desktop/python/umbod_desktop/models.py +++ b/desktop/python/umbod_desktop/models.py @@ -1,40 +1,7 @@ """Validated host requests contain credential references, never secret values.""" -from typing import Literal from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator -from umbod.core.connectors.downstream_mcp.models import EndpointUrl - - -class DesktopServer(BaseModel): - model_config = ConfigDict(extra="forbid") - id: str = "" - host_id: str = "" - name: str = Field(min_length=1, max_length=200) - transport: Literal["stdio", "http"] - command: str = "" - args: list[str] = Field(default_factory=list) - url: str = "" - oauth: bool = False - oauth_scopes: str = "" - reconnect: bool = Field(default=False, exclude=True) - headers: dict[str, str] = Field(default_factory=dict) - env: dict[str, str] = Field(default_factory=dict) - - @model_validator(mode="after") - def validate_transport(self): - if self.transport == "stdio" and not self.command.strip(): - raise ValueError("A local executable is required") - if self.transport == "http": - TypeAdapter(EndpointUrl).validate_python(self.url) - if any( - not reference.startswith("upstream:") - for reference in (*self.headers.values(), *self.env.values()) - ): - raise ValueError("Credentials must be upstream Keychain references") - return self - def loopback_origin(address: str) -> str: parsed = urlsplit("http://" + address) diff --git a/desktop/python/umbod_desktop/persistence.py b/desktop/python/umbod_desktop/persistence.py new file mode 100644 index 0000000..8e0b1ef --- /dev/null +++ b/desktop/python/umbod_desktop/persistence.py @@ -0,0 +1,21 @@ +import json +import os +import tempfile +from pathlib import Path + + +def atomic_json(path: Path, value: object) -> None: + atomic_bytes(path, json.dumps(value).encode()) + + +def atomic_bytes(path: Path, value: bytes) -> None: + fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/desktop/python/umbod_desktop/ports.py b/desktop/python/umbod_desktop/ports.py new file mode 100644 index 0000000..851aa63 --- /dev/null +++ b/desktop/python/umbod_desktop/ports.py @@ -0,0 +1,51 @@ +from typing import Any, Protocol + +from .contracts import ( + ConnectorBinding, + ConnectorCreation, + ConnectorDefinition, + InvocationCompletion, + PreparedUpstream, + ProxyConfiguration, + ToolGrant, + ToolSelection, + UpstreamInput, +) + + +class CredentialVault(Protocol): + def get(self, reference: str) -> str | None: ... + def set(self, reference: str, value: str) -> None: ... + def delete(self, reference: str) -> None: ... + + +class MetadataState(Protocol): + data: dict[str, Any] + + def save(self) -> None: ... + + +class ConnectorAdministration(Protocol): + async def inventory(self) -> tuple[ConnectorDefinition, ...]: ... + async def create(self, command: ConnectorCreation) -> ConnectorDefinition: ... + async def discover(self, connector_id: str) -> None: ... + async def tool_names(self, connector_id: str) -> tuple[str, ...]: ... + async def enable_tools(self, selection: ToolSelection) -> None: ... + async def ensure_group(self, group_id: str) -> None: ... + async def grant_tools(self, grant: ToolGrant) -> None: ... + async def publish(self, connector_id: str) -> None: ... + + +class ToolNamingPolicy(Protocol): + def readable_prefix(self, display_name: str) -> str: ... + def collision_key(self, prefix: str) -> str: ... + + +class InvocationRecorder(Protocol): + def record(self, completion: InvocationCompletion) -> None: ... + + +class UpstreamPreparation(Protocol): + async def prepare(self, server: UpstreamInput) -> PreparedUpstream: ... + def proxy_configuration(self, host_id: str) -> ProxyConfiguration: ... + def bind(self, binding: ConnectorBinding) -> None: ... diff --git a/desktop/python/umbod_desktop/runtime.py b/desktop/python/umbod_desktop/runtime.py index ccf8d27..ef1e7b7 100644 --- a/desktop/python/umbod_desktop/runtime.py +++ b/desktop/python/umbod_desktop/runtime.py @@ -2,43 +2,57 @@ import os import sys +from collections.abc import AsyncIterator, Callable from contextlib import AsyncExitStack, asynccontextmanager -from dataclasses import replace from pathlib import Path +from typing import Any from fastapi import FastAPI, HTTPException, Request from starlette.applications import Starlette from starlette.routing import Route -from umbod.core.connectors.downstream_mcp.oauth.service import OAuthService -from umbod.core.permissions import InMemoryGroupConnectorToolPermissions -from umbod.mcp.public_app import ( - build_starlette_app, - create_runtime_context, - load_configured_public_app_config, +from starlette.types import ASGIApp +from umbod.hosting import ( + InMemoryGroupConnectorToolPermissions, + OAuthService, + create_mcp_app, + create_rest_app, ) -from umbod.rest.main import create_app +from .backend_integration.administration import BackendAdministration +from .backend_integration.authentication import ( + DesktopAdminVerifier, + DesktopAgentVerifier, +) +from .backend_integration.configuration import desktop_settings +from .backend_integration.naming import BackendToolNamingPolicy +from .backend_integration.oauth import ( + DesktopOAuthHost, + callback_transport, +) +from .backend_integration.transports import DesktopUpstreams +from .backend_integration.usage import DesktopUsageSink +from .backward_compatibility.adapters.administration import BackendPrefixRepair +from .backward_compatibility.adapters.oauth import KeychainLegacyAuthorizationImporter +from .backward_compatibility.authentication import LegacyAgentCredentials +from .backward_compatibility.migration import DesktopMigration +from .backward_compatibility.profile import saved_agent_hashes +from .backward_compatibility.tool_names import DesktopToolNames from .client_setup import ClientSetup -from .configuration import desktop_settings -from .legacy import saved_agent_hashes -from .migration import DesktopMigration -from .oauth import DesktopOAuthHost, callback_transport +from .contracts import UpstreamInput from .security import ( CredentialVault, - DesktopAdminVerifier, - DesktopAgentVerifier, LoopbackAuthentication, persistent_secret, ) from .settings import DesktopSettings from .storage import DesktopState, atomic_json -from .upstreams import DesktopUpstreams -from .usage import DesktopUsageSink +from .usage import DesktopUsageRecorder async def create_desktop_app( - directory: Path, profile: str, origin: str, vault: CredentialVault, shutdown -): + directory: Path, profile: str, origin: str, vault: CredentialVault, + shutdown: Callable[[], None], +) -> tuple[ASGIApp, str]: declaration = directory / "native-connectors.json" atomic_json(declaration, {"connectors": []}) os.environ["UMBOD_CONNECTOR_DEPLOYMENT_CONFIGURATION_PATH"] = str(declaration) @@ -52,8 +66,9 @@ async def create_desktop_app( admin_token = persistent_secret(vault, "python:admin-token") agent_token = persistent_secret(vault, "python:agent-token") upstream_token = persistent_secret(vault, "python:upstream-token") - settings = desktop_settings(directory, origin, root) state = DesktopState(directory) + effective_settings = state.settings + settings = desktop_settings(directory, origin, root, effective_settings) if state.data.get("profile", profile) != profile: raise ValueError("Desktop profile does not match its data directory") state.data["profile"] = profile @@ -69,32 +84,34 @@ async def create_desktop_app( routes=[Route("/desktop/oauth/callback/{nonce}", callback), Route("/desktop/oauth/callback", callback)] ) - api = create_app(settings, admin_verifier=DesktopAdminVerifier(admin_token), oauth_service=oauth) - migration = DesktopMigration(directory, state, upstreams, api, admin_token) - config = await load_configured_public_app_config(settings) - config = replace( - config, - invocation_log_sink=DesktopUsageSink(state), + api = create_rest_app(settings, connector_registrations=None, admin_verifier=DesktopAdminVerifier(admin_token), oauth_service=oauth) + administration = BackendAdministration(api, admin_token) + tool_names = DesktopToolNames(administration, BackendToolNamingPolicy(), BackendPrefixRepair(administration.client)) + migration = DesktopMigration( + directory, state, upstreams, administration, tool_names, + KeychainLegacyAuthorizationImporter(vault), + ) + mcp = await create_mcp_app( + settings, + invocation_log_sink=DesktopUsageSink(DesktopUsageRecorder(state)), auth_provider=DesktopAgentVerifier( - agent_token, tuple(state.data["legacy_agent_hashes"]) + agent_token, LegacyAgentCredentials(tuple(state.data["legacy_agent_hashes"])) ), group_permission_runtime_state=InMemoryGroupConnectorToolPermissions( group_claim_fields=("groups",) ), ) - context = await create_runtime_context(config) - mcp = await build_starlette_app(context) @asynccontextmanager - async def lifespan(app): + async def lifespan(app: FastAPI) -> AsyncIterator[None]: async with AsyncExitStack() as stack: await stack.enter_async_context(api.router.lifespan_context(api)) - await migration.tool_names.repair(state) + await tool_names.repair(state) await stack.enter_async_context(mcp.router.lifespan_context(mcp)) await upstreams.restore() stack.push_async_callback(upstreams.close) stack.push_async_callback(oauth.close) - stack.push_async_callback(migration.close) + stack.push_async_callback(administration.close) yield host = FastAPI(docs_url=None, openapi_url=None, lifespan=lifespan) @@ -114,11 +131,6 @@ async def health(): else "development", } - effective_settings = DesktopSettings( - connector_tool_exposure_mode=settings.mcp.connector_tool_exposure_mode, - mcp_administrator_enabled=settings.feature_toggles.mcp_administrator_enabled, - ) - @host.get("/desktop/settings") async def read_settings(): pending = state.settings @@ -152,9 +164,9 @@ async def desktop_state(): return state.data @host.post("/desktop/upstreams") - async def prepare(request: Request): + async def prepare(request: Request) -> dict[str, Any]: try: - return await upstreams.prepare(await request.json()) + return (await upstreams.prepare(UpstreamInput(await request.json()))).model_dump() except (ValueError, KeyError): raise HTTPException( 422, "Invalid desktop connector or missing credential" diff --git a/desktop/python/umbod_desktop/security.py b/desktop/python/umbod_desktop/security.py index 59a22a9..6d03951 100644 --- a/desktop/python/umbod_desktop/security.py +++ b/desktop/python/umbod_desktop/security.py @@ -1,21 +1,12 @@ """Single-user authentication and OS-vault credentials; no development fallback.""" -import hashlib import secrets -from typing import Protocol -from fastmcp.server.auth import AccessToken, TokenVerifier from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send -from umbod.rest.authentication.authorization import JwtClaims, JwtVerificationError -DESKTOP_GROUP = "desktop-owner" - - -class CredentialVault(Protocol): - def get(self, reference: str) -> str | None: ... - def set(self, reference: str, value: str) -> None: ... - def delete(self, reference: str) -> None: ... +from .contracts import DESKTOP_GROUP as DESKTOP_GROUP +from .ports import CredentialVault as CredentialVault class KeychainVault: @@ -48,39 +39,6 @@ def persistent_secret(vault: CredentialVault, reference: str) -> str: return value -class DesktopAdminVerifier: - def __init__(self, token: str) -> None: - self.token = token - - def verify(self, token: str) -> JwtClaims: - if not secrets.compare_digest(token.encode(), self.token.encode()): - raise JwtVerificationError("Invalid desktop credential") - return JwtClaims( - claims={"sub": "desktop-owner", "name": "You", "groups": [DESKTOP_GROUP]} - ) - - -class DesktopAgentVerifier(TokenVerifier): - def __init__(self, token: str, legacy_hashes: tuple[str, ...] = ()) -> None: - super().__init__() - self.token = token - self.legacy_hashes = legacy_hashes - - async def verify_token(self, token: str) -> AccessToken | None: - digest = hashlib.sha256(token.encode()).hexdigest() - if not secrets.compare_digest(token.encode(), self.token.encode()) and not any( - secrets.compare_digest(digest, saved) for saved in self.legacy_hashes - ): - return None - return AccessToken( - token=token, - client_id="desktop-agent", - scopes=[], - subject="desktop-owner", - claims={"sub": "desktop-owner", "groups": [DESKTOP_GROUP]}, - ) - - class LoopbackAuthentication: """Protect every host route, including health and internal system routes.""" diff --git a/desktop/python/umbod_desktop/storage.py b/desktop/python/umbod_desktop/storage.py index fdc4e52..0febc54 100644 --- a/desktop/python/umbod_desktop/storage.py +++ b/desktop/python/umbod_desktop/storage.py @@ -1,32 +1,15 @@ """Owner-only durable host metadata, separate from shared connector storage.""" import json -import os -import tempfile from pathlib import Path from pydantic import ValidationError +from .persistence import atomic_bytes as atomic_bytes +from .persistence import atomic_json as atomic_json from .settings import DesktopSettings -def atomic_json(path: Path, value: object) -> None: - atomic_bytes(path, json.dumps(value).encode()) - - -def atomic_bytes(path: Path, value: bytes) -> None: - fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - try: - with os.fdopen(fd, "wb") as stream: - stream.write(value) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, path) - finally: - if os.path.exists(temporary): - os.unlink(temporary) - - class DesktopState: def __init__(self, directory: Path) -> None: self.path = directory / "desktop-v2.json" diff --git a/desktop/python/umbod_desktop/tool_names.py b/desktop/python/umbod_desktop/tool_names.py deleted file mode 100644 index 9603ad8..0000000 --- a/desktop/python/umbod_desktop/tool_names.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Repair desktop-generated namespaces through the shared administration API.""" - -import logging -import re - -import httpx -from umbod.core.capabilities.tools.names import mangle_public_tool_name -from umbod.core.connectors.downstream_mcp.models import default_tool_name_prefix - -_LOGGER = logging.getLogger(__name__) -_GENERATED_PREFIX = re.compile( - r"(?:desktop_[0-9a-f]{8}(?:_[0-9a-f]{4}){3}_[0-9a-f]{12}|import_[0-9a-f]{24})" -) - - -class DesktopToolNames: - def __init__(self, request): - self.request = request - - async def inventory(self): - listed = await self.request("GET", "/connectors/mcp") - return [ - await self.request("GET", f"/connectors/mcp/{item['connector_id']}") - for item in listed["connectors"] - ] - - @staticmethod - def available_prefix(display_name, definitions): - # Use the backend's naming policy, not a second desktop normalizer. - base = default_tool_name_prefix(display_name) - reserved = { - mangle_public_tool_name(item["tool_name_prefix"], "") - for item in definitions - } - candidate, suffix = base, 2 - while mangle_public_tool_name(candidate, "") in reserved: - candidate = f"{base}_{suffix}" - suffix += 1 - return candidate - - async def repair(self, state): - """Retry safely on startup; never recreate connectors or rewrite grants.""" - managed_ids = { - item.get("id") for item in state.data.get("servers", {}).values() - } - if not managed_ids: - return - try: - definitions = await self.inventory() - for definition in sorted(definitions, key=lambda item: item["connector_id"]): - prefix = definition["tool_name_prefix"] - if ( - definition["connector_id"] not in managed_ids - or not _GENERATED_PREFIX.fullmatch(prefix) - or definition["public_path"].removeprefix("/mcp/proxies/") - .replace("-", "_") != prefix - ): - continue - candidate = self.available_prefix(definition["display_name"], definitions) - try: - await self.request( - "PATCH", f"/connectors/mcp/{definition['connector_id']}", - {"tool_name_prefix": candidate}, - ) - except httpx.HTTPStatusError: - # The shared API also checks cross-kind/operation collisions. - # Preserve the working name if it rejects this candidate. - _LOGGER.warning( - "Desktop tool namespace repair deferred for connector %s", - definition["connector_id"], - ) - continue - definition["tool_name_prefix"] = candidate - except httpx.HTTPError: - # A failed metadata repair must not prevent existing tools from working. - _LOGGER.warning("Desktop tool namespace repair deferred; retry on restart") diff --git a/desktop/python/umbod_desktop/usage.py b/desktop/python/umbod_desktop/usage.py index 26f8efa..404382f 100644 --- a/desktop/python/umbod_desktop/usage.py +++ b/desktop/python/umbod_desktop/usage.py @@ -1,25 +1,19 @@ -"""Aggregate completed invocations without storing arguments or results.""" - import time -from .storage import DesktopState +from .contracts import InvocationCompletion +from .ports import MetadataState -class DesktopUsageSink: - def __init__(self, state: DesktopState) -> None: +class DesktopUsageRecorder: + def __init__(self, state: MetadataState) -> None: self.state = state - def record_invocation_completed(self, event) -> None: - tool = event.started.tool - if tool.connector_id is None or tool.operation_name is None: - return - if getattr(event, "failure_category", None) in ("authorization", "validation"): - return - identity = f"{tool.connector_id}.{tool.operation_name}" + def record(self, completion: InvocationCompletion) -> None: + identity = f"{completion.connector_id}.{completion.operation_name}" usage = self.state.data.setdefault("tool_usage", {}).setdefault( identity, {"succeeded": 0, "failed": 0} ) - usage["succeeded" if event.outcome == "success" else "failed"] += 1 + usage["succeeded" if completion.outcome == "success" else "failed"] += 1 day = str(int(time.time()) // 86400) daily = self.state.data.setdefault("daily_calls", {}) daily[day] = daily.get(day, 0) + 1 diff --git a/desktop/scripts/build.sh b/desktop/scripts/build.sh index a5c51b0..ad039f4 100755 --- a/desktop/scripts/build.sh +++ b/desktop/scripts/build.sh @@ -9,10 +9,11 @@ case "$CHANNEL" in release) APP_NAME="Umbod"; BUNDLE_ID="app.umbod.desktop"; APP="$OUTPUT/Umbod-$VERSION.app" ;; *) echo "Usage: scripts/build.sh [dev|release]" >&2; exit 1 ;; esac +bash scripts/check-macos-toolchain.sh FINAL_APP="$APP" APP="${FINAL_APP}.building-$(python3 -c 'import uuid; print(uuid.uuid4())')" trap 'rm -rf "$APP"' EXIT -swift build --package-path native -c release --product Umbod +/usr/bin/xcrun --toolchain XcodeDefault --sdk macosx swift build --package-path native -c release --product Umbod mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources" scripts/bundle-python.sh "$APP/Contents/Resources" python3 -c 'import uuid; print(uuid.uuid4())' > "$APP/Contents/Resources/runtime-id" @@ -32,7 +33,7 @@ chmod +x "$APP/Contents/MacOS/umbod-desktop" # Existing agent configurations can keep the old command path; no Rust is bundled. cp "$APP/Contents/MacOS/umbod-desktop" "$APP/Contents/MacOS/umbod-gateway" cp native/.build/release/Umbod "$APP/Contents/MacOS/Umbod" -swift scripts/icon.swift "$OUTPUT/Umbod-$CHANNEL.iconset" "$CHANNEL" +/usr/bin/xcrun --toolchain XcodeDefault --sdk macosx swift scripts/icon.swift "$OUTPUT/Umbod-$CHANNEL.iconset" "$CHANNEL" iconutil -c icns "$OUTPUT/Umbod-$CHANNEL.iconset" -o "$APP/Contents/Resources/UmbodBrand.icns" rm -f "$APP/Contents/MacOS/umbod-profile" printf '%s\n' "$CHANNEL" > "$APP/Contents/Resources/umbod-profile" diff --git a/desktop/scripts/bundle-python.sh b/desktop/scripts/bundle-python.sh index 8fbf6f5..fc80081 100755 --- a/desktop/scripts/bundle-python.sh +++ b/desktop/scripts/bundle-python.sh @@ -15,7 +15,7 @@ uv export --quiet --project "$ROOT/desktop/python" --frozen --no-dev --no-editab # Select wheels for macOS 14 even when building on a newer OS. # Fresh source builds must not reuse wheels compiled for the build host. # Export covers the full lock; development tool.uv.sources must not restore editable installs. -(cd "$ROOT/desktop/python" && MACOSX_DEPLOYMENT_TARGET=14.0 uv pip install --no-cache --python-platform aarch64-apple-darwin --python "$DEST/python/bin/python3" --reinstall-package umbod-desktop --reinstall-package umbod-core --reinstall-package umbod --break-system-packages --no-editable --no-sources --requirements "$DEST/requirements.txt") +(cd "$ROOT/desktop/python" && MACOSX_DEPLOYMENT_TARGET=14.0 uv pip install --no-cache --python-platform aarch64-apple-darwin --python "$DEST/python/bin/python3" --reinstall-package umbod-desktop --reinstall-package umbod-core --reinstall-package umbod --break-system-packages --no-sources --requirements "$DEST/requirements.txt") rm "$DEST/requirements.txt" "$DEST/python/bin/python3" -I -B "$ROOT/desktop/scripts/verify-python-bundle.py" "$DEST/python/bin/python3" -I -B "$ROOT/desktop/scripts/dependency-notices.py" python "$DEST" diff --git a/desktop/scripts/check-macos-toolchain.sh b/desktop/scripts/check-macos-toolchain.sh new file mode 100755 index 0000000..4bf41a7 --- /dev/null +++ b/desktop/scripts/check-macos-toolchain.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -euo pipefail + +host_tool() { + /usr/bin/uname "$@" +} + +apple_tool() { + local tool="$1" + shift + "/usr/bin/$tool" "$@" +} + +toolchain_error() { + printf 'macOS desktop toolchain: %s\n' "$1" >&2 + printf 'Install full Xcode (not only Command Line Tools), open it to complete first launch and license setup, then select it with DEVELOPER_DIR or sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer.\n' >&2 + return 1 +} + +check_macos_toolchain() ( + local xcode_version swift swiftc sdk swift_version major + [[ "$(host_tool -s)" == Darwin ]] || { toolchain_error 'Darwin is required.'; return 1; } + [[ "$(host_tool -m)" == arm64 ]] || { toolchain_error 'Apple Silicon (arm64) is required.'; return 1; } + if ! xcode_version="$(apple_tool xcodebuild -version 2>&1)"; then + printf '%s\n' "$xcode_version" >&2 + case "$xcode_version" in + *[Cc]ommand[Ll]ine[Tt]ools*|*command\ line\ tools*) toolchain_error 'Command Line Tools are selected; full Xcode is required.' ;; + *) toolchain_error 'Selected Xcode is unavailable or requires license/first-launch setup.' ;; + esac + return 1 + fi + [[ "$xcode_version" == Xcode\ * ]] || { toolchain_error 'Full Xcode must be selected.'; return 1; } + if ! apple_tool xcodebuild -checkFirstLaunchStatus; then + toolchain_error 'Full Xcode first-launch setup is incomplete; open Xcode to finish setup.' + return 1 + fi + swift="$(apple_tool xcrun --toolchain XcodeDefault --sdk macosx --find swift)" || { toolchain_error 'Cannot locate XcodeDefault swift.'; return 1; } + swiftc="$(apple_tool xcrun --toolchain XcodeDefault --sdk macosx --find swiftc)" || { toolchain_error 'Cannot locate XcodeDefault swiftc.'; return 1; } + sdk="$(apple_tool xcrun --sdk macosx --show-sdk-path)" || { toolchain_error 'Cannot locate the macOS SDK.'; return 1; } + case "$swift" in + /nix/store/*) toolchain_error 'Swift must come from full Xcode.'; return 1 ;; + /*.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift) ;; + *) toolchain_error 'Swift must come from full Xcode.'; return 1 ;; + esac + [[ "$swiftc" == "${swift}c" ]] || { toolchain_error 'swiftc must use the same XcodeDefault toolchain as swift.'; return 1; } + case "$sdk" in + /nix/store/*) toolchain_error 'SDK must come from full Xcode.'; return 1 ;; + /*.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk) ;; + *) toolchain_error 'SDK must come from full Xcode.'; return 1 ;; + esac + swift_version="$(apple_tool xcrun --toolchain XcodeDefault --sdk macosx swift --version)" || { toolchain_error 'Cannot run Swift.'; return 1; } + if [[ "$swift_version" =~ Swift\ version\ ([0-9]+)\. ]]; then + major="${BASH_REMATCH[1]}" + else + toolchain_error 'Cannot determine Swift version.' + return 1 + fi + (( major >= 6 )) || { toolchain_error 'Swift 6 or newer is required.'; return 1; } + temporary="$(mktemp -d "${TMPDIR:-/tmp}/umbod-toolchain.XXXXXX")" + trap 'rm -rf "$temporary"' EXIT + mkdir -p "$temporary/modules" + printf '%s\n' 'import SwiftUI' '@MainActor struct ToolchainProbe: View {' ' @State private var value = false' ' var body: some View { Text(value ? "on" : "off") }' '}' > "$temporary/Probe.swift" + if ! apple_tool xcrun --toolchain XcodeDefault --sdk macosx swiftc -swift-version 6 -sdk "$sdk" -target arm64-apple-macosx14.0 -module-cache-path "$temporary/modules" -parse-as-library -typecheck "$temporary/Probe.swift"; then + toolchain_error 'SwiftUI @State macro typecheck failed with the selected Xcode SDK.' + return 1 + fi + printf 'macOS desktop toolchain: Xcode Swift %s, SwiftUI ready.\n' "$major" +) + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + check_macos_toolchain +fi diff --git a/desktop/scripts/install-smoke.py b/desktop/scripts/install-smoke.py index dd6f4a9..6aa2233 100644 --- a/desktop/scripts/install-smoke.py +++ b/desktop/scripts/install-smoke.py @@ -7,7 +7,7 @@ import tempfile -def main(): +def main() -> None: desktop = Path(__file__).resolve().parents[1] archive = Path(sys.argv[1]).resolve() with tempfile.TemporaryDirectory(prefix="umbod-test-install-") as temporary: @@ -24,7 +24,7 @@ def main(): subprocess.run([str(app / "Contents/Resources/python/bin/python3"), "-I", "-B", str(desktop / "scripts/verify-python-bundle.py")], check=True) environment = {**os.environ, "UMBOD_TEST_APP": str(app)} # Swift drives the tests; every backend and agent process uses the installed interpreter. - subprocess.run(["swift", "run", "--package-path", str(desktop / "native"), "UmbodTests"], env=environment, check=True) + subprocess.run(["/usr/bin/xcrun", "--toolchain", "XcodeDefault", "--sdk", "macosx", "swift", "run", "--package-path", str(desktop / "native"), "UmbodTests"], env=environment, check=True) subprocess.run(["codesign", "--verify", "--deep", "--strict", str(app)], check=True) print("PASS: DMG install, packaged agent flow, and persistence") diff --git a/desktop/scripts/prepare-macos-acceptance.sh b/desktop/scripts/prepare-macos-acceptance.sh index 9ab67dc..de00342 100755 --- a/desktop/scripts/prepare-macos-acceptance.sh +++ b/desktop/scripts/prepare-macos-acceptance.sh @@ -2,8 +2,9 @@ set -euo pipefail cd "$(dirname "$0")/.." DEST="${1:?test kit output directory required}" +bash scripts/check-macos-toolchain.sh mkdir -p "$DEST/scripts" "$DEST/tests/fixtures" -swift build --package-path native --product UmbodTests +/usr/bin/xcrun --toolchain XcodeDefault --sdk macosx swift build --package-path native --product UmbodTests cp native/.build/debug/UmbodTests "$DEST/UmbodTests" cp scripts/{accept-macos-artifact.sh,audit-macos-bundle.py,verify-python-bundle.py,fixture_stdio.py} "$DEST/scripts/" cp tests/fixtures/update-manifest.json "$DEST/tests/fixtures/" diff --git a/desktop/scripts/record-macos-build.py b/desktop/scripts/record-macos-build.py index 59af8f2..4f5b462 100644 --- a/desktop/scripts/record-macos-build.py +++ b/desktop/scripts/record-macos-build.py @@ -10,7 +10,7 @@ app = Path(sys.argv[1]) resources = app / 'Contents/Resources' -def output(*args): +def output(*args: str) -> str: return subprocess.check_output(args, text=True, cwd=root).strip() metadata = { @@ -23,8 +23,8 @@ def output(*args): 'python': output(str(resources / 'python/bin/python3'), '-I', '-B', '--version'), 'pythonBuild': (resources / 'python/BUILD').read_text().strip(), 'uv': output('uv', '--version'), - 'swift': output('swift', '--version'), - 'xcode': output('xcodebuild', '-version'), + 'swift': output('/usr/bin/xcrun', '--toolchain', 'XcodeDefault', '--sdk', 'macosx', 'swift', '--version'), + 'xcode': output('/usr/bin/xcodebuild', '-version'), 'lockSha256': hashlib.sha256((root / 'desktop/python/uv.lock').read_bytes()).hexdigest(), } (resources / 'build-metadata.json').write_text(json.dumps(metadata, indent=2) + '\n') diff --git a/desktop/scripts/verify-python-bundle.py b/desktop/scripts/verify-python-bundle.py index 73e876f..c3fed6f 100644 --- a/desktop/scripts/verify-python-bundle.py +++ b/desktop/scripts/verify-python-bundle.py @@ -28,11 +28,19 @@ def verify_installation(prefix, paths, distributions): require_bundled(pth.parent / line, prefix) -def main(): +def main() -> None: prefix = Path(sys.prefix) verify_installation(prefix, sys.path, importlib.metadata.distributions()) # Exercise the real CLI import graph as well as REST/MCP entry points. - for name in ("umbod_desktop.main", "umbod.core", "umbod.rest.main", "umbod.mcp.public_app", "umbod_sdk"): + for name in ( + "umbod_desktop.main", + "umbod_desktop.runtime", + "umbod.hosting", + "umbod.core", + "umbod.rest.main", + "umbod.mcp.public_app", + "umbod_sdk", + ): module = importlib.import_module(name) if module.__file__: require_bundled(module.__file__, prefix) diff --git a/desktop/scripts/verify.sh b/desktop/scripts/verify.sh index f4e9379..ba01c27 100755 --- a/desktop/scripts/verify.sh +++ b/desktop/scripts/verify.sh @@ -1,6 +1,7 @@ #!/bin/bash set -euo pipefail cd "$(dirname "$0")/.." +bash scripts/check-macos-toolchain.sh (cd python && uv run --locked ruff check . && uv run --locked pytest -q) bash scripts/build.sh dev -swift run --package-path native UmbodTests +/usr/bin/xcrun --toolchain XcodeDefault --sdk macosx swift run --package-path native UmbodTests diff --git a/desktop/tests/test_macos_toolchain.py b/desktop/tests/test_macos_toolchain.py new file mode 100644 index 0000000..5fac043 --- /dev/null +++ b/desktop/tests/test_macos_toolchain.py @@ -0,0 +1,191 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "desktop/scripts/check-macos-toolchain.sh" +HARNESS = r''' +source "$1" +host_tool() { + case "$1" in + -s) printf '%s\n' "$HOST_SYSTEM" ;; + -m) printf '%s\n' "$HOST_ARCH" ;; + esac +} +apple_tool() { + printf '%s\n' "$DEVELOPER_DIR" "$@" >> "$CALL_LOG" + local tool="$1" + shift + if [[ "$tool" == xcodebuild ]]; then + if [[ "$SCENARIO" == clt ]]; then + printf 'requires Xcode, active developer directory is CommandLineTools\n' >&2 + return 1 + fi + if [[ "$SCENARIO" == license ]]; then + printf 'license agreement not accepted\n' >&2 + return 1 + fi + if [[ "$1" == -checkFirstLaunchStatus ]]; then + [[ "$SCENARIO" != setup ]] + return + fi + printf 'Xcode 26.0\nBuild version 17A\n' + return + fi + case "$*" in + *'--find swiftc') + [[ "$SCENARIO" != missing-swiftc ]] || return 1 + printf '%s/Toolchains/XcodeDefault.xctoolchain/usr/bin/swiftc\n' "$DEVELOPER_DIR" ;; + *'--find swift') + [[ "$SCENARIO" != missing-swift ]] || return 1 + if [[ "$SCENARIO" == nix-compiler ]]; then + printf '/nix/store/swift/bin/swift\n' + else + printf '%s/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift\n' "$DEVELOPER_DIR" + fi ;; + *'--show-sdk-path') + [[ "$SCENARIO" != missing-sdk ]] || return 1 + if [[ "$SCENARIO" == clt-sdk ]]; then + printf '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\n' + else + printf '%s/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk\n' "$DEVELOPER_DIR" + fi ;; + *'swift --version') + if [[ "$SCENARIO" == old ]]; then + printf 'Apple Swift version 5.10\n' + else + printf 'Apple Swift version 6.2\n' + fi ;; + *'swiftc '*) + cp "${@: -1}" "$PROBE_COPY" + printf '%s\n' "${@: -1}" > "$PROBE_PATH" + if [[ "$SCENARIO" == probe ]]; then + printf 'SwiftUIMacros plugin unavailable\n' >&2 + return 1 + fi ;; + *) return 1 ;; + esac +} +check_macos_toolchain +''' + + +class MacOSToolchainTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory(prefix="umbod-toolchain-test-") + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.script = self.directory / "standalone toolchain.sh" + self.script.write_text(SCRIPT.read_text()) + self.calls = self.directory / "calls" + self.probe_copy = self.directory / "probe-copy" + self.probe_path = self.directory / "probe-path" + self.developer = "/Applications/Xcode Stable.app/Contents/Developer" + + def run_preflight( + self, scenario: str = "valid", system: str = "Darwin", architecture: str = "arm64" + ) -> subprocess.CompletedProcess[str]: + environment = { + **os.environ, + "SCENARIO": scenario, + "HOST_SYSTEM": system, + "HOST_ARCH": architecture, + "DEVELOPER_DIR": self.developer, + "CALL_LOG": str(self.calls), + "PROBE_COPY": str(self.probe_copy), + "PROBE_PATH": str(self.probe_path), + "TMPDIR": str(self.directory), + } + return subprocess.run( + ["/bin/bash", "-c", HARNESS, "toolchain-test", str(self.script)], + cwd=self.directory, + env=environment, + text=True, + capture_output=True, + check=False, + ) + + def test_rejects_unsupported_hosts_before_apple_tools(self) -> None: + for system, architecture, diagnostic in [ + ("Linux", "arm64", "Darwin is required"), + ("Darwin", "x86_64", "arm64"), + ]: + with self.subTest(system=system, architecture=architecture): + result = self.run_preflight(system=system, architecture=architecture) + self.assertNotEqual(result.returncode, 0) + self.assertIn(diagnostic, result.stderr) + self.assertFalse(self.calls.exists()) + + def test_reports_selection_license_and_setup_failures(self) -> None: + for scenario, diagnostic in [ + ("clt", "Command Line Tools are selected"), + ("license", "license agreement not accepted"), + ("setup", "first-launch setup is incomplete"), + ]: + with self.subTest(scenario=scenario): + result = self.run_preflight(scenario) + self.assertNotEqual(result.returncode, 0) + self.assertIn(diagnostic, result.stderr) + self.assertIn("DEVELOPER_DIR", result.stderr) + self.assertFalse(self.probe_path.exists()) + + def test_rejects_old_swift_and_missing_or_foreign_toolchain_inputs(self) -> None: + for scenario, diagnostic in [ + ("old", "Swift 6 or newer"), + ("missing-swift", "Cannot locate XcodeDefault swift"), + ("missing-swiftc", "Cannot locate XcodeDefault swiftc"), + ("missing-sdk", "Cannot locate the macOS SDK"), + ("nix-compiler", "Swift must come from full Xcode"), + ("clt-sdk", "SDK must come from full Xcode"), + ]: + with self.subTest(scenario=scenario): + result = self.run_preflight(scenario) + self.assertNotEqual(result.returncode, 0) + self.assertIn(diagnostic, result.stderr) + self.assertFalse(self.probe_path.exists()) + + def test_valid_xcode_preserves_selection_and_probes_macros_with_exact_flags(self) -> None: + result = self.run_preflight() + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls.read_text().splitlines() + self.assertEqual(calls.count(self.developer), 7) + self.assertIn("-checkFirstLaunchStatus", calls) + probe = Path(self.probe_path.read_text().strip()) + expected = [ + "xcrun", "--toolchain", "XcodeDefault", "--sdk", "macosx", "swiftc", + "-swift-version", "6", "-sdk", + self.developer + "/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", + "-target", "arm64-apple-macosx14.0", "-module-cache-path", + str(probe.parent / "modules"), "-parse-as-library", "-typecheck", str(probe), + ] + self.assertEqual(calls[-len(expected):], expected) + self.assertIn("@MainActor struct ToolchainProbe: View", self.probe_copy.read_text()) + self.assertIn("@State private var value = false", self.probe_copy.read_text()) + self.assertIn('Text(value ? "on" : "off")', self.probe_copy.read_text()) + self.assertFalse(probe.parent.exists()) + + def test_probe_failure_retains_diagnostics_and_removes_temporary_files(self) -> None: + result = self.run_preflight("probe") + self.assertNotEqual(result.returncode, 0) + self.assertIn("SwiftUIMacros plugin unavailable", result.stderr) + self.assertIn("SwiftUI @State macro typecheck failed", result.stderr) + probe = Path(self.probe_path.read_text().strip()) + self.assertFalse(probe.parent.exists()) + + def test_nix_wiring_keeps_default_shell_and_limits_desktop_to_arm_darwin(self) -> None: + flake = (ROOT / "flake.nix").read_text() + self.assertIn("default = pkgs.mkShell shellInputs;", flake) + self.assertIn('optionalAttrs (system == "aarch64-darwin")', flake) + self.assertIn("desktop = pkgs.mkShellNoCC (shellInputs // {", flake) + self.assertIn("${pkgs.bash}/bin/bash ${./desktop/scripts/check-macos-toolchain.sh} || exit $?", flake) + self.assertIn('UV_PYTHON = "${pkgs.python314}/bin/python3.14";', flake) + self.assertIn('UV_PYTHON_DOWNLOADS = "never";', flake) + for package in ["swift", "clang", "apple-sdk"]: + self.assertNotIn("pkgs." + package, flake) + + +if __name__ == "__main__": + unittest.main() diff --git a/flake.nix b/flake.nix index 90486c2..e2f8b3d 100644 --- a/flake.nix +++ b/flake.nix @@ -11,8 +11,7 @@ devShells = nixpkgs.lib.genAttrs systems (system: let pkgs = import nixpkgs { inherit system; }; - in { - default = pkgs.mkShell { + shellInputs = { packages = with pkgs; [ bashInteractive git @@ -33,6 +32,14 @@ UV_PYTHON = "${pkgs.python314}/bin/python3.14"; UV_PYTHON_DOWNLOADS = "never"; }; + in { + default = pkgs.mkShell shellInputs; + } // nixpkgs.lib.optionalAttrs (system == "aarch64-darwin") { + desktop = pkgs.mkShellNoCC (shellInputs // { + shellHook = '' + ${pkgs.bash}/bin/bash ${./desktop/scripts/check-macos-toolchain.sh} || exit $? + ''; + }); }); }; } diff --git a/umbod/api/README.md b/umbod/api/README.md index 128da82..688347c 100644 --- a/umbod/api/README.md +++ b/umbod/api/README.md @@ -2,6 +2,10 @@ Minimal FastAPI, FastMCP, and Typer scaffold for Umbod. +## Hosting boundary + +[`umbod.hosting`](src/umbod/hosting/README.md) is the supported Python application-composition API. Desktop embeds its REST and MCP applications in one process; Kubernetes runs separate REST and MCP services through the same API. Its contract documents supported injections, resource ownership, compatibility checks, and remaining HTTP and persistence coupling. + ## Telemetry ingestion The REST boundary exposes authenticated OTLP JSON logs, metrics, and traces under `/v1`. Accepted exports use the shared structured stdout logger with checked write acknowledgements; ingestion is disabled by default. See [telemetry ingestion](../docs/reference/telemetry-ingestion.md) for operator settings and deployment paths, and `src/umbod/rest/telemetry/README.md` for module responsibilities. diff --git a/umbod/api/src/umbod/cli/README.md b/umbod/api/src/umbod/cli/README.md index 900e0ae..0ce0d4a 100644 --- a/umbod/api/src/umbod/cli/README.md +++ b/umbod/api/src/umbod/cli/README.md @@ -4,6 +4,10 @@ **Read when working with:** Command-line arguments, service startup commands, configuration validation, or terminal output. +## Service composition + +REST targets `umbod.hosting.rest_service:create_app` with the Uvicorn factory flag. MCP targets `umbod.hosting.mcp_service:app`. Both use the backend-owned [hosting API](../hosting/README.md); process configuration, reload and port behavior remain unchanged. + ## Modules ### `api.py` diff --git a/umbod/api/src/umbod/cli/api.py b/umbod/api/src/umbod/cli/api.py index 4ab2ee8..7f46162 100644 --- a/umbod/api/src/umbod/cli/api.py +++ b/umbod/api/src/umbod/cli/api.py @@ -47,7 +47,7 @@ def __call__(self, **kwargs: Any) -> None: configure_logging(settings.runtime.app_name, settings.runtime.log_level) configured_port = settings.rest.port if kwargs["port"] is None else int(kwargs["port"]) uvicorn.run( - "umbod.rest.main:create_app", + "umbod.hosting.rest_service:create_app", host=str(kwargs["host"]), port=configured_port, reload=bool(kwargs["reload"]), diff --git a/umbod/api/src/umbod/cli/mcp.py b/umbod/api/src/umbod/cli/mcp.py index f1e7db4..44fa6f0 100644 --- a/umbod/api/src/umbod/cli/mcp.py +++ b/umbod/api/src/umbod/cli/mcp.py @@ -47,7 +47,7 @@ def __call__(self, **kwargs: Any) -> None: configure_logging(settings.runtime.app_name, settings.runtime.log_level) configured_port = settings.mcp.port if kwargs["port"] is None else int(kwargs["port"]) uvicorn.run( - "umbod.mcp.main:app", + "umbod.hosting.mcp_service:app", host=str(kwargs["host"]), port=configured_port, reload=bool(kwargs["reload"]), diff --git a/umbod/api/src/umbod/hosting/README.md b/umbod/api/src/umbod/hosting/README.md new file mode 100644 index 0000000..2277427 --- /dev/null +++ b/umbod/api/src/umbod/hosting/README.md @@ -0,0 +1,25 @@ +# Hosting + +Backend-owned composition API shared by the macOS host and the separate Kubernetes REST and MCP services. This package changes composition ownership, not routes, authentication defaults, transports, or deployment topology. + +## Supported API + +Import host-facing functions and contracts from `umbod.hosting`. The synchronous `create_rest_app` returns FastAPI; asynchronous `create_mcp_app` returns Starlette. Both require an explicit `AppConfig`. Importing this package does not load environment configuration or construct applications. + +Supported contract reexports are `AppConfig`, `ConnectorRegistration`, `JwtVerifier`, `JwtClaims`, `JwtVerificationError`, `OAuthService`, `MCPAuthProvider`, `InMemoryGroupConnectorToolPermissions`, `McpToolInvocationLogSink`, `McpToolInvocationLogEvent`, `AccessToken`, and `TokenVerifier`. These retain the underlying types' identity. Annotations describe contracts; they do not check protocol conformance at runtime. + +REST accepts connector registrations, admin verifier and OAuth service hooks. MCP accepts auth provider, group permission runtime state and completed-invocation sink hooks. Every hook is a required keyword-only argument: callers explicitly supply None for configured/default selection or supply an override. Optional types deliberately model that choice at the host composition boundary. None does not disable configured authentication or logging. An empty REST registration sequence remains an explicit override. + +`applications.py` assembles existing backend applications; `contracts.py` owns supported reexports. Internal MCP configuration and runtime contexts are not host contracts. Hosts should not import hosting implementation submodules. Changes to the documented signatures, reexports, override semantics or resource ownership require explicit compatibility review and migration guidance; internal backend modules may evolve independently. + +## Ownership and lifecycle + +The host owns loopback sockets, credentials and identity policy, native transport adapters, migration, local metadata and aggregate usage, and supplied OAuthService shutdown. Backend app lifespans own persistence, metrics listeners, MCP workers and existing synchronization tasks. Hosts must enter both returned ASGI lifespans. Desktop retains REST construction, migration setup, MCP construction, then REST lifespan, tool-name repair, MCP lifespan and upstream restore. + +MCP composition closes acquired persistence on configuration acquisition or later construction failure, including cancellation. Successful composition leaves shutdown to the existing lifespan. This is not a general rollback guarantee for all constructed collaborators. REST delegates unchanged to its legacy factory: persistence is created before startup validation and later assembly, and those construction failures are not covered by its lifespan shutdown. This API does not claim REST construction rollback. + +## Services and remaining coupling + +`rest_service.py` is the zero-argument Uvicorn factory; it loads .env, configures logging and constructs REST. `mcp_service.py` exports a lightweight Starlette shell that is safe to import inside an active event loop. Its ASGI lifespan loads .env, configures logging, awaits MCP composition in that loop and enters the returned application's lifespan. It mounts the application for HTTP forwarding only while that lifespan is active and removes its mount on exit. Import does not load settings or construct the backend MCP application. Construction and startup failures propagate through ASGI startup. CLI services remain separate processes. The legacy REST factory retains its existing invocation forms; legacy MCP main exports the same shell object, and legacy MCP production factories delegate to hosting composition. Kubernetes chart wiring is unchanged. + +Explicit settings avoid operator .env loading, but native connector deployment declarations still come from `UMBOD_CONNECTOR_DEPLOYMENT_CONFIGURATION_PATH`. Shared settings still expose HTTP endpoints, persistence choices and OAuth storage details. Backend synchronization can still use system HTTP endpoints or SQL events. Desktop confines shared backend dependencies to `umbod_desktop/backend_integration` and hosting-root runtime composition. Its migration, naming repair and usage recorder depend on desktop-owned DTOs and semantic protocols, including a shared metadata protocol, not HTTP clients or backend models. The administration adapter owns authenticated `/api/admin` JSON translation and runtime-owned client closure. Exact residual imports remain limited to the naming functions, secret derivation, EndpointUrl, OAuth authorization storage/connection and runtime provider symbols documented in the desktop integration index. Both desktop and hosting tests independently scan nested/relative imports and reject unsupported file/module/symbol combinations; synthetic violations protect the scanners. Desktop tests also enforce transitive use-case independence. Legacy source/backup filesystem I/O remains an explicitly incremental desktop-local boundary. This facade does not remove those residual OAuth/HTTP/configuration couplings or introduce a separate desktop backend. diff --git a/umbod/api/src/umbod/hosting/__init__.py b/umbod/api/src/umbod/hosting/__init__.py new file mode 100644 index 0000000..e51f1cf --- /dev/null +++ b/umbod/api/src/umbod/hosting/__init__.py @@ -0,0 +1,32 @@ +from umbod.hosting.applications import create_mcp_app, create_rest_app +from umbod.hosting.contracts import ( + AccessToken, + AppConfig, + ConnectorRegistration, + InMemoryGroupConnectorToolPermissions, + JwtClaims, + JwtVerificationError, + JwtVerifier, + MCPAuthProvider, + McpToolInvocationLogEvent, + McpToolInvocationLogSink, + OAuthService, + TokenVerifier, +) + +__all__ = [ + "AccessToken", + "AppConfig", + "ConnectorRegistration", + "InMemoryGroupConnectorToolPermissions", + "JwtClaims", + "JwtVerificationError", + "JwtVerifier", + "MCPAuthProvider", + "McpToolInvocationLogEvent", + "McpToolInvocationLogSink", + "OAuthService", + "TokenVerifier", + "create_mcp_app", + "create_rest_app" +] diff --git a/umbod/api/src/umbod/hosting/applications.py b/umbod/api/src/umbod/hosting/applications.py new file mode 100644 index 0000000..2db0570 --- /dev/null +++ b/umbod/api/src/umbod/hosting/applications.py @@ -0,0 +1,57 @@ +from dataclasses import replace +from typing import Optional, Sequence + +from fastapi import FastAPI +from starlette.applications import Starlette + +from umbod.hosting.contracts import ( + AppConfig, + ConnectorRegistration, + InMemoryGroupConnectorToolPermissions, + JwtVerifier, + MCPAuthProvider, + McpToolInvocationLogSink, + OAuthService, +) +from umbod.mcp.public_app import ( + build_starlette_app, + create_runtime_context, + load_configured_public_app_config, +) +from umbod.rest.main import create_app as _create_rest_app + + +def create_rest_app( + settings: AppConfig, + *, + connector_registrations: Optional[Sequence[ConnectorRegistration]], + admin_verifier: Optional[JwtVerifier], + oauth_service: Optional[OAuthService], +) -> FastAPI: + return _create_rest_app( + settings, + connector_registrations=connector_registrations, + admin_verifier=admin_verifier, + oauth_service=oauth_service, + ) + + +async def create_mcp_app( + settings: AppConfig, + *, + auth_provider: Optional[MCPAuthProvider], + group_permission_runtime_state: Optional[InMemoryGroupConnectorToolPermissions], + invocation_log_sink: Optional[McpToolInvocationLogSink], +) -> Starlette: + config = await load_configured_public_app_config(settings) + try: + if auth_provider is not None: + config = replace(config, auth_provider=auth_provider) + if group_permission_runtime_state is not None: + config = replace(config, group_permission_runtime_state=group_permission_runtime_state) + if invocation_log_sink is not None: + config = replace(config, invocation_log_sink=invocation_log_sink) + return await build_starlette_app(await create_runtime_context(config)) + except BaseException: + await config.persistence_runtime.shutdown() + raise diff --git a/umbod/api/src/umbod/hosting/contracts.py b/umbod/api/src/umbod/hosting/contracts.py new file mode 100644 index 0000000..0341748 --- /dev/null +++ b/umbod/api/src/umbod/hosting/contracts.py @@ -0,0 +1,24 @@ +from fastmcp.server.auth import AccessToken, TokenVerifier +from umbod.config import AppConfig +from umbod.core.connectors.downstream_mcp.oauth.service import OAuthService +from umbod.core.connectors.native.registry import ConnectorRegistration +from umbod.core.permissions import InMemoryGroupConnectorToolPermissions +from umbod.mcp.auth.factory import MCPAuthProvider +from umbod.mcp.logging import McpToolInvocationLogSink +from umbod.mcp.logging.invocation.event import McpToolInvocationLogEvent +from umbod.rest.authentication.authorization import JwtClaims, JwtVerificationError, JwtVerifier + +__all__ = [ + "AccessToken", + "AppConfig", + "ConnectorRegistration", + "InMemoryGroupConnectorToolPermissions", + "JwtClaims", + "JwtVerificationError", + "JwtVerifier", + "MCPAuthProvider", + "McpToolInvocationLogEvent", + "McpToolInvocationLogSink", + "OAuthService", + "TokenVerifier" +] diff --git a/umbod/api/src/umbod/hosting/mcp_service.py b/umbod/api/src/umbod/hosting/mcp_service.py new file mode 100644 index 0000000..e60574d --- /dev/null +++ b/umbod/api/src/umbod/hosting/mcp_service.py @@ -0,0 +1,26 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from starlette.applications import Starlette +from starlette.routing import Mount + +from umbod.config import load_app_config +from umbod.hosting import create_mcp_app +from umbod.logging import configure_logging + + +@asynccontextmanager +async def lifespan(service: Starlette) -> AsyncIterator[None]: + settings = load_app_config(".env") + configure_logging(settings.runtime.app_name, settings.runtime.log_level) + application = await create_mcp_app(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) + async with application.router.lifespan_context(application): + mount = Mount("/", app=application) + service.router.routes.append(mount) + try: + yield + finally: + service.router.routes.remove(mount) + + +app = Starlette(lifespan=lifespan) diff --git a/umbod/api/src/umbod/hosting/rest_service.py b/umbod/api/src/umbod/hosting/rest_service.py new file mode 100644 index 0000000..d765ae8 --- /dev/null +++ b/umbod/api/src/umbod/hosting/rest_service.py @@ -0,0 +1,11 @@ +from fastapi import FastAPI + +from umbod.config import load_app_config +from umbod.hosting import create_rest_app +from umbod.logging import configure_logging + + +def create_app() -> FastAPI: + settings = load_app_config(".env") + configure_logging(settings.runtime.app_name, settings.runtime.log_level) + return create_rest_app(settings, connector_registrations=None, admin_verifier=None, oauth_service=None) diff --git a/umbod/api/src/umbod/mcp/README.md b/umbod/api/src/umbod/mcp/README.md index 4a5720b..4abe3f1 100644 --- a/umbod/api/src/umbod/mcp/README.md +++ b/umbod/api/src/umbod/mcp/README.md @@ -54,4 +54,6 @@ File-input connector tools keep uploaded bytes only for the current app-backend ## Embedded host composition -`PublicAppConfig.invocation_log_sink` optionally supplies a deployment-owned completed-invocation sink; the normal structured logger remains the default. `live_permission_updates.py` reads group permissions from the shared database when SQL messaging is selected, or the system HTTP API for HTTP messaging. This allows embedded REST/MCP applications to synchronize permissions without exposing an unauthenticated internal API. +Supported hosts use [`umbod.hosting`](../hosting/README.md) and `create_mcp_app`; `PublicAppConfig` and runtime contexts remain backend internals. `main.py` delegates to `umbod.hosting.mcp_service:app`, the separate service bootstrap. + +The `invocation_log_sink` hook on `create_mcp_app` supplies a deployment-owned completed-invocation sink; the normal structured logger remains the default. `live_permission_updates.py` reads group permissions from the shared database when SQL messaging is selected, or the system HTTP API for HTTP messaging. This allows embedded REST/MCP applications to synchronize permissions without exposing an unauthenticated internal API. diff --git a/umbod/api/src/umbod/mcp/main.py b/umbod/api/src/umbod/mcp/main.py index 83c14d0..213889f 100644 --- a/umbod/api/src/umbod/mcp/main.py +++ b/umbod/api/src/umbod/mcp/main.py @@ -1,9 +1 @@ -import asyncio - -from umbod.config import load_app_config -from umbod.logging import configure_logging -from umbod.mcp.public_app import create_configured_production_app - -settings = load_app_config(".env") -configure_logging(settings.runtime.app_name, settings.runtime.log_level) -app = asyncio.run(create_configured_production_app(settings)) +from umbod.hosting.mcp_service import app as app diff --git a/umbod/api/src/umbod/mcp/public_app.py b/umbod/api/src/umbod/mcp/public_app.py index 028e43b..edc9cba 100644 --- a/umbod/api/src/umbod/mcp/public_app.py +++ b/umbod/api/src/umbod/mcp/public_app.py @@ -266,14 +266,18 @@ async def load_configured_public_app_config(settings: AppConfig) -> PublicAppCon private_api_base_url = settings.endpoints.private_api_base_url.strip() api_base_url = settings.endpoints.api_base_url.strip() persistence_runtime = ConfiguredPersistenceRuntimeProvider(settings.connector_store).create() - await persistence_runtime.readiness.ensure_ready() - if isinstance(settings.connector_store, InMemoryConnectorStoreConfig): - configured_source_strategy = HttpConnectorRuntimeStateSourceStrategy() - elif isinstance(settings.connector_store, SQLiteConnectorStoreConfig): - configured_source_strategy = StoreBackedConnectorRuntimeStateSourceStrategy() - else: - raise TypeError('unsupported connector store configuration') - return PublicAppConfig(settings=settings, connector_runtime=await DefaultConnectorRuntimeFactory(load_connector_plugins, partial(create_encrypted_connector_configuration_store, persistence_runtime.database), ConfiguredConnectorPublishingStoreFactory(persistence_runtime.database)).create(settings), source_stores=ConnectorRuntimeSourceStores(), configured_source_strategy=configured_source_strategy, stateless_http=settings.mcp.stateless_http, api_base_url=private_api_base_url or api_base_url or None, persistence_runtime=persistence_runtime) + try: + await persistence_runtime.readiness.ensure_ready() + if isinstance(settings.connector_store, InMemoryConnectorStoreConfig): + configured_source_strategy = HttpConnectorRuntimeStateSourceStrategy() + elif isinstance(settings.connector_store, SQLiteConnectorStoreConfig): + configured_source_strategy = StoreBackedConnectorRuntimeStateSourceStrategy() + else: + raise TypeError('unsupported connector store configuration') + return PublicAppConfig(settings=settings, connector_runtime=await DefaultConnectorRuntimeFactory(load_connector_plugins, partial(create_encrypted_connector_configuration_store, persistence_runtime.database), ConfiguredConnectorPublishingStoreFactory(persistence_runtime.database)).create(settings), source_stores=ConnectorRuntimeSourceStores(), configured_source_strategy=configured_source_strategy, stateless_http=settings.mcp.stateless_http, api_base_url=private_api_base_url or api_base_url or None, persistence_runtime=persistence_runtime) + except BaseException: + await persistence_runtime.shutdown() + raise async def create_runtime_context(config: PublicAppConfig) -> PublicAppRuntimeContext: install_umbod_consent_screen() @@ -335,11 +339,14 @@ async def build_starlette_app(context: PublicAppRuntimeContext) -> Starlette: return Starlette(routes=[Route(_HEALTH_ROUTE, health_response, methods=['GET']), Route('/.well-known/mcp/server-card.json', lambda _request: JSONResponse(server_card, headers=_CORS_HEADERS), methods=['GET']), Route(_LOGO_ROUTE, lambda _request: FileResponse(_LOGO_FILE, media_type='image/png'), methods=['GET']), Mount('/', app=DynamicPublicMcpDispatcher(context.http_app, downstream_stores.definitions, downstream_stores.publishing))], lifespan=_create_public_http_lifespan(context)) async def create_production_app() -> Starlette: - return await create_configured_production_app(load_app_config('.env')) + from umbod.hosting import create_mcp_app + + return await create_mcp_app(load_app_config('.env'), auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) async def create_configured_production_app(settings: AppConfig) -> Starlette: - config = await load_configured_public_app_config(settings) - return await build_starlette_app(await create_runtime_context(config)) + from umbod.hosting import create_mcp_app + + return await create_mcp_app(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) def runtime_state_synchronizer_options(context: PublicAppRuntimeContext) -> ConnectorRuntimeStateSynchronizerOptions | None: if context.api_base_url is None: diff --git a/umbod/api/src/umbod/rest/README.md b/umbod/api/src/umbod/rest/README.md index 44b6169..7c07dd1 100644 --- a/umbod/api/src/umbod/rest/README.md +++ b/umbod/api/src/umbod/rest/README.md @@ -48,7 +48,7 @@ Setup is compensated, not transactional: failures after creation attempt both cr Downstream MCP responses also describe saved `oauth` authorizations. Shared OAuth administration lives in `connectors/downstream_mcp/oauth.py` under `/api/admin/connectors/mcp/oauth`: `GET /status`, `GET /{connector_id}`, and `POST /{connector_id}/{discover,begin,cancel,refresh,logout}`. Begin accepts optional `client_id` and `scopes`; omitted scopes preserve the saved choice. Details expose saved authorization, granted/requested/available scopes, registration mode, bounded redacted diagnostics, and separate authorization/discovery outcomes. Admin authentication and cookie safeguards apply to every route. -Deployment composition supplies `create_app(oauth_service=OAuthService(host))`. The host implements shared storage, connector lookup, callback addresses, and transport reconnection ports; it does not implement OAuth policy. Desktop supplies Keychain and loopback adapters, including lookup of local drafts. The composing host must close the service during shutdown. Existing callback paths remain stable for registered clients. +Deployment composition supplies an `OAuthService` through `umbod.hosting.create_rest_app`. The host implements shared storage, connector lookup, callback addresses, and transport reconnection ports; it does not implement OAuth policy. Desktop supplies Keychain and loopback adapters, including lookup of local drafts. The composing host must close the service during shutdown. Existing callback paths remain stable for registered clients. Without a configured host, these routes return `503` with `detail.code=oauth_host_unavailable`. This change does not enable browser sign-in in multi-worker Enterprise deployments: credential ownership and callback/session coordination must be supplied explicitly before enabling it. Saved server authorizations continue to use shared runtime/storage support in `core/connectors/downstream_mcp/adapters/fastmcp/oauth.py`, with the same public-client policy as Desktop. @@ -78,4 +78,6 @@ Without a configured host, these routes return `503` with `detail.code=oauth_hos ## Embedded host composition -`create_app(settings, admin_verifier=...)` supplies the same deployment verifier to authentication middleware and the request-scoped current-user dependency. Normal verifier selection and profile enrichment remain unchanged. The macOS host owns its loopback bearer-token policy outside this package. +Supported hosts use [`umbod.hosting`](../hosting/README.md), including `create_rest_app`. `main.py:create_app` remains the unchanged legacy compatibility entrypoint. The service factory is `umbod.hosting.rest_service:create_app`. + +The `admin_verifier` hook on `create_rest_app` supplies the same deployment verifier to authentication middleware and the request-scoped current-user dependency. Normal verifier selection and profile enrichment remain unchanged. The macOS host owns its loopback bearer-token policy outside this package. diff --git a/umbod/api/tests/hosting/test_applications.py b/umbod/api/tests/hosting/test_applications.py new file mode 100644 index 0000000..2ef294e --- /dev/null +++ b/umbod/api/tests/hosting/test_applications.py @@ -0,0 +1,228 @@ +import asyncio +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock + +import httpx +import pytest +from starlette.testclient import TestClient + +from umbod.hosting import ( + AppConfig, + InMemoryGroupConnectorToolPermissions, + JwtClaims, + JwtVerificationError, + McpToolInvocationLogEvent, + create_mcp_app, + create_rest_app, +) +from umbod.hosting import applications +from umbod.mcp import public_app + + +class OwnerVerifier: + def __init__(self, owner: str) -> None: + self.owner = owner + + def verify(self, token: str) -> JwtClaims: + if token != self.owner: + raise JwtVerificationError("Invalid token") + return JwtClaims(claims={"sub": self.owner, "groups": ["owners"]}) + + +class UsageSink: + def record_invocation_completed(self, event: McpToolInvocationLogEvent) -> None: + pass + + +@pytest.fixture +def settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> AppConfig: + declaration = tmp_path / "connectors.json" + declaration.write_text('{"connectors": []}') + monkeypatch.setenv("UMBOD_CONNECTOR_DEPLOYMENT_CONFIGURATION_PATH", str(declaration)) + return AppConfig.model_validate({ + "rest": {"metrics_enabled": False}, + "mcp": {"metrics_enabled": False, "auth_mode": "none"}, + "admin_authentication": { + "mode": "jwt", "membership_claim": "groups", + "required_membership": "owners", + }, + }) + + +def test_explicit_rest_settings_and_verifiers_are_isolated( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("UMBOD_ADMIN_AUTH_MODE", "disabled") + for owner in ("first", "second"): + selected_settings = settings.model_copy(update={ + "admin_authentication": settings.admin_authentication.model_copy(update={ + "jwt_header_name": f"X-{owner}", + }), + }) + app = create_rest_app(selected_settings, connector_registrations=[], admin_verifier=OwnerVerifier(owner), oauth_service=None) + with TestClient(app) as client: + response = client.get("/api/admin/users", headers={f"X-{owner}": owner}) + assert response.status_code == 200 + assert response.json()["id"] == owner + assert client.get("/api/admin/users", headers={f"X-{owner}": "wrong"}).status_code == 401 + assert client.get("/api/admin/users", headers={"X-other": owner}).status_code == 401 + assert client.get("/api/system/health").status_code == 200 + assert client.get("/admin/users").status_code == 404 + + +@pytest.mark.asyncio +async def test_actual_mcp_app_serves_health_and_enters_lifespan( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(public_app, "_create_public_polling_synchronizers", AsyncMock(return_value=(None,) * 5)) + create_runtime = public_app.ConfiguredPersistenceRuntimeProvider.create + shutdown = AsyncMock() + + def create_persistence(provider: public_app.ConfiguredPersistenceRuntimeProvider) -> Any: + runtime = create_runtime(provider) + shutdown.side_effect = runtime.shutdown + return SimpleNamespace(database=runtime.database, readiness=runtime.readiness, shutdown=shutdown) + + monkeypatch.setattr(public_app.ConfiguredPersistenceRuntimeProvider, "create", create_persistence) + app = await create_mcp_app(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) + shutdown.assert_not_awaited() + async with app.router.lifespan_context(app): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client: + assert (await client.get("/system/health")).json() == {"status": "ok"} + assert (await client.get("/.well-known/mcp/server-card.json")).status_code == 200 + shutdown.assert_awaited_once() + + +@dataclass(frozen=True) +class ConfigDouble: + persistence_runtime: Any + auth_provider: Any + group_permission_runtime_state: Any + invocation_log_sink: Any + + +@pytest.mark.asyncio +@pytest.mark.parametrize("override", [False, True]) +async def test_mcp_overrides_preserve_configured_defaults( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch, override: bool +) -> None: + configured = ConfigDouble(AsyncMock(), object(), object(), object()) + load = AsyncMock(return_value=configured) + context = AsyncMock(return_value=object()) + build = AsyncMock(return_value=object()) + monkeypatch.setattr(applications, "load_configured_public_app_config", load) + monkeypatch.setattr(applications, "create_runtime_context", context) + monkeypatch.setattr(applications, "build_starlette_app", build) + permissions = InMemoryGroupConnectorToolPermissions(group_claim_fields=("groups",)) + sink = UsageSink() + auth = object() + arguments = dict(auth_provider=auth, group_permission_runtime_state=permissions, invocation_log_sink=sink) if override else dict(auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) + result = await create_mcp_app(settings, **arguments) + selected = context.await_args.args[0] + assert selected.auth_provider is (auth if override else configured.auth_provider) + assert selected.group_permission_runtime_state is (permissions if override else configured.group_permission_runtime_state) + assert selected.invocation_log_sink is (sink if override else configured.invocation_log_sink) + assert result is build.return_value + load.assert_awaited_once_with(settings) + configured.persistence_runtime.shutdown.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ["context", "app"]) +@pytest.mark.parametrize("failure", [RuntimeError, asyncio.CancelledError]) +async def test_mcp_construction_failure_closes_persistence( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch, stage: str, failure: type[BaseException] +) -> None: + persistence = AsyncMock() + config = ConfigDouble(persistence, None, None, None) + monkeypatch.setattr(applications, "load_configured_public_app_config", AsyncMock(return_value=config)) + monkeypatch.setattr(applications, "create_runtime_context", AsyncMock(side_effect=failure() if stage == "context" else None)) + monkeypatch.setattr(applications, "build_starlette_app", AsyncMock(side_effect=failure())) + with pytest.raises(failure): + await create_mcp_app(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) + persistence.shutdown.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ["readiness", "runtime"]) +@pytest.mark.parametrize("failure", [RuntimeError, asyncio.CancelledError]) +async def test_configuration_acquisition_failure_closes_persistence( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch, stage: str, failure: type[BaseException] +) -> None: + persistence = AsyncMock() + persistence.readiness.ensure_ready.side_effect = failure() if stage == "readiness" else None + monkeypatch.setattr(public_app.ConfiguredPersistenceRuntimeProvider, "create", lambda self: persistence) + monkeypatch.setattr(public_app.DefaultConnectorRuntimeFactory, "create", AsyncMock(side_effect=failure())) + with pytest.raises(failure): + await create_mcp_app(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) + persistence.shutdown.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ["readiness", "runtime"]) +async def test_task_cancellation_during_acquisition_closes_persistence( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch, stage: str +) -> None: + entered = asyncio.Event() + blocked = asyncio.Event() + persistence = AsyncMock() + + async def wait_for_cancellation(*arguments: object) -> None: + entered.set() + await blocked.wait() + + monkeypatch.setattr(public_app.ConfiguredPersistenceRuntimeProvider, "create", lambda self: persistence) + if stage == "readiness": + persistence.readiness.ensure_ready.side_effect = wait_for_cancellation + else: + monkeypatch.setattr(public_app.DefaultConnectorRuntimeFactory, "create", wait_for_cancellation) + task = asyncio.create_task(create_mcp_app(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None)) + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + persistence.shutdown.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [False, True]) +async def test_rest_lifespan_does_not_close_host_owned_oauth( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch, failure: bool +) -> None: + oauth = SimpleNamespace(close=AsyncMock()) + persistence = AsyncMock() + if failure: + persistence.readiness.ensure_ready.side_effect = RuntimeError("readiness failed") + create_runtime = public_app.ConfiguredPersistenceRuntimeProvider.create + + def create_persistence(provider: public_app.ConfiguredPersistenceRuntimeProvider) -> Any: + runtime = create_runtime(provider) + return SimpleNamespace(database=runtime.database, readiness=persistence.readiness, shutdown=persistence.shutdown) + + monkeypatch.setattr(public_app.ConfiguredPersistenceRuntimeProvider, "create", create_persistence) + app = create_rest_app(settings, connector_registrations=[], admin_verifier=OwnerVerifier("owner"), oauth_service=oauth) + if failure: + with pytest.raises(RuntimeError, match="readiness failed"): + async with app.router.lifespan_context(app): + pytest.fail("Startup should fail") + else: + async with app.router.lifespan_context(app): + oauth.close.assert_not_awaited() + oauth.close.assert_not_awaited() + persistence.shutdown.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_legacy_mcp_factories_delegate_to_hosting( + settings: AppConfig, monkeypatch: pytest.MonkeyPatch +) -> None: + build = AsyncMock(return_value=object()) + monkeypatch.setattr("umbod.hosting.create_mcp_app", build) + monkeypatch.setattr(public_app, "load_app_config", lambda path: settings) + assert await public_app.create_production_app() is build.return_value + assert await public_app.create_configured_production_app(settings) is build.return_value + assert build.await_count == 2 + build.assert_awaited_with(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) diff --git a/umbod/api/tests/hosting/test_boundary.py b/umbod/api/tests/hosting/test_boundary.py new file mode 100644 index 0000000..1521a94 --- /dev/null +++ b/umbod/api/tests/hosting/test_boundary.py @@ -0,0 +1,210 @@ +import ast +import inspect +import subprocess +import sys +from pathlib import Path + +import pytest + +import umbod.hosting as hosting + +ROOT = Path(__file__).resolve().parents[4] + + +def test_public_exports_and_explicit_keyword_only_signatures() -> None: + expected_exports = { + "AccessToken", "AppConfig", "ConnectorRegistration", + "InMemoryGroupConnectorToolPermissions", "JwtClaims", "JwtVerificationError", + "JwtVerifier", "MCPAuthProvider", "McpToolInvocationLogEvent", + "McpToolInvocationLogSink", "OAuthService", "TokenVerifier", + "create_mcp_app", "create_rest_app", + } + assert set(hosting.__all__) == expected_exports + for name in expected_exports: + assert getattr(hosting, name) is not None + assert not hasattr(hosting, "PublicAppConfig") + assert not hasattr(hosting, "PublicAppRuntimeContext") + for factory, expected_names in ( + (hosting.create_rest_app, ("settings", "connector_registrations", "admin_verifier", "oauth_service")), + (hosting.create_mcp_app, ("settings", "auth_provider", "group_permission_runtime_state", "invocation_log_sink")), + ): + parameters = list(inspect.signature(factory).parameters.values()) + assert tuple(parameter.name for parameter in parameters) == expected_names + assert parameters[0].kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + assert parameters[0].default is inspect.Parameter.empty + assert all(parameter.kind == inspect.Parameter.KEYWORD_ONLY for parameter in parameters[1:]) + assert all(parameter.default is inspect.Parameter.empty for parameter in parameters[1:]) + + +CORE = { + "contracts", "ports", "usage", "backward_compatibility.contracts", + "backward_compatibility.ports", "backward_compatibility.migration", + "backward_compatibility.tool_names", +} +HOSTING = { + "AccessToken", "AppConfig", "ConnectorRegistration", + "InMemoryGroupConnectorToolPermissions", "JwtClaims", "JwtVerificationError", + "JwtVerifier", "MCPAuthProvider", "McpToolInvocationLogEvent", + "McpToolInvocationLogSink", "OAuthService", "TokenVerifier", + "create_mcp_app", "create_rest_app", +} +RESIDUAL = { + "umbod_desktop.backward_compatibility.adapters.oauth": { + "umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth": {"AuthorizationDocument"}, + }, + "umbod_desktop.backend_integration.configuration": { + "umbod.config.secret_derivation": {"derive_secret"}, + }, + "umbod_desktop.backend_integration.naming": { + "umbod.core.capabilities.tools.names": {"mangle_public_tool_name"}, + "umbod.core.connectors.downstream_mcp.models": {"default_tool_name_prefix"}, + }, + "umbod_desktop.backend_integration.models": { + "umbod.core.connectors.downstream_mcp.models": {"EndpointUrl"}, + }, + "umbod_desktop.backend_integration.oauth": { + "umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth": { + "AuthorizationDocument", "ConnectorOAuthStorage", "SignInRequired", + }, + "umbod.core.connectors.downstream_mcp.oauth.ports": {"OAuthConnection"}, + }, + "umbod_desktop.backend_integration.transports": { + "umbod.core.connectors.downstream_mcp.oauth.provider": { + "RuntimeOAuth", "response_authentication_status", + }, + }, +} + + +def imports(source: str, module: str, is_package: bool) -> list[tuple[str, tuple[str, ...]]]: + found = [] + package = module.split(".") if is_package else module.split(".")[:-1] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + found.extend((alias.name, ()) for alias in node.names) + elif isinstance(node, ast.ImportFrom): + base = node.module or "" + if node.level: + prefix = package[:len(package) - node.level + 1] + base = ".".join((*prefix, *base.split("."))).rstrip(".") + found.append((base, tuple(alias.name for alias in node.names))) + return found + + +def violations(source: str, module: str, is_package: bool = False) -> list[str]: + failures = [] + adapter = module.startswith("umbod_desktop.backend_integration") + runtime = module == "umbod_desktop.runtime" + core = module.removeprefix("umbod_desktop.") in CORE + for target, names in imports(source, module, is_package): + candidates = (target, *(f"{target}.{name}" for name in names)) + if not module.startswith("umbod_desktop.backward_compatibility") and module not in { + "umbod_desktop.main", "umbod_desktop.runtime", + } and any(candidate == "umbod_desktop.backward_compatibility" or candidate.startswith("umbod_desktop.backward_compatibility.") for candidate in candidates): + failures.append(target) + if target == "umbod" or target.startswith("umbod."): + permitted = (adapter or runtime) and target == "umbod.hosting" and bool(names) and set(names) <= HOSTING + permitted = permitted or (bool(names) and target in RESIDUAL.get(module, {}) and set(names) <= RESIDUAL[module][target]) + if not permitted: + failures.append(target) + if not adapter and not runtime and any( + candidate == "umbod_desktop.backend_integration" or candidate.startswith("umbod_desktop.backend_integration.") + for candidate in candidates + ): + failures.append(target) + if core and any(candidate in { + "umbod_desktop.runtime", "umbod_desktop.storage", "umbod_desktop.security", + } for candidate in candidates): + failures.append(target) + if core and target.split(".")[0] in {"httpx", "httpx2", "fastapi", "starlette", "keyring"}: + failures.append(target) + if not adapter and module not in {"umbod_desktop.backward_compatibility.adapters.legacy_host", "umbod_desktop.backward_compatibility.adapters.administration"} and target.split(".")[0] == "httpx": + failures.append(target) + if module not in {"umbod_desktop.backend_integration.administration", "umbod_desktop.backward_compatibility.adapters.administration"} and any( + isinstance(node, ast.Constant) and isinstance(node.value, str) and "/api/admin" in node.value + for node in ast.walk(ast.parse(source)) + ): + failures.append("administration HTTP path") + return failures + + +def transitive_violations(sources: dict[str, tuple[str, bool]], root: str) -> list[str]: + pending = [root] + visited: set[str] = set() + failures = [] + while pending: + module = pending.pop() + if module in visited: + continue + visited.add(module) + source, is_package = sources[module] + for target, names in imports(source, module, is_package): + candidates = (target, *(f"{target}.{name}" for name in names)) + for candidate in candidates: + if candidate in sources: + pending.append(candidate) + if target.split(".")[0] in {"umbod", "httpx", "httpx2", "fastapi", "starlette", "keyring"}: + failures.append(target) + if any(candidate in { + "umbod_desktop.runtime", "umbod_desktop.storage", "umbod_desktop.security", + } or candidate.startswith("umbod_desktop.backend_integration") for candidate in candidates): + failures.append(target) + return failures + + +@pytest.mark.parametrize("path", sorted((ROOT / "desktop/python/umbod_desktop").rglob("*.py"))) +def test_desktop_uses_only_supported_hosting_imports(path: Path) -> None: + package = ROOT / "desktop/python/umbod_desktop" + parts = path.relative_to(package).with_suffix("").parts + is_package = parts[-1] == "__init__" + module = ".".join(("umbod_desktop", *(parts[:-1] if is_package else parts))) + assert violations(path.read_text(), module, is_package) == [] + + +@pytest.mark.parametrize("source,module,is_package", [ + ("def run():\n import umbod.rest.dependencies", "umbod_desktop.runtime", False), + ("from umbod.hosting.contracts import AppConfig", "umbod_desktop.backend_integration.configuration", False), + ("from umbod.core.connectors.downstream_mcp.models import Connector", "umbod_desktop.backend_integration.models", False), + ("from umbod.core.connectors.downstream_mcp.models import EndpointUrl", "umbod_desktop.backend_integration.naming", False), + ("from umbod.core.connectors.downstream_mcp.oauth.provider import RuntimeOAuth", "umbod_desktop.backend_integration.usage", False), + ("import umbod.core.capabilities.tools.names", "umbod_desktop.backend_integration.naming", False), + ("from ..backend_integration.naming import BackendToolNamingPolicy", "umbod_desktop.backward_compatibility.migration", False), + ("from .backward_compatibility import migration", "umbod_desktop.endpoint", False), + ("from ..backward_compatibility.authentication import LegacyAgentCredentials", "umbod_desktop.backend_integration.authentication", False), + ("from umbod.core.connectors.downstream_mcp.adapters.fastmcp.oauth import ConnectorOAuthStorage", "umbod_desktop.backward_compatibility.adapters.oauth", False), + ("import httpx", "umbod_desktop.backward_compatibility.profile", False), + ("from .. import backend_integration", "umbod_desktop.backward_compatibility.tool_names", False), + ("from umbod_desktop import backend_integration", "umbod_desktop.usage", False), + ("from ..runtime import create_desktop_app", "umbod_desktop.backward_compatibility.migration", False), + ("from . import runtime", "umbod_desktop.ports", False), + ("from .storage import DesktopState", "umbod_desktop.usage", False), + ("from .security import CredentialVault", "umbod_desktop.ports", False), + ("from starlette.types import ASGIApp", "umbod_desktop.contracts", False), + ("import httpx", "umbod_desktop.backward_compatibility.migration", False), + ('path = "/api/admin/connectors/mcp"', "umbod_desktop.legacy", False), + ("from ..backend_integration import naming", "umbod_desktop.nested", True), +]) +def test_checker_rejects_forbidden_synthetic_imports(source: str, module: str, is_package: bool) -> None: + assert violations(source, module, is_package) + + +def test_checker_resolves_nested_relative_imports() -> None: + assert imports("def factory():\n from ..contracts import ConnectorDefinition", "umbod_desktop.backend_integration.administration", False) == [ + ("umbod_desktop.contracts", ("ConnectorDefinition",)) + ] + assert violations("from ..contracts import ConnectorDefinition", "umbod_desktop.backend_integration.administration") == [] + assert violations("from umbod.hosting import AppConfig", "umbod_desktop.backend_integration.configuration") == [] + + +def test_import_does_not_read_environment_or_create_applications(tmp_path: Path) -> None: + program = ( + "import os; before = dict(os.environ); " + "import umbod.config as config; " + "config.load_app_config = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError('config loaded')); " + "import umbod.hosting; import sys; " + "assert dict(os.environ) == before; " + "assert 'umbod.hosting.mcp_service' not in sys.modules; " + "assert 'umbod.hosting.rest_service' not in sys.modules" + ) + result = subprocess.run([sys.executable, "-c", program], cwd=tmp_path, capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/umbod/api/tests/hosting/test_services.py b/umbod/api/tests/hosting/test_services.py new file mode 100644 index 0000000..f69f906 --- /dev/null +++ b/umbod/api/tests/hosting/test_services.py @@ -0,0 +1,170 @@ +import importlib +import subprocess +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from unittest.mock import AsyncMock, Mock + +import httpx +import pytest +import uvicorn +from fastapi import FastAPI +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.routing import Route +from typer.testing import CliRunner + +import umbod.hosting as hosting +from umbod.config import AppConfig +from umbod.hosting import rest_service + + +def test_rest_service_loads_environment_and_configures_logging( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = AppConfig() + load = Mock(return_value=settings) + configure = Mock() + build = Mock(return_value=FastAPI()) + monkeypatch.setattr(rest_service, "load_app_config", load) + monkeypatch.setattr(rest_service, "configure_logging", configure) + monkeypatch.setattr(rest_service, "create_rest_app", build) + assert rest_service.create_app() is build.return_value + load.assert_called_once_with(".env") + configure.assert_called_once_with(settings.runtime.app_name, settings.runtime.log_level) + build.assert_called_once_with(settings, connector_registrations=None, admin_verifier=None, oauth_service=None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [False, True]) +async def test_mcp_service_defers_composition_and_owns_forwarding_lifespan( + monkeypatch: pytest.MonkeyPatch, failure: bool +) -> None: + events: list[str] = [] + + @asynccontextmanager + async def lifespan(application: Starlette) -> AsyncIterator[None]: + events.append("start") + try: + yield + finally: + events.append("stop") + + async def health(request: Request) -> JSONResponse: + return JSONResponse({"status": "ok"}) + + application = Starlette(routes=[Route("/system/health", health)], lifespan=lifespan) + settings = AppConfig() + load = Mock(return_value=settings) + configure = Mock() + build = AsyncMock(return_value=application, side_effect=RuntimeError("construction failed") if failure else None) + monkeypatch.setattr("umbod.config.load_app_config", load) + monkeypatch.setattr("umbod.logging.configure_logging", configure) + monkeypatch.setattr(hosting, "create_mcp_app", build) + for name in ("umbod.hosting.mcp_service", "umbod.mcp.main"): + monkeypatch.delitem(sys.modules, name, raising=False) + try: + config = uvicorn.Config("umbod.hosting.mcp_service:app", log_config=None) + config.load() + service = importlib.import_module("umbod.hosting.mcp_service") + legacy = importlib.import_module("umbod.mcp.main") + assert legacy.app is service.app + build.assert_not_called() + load.assert_not_called() + configure.assert_not_called() + original_routes = list(service.app.routes) + if failure: + with pytest.raises(RuntimeError, match="construction failed"): + async with service.app.router.lifespan_context(service.app): + pytest.fail("Startup should fail") + assert events == [] + else: + async with service.app.router.lifespan_context(service.app): + assert events == ["start"] + async with httpx.AsyncClient(transport=httpx.ASGITransport(service.app), base_url="http://test") as client: + assert (await client.get("/system/health")).json() == {"status": "ok"} + assert events == ["start", "stop"] + assert service.app.routes == original_routes + load.assert_called_once_with(".env") + configure.assert_called_once_with(settings.runtime.app_name, settings.runtime.log_level) + build.assert_awaited_once_with(settings, auth_provider=None, group_permission_runtime_state=None, invocation_log_sink=None) + finally: + sys.modules.pop("umbod.hosting.mcp_service", None) + sys.modules.pop("umbod.mcp.main", None) + + +def test_fresh_process_uvicorn_serves_mcp_public_health(tmp_path: Path) -> None: + program = ''' +import asyncio +import socket +from unittest.mock import AsyncMock +import httpx +import uvicorn +import umbod.config +from umbod.mcp import public_app + +async def verify() -> None: + settings = umbod.config.AppConfig.model_validate({ + "mcp": {"auth_mode": "none", "metrics_enabled": False}, + "connector_store": {"type": "sqlite", "sqlite_path": "backend.sqlite3"}, + }) + umbod.config.load_app_config = lambda path: settings + public_app._create_public_polling_synchronizers = AsyncMock(return_value=(None,) * 5) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(128) + origin = f"http://127.0.0.1:{listener.getsockname()[1]}" + server = uvicorn.Server(uvicorn.Config("umbod.hosting.mcp_service:app", log_level="error")) + task = asyncio.create_task(server.serve(sockets=[listener])) + try: + async with asyncio.timeout(15): + while not server.started: + if task.done(): + await task + raise AssertionError("Server stopped before startup") + await asyncio.sleep(0.01) + async with httpx.AsyncClient(base_url=origin) as client: + response = await client.get("/system/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + finally: + server.should_exit = True + await task + +asyncio.run(verify()) +''' + declaration = tmp_path / "connectors.json" + declaration.write_text('{"connectors": []}') + import os + + environment = dict(os.environ, UMBOD_CONNECTOR_DEPLOYMENT_CONFIGURATION_PATH=str(declaration)) + result = subprocess.run([sys.executable, "-c", program], cwd=tmp_path, env=environment, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_rest_cli_uses_hosting_factory(monkeypatch: pytest.MonkeyPatch) -> None: + from umbod.cli import api + + run = Mock() + monkeypatch.setattr(api, "load_app_config", Mock(return_value=AppConfig())) + monkeypatch.setattr(api, "configure_logging", Mock()) + monkeypatch.setattr(api.uvicorn, "run", run) + result = CliRunner().invoke(api.app, []) + assert result.exit_code == 0, result.output + assert run.call_args.args == ("umbod.hosting.rest_service:create_app",) + assert run.call_args.kwargs["factory"] is True + + +def test_legacy_rest_factory_accepts_positional_and_keyword_settings( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from umbod.rest.main import create_app + + declaration = tmp_path / "connectors.json" + declaration.write_text('{"connectors": []}') + monkeypatch.setenv("UMBOD_CONNECTOR_DEPLOYMENT_CONFIGURATION_PATH", str(declaration)) + settings = AppConfig() + assert isinstance(create_app(settings, []), FastAPI) + assert isinstance(create_app(settings=settings, connector_registrations=[]), FastAPI) diff --git a/umbod/api/tests/umbod/test_mcp_cli.py b/umbod/api/tests/umbod/test_mcp_cli.py index 1e9ef69..c96035f 100644 --- a/umbod/api/tests/umbod/test_mcp_cli.py +++ b/umbod/api/tests/umbod/test_mcp_cli.py @@ -38,4 +38,4 @@ def test_serve_preserves_explicit_cli_port(monkeypatch: MonkeyPatch) -> None: def test_serve_starts_the_mcp_application(monkeypatch: MonkeyPatch) -> None: invocation = _run(monkeypatch, configured_port=9100) - assert invocation["args"] == ("umbod.mcp.main:app",) + assert invocation["args"] == ("umbod.hosting.mcp_service:app",)