From 48a573ab2c3e765a22826b2650e991e99e90f196 Mon Sep 17 00:00:00 2001 From: Evgeny Shurakov Date: Thu, 3 Sep 2026 09:29:26 +0200 Subject: [PATCH 1/3] feat(cloud-agent): show passive sandbox status --- .specs/cloud-agent-session.md | 138 +- .../cloud-agent-next/ChatHeader.tsx | 11 + .../cloud-agent-next/CloudChatPage.tsx | 10 + .../SandboxStatusIndicator.tsx | 314 ++++ .../cloud-agent-next/sandbox-status.test.ts | 535 ++++++ .../cloud-agent-next/sandbox-status.ts | 181 ++ .../cloud-agent-client.test.ts | 135 ++ .../cloud-agent-next/cloud-agent-client.ts | 45 + .../lib/cloud-agent/session-ownership.test.ts | 33 +- .../routers/cloud-agent-next-router.test.ts | 140 +- .../src/routers/cloud-agent-next-router.ts | 17 + .../routers/cloud-agent-next-schemas.test.ts | 201 +++ .../src/routers/cloud-agent-next-schemas.ts | 18 + ...ganization-cloud-agent-next-router.test.ts | 263 ++- .../organization-cloud-agent-next-router.ts | 34 +- .../e2e/cloud-agent-sandbox-status.spec.ts | 1483 +++++++++++++++++ .../src/middleware/balance.test.ts | 119 +- .../src/persistence/SandboxControl.ts | 131 +- services/cloud-agent-next/src/router.test.ts | 221 ++- .../src/router/handlers/session-management.ts | 39 + .../cloud-agent-next/src/router/schemas.ts | 10 + .../src/sandbox-control/durable-state.test.ts | 58 + .../src/sandbox-control/durable-state.ts | 98 +- .../src/sandbox-control/frames.test.ts | 25 + .../src/sandbox-control/frames.ts | 4 +- .../src/sandbox-control/lifecycle.test.ts | 282 +++- .../src/sandbox-control/socket.test.ts | 511 +++++- .../src/sandbox-control/socket.ts | 109 +- .../sandbox-control/status-projection.test.ts | 429 ++++- .../src/sandbox-control/status-projection.ts | 132 ++ .../src/sandbox-session/SandboxSession.ts | 51 +- .../terminal-lifecycle.test.ts | 20 + .../src/sandbox-session/terminal-lifecycle.ts | 10 +- .../src/shared/sandbox-control-protocol.ts | 22 + .../src/shared/sandbox-status.test.ts | 335 ++++ .../src/shared/sandbox-status.ts | 142 ++ .../test/integration/sandbox-control.test.ts | 696 +++++++- .../integration/worktree-deletion.test.ts | 15 +- .../worktree-credential-refresh.test.ts | 22 +- .../control/sandbox-control-client.test.ts | 209 ++- .../src/control/sandbox-control-client.ts | 15 +- .../src/control/sandbox-control-handlers.ts | 7 +- .../src/control/worktree-runtime.test.ts | 116 +- .../wrapper/src/control/worktree-runtime.ts | 25 + 44 files changed, 7305 insertions(+), 106 deletions(-) create mode 100644 apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx create mode 100644 apps/web/src/components/cloud-agent-next/sandbox-status.test.ts create mode 100644 apps/web/src/components/cloud-agent-next/sandbox-status.ts create mode 100644 apps/web/tests/e2e/cloud-agent-sandbox-status.spec.ts create mode 100644 services/cloud-agent-next/src/shared/sandbox-status.test.ts create mode 100644 services/cloud-agent-next/src/shared/sandbox-status.ts diff --git a/.specs/cloud-agent-session.md b/.specs/cloud-agent-session.md index 3e8b08f228..d0863b6f43 100644 --- a/.specs/cloud-agent-session.md +++ b/.specs/cloud-agent-session.md @@ -8,12 +8,14 @@ itself, without reading logs. It does not prescribe implementation and it carries no test mechanics: stack setup, account provisioning, and the mapping from a rule to a wire event belong to the `test-cloud-agent` skill. Any backend serving this chat MUST satisfy this surface; the user MUST NOT be able to tell -which implementation they are on. +which implementation they are on beyond the bounded runtime-metadata exception +specified under Sandbox Status. All other runtime and infrastructure privacy +requirements remain in force. ## Status -Draft -- created 2026-08-24, revised 2026-08-26 for worktree navigation, grouped -browser-chat tabs, and their filesystem durability boundary. +Draft -- created 2026-08-24, revised 2026-08-27 to define the passive sandbox +status contract alongside shared worktree navigation and durability. ## Conventions @@ -182,6 +184,108 @@ repository. Incomplete cleanup MUST remain recoverable, MUST NOT report success, and MUST NOT permit late creation or ingestion to resurrect the worktree. +### Sandbox Status + +1. Eligible owned control-plane Cloud Agent web chats MUST show a compact, + icon-only sandbox lifecycle control in the header. A fixed box icon MUST + identify the sandbox, with a small bottom-right static badge for its state. + Badge shapes and accessible labels MUST distinguish states without relying + only on color or a loading spinner. Details MUST be available through + hover, keyboard focus, and touch/click, with visible focus and Escape + dismissal. Sandbox lifecycle MUST remain distinct from agent progress and + compute-billing phase; observing it MUST NOT change the composer, terminal, + or turn behavior. +2. The public snapshot MUST contain only `status`, `provider`, `observedAt`, + `detailCode`, `inactivityTimeoutMs`, `estimatedSleepAt`, and an OPTIONAL + `runtime` object. Status MUST be one of `active`, `sleeping`, `starting`, + `stopping`, `error`, `unreachable`, or `unknown`. Provider MUST be exactly + `Cloudflare`, `Vercel`, or `Unknown`, derived from authoritative stored + provider information rather than an arbitrary provider string. When present, + `runtime` MUST contain only nullable `sandboxType`, `kiloCliVersion`, + `wrapperVersion`, `startedAt`, and `stoppedAt` fields. Sandbox types MUST be + bounded to shared, isolated-small, isolated-standard, code-review, + devcontainer, or unknown. Versions MUST be bounded, validated values reported + by the relevant runtime, not inferred from the current deployment. Runtime + identifiers, owner identifiers, infrastructure addresses or regions, + credentials, and raw backend errors MUST NOT appear in status responses or + details. Unknown response fields MUST NOT survive validation. Validation + errors MUST NOT be forwarded as raw backend diagnostics. +3. Detail codes MUST be bounded and consistent with status. The safe meanings + are: + + | Status | Detail code | Meaning | + |---|---|---| + | `active` | `sandbox_ready` | The sandbox is active. | + | `sleeping` | `sandbox_stopped` | The sandbox is sleeping; a message can resume it. | + | `starting` | `sandbox_starting` | The sandbox is starting. | + | `stopping` | `sandbox_stopping` | The sandbox is stopping. | + | `error` | `sandbox_failed` | The sandbox encountered an error; a message can retry. | + | `unreachable` | `connection_unavailable` | The sandbox connection is unavailable; its current state cannot be confirmed. | + | `unknown` | `insufficient_evidence` | There is not enough information to confirm the current state. | + | `unknown` | `status_unavailable` | Sandbox status is temporarily unavailable; this does not mean the sandbox failed. | + + Loading a snapshot MUST use the static `unknown` presentation, never + authoritative `starting`. Failure of the observation service MUST use + `status_unavailable`, not + `sandbox_failed`, and MUST remain non-blocking. Failed, paused, unauthorized, + or stale observations MUST NOT retain current-looking active status or a + sleep estimate. A snapshot from another session or personal/organization + context MUST NOT be displayed. +4. Observation MUST be passive, including first access after reconstruction + and repeated polling. It MUST NOT wake, start, probe, extend, keep alive, + bill, or otherwise mutate a sandbox. It MUST NOT trigger recovery, repair + lifecycle records, change alarms, request wrapper frames, or initialize + operational work indirectly. It MAY use validated stored state and evidence + already received on the current connection. Existing session authorization, + exact organization isolation, and billing rules MUST remain unchanged; + reading status MUST NOT require credits or admit paid work. +5. Status observation MUST accept only valid control-plane `workspace_` + references. Legacy `agent_`, unresolved, demo, remote, unrelated, and + read-only sessions MUST NOT show the indicator or request its status. + Legacy and malformed references MUST be rejected before any legacy runtime + lookup. This does not change the Read-only rules for historical sessions. +6. For control-plane `workspace_` sessions, confirmed stopped, creating, + stopping, failed, and unknown physical states correspond to `sleeping`, + `starting`, `stopping`, `error`, and `unknown`. A missing physical record MUST + yield `unknown`, not synthetic `sleeping`. Running requires fresh readiness + from the unique current open, handshaken connection to establish `active`; + explicit current not-ready evidence means `starting`, while a disconnected + or expired connection means `unreachable`. Historical readiness or provider + instance identity alone MUST NOT establish current readiness. +7. `observedAt` MUST be the snapshot creation time in Unix epoch milliseconds, + not a claim that the sandbox was probed. Timestamps MUST be finite, + nonnegative whole milliseconds within the representable date range. The + applicable `inactivityTimeoutMs` MUST be a finite positive whole-millisecond + duration, or `null` when unknown. It MUST describe the real inactivity policy, + not wrapper retention or a provider's maximum lifetime. The current policy + is 5 minutes for control-plane idle stop; a policy MUST NOT be guessed for an + unsupported provider or session. +8. `estimatedSleepAt` MUST be `null` unless an active sandbox has a known + inactivity policy, a valid future idle-stop deadline, and coherent fresh + sandbox-wide idle evidence. Busy, finalizing, stalled, incomplete, stale, or + mismatched activity evidence MUST suppress the estimate. A stored deadline + alone is insufficient. Supported estimates MUST be presented as approximate + remaining minutes, rounded up and conditional on continued inactivity, never + exact promises. The popup MUST reveal the estimate only after two minutes of + the confirmed idle window, derived from the returned deadline and inactivity + policy. This display delay MUST NOT change + the actual idle-stop policy. The toolbar MUST show `Sleeping soon` in the + final 60 seconds of a supported deadline. Expiry of an estimate MUST NOT + locally invent `sleeping`; new authoritative evidence is required. +9. Details MUST group the current text status, runtime information, and known + lifecycle timing. Runtime information MUST identify the provider and bounded + sandbox type using plain-language labels. The click/touch popup MUST include + a Debug section, collapsed by default, with labeled control-plane mode and + reported CLI/wrapper versions. The hover tooltip MUST remain noninteractive + and omit Debug. Compact runtime codes MUST NOT be shown. Missing versions + MUST display Unknown rather than guessed values. Debug MUST NOT contain a + unique sandbox or provider identifier. + Known physical start/stop dates MUST use local display times; unknown dates + MUST be omitted. Creation requests, wrapper readiness, chat creation, and + billing intervals MUST NOT substitute for physical lifecycle dates. Retained + runtime metadata MUST be isolated to its allocation and MUST NOT carry into a + replacement. Status observation MUST NOT collect or persist new metadata. + ### Persistence 1. Refresh MUST restore the transcript, the preparation history that was shown, @@ -215,9 +319,12 @@ repository. Cloud administration surfaces beyond the existing chat sidebar (MCP Gateway, triggers, webhooks), profile and organization administration, the mobile app, -and the CLI. Also out of scope, and deliberately so: which sandbox provider -runs the environment, the shape of session ids, and the streaming wire protocol. -A tester MUST NOT be able to tell those apart from the chat. +and the CLI. +The bounded provider, sandbox type, debug metadata, and known lifecycle dates in +Sandbox Status are a narrow exception to runtime opacity. They do not permit +arbitrary provider names, runtime identifiers, infrastructure addresses, +credentials, or raw errors. Session-id formats and the streaming wire protocol +remain implementation details that MUST NOT be exposed in status details. ## Not Yet Implemented @@ -234,6 +341,16 @@ The following use SHOULD and are not enforced today: ## Changelog +### 2026-09-02 -- Structured sandbox details + +- Made the toolbar icon-only with a fixed sandbox icon, static status badges, + and a final-minute sleep warning. +- Grouped details into status, runtime, and timing; kept bounded reported + versions in a collapsed Debug section instead of a compact runtime code. +- Showed approximate remaining minutes if inactive after two minutes idle, + without changing the five-minute stop policy. Unknown lifecycle dates remain + omitted. + ### 2026-09-01 -- Shared worktree auto-commit - Restored automatic commit and push for shared worktrees, enabled by default, @@ -249,6 +366,15 @@ The following use SHOULD and are not enforced today: - Defined transcript and group recovery separately from physical filesystem durability without changing terminal or permission requirements. +### 2026-08-27 -- Sandbox status contract + +- Added a bounded passive lifecycle snapshot and distinct observation-failure + details for control-plane web sessions only. +- Required supported inactivity policies and explicitly approximate optional + sleep times. +- Allowed only a bounded human-readable provider label, preserving all other + runtime, infrastructure, credential, and error privacy guarantees. + ### 2026-08-24 -- Audit revision - Corrected: a failing setup command fails the turn on a rebuild too, not only diff --git a/apps/web/src/components/cloud-agent-next/ChatHeader.tsx b/apps/web/src/components/cloud-agent-next/ChatHeader.tsx index fa1651c326..5e02772a67 100644 --- a/apps/web/src/components/cloud-agent-next/ChatHeader.tsx +++ b/apps/web/src/components/cloud-agent-next/ChatHeader.tsx @@ -18,6 +18,7 @@ import { SoundToggleButton } from '@/components/shared/SoundToggleButton'; import { FeedbackDialog } from './FeedbackDialog'; import { buildRepoBrowseUrl, detectGitPlatform } from './utils/git-utils'; import { useTRPC } from '@/lib/trpc/utils'; +import { SandboxStatusIndicator } from './SandboxStatusIndicator'; export function computeBillingRefetchInterval( sessionActive: boolean, @@ -45,6 +46,7 @@ type ChatHeaderProps = { onToggleSound?: () => void; sessionTitle?: string; sessionActive: boolean; + sandboxStatusEligible?: boolean; }; export function ChatHeader({ @@ -64,6 +66,7 @@ export function ChatHeader({ organizationId, sessionTitle, sessionActive, + sandboxStatusEligible = false, }: ChatHeaderProps) { const [showActionsDialog, setShowActionsDialog] = useState(false); const moreOptionsTriggerRef = useRef(null); @@ -124,6 +127,14 @@ export function ChatHeader({ repository={repository} />
+ {sandboxStatusEligible && ( + + )} {onToggleSound && ( )} diff --git a/apps/web/src/components/cloud-agent-next/CloudChatPage.tsx b/apps/web/src/components/cloud-agent-next/CloudChatPage.tsx index 00abae8678..763ccba074 100644 --- a/apps/web/src/components/cloud-agent-next/CloudChatPage.tsx +++ b/apps/web/src/components/cloud-agent-next/CloudChatPage.tsx @@ -14,6 +14,7 @@ import { useManager } from './CloudAgentProvider'; import { useWorktreeChatCreation, useWorktreeChatTabs } from './CloudSidebarLayout'; import { MobileSidebarToggle } from './MobileSidebarToggle'; import { ChatHeader } from './ChatHeader'; +import { isSandboxStatusEligible } from './sandbox-status'; import { ChatInput } from './ChatInput'; import { dedupeCustomModeOptions, @@ -950,6 +951,15 @@ export default function CloudChatPage({ soundEnabled={soundEnabled} onToggleSound={handleToggleSound} sessionActive={isStreaming || activity.type === 'busy' || activity.type === 'retrying'} + sandboxStatusEligible={isSandboxStatusEligible({ + currentUserId, + sessionId, + sessionIdFromParams, + organizationId, + activeSessionType, + isReadOnly, + fetchedSessionData, + })} /> ); diff --git a/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx b/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx new file mode 100644 index 0000000000..fdd341b066 --- /dev/null +++ b/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx @@ -0,0 +1,314 @@ +'use client'; + +import { useEffect, useId, useRef, useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { + Box, + ChevronRight, + Circle, + CircleQuestionMark, + Clock3, + Moon, + Power, + PowerOff, + TriangleAlert, + Unplug, + type LucideIcon, +} from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; +import { useTRPC } from '@/lib/trpc/utils'; +import { cn } from '@/lib/utils'; +import { + SANDBOX_STATUS_POLL_INTERVAL_MS, + sandboxStatusPresentation, + type SandboxStatusPresentation, +} from './sandbox-status'; + +const statusBadgeIcons = { + active: Circle, + sleeping: Moon, + 'sleeping-soon': Clock3, + starting: Power, + stopping: PowerOff, + error: TriangleAlert, + unreachable: Unplug, + unknown: CircleQuestionMark, +} satisfies Record; + +function SandboxStatusDetails({ view }: { view: SandboxStatusPresentation }) { + const hasLifecycleDates = view.startedAt !== null || view.stoppedAt !== null; + const hasTiming = hasLifecycleDates || view.estimatedSleepAt !== null; + + return ( +
+
+
+ Sandbox + {view.label} +
+ {view.status !== 'active' &&

{view.detail}

} +
+
+

Runtime

+
+
Provider
+
{view.provider}
+
Sandbox type
+
{view.sandboxType}
+
+
+ {hasTiming && ( +
+
+

Timing

+ {hasLifecycleDates && Local time} +
+
+ {view.startedAt !== null && ( + <> +
Started
+
+ +
+ + )} + {view.stoppedAt !== null && ( + <> +
Stopped
+
+ +
+ + )} + {view.estimatedSleepAt !== null && ( + <> +
Sleeps in
+
+ +
+ + )} +
+
+ )} +
+ ); +} + +export function SandboxStatusIndicator({ + cloudAgentSessionId, + organizationId, + sessionActive, +}: { + cloudAgentSessionId: string; + organizationId?: string; + sessionActive: boolean; +}) { + const trpc = useTRPC(); + const [observation, setObservation] = useState({ + enabled: false, + initialized: false, + freshAfter: 0, + }); + const [clock, setClock] = useState(0); + const [activity, setActivity] = useState({ active: sessionActive, changedAt: 0 }); + const [popoverOpen, setPopoverOpen] = useState(false); + const [tooltipOpen, setTooltipOpen] = useState(false); + const triggerRef = useRef(null); + const detailsId = useId(); + + if (activity.active !== sessionActive) { + setActivity({ active: sessionActive, changedAt: Date.now() }); + } + + useEffect(() => { + const update = () => { + setObservation({ + enabled: document.visibilityState === 'visible' && navigator.onLine, + initialized: true, + freshAfter: Date.now(), + }); + }; + update(); + document.addEventListener('visibilitychange', update); + window.addEventListener('offline', update); + window.addEventListener('online', update); + return () => { + document.removeEventListener('visibilitychange', update); + window.removeEventListener('offline', update); + window.removeEventListener('online', update); + }; + }, []); + + const query = useQuery({ + ...(organizationId + ? trpc.organizations.cloudAgentNext.getSandboxStatus.queryOptions({ + organizationId, + cloudAgentSessionId, + }) + : trpc.cloudAgentNext.getSandboxStatus.queryOptions({ cloudAgentSessionId })), + enabled: observation.enabled, + refetchInterval: observation.enabled ? SANDBOX_STATUS_POLL_INTERVAL_MS : false, + refetchIntervalInBackground: false, + refetchOnWindowFocus: 'always', + refetchOnReconnect: 'always', + refetchOnMount: 'always', + staleTime: 0, + gcTime: 0, + retry: false, + throwOnError: false, + placeholderData: undefined, + }); + + useEffect(() => { + const refresh = () => { + if (document.visibilityState === 'visible' && navigator.onLine) { + void query.refetch({ cancelRefetch: false }); + } + }; + window.addEventListener('focus', refresh); + return () => window.removeEventListener('focus', refresh); + }, [query.refetch]); + + const now = Math.max(clock, Date.now()); + const view = sandboxStatusPresentation({ + data: query.data, + observation: !observation.initialized + ? 'checking' + : !observation.enabled || query.fetchStatus === 'paused' + ? 'paused' + : query.isError + ? 'unavailable' + : !query.isFetchedAfterMount || query.isPending + ? 'checking' + : 'observing', + dataUpdatedAt: query.dataUpdatedAt, + freshAfter: observation.freshAfter, + estimateAfter: activity.changedAt, + sessionActive, + now, + }); + + useEffect(() => { + const deadline = view.nextChangeAt; + if (deadline === null) return; + const timer = setTimeout( + () => setClock(current => Math.max(current, deadline, Date.now())), + Math.max(0, deadline - Date.now()) + ); + return () => clearTimeout(timer); + }, [view.nextChangeAt]); + + const StatusBadgeIcon = statusBadgeIcons[view.status]; + + return ( + { + setTooltipOpen(false); + setPopoverOpen(open); + }} + > + + + + + + + {!popoverOpen && ( + + + + )} + + {popoverOpen && ( + triggerRef.current?.focus()} + onKeyDown={event => { + if ( + event.key === 'Tab' && + !event.altKey && + !event.ctrlKey && + !event.metaKey && + (event.shiftKey || event.target !== event.currentTarget) + ) { + triggerRef.current?.focus(); + setPopoverOpen(false); + } + }} + > +
+ +
+
+ + +
+
Execution
+
Control plane
+
Kilo CLI
+
+ {view.kiloCliVersion ?? 'Unknown'} +
+
Wrapper
+
+ {view.wrapperVersion ?? 'Unknown'} +
+
+
+ + )} + + ); +} diff --git a/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts b/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts new file mode 100644 index 0000000000..b2094dce47 --- /dev/null +++ b/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts @@ -0,0 +1,535 @@ +import { describe, expect, it } from '@jest/globals'; +import type { CloudAgentSessionId, KiloSessionId } from '@kilocode/cloud-agent-sdk'; +import type { SandboxStatusSnapshot } from '@/routers/cloud-agent-next-schemas'; +import { isSandboxStatusEligible, sandboxStatusPresentation } from './sandbox-status'; + +const now = 1_800_000_000_000; +const snapshot: SandboxStatusSnapshot = { + status: 'active', + provider: 'Cloudflare', + observedAt: now, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: now + 180_000, +}; +const runtime = { + sandboxType: 'isolated-standard', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + startedAt: now - 600_000, + stoppedAt: null, +} satisfies NonNullable; +const observation = { + data: snapshot, + observation: 'observing' as const, + dataUpdatedAt: now, + freshAfter: now, + estimateAfter: 0, + sessionActive: false, + now, +}; + +const sessionId = 'workspace_00000000-0000-4000-8000-000000000001' as CloudAgentSessionId; +const kiloSessionId = 'ses_status_first' as KiloSessionId; +const eligible = { + currentUserId: 'owner', + sessionId, + sessionIdFromParams: kiloSessionId, + activeSessionType: 'cloud-agent' as const, + isReadOnly: false, + fetchedSessionData: { kiloSessionId, cloudAgentSessionId: sessionId, organizationId: null }, +}; + +describe('sandbox status eligibility', () => { + it('requires a resolved owned workspace in the exact route scope', () => { + expect(isSandboxStatusEligible(eligible)).toBe(true); + expect( + isSandboxStatusEligible({ + ...eligible, + organizationId: 'org-1', + fetchedSessionData: { ...eligible.fetchedSessionData, organizationId: 'org-1' }, + }) + ).toBe(true); + }); + + it.each([ + { currentUserId: undefined }, + { sessionId: null }, + { sessionId: 'Starting session…' }, + { sessionId: 'workspace_invalid' }, + { sessionId: 'agent_00000000-0000-4000-8000-000000000001' }, + { activeSessionType: null }, + { activeSessionType: 'remote' as const }, + { activeSessionType: 'read-only' as const }, + { isReadOnly: true }, + { fetchedSessionData: null }, + { sessionIdFromParams: 'ses_other' }, + { organizationId: 'org-other' }, + { fetchedSessionData: { ...eligible.fetchedSessionData, organizationId: 'org-other' } }, + { fetchedSessionData: { ...eligible.fetchedSessionData, cloudAgentSessionId: null } }, + ])('excludes unresolved, unrelated, and unsupported contexts: %j', override => { + expect(isSandboxStatusEligible({ ...eligible, ...override })).toBe(false); + }); +}); + +describe('sandbox status presentation', () => { + it.each([ + ['active', 'sandbox_ready', 'Active'], + ['sleeping', 'sandbox_stopped', 'Sleeping'], + ['starting', 'sandbox_starting', 'Starting'], + ['stopping', 'sandbox_stopping', 'Stopping'], + ['error', 'sandbox_failed', 'Error'], + ['unreachable', 'connection_unavailable', 'Unreachable'], + ['unknown', 'insufficient_evidence', 'Unknown'], + ] as const)( + 'presents authoritative %s lifecycle without raw diagnostics', + (status, detailCode, label) => { + const view = sandboxStatusPresentation({ + ...observation, + data: { + ...snapshot, + status, + detailCode, + estimatedSleepAt: null, + privateRuntime: 'PRIVATE_SENTINEL', + }, + }); + expect(view).toMatchObject({ + status, + label, + provider: 'Cloudflare', + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + expect(JSON.stringify(view)).not.toContain('PRIVATE_SENTINEL'); + } + ); + + it.each(['Cloudflare', 'Vercel', 'Unknown'] as const)( + 'uses the bounded %s provider', + provider => { + expect( + sandboxStatusPresentation({ ...observation, data: { ...snapshot, provider } }).provider + ).toBe(provider); + } + ); + + it.each([ + [180_000, 1], + [300_000, 3], + [600_000, 8], + ])( + 'reveals the estimate after two minutes using the returned %d ms policy', + (inactivityTimeoutMs, sleepMinutesRemaining) => { + const data = { + ...snapshot, + inactivityTimeoutMs, + estimatedSleepAt: now + inactivityTimeoutMs - 120_000 + 1, + }; + expect(sandboxStatusPresentation({ ...observation, data })).toMatchObject({ + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: now + 1, + }); + expect(sandboxStatusPresentation({ ...observation, data, now: now + 1 })).toMatchObject({ + estimatedSleepAt: data.estimatedSleepAt, + sleepMinutesRemaining, + }); + } + ); + + it('hides timing for a new idle window and an unknown policy', () => { + expect( + sandboxStatusPresentation({ + ...observation, + data: { ...snapshot, estimatedSleepAt: now + 300_000 }, + }) + ).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: now + 15_000, + }); + expect( + sandboxStatusPresentation({ + ...observation, + data: { ...snapshot, inactivityTimeoutMs: null, estimatedSleepAt: null }, + }) + ).toMatchObject({ status: 'active', estimatedSleepAt: null, sleepMinutesRemaining: null }); + }); + + it.each([ + [180_000, 3], + [120_001, 3], + [120_000, 2], + [119_999, 2], + [60_001, 2], + [60_000, 1], + [59_999, 1], + [1, 1], + ])('rounds %d ms remaining up to %d minutes', (remainingMs, sleepMinutesRemaining) => { + expect( + sandboxStatusPresentation({ + ...observation, + data: { ...snapshot, estimatedSleepAt: now + remainingMs }, + }) + ).toMatchObject({ + estimatedSleepAt: now + remainingMs, + sleepMinutesRemaining, + }); + }); + + it('schedules countdown changes before freshness expires without new polling evidence', () => { + const data = { ...snapshot, estimatedSleepAt: now + 121_000 }; + expect(sandboxStatusPresentation({ ...observation, data })).toMatchObject({ + status: 'active', + sleepMinutesRemaining: 3, + nextChangeAt: now + 1_000, + }); + expect(sandboxStatusPresentation({ ...observation, data, now: now + 999 })).toMatchObject({ + sleepMinutesRemaining: 3, + nextChangeAt: now + 1_000, + }); + expect(sandboxStatusPresentation({ ...observation, data, now: now + 1_000 })).toMatchObject({ + status: 'active', + estimatedSleepAt: data.estimatedSleepAt, + sleepMinutesRemaining: 2, + nextChangeAt: now + 15_000, + }); + }); + + it('expires freshness when it coincides with a countdown boundary', () => { + const data = { ...snapshot, estimatedSleepAt: now + 135_000 }; + expect(sandboxStatusPresentation({ ...observation, data })).toMatchObject({ + status: 'active', + sleepMinutesRemaining: 3, + nextChangeAt: now + 15_000, + }); + expect(sandboxStatusPresentation({ ...observation, data, now: now + 15_000 })).toMatchObject({ + status: 'unknown', + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: null, + }); + }); + + it('uses unknown for initial loading without a separate loading state', () => { + expect(sandboxStatusPresentation({ ...observation, observation: 'checking' })).toMatchObject({ + status: 'unknown', + label: 'Unknown', + kiloCliVersion: null, + wrapperVersion: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + }); + + it('enters sleeping soon at the final-minute boundary', () => { + const data = { ...snapshot, estimatedSleepAt: now + 60_001 }; + expect(sandboxStatusPresentation({ ...observation, data })).toMatchObject({ + status: 'active', + sleepMinutesRemaining: 2, + nextChangeAt: now + 1, + }); + expect(sandboxStatusPresentation({ ...observation, data, now: now + 1 })).toMatchObject({ + status: 'sleeping-soon', + label: 'Sleeping soon', + estimatedSleepAt: now + 60_001, + sleepMinutesRemaining: 1, + nextChangeAt: now + 15_000, + }); + }); + + it('cancels sleeping soon when activity invalidates the deadline', () => { + const data = { ...snapshot, estimatedSleepAt: now + 60_000 }; + for (const override of [{ sessionActive: true }, { estimateAfter: now + 1 }]) { + expect(sandboxStatusPresentation({ ...observation, data, ...override })).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: now + 15_000, + }); + } + }); + + it('requires fresh idle evidence after activity before restoring the countdown', () => { + const data = { ...snapshot, estimatedSleepAt: now + 121_000 }; + expect(sandboxStatusPresentation({ ...observation, data })).toMatchObject({ + sleepMinutesRemaining: 3, + nextChangeAt: now + 1_000, + }); + for (const override of [{ sessionActive: true }, { estimateAfter: now + 1 }]) { + expect(sandboxStatusPresentation({ ...observation, data, ...override })).toMatchObject({ + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: now + 15_000, + }); + } + expect( + sandboxStatusPresentation({ + ...observation, + data: { ...data, observedAt: now + 1_000 }, + dataUpdatedAt: now + 1_000, + estimateAfter: now + 1, + now: now + 1_000, + }) + ).toMatchObject({ + estimatedSleepAt: data.estimatedSleepAt, + sleepMinutesRemaining: 2, + nextChangeAt: now + 16_000, + }); + }); + + it.each(['paused', 'unavailable'] as const)( + 'suppresses all cached current evidence when %s', + state => { + expect(sandboxStatusPresentation({ ...observation, observation: state })).toMatchObject({ + status: 'unknown', + provider: 'Unknown', + kiloCliVersion: null, + wrapperVersion: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: null, + }); + } + ); + + it('distinguishes observation failure from confirmed sandbox failure', () => { + const unavailable = sandboxStatusPresentation({ ...observation, observation: 'unavailable' }); + const failed = sandboxStatusPresentation({ + ...observation, + data: { ...snapshot, status: 'error', detailCode: 'sandbox_failed', estimatedSleepAt: null }, + }); + expect(unavailable.detail).toContain('does not mean the sandbox failed'); + expect(failed.detail).toContain('encountered an error'); + }); + + it.each([ + { now: now + 15_000 }, + { dataUpdatedAt: now - 15_000, freshAfter: 0 }, + { data: { ...snapshot, runtime, observedAt: now - 15_000 }, freshAfter: 0 }, + { freshAfter: now + 1 }, + { data: { ...snapshot, runtime, observedAt: now + 1 } }, + ])('bounds freshness and requires fresh evidence after resuming: %j', override => { + expect( + sandboxStatusPresentation({ ...observation, data: { ...snapshot, runtime }, ...override }) + ).toMatchObject({ + status: 'unknown', + provider: 'Unknown', + sandboxType: 'Unknown', + kiloCliVersion: null, + wrapperVersion: null, + startedAt: null, + stoppedAt: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: null, + }); + }); + + it('remains current before the 15-second boundary and schedules its expiry', () => { + expect( + sandboxStatusPresentation({ + ...observation, + data: { ...snapshot, runtime }, + now: now + 14_999, + }) + ).toMatchObject({ + status: 'active', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + sleepMinutesRemaining: 3, + nextChangeAt: now + 15_000, + }); + }); + + it.each([ + { sessionActive: true }, + { estimateAfter: now + 1 }, + { data: { ...snapshot, estimatedSleepAt: null } }, + { data: { ...snapshot, provider: 'Unknown' } }, + ])('hides unsupported or invalidated sleep estimates: %j', override => { + expect(sandboxStatusPresentation({ ...observation, ...override })).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + }); + + it('expires an estimate without inventing sleeping', () => { + const data = { ...snapshot, estimatedSleepAt: now + 2_000 }; + expect(sandboxStatusPresentation({ ...observation, data })).toMatchObject({ + status: 'sleeping-soon', + estimatedSleepAt: now + 2_000, + sleepMinutesRemaining: 1, + nextChangeAt: now + 2_000, + }); + for (const elapsedMs of [2_000, 2_001]) { + expect( + sandboxStatusPresentation({ ...observation, data, now: now + elapsedMs }) + ).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: now + 15_000, + }); + } + }); + + it('exposes validated runtime versions without sandbox identities', () => { + const view = sandboxStatusPresentation({ + ...observation, + data: { + ...snapshot, + sandboxId: 'PRIVATE_SANDBOX_ID', + runtime: { ...runtime, providerRef: 'PRIVATE_PROVIDER_REF' }, + }, + }); + expect(view).toMatchObject({ + provider: 'Cloudflare', + sandboxType: 'Standard', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + startedAt: now - 600_000, + stoppedAt: null, + }); + expect(view).not.toHaveProperty('runtimeCode'); + expect(JSON.stringify(view)).not.toContain('PRIVATE_'); + }); + + it('preserves known lifecycle dates on a sleeping snapshot', () => { + expect( + sandboxStatusPresentation({ + ...observation, + data: { + ...snapshot, + status: 'sleeping', + detailCode: 'sandbox_stopped', + estimatedSleepAt: null, + provider: 'Vercel', + runtime: { + ...runtime, + sandboxType: 'isolated-small', + stoppedAt: now - 60_000, + }, + }, + }) + ).toMatchObject({ + status: 'sleeping', + provider: 'Vercel', + sandboxType: 'Small', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + startedAt: now - 600_000, + stoppedAt: now - 60_000, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + }); + + it('does not invent versions or dates for older snapshots', () => { + expect(sandboxStatusPresentation(observation)).toMatchObject({ + kiloCliVersion: null, + wrapperVersion: null, + sandboxType: 'Unknown', + startedAt: null, + stoppedAt: null, + }); + }); + + it.each([ + { kiloCliVersion: null, wrapperVersion: '2.4.0' }, + { kiloCliVersion: '7.4.20', wrapperVersion: null }, + { kiloCliVersion: null, wrapperVersion: null }, + { kiloCliVersion: '7.4.20-canary.123', wrapperVersion: '2.4.0-rc.1' }, + ])('preserves validated nullable runtime versions: %j', versions => { + expect( + sandboxStatusPresentation({ + ...observation, + data: { ...snapshot, runtime: { ...runtime, ...versions } }, + }) + ).toMatchObject({ + ...versions, + status: 'active', + sandboxType: 'Standard', + startedAt: now - 600_000, + stoppedAt: null, + }); + }); + + it('preserves runtime versions when the authoritative lifecycle is unknown', () => { + expect( + sandboxStatusPresentation({ + ...observation, + data: { + ...snapshot, + runtime, + status: 'unknown', + detailCode: 'insufficient_evidence', + estimatedSleepAt: null, + }, + }) + ).toMatchObject({ + status: 'unknown', + provider: 'Cloudflare', + sandboxType: 'Standard', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + startedAt: now - 600_000, + stoppedAt: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + }); + + it.each(['paused', 'unavailable', 'checking'] as const)( + 'does not retain cached runtime details when %s', + state => { + expect( + sandboxStatusPresentation({ + ...observation, + observation: state, + data: { ...snapshot, runtime: { ...runtime, sandboxType: 'devcontainer' } }, + }) + ).toMatchObject({ + status: 'unknown', + provider: 'Unknown', + sandboxType: 'Unknown', + kiloCliVersion: null, + wrapperVersion: null, + startedAt: null, + stoppedAt: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + } + ); + + it.each([ + null, + undefined, + { ...snapshot, provider: 'PRIVATE_SENTINEL' }, + { ...snapshot, detailCode: 'PRIVATE_SENTINEL' }, + { ...snapshot, status: 'PRIVATE_SENTINEL' }, + { ...snapshot, status: 'sleeping' }, + { ...snapshot, observedAt: NaN }, + { ...snapshot, estimatedSleepAt: Infinity }, + { ...snapshot, runtime: { ...runtime, kiloCliVersion: 'PRIVATE_SENTINEL' } }, + { ...snapshot, runtime: { ...runtime, wrapperVersion: 'PRIVATE_SENTINEL' } }, + { ...snapshot, runtime: { ...runtime, kiloCliVersion: '7.4.20\nPRIVATE_SENTINEL' } }, + { ...snapshot, runtime: { ...runtime, wrapperVersion: '2.4.0+PRIVATE_SENTINEL' } }, + { ...snapshot, runtime: { ...runtime, sandboxType: 'PRIVATE_SENTINEL' } }, + ])('fails closed for malformed data without exposing values: %j', data => { + const view = sandboxStatusPresentation({ ...observation, data }); + expect(view).toMatchObject({ + status: 'unknown', + provider: 'Unknown', + kiloCliVersion: null, + wrapperVersion: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + }); + expect(JSON.stringify(view)).not.toContain('PRIVATE_SENTINEL'); + }); +}); diff --git a/apps/web/src/components/cloud-agent-next/sandbox-status.ts b/apps/web/src/components/cloud-agent-next/sandbox-status.ts new file mode 100644 index 0000000000..537caf943b --- /dev/null +++ b/apps/web/src/components/cloud-agent-next/sandbox-status.ts @@ -0,0 +1,181 @@ +import { + baseGetSandboxStatusNextOutputSchema, + baseGetSandboxStatusNextSchema, + SANDBOX_STATUS_DETAIL_MESSAGES, + type SandboxLifecycleStatus, + type SandboxProviderLabel, +} from '@/routers/cloud-agent-next-schemas'; +import type { FetchedSessionData, ResolvedSession } from '@kilocode/cloud-agent-sdk'; + +export const SANDBOX_STATUS_POLL_INTERVAL_MS = 5_000; +export const SANDBOX_STATUS_FRESHNESS_MS = 15_000; +export const SANDBOX_SLEEP_ESTIMATE_DELAY_MS = 120_000; +export const SANDBOX_SLEEP_SOON_MS = 60_000; + +export function isSandboxStatusEligible({ + currentUserId, + sessionId, + sessionIdFromParams, + organizationId, + activeSessionType, + isReadOnly, + fetchedSessionData, +}: { + currentUserId?: string; + sessionId: string | null; + sessionIdFromParams: string | null; + organizationId?: string; + activeSessionType: ResolvedSession['type'] | null; + isReadOnly: boolean; + fetchedSessionData: Pick< + FetchedSessionData, + 'kiloSessionId' | 'cloudAgentSessionId' | 'organizationId' + > | null; +}): boolean { + return Boolean( + currentUserId && + activeSessionType === 'cloud-agent' && + !isReadOnly && + fetchedSessionData && + fetchedSessionData.kiloSessionId === sessionIdFromParams && + fetchedSessionData.cloudAgentSessionId === sessionId && + (fetchedSessionData.organizationId ?? undefined) === organizationId && + baseGetSandboxStatusNextSchema.safeParse({ cloudAgentSessionId: sessionId }).success + ); +} + +const statusLabels = { + active: 'Active', + sleeping: 'Sleeping', + starting: 'Starting', + stopping: 'Stopping', + error: 'Error', + unreachable: 'Unreachable', + unknown: 'Unknown', +} satisfies Record; + +const sandboxTypes = { + shared: 'Shared', + 'isolated-small': 'Small', + 'isolated-standard': 'Standard', + 'code-review': 'Code review', + devcontainer: 'Custom environment', + unknown: 'Unknown', +}; + +export type SandboxStatusPresentation = { + status: SandboxLifecycleStatus | 'sleeping-soon'; + label: string; + detail: string; + provider: SandboxProviderLabel; + sandboxType: string; + kiloCliVersion: string | null; + wrapperVersion: string | null; + startedAt: number | null; + stoppedAt: number | null; + estimatedSleepAt: number | null; + sleepMinutesRemaining: number | null; + nextChangeAt: number | null; +}; + +export function sandboxStatusPresentation({ + data, + observation, + dataUpdatedAt, + freshAfter, + estimateAfter, + sessionActive, + now, +}: { + data: unknown; + observation: 'checking' | 'paused' | 'unavailable' | 'observing'; + dataUpdatedAt: number; + freshAfter: number; + estimateAfter: number; + sessionActive: boolean; + now: number; +}): SandboxStatusPresentation { + const unavailable: SandboxStatusPresentation = { + status: 'unknown', + label: 'Unknown', + detail: SANDBOX_STATUS_DETAIL_MESSAGES.status_unavailable, + provider: 'Unknown', + sandboxType: 'Unknown', + kiloCliVersion: null, + wrapperVersion: null, + startedAt: null, + stoppedAt: null, + estimatedSleepAt: null, + sleepMinutesRemaining: null, + nextChangeAt: null, + }; + + if (observation === 'checking') { + return { ...unavailable, detail: 'Sandbox status is not available yet.' }; + } + if (observation === 'paused') { + return { + ...unavailable, + detail: 'Status updates are paused while this page is hidden or offline.', + }; + } + if (observation === 'unavailable') return unavailable; + + const parsed = baseGetSandboxStatusNextOutputSchema.safeParse(data); + if (!parsed.success) return unavailable; + const snapshot = parsed.data; + const evidenceAt = Math.min(snapshot.observedAt, dataUpdatedAt); + const freshUntil = evidenceAt + SANDBOX_STATUS_FRESHNESS_MS; + if (evidenceAt < freshAfter || snapshot.observedAt > now || now >= freshUntil) { + return { ...unavailable, detail: 'Sandbox status is out of date. Waiting for a fresh update.' }; + } + + const sleepDeadline = + !sessionActive && + snapshot.status === 'active' && + snapshot.provider !== 'Unknown' && + snapshot.inactivityTimeoutMs !== null && + snapshot.observedAt >= estimateAfter && + snapshot.estimatedSleepAt !== null && + snapshot.estimatedSleepAt > now + ? snapshot.estimatedSleepAt + : null; + const estimateVisibleAt = + sleepDeadline !== null && snapshot.inactivityTimeoutMs !== null + ? sleepDeadline - snapshot.inactivityTimeoutMs + SANDBOX_SLEEP_ESTIMATE_DELAY_MS + : null; + const estimatedSleepAt = + estimateVisibleAt !== null && now >= estimateVisibleAt ? sleepDeadline : null; + const sleepMinutesRemaining = + estimatedSleepAt !== null ? Math.ceil((estimatedSleepAt - now) / 60_000) : null; + const sleepingSoon = sleepDeadline !== null && sleepDeadline - now <= SANDBOX_SLEEP_SOON_MS; + const deadlines = [freshUntil]; + if (sleepDeadline !== null) { + deadlines.push(sleepDeadline); + const sleepingSoonAt = sleepDeadline - SANDBOX_SLEEP_SOON_MS; + if (sleepingSoonAt > now) deadlines.push(sleepingSoonAt); + } + if (estimateVisibleAt !== null && estimateVisibleAt > now) deadlines.push(estimateVisibleAt); + if (estimatedSleepAt !== null && sleepMinutesRemaining !== null) { + deadlines.push(estimatedSleepAt - (sleepMinutesRemaining - 1) * 60_000); + } + + const runtime = snapshot.runtime; + + return { + status: sleepingSoon ? 'sleeping-soon' : snapshot.status, + label: sleepingSoon ? 'Sleeping soon' : statusLabels[snapshot.status], + detail: sleepingSoon + ? 'The sandbox will sleep soon if inactivity continues.' + : SANDBOX_STATUS_DETAIL_MESSAGES[snapshot.detailCode], + provider: snapshot.provider, + sandboxType: sandboxTypes[runtime?.sandboxType ?? 'unknown'], + kiloCliVersion: runtime?.kiloCliVersion ?? null, + wrapperVersion: runtime?.wrapperVersion ?? null, + startedAt: runtime?.startedAt ?? null, + stoppedAt: runtime?.stoppedAt ?? null, + estimatedSleepAt, + sleepMinutesRemaining, + nextChangeAt: Math.min(...deadlines), + }; +} diff --git a/apps/web/src/lib/cloud-agent-next/cloud-agent-client.test.ts b/apps/web/src/lib/cloud-agent-next/cloud-agent-client.test.ts index a874292bfa..0977376bde 100644 --- a/apps/web/src/lib/cloud-agent-next/cloud-agent-client.test.ts +++ b/apps/web/src/lib/cloud-agent-next/cloud-agent-client.test.ts @@ -18,6 +18,9 @@ const mockHttpLink = jest.fn<(options: { url: string; headers: () => Record }) => undefined>(); const mockCaptureException = jest.fn(); +import type * as SentryModule from '@sentry/nextjs'; +import type { SandboxStatusSnapshot } from '@/routers/cloud-agent-next-schemas'; + jest.mock('@/lib/dotenvx', () => ({ getEnvVariable: jest.fn(() => 'http://cloud-agent-next'), })); @@ -54,6 +57,10 @@ jest.mock('./cloud-agent-client', () => { }; }); +const { createTRPCClient, TRPCClientError } = + jest.requireMock>('@trpc/client'); +const { captureException } = jest.requireMock>('@sentry/nextjs'); + const clientModule: { createCloudAgentNextClient: jest.Mock; createAppBuilderCloudAgentNextClient: jest.Mock; @@ -603,6 +610,134 @@ describe('CloudAgentNextClient.createWorktreeChat', () => { }); }); +describe('CloudAgentNextClient.getSandboxStatus', () => { + const cloudAgentSessionId = 'workspace_12345678-1234-4234-9234-123456789abc'; + const snapshot = { + status: 'active', + provider: 'Cloudflare', + observedAt: 1_800_000_000_000, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: 1_800_000_060_000, + } satisfies SandboxStatusSnapshot; + const query = jest.fn<(input: { cloudAgentSessionId: string }) => Promise>(); + const { CloudAgentNextClient } = + jest.requireActual('./cloud-agent-client'); + let client: InstanceType; + + beforeEach(() => { + jest.clearAllMocks(); + query.mockReset().mockResolvedValue(snapshot); + jest.mocked(createTRPCClient).mockReturnValue({ getSandboxStatus: { query } } as never); + client = new CloudAgentNextClient('test-token'); + }); + + it('uses the existing transport and returns only the validated public snapshot', async () => { + query.mockResolvedValue({ + ...snapshot, + sandboxId: 'private-sandbox', + providerInstanceId: 'private-instance', + ownerId: 'private-owner', + headers: { authorization: 'private-token' }, + error: 'private-diagnostics', + }); + const response = await client.getSandboxStatus(cloudAgentSessionId); + expect(query).toHaveBeenCalledWith({ cloudAgentSessionId }); + expect(response).toEqual(snapshot); + expect(JSON.stringify(response)).not.toContain('private'); + }); + + it.each([ + null, + {}, + { ...snapshot, status: 'private-status' }, + { ...snapshot, provider: 'private-provider' }, + { ...snapshot, detailCode: 'private-details' }, + { ...snapshot, observedAt: Infinity }, + { ...snapshot, inactivityTimeoutMs: 0 }, + { ...snapshot, estimatedSleepAt: snapshot.observedAt }, + ])('bounds malformed Worker data without reporting raw diagnostics: %j', async response => { + query.mockResolvedValue(response); + const result = await client.getSandboxStatus(cloudAgentSessionId); + expect(result).toEqual({ + status: 'unknown', + provider: 'Unknown', + observedAt: expect.any(Number), + detailCode: 'status_unavailable', + inactivityTimeoutMs: null, + estimatedSleepAt: null, + }); + expect(captureException).not.toHaveBeenCalled(); + }); + + it.each([ + new Error('private-network-diagnostics'), + new SyntaxError('private-upstream-html is not valid JSON'), + new TRPCClientError('private-infrastructure-error'), + ])('makes transport failures observation-unavailable without logging the cause', async error => { + const log = jest.spyOn(console, 'error').mockImplementation(() => undefined); + try { + query.mockRejectedValue(error); + const result = await client.getSandboxStatus(cloudAgentSessionId); + expect(result).toMatchObject({ status: 'unknown', detailCode: 'status_unavailable' }); + expect(JSON.stringify(result)).not.toContain('private'); + expect(captureException).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + }); + + it.each([ + { data: { code: 'UNAUTHORIZED', httpStatus: 401 }, expectedCode: 'UNAUTHORIZED' }, + { data: { code: 'FORBIDDEN', httpStatus: 403 }, expectedCode: 'FORBIDDEN' }, + { data: { code: 'NOT_FOUND', httpStatus: 404 }, expectedCode: 'FORBIDDEN' }, + { data: { httpStatus: 401 }, expectedCode: 'UNAUTHORIZED' }, + { data: { httpStatus: 403 }, expectedCode: 'FORBIDDEN' }, + { data: { httpStatus: 404 }, expectedCode: 'FORBIDDEN' }, + ])('preserves a sanitized denial for $data', async ({ data, expectedCode }) => { + query.mockRejectedValue( + Object.assign(new TRPCClientError('private-access-diagnostics'), { data }) + ); + await expect(client.getSandboxStatus(cloudAgentSessionId)).rejects.toMatchObject({ + code: expectedCode, + message: + expectedCode === 'UNAUTHORIZED' + ? 'Authentication required' + : 'Session not found or access denied', + cause: undefined, + }); + expect(captureException).not.toHaveBeenCalled(); + }); + + it('does not turn unavailable Worker authorization storage into a successful snapshot', async () => { + query.mockRejectedValue( + Object.assign(new TRPCClientError('private-authorization-storage-diagnostics'), { + data: { code: 'SERVICE_UNAVAILABLE', httpStatus: 503 }, + }) + ); + await expect(client.getSandboxStatus(cloudAgentSessionId)).rejects.toMatchObject({ + code: 'SERVICE_UNAVAILABLE', + message: 'Sandbox status is temporarily unavailable', + cause: undefined, + }); + expect(captureException).not.toHaveBeenCalled(); + }); + + it('also preserves denials supplied on the tRPC error shape', async () => { + query.mockRejectedValue( + Object.assign(new TRPCClientError('private-access-diagnostics'), { + shape: { data: { code: 'FORBIDDEN', httpStatus: 403 } }, + }) + ); + await expect(client.getSandboxStatus(cloudAgentSessionId)).rejects.toMatchObject({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + cause: undefined, + }); + }); +}); + describe('closeCloudAgentOrgStreams', () => { const originalFetch = global.fetch; diff --git a/apps/web/src/lib/cloud-agent-next/cloud-agent-client.ts b/apps/web/src/lib/cloud-agent-next/cloud-agent-client.ts index 2f752d9ce5..748768f396 100644 --- a/apps/web/src/lib/cloud-agent-next/cloud-agent-client.ts +++ b/apps/web/src/lib/cloud-agent-next/cloud-agent-client.ts @@ -12,6 +12,10 @@ import { INTERNAL_API_SECRET } from '@/lib/config.server'; import { parseCustomerBillingFailure } from '@kilocode/cloud-agent-sdk'; import type { CloudAgentWorktreeId } from '@kilocode/session-ingest-contracts'; import type { SendMessagePayload } from './types.js'; +import { + SandboxStatusSnapshotSchema, + type SandboxStatusSnapshot, +} from '../../../../../services/cloud-agent-next/src/shared/sandbox-status'; export type { SendMessagePayload } from './types.js'; /** @@ -578,6 +582,9 @@ type CloudAgentNextTRPCClient = { getSession: { query: (input: GetSessionInput) => Promise; }; + getSandboxStatus: { + query: (input: GetSessionInput) => Promise; + }; getComputeBillingStatus: { query: (input: GetSessionInput) => Promise; }; @@ -797,6 +804,44 @@ export class CloudAgentNextClient { } } + async getSandboxStatus(cloudAgentSessionId: string): Promise { + try { + return SandboxStatusSnapshotSchema.parse( + await this.client.getSandboxStatus.query({ cloudAgentSessionId }) + ); + } catch (error) { + if (error instanceof TRPCClientError) { + const code = error.data?.code ?? error.shape?.data?.code; + const httpStatus = error.data?.httpStatus ?? error.shape?.data?.httpStatus; + if (code === 'UNAUTHORIZED' || httpStatus === 401) { + throw new TRPCError({ code: 'UNAUTHORIZED', message: 'Authentication required' }); + } + if ( + code === 'FORBIDDEN' || + code === 'NOT_FOUND' || + httpStatus === 403 || + httpStatus === 404 + ) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'Session not found or access denied' }); + } + if (code === 'SERVICE_UNAVAILABLE' || httpStatus === 503) { + throw new TRPCError({ + code: 'SERVICE_UNAVAILABLE', + message: 'Sandbox status is temporarily unavailable', + }); + } + } + return { + status: 'unknown', + provider: 'Unknown', + observedAt: Date.now(), + detailCode: 'status_unavailable', + inactivityTimeoutMs: null, + estimatedSleepAt: null, + }; + } + } + async getComputeBillingStatus(cloudAgentSessionId: string): Promise { try { return await this.client.getComputeBillingStatus.query({ cloudAgentSessionId }); diff --git a/apps/web/src/lib/cloud-agent/session-ownership.test.ts b/apps/web/src/lib/cloud-agent/session-ownership.test.ts index 7370c4ef82..d7e95cb6a0 100644 --- a/apps/web/src/lib/cloud-agent/session-ownership.test.ts +++ b/apps/web/src/lib/cloud-agent/session-ownership.test.ts @@ -18,9 +18,9 @@ import { } from './session-ownership'; const PERSONAL_SESSION_ID = 'ses_access_personal_1234567890'; -const PERSONAL_CLOUD_AGENT_SESSION_ID = 'agent_access_personal'; +const PERSONAL_CLOUD_AGENT_SESSION_ID = 'workspace_12345678-1234-4234-9234-123456789abc'; const ORGANIZATION_SESSION_ID = 'ses_access_organization_123456'; -const ORGANIZATION_CLOUD_AGENT_SESSION_ID = 'agent_access_organization'; +const ORGANIZATION_CLOUD_AGENT_SESSION_ID = 'workspace_87654321-4321-4321-8321-cba987654321'; let owner: User; let otherMember: User; @@ -161,6 +161,27 @@ describe('Cloud Agent session ownership', () => { ).resolves.toBeNull(); }); + it('does not allow a personal session through organization scope', async () => { + await expect( + verifyOrgOwnsSessionV2ByCloudAgentId( + db, + organization.id, + owner.id, + PERSONAL_CLOUD_AGENT_SESSION_ID + ) + ).resolves.toBeNull(); + }); + + it('denies missing control-plane sessions in either scope', async () => { + const missingId = 'workspace_aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'; + await expect( + verifyUserOwnsSessionV2ByCloudAgentId(db, owner.id, missingId) + ).resolves.toBeNull(); + await expect( + verifyOrgOwnsSessionV2ByCloudAgentId(db, organization.id, owner.id, missingId) + ).resolves.toBeNull(); + }); + it('denies access after removal and restores it after rejoining', async () => { await db .delete(organization_memberships) @@ -232,6 +253,14 @@ describe('Cloud Agent session ownership', () => { .set({ deleted_at: new Date().toISOString() }) .where(eq(organizations.id, organization.id)); + await expect( + verifyOrgOwnsSessionV2ByCloudAgentId( + db, + organization.id, + owner.id, + ORGANIZATION_CLOUD_AGENT_SESSION_ID + ) + ).resolves.toBeNull(); await expect( queryAccessibleCloudAgentSession(db, { kiloUserId: owner.id, diff --git a/apps/web/src/routers/cloud-agent-next-router.test.ts b/apps/web/src/routers/cloud-agent-next-router.test.ts index e1802d4520..1f6374c87c 100644 --- a/apps/web/src/routers/cloud-agent-next-router.test.ts +++ b/apps/web/src/routers/cloud-agent-next-router.test.ts @@ -3,7 +3,12 @@ import type { User } from '@kilocode/db/schema'; import type { createWorktreeChat as CreateWorktreeChat } from '@/lib/cloud-agent-next/worktree-chat'; import type * as MinimumVersionModule from '@/lib/trpc/min-version'; import type { z } from 'zod'; -import type { personalPrepareSessionNextSchema } from '@/routers/cloud-agent-next-schemas'; +import type { + personalPrepareSessionNextSchema, + SandboxStatusSnapshot, +} from '@/routers/cloud-agent-next-schemas'; +import { TRPCError } from '@trpc/server'; +import type { verifyUserOwnsSessionV2ByCloudAgentId } from '@/lib/cloud-agent/session-ownership'; type AttachmentReference = { path: string; files: string[] }; @@ -58,12 +63,15 @@ const mockCreateWorktreeChat = jest.fn(); const mockCancelQueuedMessage = jest.fn<(input: { sessionId: string; messageId: string }) => Promise<{ dropped: boolean }>>(); +const mockGetSandboxStatus = + jest.fn<(cloudAgentSessionId: string) => Promise>(); -const mockCreateCloudAgentNextClient = jest.fn(() => ({ +const mockCreateCloudAgentNextClient = jest.fn((_authToken: string) => ({ prepareSession: mockPrepareSession, sendMessage: mockSendMessage, getSession: mockGetSession, cancelQueuedMessage: mockCancelQueuedMessage, + getSandboxStatus: mockGetSandboxStatus, })); const mockCreateCloudAgentNextClientForModel = jest.fn( @@ -83,7 +91,7 @@ const mockComputeCloudAgentNextBalanceCheckEligibility = jest.fn< const mockIsFeatureFlagEnabledOrDevelopment = jest.fn<(flagName: string, distinctId: string) => Promise>(); const mockVerifyUserOwnsSessionV2ByCloudAgentId = - jest.fn<() => Promise<{ kiloSessionId: string } | null>>(); + jest.fn(); const mockGetBalanceForUser = jest.fn<(user: User) => Promise<{ balance: number }>>(); const mockFetchGitHubRepositoriesForUser = jest.fn< ( @@ -200,6 +208,7 @@ let createCaller: (ctx: { user: User; headersList?: Headers }) => { }) => Promise; getAttachmentDownloadUrl: (input: { messageUuid: string; filename: string }) => Promise; cancelQueuedMessage: (input: { sessionId: string; messageId: string }) => Promise; + getSandboxStatus: (input: { cloudAgentSessionId: string }) => Promise; checkEligibility: () => Promise<{ balance: number; minBalance: number; @@ -461,6 +470,131 @@ describe('cloudAgentNextRouter.cancelQueuedMessage', () => { }); }); +describe('cloudAgentNextRouter.getSandboxStatus', () => { + const cloudAgentSessionId = 'workspace_12345678-1234-4234-9234-123456789abc'; + const user = { id: 'oauth/provider:status-owner', is_admin: false } as User; + const snapshot = { + status: 'sleeping', + provider: 'Vercel', + observedAt: 1_800_000_000_000, + detailCode: 'sandbox_stopped', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: null, + } satisfies SandboxStatusSnapshot; + + beforeEach(() => { + jest.clearAllMocks(); + mockVerifyUserOwnsSessionV2ByCloudAgentId.mockReset().mockResolvedValue({ + kiloSessionId: 'ses_12345678901234567890123456', + }); + mockGetSandboxStatus.mockReset().mockResolvedValue(snapshot); + }); + + it('authorizes personal creator access before contacting the Worker without balance checks', async () => { + mockVerifyUserOwnsSessionV2ByCloudAgentId.mockImplementationOnce(async () => { + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + return { kiloSessionId: 'ses_12345678901234567890123456' }; + }); + await expect(createCaller({ user }).getSandboxStatus({ cloudAgentSessionId })).resolves.toEqual( + snapshot + ); + expect(mockVerifyUserOwnsSessionV2ByCloudAgentId).toHaveBeenCalledWith( + expect.anything(), + user.id, + cloudAgentSessionId + ); + expect(mockGetSandboxStatus).toHaveBeenCalledWith(cloudAgentSessionId); + expect(mockCreateCloudAgentNextClient).toHaveBeenCalledWith('cloud-agent-token'); + expect(mockCreateCloudAgentNextClientForModel).not.toHaveBeenCalled(); + expect(mockComputeCloudAgentNextBalanceCheckEligibility).not.toHaveBeenCalled(); + expect(mockGetBalanceForUser).not.toHaveBeenCalled(); + }); + + it.each([ + 'agent_12345678-1234-4234-9234-123456789abc', + 'ses_12345678901234567890123456', + 'workspace_', + 'workspace_pending', + 'workspace_../../private', + `${cloudAgentSessionId} `, + '', + ])('rejects invalid reference %s before ownership or Worker lookup', async invalidId => { + await expect( + createCaller({ user }).getSandboxStatus({ cloudAgentSessionId: invalidId }) + ).rejects.toMatchObject({ + code: 'BAD_REQUEST', + }); + expect(mockVerifyUserOwnsSessionV2ByCloudAgentId).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + }); + + it.each([ + 'provider', + 'ownerId', + 'userId', + 'organizationId', + 'sandboxId', + 'observedAt', + 'inactivityTimeoutMs', + 'estimatedSleepAt', + ])('rejects caller override %s', async field => { + const input = { cloudAgentSessionId, [field]: 'private-override' }; + await expect(createCaller({ user }).getSandboxStatus(input)).rejects.toMatchObject({ + code: 'BAD_REQUEST', + }); + expect(mockVerifyUserOwnsSessionV2ByCloudAgentId).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + }); + + it('keeps inaccessible personal or organization sessions denied before status', async () => { + mockVerifyUserOwnsSessionV2ByCloudAgentId.mockResolvedValueOnce(null); + await expect( + createCaller({ user }).getSandboxStatus({ cloudAgentSessionId }) + ).rejects.toMatchObject({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + }); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + }); + + it('fails closed without leaking unavailable authorization storage diagnostics', async () => { + mockVerifyUserOwnsSessionV2ByCloudAgentId.mockRejectedValueOnce( + new Error('private-database-diagnostics') + ); + await expect( + createCaller({ user }).getSandboxStatus({ cloudAgentSessionId }) + ).rejects.toMatchObject({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + cause: undefined, + }); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + }); + + it.each(['UNAUTHORIZED', 'FORBIDDEN'] as const)('preserves Worker %s denials', async code => { + mockGetSandboxStatus.mockRejectedValueOnce( + new TRPCError({ code, message: 'Session access denied' }) + ); + await expect( + createCaller({ user }).getSandboxStatus({ cloudAgentSessionId }) + ).rejects.toMatchObject({ code }); + }); + + it('strips unexpected client fields at the web output boundary', async () => { + mockGetSandboxStatus.mockResolvedValueOnce({ + ...snapshot, + credentials: 'private-token', + sandboxId: 'private-runtime', + } as SandboxStatusSnapshot); + const response = await createCaller({ user }).getSandboxStatus({ cloudAgentSessionId }); + expect(response).toEqual(snapshot); + expect(JSON.stringify(response)).not.toContain('private'); + }); +}); + describe('cloudAgentNextRouter helper procedures', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/apps/web/src/routers/cloud-agent-next-router.ts b/apps/web/src/routers/cloud-agent-next-router.ts index bb2f2274da..13b4147f8b 100644 --- a/apps/web/src/routers/cloud-agent-next-router.ts +++ b/apps/web/src/routers/cloud-agent-next-router.ts @@ -29,6 +29,8 @@ import { baseCancelQueuedMessageNextSchema, baseGetSessionNextSchema, baseGetSessionNextOutputSchema, + baseGetSandboxStatusNextSchema, + baseGetSandboxStatusNextOutputSchema, baseAnswerQuestionNextSchema, baseRejectQuestionNextSchema, baseAnswerPermissionNextSchema, @@ -539,6 +541,21 @@ export const cloudAgentNextRouter = createTRPCRouter({ return await client.getSession(input.cloudAgentSessionId); }), + getSandboxStatus: baseProcedure + .input(baseGetSandboxStatusNextSchema) + .output(baseGetSandboxStatusNextOutputSchema) + .query(async ({ ctx, input }) => { + await assertUserOwnsSession(ctx.user.id, input.cloudAgentSessionId).catch(() => { + throw new TRPCError({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + }); + }); + return await createCloudAgentNextClient(generateCloudAgentToken(ctx.user)).getSandboxStatus( + input.cloudAgentSessionId + ); + }), + getComputeBillingStatus: baseProcedure .input(baseGetSessionNextSchema) .query(async ({ ctx, input }) => { diff --git a/apps/web/src/routers/cloud-agent-next-schemas.test.ts b/apps/web/src/routers/cloud-agent-next-schemas.test.ts index 05765aefa2..0724599988 100644 --- a/apps/web/src/routers/cloud-agent-next-schemas.test.ts +++ b/apps/web/src/routers/cloud-agent-next-schemas.test.ts @@ -2,8 +2,12 @@ import { describe, expect, it } from '@jest/globals'; import { baseCreateWorktreeChatNextOutputSchema, baseCreateWorktreeChatNextSchema, + baseGetSandboxStatusNextOutputSchema, + baseGetSandboxStatusNextSchema, basePrepareSessionNextSchema, baseCancelQueuedMessageNextSchema, + SANDBOX_STATUS_DETAIL_MESSAGES, + type SandboxStatusSnapshot, cloudAgentGetAttachmentDownloadUrlSchema, cloudAgentGetAttachmentUploadUrlSchema, cloudAgentRelaxedAttachmentFilenameSchema, @@ -13,6 +17,203 @@ const MESSAGE_UUID = '12345678-1234-4234-9234-123456789abc'; const ATTACHMENT_ID = '87654321-4321-4321-8321-cba987654321'; const KILO_SESSION_ID = 'ses_12345678901234567890123456'; +const sandboxSnapshot = { + status: 'active', + provider: 'Cloudflare', + observedAt: 1_800_000_000_000, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: null, +} satisfies SandboxStatusSnapshot; + +const sandboxLifecycleCases = [ + { status: 'active', detailCode: 'sandbox_ready' }, + { status: 'sleeping', detailCode: 'sandbox_stopped' }, + { status: 'starting', detailCode: 'sandbox_starting' }, + { status: 'stopping', detailCode: 'sandbox_stopping' }, + { status: 'error', detailCode: 'sandbox_failed' }, + { status: 'unreachable', detailCode: 'connection_unavailable' }, + { status: 'unknown', detailCode: 'insufficient_evidence' }, + { status: 'unknown', detailCode: 'status_unavailable' }, +] satisfies Pick[]; + +describe('baseGetSandboxStatusNextOutputSchema', () => { + it.each(sandboxLifecycleCases)('accepts $status with $detailCode', lifecycle => { + for (const provider of ['Cloudflare', 'Vercel', 'Unknown']) { + const response = { ...sandboxSnapshot, ...lifecycle, provider }; + expect(baseGetSandboxStatusNextOutputSchema.parse(response)).toEqual(response); + } + }); + + it('strips runtime identity, infrastructure, credentials, and raw errors before serialization', () => { + const response = baseGetSandboxStatusNextOutputSchema.parse({ + ...sandboxSnapshot, + cloudAgentSessionId: 'agent_private-session', + sandboxId: 'usr-private-sandbox', + providerInstanceId: 'private-instance', + wrapperRunId: 'private-wrapper', + region: 'private-region', + url: 'https://private-runtime.invalid', + credentials: { token: 'private-credential' }, + headers: { Authorization: 'Bearer private-credential' }, + error: { message: 'private-provider-error', stack: 'private-stack' }, + message: 'private-provider-error', + }); + expect(response).toStrictEqual(sandboxSnapshot); + expect(JSON.stringify(response)).not.toContain('private'); + }); + + it.each(['status', 'provider', 'detailCode'] as const)('rejects arbitrary text in %s', field => { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ + ...sandboxSnapshot, + [field]: 'private-provider-error', + }).success + ).toBe(false); + }); + + it('does not interpret loading as an authoritative starting state', () => { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ + ...sandboxSnapshot, + status: 'loading', + detailCode: 'sandbox_starting', + }).success + ).toBe(false); + }); + + it('keeps an observation outage unknown, with distinct safe copy from sandbox failure', () => { + const response = baseGetSandboxStatusNextOutputSchema.parse({ + ...sandboxSnapshot, + status: 'unknown', + detailCode: 'status_unavailable', + inactivityTimeoutMs: null, + }); + expect(SANDBOX_STATUS_DETAIL_MESSAGES[response.detailCode]).toBe( + 'Sandbox status is temporarily unavailable. This does not mean the sandbox failed.' + ); + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ ...response, status: 'error' }).success + ).toBe(false); + expect(SANDBOX_STATUS_DETAIL_MESSAGES.sandbox_failed).not.toBe( + SANDBOX_STATUS_DETAIL_MESSAGES[response.detailCode] + ); + }); + + it.each(['observedAt', 'inactivityTimeoutMs', 'estimatedSleepAt'] as const)( + 'rejects invalid numeric values in %s', + field => { + for (const value of [NaN, Infinity, -Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER + 1, '123']) { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ ...sandboxSnapshot, [field]: value }) + .success + ).toBe(false); + } + } + ); + + it('requires a positive inactivity timeout when present', () => { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ ...sandboxSnapshot, inactivityTimeoutMs: 0 }) + .success + ).toBe(false); + }); + + it('allows null timing but not omitted timing fields', () => { + const response = { ...sandboxSnapshot, inactivityTimeoutMs: null }; + expect(baseGetSandboxStatusNextOutputSchema.parse(response)).toEqual(response); + for (const field of ['inactivityTimeoutMs', 'estimatedSleepAt']) { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ ...response, [field]: undefined }).success + ).toBe(false); + } + }); + + it('preserves a supported future approximate estimate', () => { + const response = { + ...sandboxSnapshot, + estimatedSleepAt: sandboxSnapshot.observedAt + 60_000, + }; + expect(baseGetSandboxStatusNextOutputSchema.parse(response)).toEqual(response); + }); + + it.each(sandboxLifecycleCases.filter(lifecycle => lifecycle.status !== 'active'))( + 'rejects an estimate for $status/$detailCode', + lifecycle => { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ + ...sandboxSnapshot, + ...lifecycle, + estimatedSleepAt: sandboxSnapshot.observedAt + 60_000, + }).success + ).toBe(false); + } + ); + + it('rejects estimates that are expired or lack a supported policy', () => { + for (const estimatedSleepAt of [sandboxSnapshot.observedAt - 1, sandboxSnapshot.observedAt]) { + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ ...sandboxSnapshot, estimatedSleepAt }) + .success + ).toBe(false); + } + expect( + baseGetSandboxStatusNextOutputSchema.safeParse({ + ...sandboxSnapshot, + estimatedSleepAt: sandboxSnapshot.observedAt + 60_000, + inactivityTimeoutMs: null, + }).success + ).toBe(false); + }); +}); + +describe('baseGetSandboxStatusNextSchema', () => { + it('accepts a control-plane session reference without runtime overrides', () => { + const input = { cloudAgentSessionId: `workspace_${MESSAGE_UUID}` }; + expect(baseGetSandboxStatusNextSchema.parse(input)).toEqual(input); + }); + + it.each([ + `agent_${MESSAGE_UUID}`, + `sess_${MESSAGE_UUID}`, + KILO_SESSION_ID, + MESSAGE_UUID, + 'workspace_', + 'workspace_pending', + `workspace_${MESSAGE_UUID} `, + `workspace_${MESSAGE_UUID}\n`, + ` workspace_${MESSAGE_UUID}`, + 'workspace_../../private', + 'workspace_zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz', + ])('rejects legacy, unrelated, or malformed reference %s', cloudAgentSessionId => { + expect(baseGetSandboxStatusNextSchema.safeParse({ cloudAgentSessionId }).success).toBe(false); + }); + + it.each([ + 'provider', + 'sandboxId', + 'providerInstanceId', + 'userId', + 'orgId', + 'estimatedSleepAt', + 'inactivityTimeoutMs', + ])('rejects caller-supplied %s', field => { + expect( + baseGetSandboxStatusNextSchema.safeParse({ + cloudAgentSessionId: `workspace_${MESSAGE_UUID}`, + [field]: 'private-override', + }).success + ).toBe(false); + }); + + it('requires a nonempty session reference', () => { + expect(baseGetSandboxStatusNextSchema.safeParse({}).success).toBe(false); + expect(baseGetSandboxStatusNextSchema.safeParse({ cloudAgentSessionId: '' }).success).toBe( + false + ); + }); +}); + describe('cloudAgentGetAttachmentUploadUrlSchema', () => { it('preserves the legacy 9-MIME contract when extension is absent', () => { const result = cloudAgentGetAttachmentUploadUrlSchema.safeParse({ diff --git a/apps/web/src/routers/cloud-agent-next-schemas.ts b/apps/web/src/routers/cloud-agent-next-schemas.ts index 2368658584..b3b06e577a 100644 --- a/apps/web/src/routers/cloud-agent-next-schemas.ts +++ b/apps/web/src/routers/cloud-agent-next-schemas.ts @@ -1,4 +1,5 @@ import * as z from 'zod'; +import { SandboxStatusSessionIdSchema } from '../../../../services/cloud-agent-next/src/shared/sandbox-status'; import { cloudAgentWorktreeIdSchema, sessionIdSchema as kiloSessionIdSchema, @@ -17,6 +18,17 @@ import { CLOUD_AGENT_IMAGE_MAX_SIZE_BYTES, } from '@/lib/cloud-agent/constants'; +export { + SandboxStatusSnapshotSchema as baseGetSandboxStatusNextOutputSchema, + SANDBOX_STATUS_DETAIL_MESSAGES, +} from '../../../../services/cloud-agent-next/src/shared/sandbox-status'; +export type { + SandboxLifecycleStatus, + SandboxProviderLabel, + SandboxStatusDetailCode, + SandboxStatusSnapshot, +} from '../../../../services/cloud-agent-next/src/shared/sandbox-status'; + /** * Shared schemas for cloud-agent-next routers * @@ -548,6 +560,12 @@ export const baseGetSessionNextSchema = z.object({ cloudAgentSessionId: z.string(), }); +export const baseGetSandboxStatusNextSchema = z + .object({ + cloudAgentSessionId: SandboxStatusSessionIdSchema, + }) + .strict(); + export const cloudAgentTerminalSizeSchema = z.object({ cols: z.number().int().min(2).max(500), rows: z.number().int().min(2).max(200), diff --git a/apps/web/src/routers/organizations/organization-cloud-agent-next-router.test.ts b/apps/web/src/routers/organizations/organization-cloud-agent-next-router.test.ts index 95e4a9835b..3f74353f4b 100644 --- a/apps/web/src/routers/organizations/organization-cloud-agent-next-router.test.ts +++ b/apps/web/src/routers/organizations/organization-cloud-agent-next-router.test.ts @@ -1,14 +1,23 @@ import { describe, expect, it, jest, beforeAll, beforeEach } from '@jest/globals'; +import { inspect } from 'node:util'; +import { DrizzleQueryError } from 'drizzle-orm'; +import { db } from '@/lib/drizzle'; import type * as TrpcInitModule from '@/lib/trpc/init'; import type { createWorktreeChat as CreateWorktreeChat } from '@/lib/cloud-agent-next/worktree-chat'; import type * as MinimumVersionModule from '@/lib/trpc/min-version'; +import type * as OrganizationUtilsModule from '@/routers/organizations/utils'; +import { fetchRequestHandler } from '@trpc/server/adapters/fetch'; import type * as ZodModule from 'zod'; import type { z } from 'zod'; import type { User } from '@kilocode/db/schema'; import type * as BitbucketIntegrationHelpers from '@/lib/cloud-agent/bitbucket-integration-helpers'; import type { BitbucketOrganizationRepositoryListResult } from '@/lib/cloud-agent/bitbucket-integration-helpers'; import { TRPCError } from '@trpc/server'; -import type { basePrepareSessionNextSchema } from '@/routers/cloud-agent-next-schemas'; +import type { verifyOrgOwnsSessionV2ByCloudAgentId } from '@/lib/cloud-agent/session-ownership'; +import type { + basePrepareSessionNextSchema, + SandboxStatusSnapshot, +} from '@/routers/cloud-agent-next-schemas'; const ORGANIZATION_ID = '9a283301-b75d-4375-a1ba-e319a02e18b7'; @@ -62,12 +71,15 @@ const mockCreateWorktreeChat = jest.fn(); const mockCancelQueuedMessage = jest.fn<(input: { sessionId: string; messageId: string }) => Promise<{ dropped: boolean }>>(); +const mockGetSandboxStatus = + jest.fn<(cloudAgentSessionId: string) => Promise>(); -const mockCreateCloudAgentNextClient = jest.fn(() => ({ +const mockCreateCloudAgentNextClient = jest.fn((_authToken: string) => ({ prepareSession: mockPrepareSession, sendMessage: mockSendMessage, getSession: mockGetSession, cancelQueuedMessage: mockCancelQueuedMessage, + getSandboxStatus: mockGetSandboxStatus, })); const mockCreateCloudAgentNextClientForModel = jest.fn( @@ -87,7 +99,7 @@ const mockComputeCloudAgentNextBalanceCheckEligibility = jest.fn< const mockIsFeatureFlagEnabledOrDevelopment = jest.fn<(flagName: string, distinctId: string) => Promise>(); const mockVerifyOrgOwnsSessionV2ByCloudAgentId = - jest.fn<() => Promise<{ kiloSessionId: string } | null>>(); + jest.fn(); const mockFetchBitbucketRepositoriesForOrganization = jest.fn< ( @@ -130,7 +142,8 @@ const mockOrderRepositoriesByUsage = gitlabInstanceUrl?: string; }) => Promise >(); -const mockEnsureOrganizationAccess = jest.fn<(userId: string, organizationId: string) => void>(); +const mockEnsureOrganizationAccess = + jest.fn(); jest.mock('@/lib/tokens', () => ({ generateCloudAgentToken: jest.fn(() => 'cloud-agent-token'), @@ -201,11 +214,12 @@ jest.mock('@/routers/organizations/utils', () => { const organizationProcedure = trpcInit.baseProcedure .input(zod.object({ organizationId: zod.uuid() })) .use(async ({ ctx, input, next }) => { - mockEnsureOrganizationAccess(ctx.user.id, input.organizationId); + await mockEnsureOrganizationAccess(ctx, input.organizationId); return next(); }); return { + ensureOrganizationAccess: mockEnsureOrganizationAccess, organizationMemberProcedure: organizationProcedure, organizationMemberMutationProcedure: organizationProcedure, }; @@ -253,6 +267,10 @@ let createCaller: (ctx: { user: User; headersList?: Headers }) => { organizationId: string; forceRefresh?: boolean; }) => Promise; + getSandboxStatus: (input: { + organizationId: string; + cloudAgentSessionId: string; + }) => Promise; checkEligibility: (input: { organizationId: string }) => Promise<{ balance: number; minBalance: number; @@ -297,7 +315,215 @@ beforeAll(async () => { }); beforeEach(() => { - mockEnsureOrganizationAccess.mockImplementation(() => undefined); + mockEnsureOrganizationAccess.mockReset().mockResolvedValue('member'); +}); + +describe('organizationCloudAgentNextRouter.getSandboxStatus', () => { + const cloudAgentSessionId = 'workspace_12345678-1234-4234-9234-123456789abc'; + const user = { id: 'oauth/provider:status-owner', is_admin: false } as User; + const input = { organizationId: ORGANIZATION_ID, cloudAgentSessionId }; + const snapshot = { + status: 'active', + provider: 'Cloudflare', + observedAt: 1_800_000_000_000, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: 1_800_000_060_000, + } satisfies SandboxStatusSnapshot; + + beforeEach(() => { + jest.clearAllMocks(); + mockVerifyOrgOwnsSessionV2ByCloudAgentId.mockReset().mockResolvedValue({ + kiloSessionId: 'ses_12345678901234567890123456', + }); + mockGetSandboxStatus.mockReset().mockResolvedValue(snapshot); + }); + + it('requires exact organization creator access before observation without paid execution checks', async () => { + mockVerifyOrgOwnsSessionV2ByCloudAgentId.mockImplementationOnce(async () => { + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + return { kiloSessionId: 'ses_12345678901234567890123456' }; + }); + await expect(createCaller({ user }).getSandboxStatus(input)).resolves.toEqual(snapshot); + expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith( + expect.objectContaining({ user }), + ORGANIZATION_ID + ); + expect(mockVerifyOrgOwnsSessionV2ByCloudAgentId).toHaveBeenCalledWith( + expect.anything(), + ORGANIZATION_ID, + user.id, + cloudAgentSessionId + ); + expect(mockGetSandboxStatus).toHaveBeenCalledWith(cloudAgentSessionId); + expect(mockCreateCloudAgentNextClient).toHaveBeenCalledWith('cloud-agent-token'); + expect(mockCreateCloudAgentNextClientForModel).not.toHaveBeenCalled(); + expect(mockComputeCloudAgentNextBalanceCheckEligibility).not.toHaveBeenCalled(); + expect(mockGetBalanceForOrganizationUser).not.toHaveBeenCalled(); + }); + + it.each([ + 'agent_12345678-1234-4234-9234-123456789abc', + 'ses_12345678901234567890123456', + 'workspace_', + 'workspace_pending', + 'workspace_../../private', + `${cloudAgentSessionId} `, + '', + ])('rejects invalid reference %s before session or Worker lookup', async invalidId => { + await expect( + createCaller({ user }).getSandboxStatus({ ...input, cloudAgentSessionId: invalidId }) + ).rejects.toMatchObject({ + code: 'BAD_REQUEST', + }); + expect(mockVerifyOrgOwnsSessionV2ByCloudAgentId).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + }); + + it.each([ + 'provider', + 'ownerId', + 'userId', + 'orgId', + 'sandboxId', + 'providerInstanceId', + 'observedAt', + 'inactivityTimeoutMs', + 'estimatedSleepAt', + ])('rejects caller override %s', async field => { + await expect( + createCaller({ user }).getSandboxStatus({ ...input, [field]: 'private-override' }) + ).rejects.toMatchObject({ code: 'BAD_REQUEST' }); + expect(mockVerifyOrgOwnsSessionV2ByCloudAgentId).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + }); + + it('denies removed members before querying session status', async () => { + mockEnsureOrganizationAccess.mockImplementationOnce(() => { + throw new TRPCError({ code: 'UNAUTHORIZED', message: 'Organization access denied' }); + }); + await expect(createCaller({ user }).getSandboxStatus(input)).rejects.toMatchObject({ + code: 'UNAUTHORIZED', + }); + expect(mockVerifyOrgOwnsSessionV2ByCloudAgentId).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + }); + + it('requires session ownership even when organization membership succeeds', async () => { + mockVerifyOrgOwnsSessionV2ByCloudAgentId.mockResolvedValueOnce(null); + await expect(createCaller({ user }).getSandboxStatus(input)).rejects.toMatchObject({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + }); + expect(mockEnsureOrganizationAccess).toHaveBeenCalledTimes(1); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + }); + + it.each(['development', 'production'] as const)( + 'sanitizes early membership failures in %s responses and error reporting', + async nodeEnv => { + const { ensureOrganizationAccess } = jest.requireActual( + '@/routers/organizations/utils' + ); + const { organizationCloudAgentNextRouter } = + await import('./organization-cloud-agent-next-router'); + const env = jest.replaceProperty(process.env, 'NODE_ENV', nodeEnv); + const log = jest.spyOn(console, 'log').mockImplementation(() => undefined); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + const errorLog = jest.spyOn(console, 'error').mockImplementation(() => undefined); + const select = jest.spyOn(db, 'select').mockImplementationOnce(() => { + throw new DrizzleQueryError( + 'select private-membership-query from organization_memberships', + ['private-membership-parameter'], + new Error('private-membership-database-cause') + ); + }); + mockEnsureOrganizationAccess.mockImplementationOnce(ensureOrganizationAccess); + const onError = jest.fn<({ error }: { error: TRPCError }) => void>(); + + try { + const url = new URL('http://localhost/api/trpc/getSandboxStatus'); + url.searchParams.set('input', JSON.stringify(input)); + const response = await fetchRequestHandler({ + endpoint: '/api/trpc', + req: new Request(url), + router: organizationCloudAgentNextRouter, + createContext: async () => ({ user }), + onError, + }); + const body = await response.text(); + + expect(response.status).toBe(403); + expect(JSON.parse(body)).toMatchObject({ + error: { + message: 'Session not found or access denied', + data: { code: 'FORBIDDEN' }, + }, + }); + expect(onError).toHaveBeenCalledTimes(1); + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ + error: expect.objectContaining({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + cause: undefined, + }), + }) + ); + expect( + inspect( + [body, onError.mock.calls, log.mock.calls, warn.mock.calls, errorLog.mock.calls], + { + depth: null, + } + ) + ).not.toContain('private-membership'); + expect(select).toHaveBeenCalledTimes(1); + expect(mockVerifyOrgOwnsSessionV2ByCloudAgentId).not.toHaveBeenCalled(); + expect(mockCreateCloudAgentNextClient).not.toHaveBeenCalled(); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + } finally { + select.mockRestore(); + errorLog.mockRestore(); + warn.mockRestore(); + log.mockRestore(); + env.restore(); + } + } + ); + + it('fails closed without leaking unavailable authorization storage diagnostics', async () => { + mockVerifyOrgOwnsSessionV2ByCloudAgentId.mockRejectedValueOnce( + new Error('private-database-diagnostics') + ); + await expect(createCaller({ user }).getSandboxStatus(input)).rejects.toMatchObject({ + code: 'FORBIDDEN', + message: 'Session not found or access denied', + cause: undefined, + }); + expect(mockGetSandboxStatus).not.toHaveBeenCalled(); + }); + + it.each(['UNAUTHORIZED', 'FORBIDDEN'] as const)('preserves Worker %s denials', async code => { + mockGetSandboxStatus.mockRejectedValueOnce( + new TRPCError({ code, message: 'Session access denied' }) + ); + await expect(createCaller({ user }).getSandboxStatus(input)).rejects.toMatchObject({ code }); + }); + + it('strips unexpected client fields at the organization output boundary', async () => { + mockGetSandboxStatus.mockResolvedValueOnce({ + ...snapshot, + credentials: 'private-token', + sandboxId: 'private-runtime', + } as SandboxStatusSnapshot); + const response = await createCaller({ user }).getSandboxStatus(input); + expect(response).toEqual(snapshot); + expect(JSON.stringify(response)).not.toContain('private'); + }); }); describe('organizationCloudAgentNextRouter attachment forwarding', () => { @@ -507,7 +733,7 @@ describe('organizationCloudAgentNextRouter attachment forwarding', () => { describe('organizationCloudAgentNextRouter.cancelQueuedMessage', () => { beforeEach(() => { jest.clearAllMocks(); - mockEnsureOrganizationAccess.mockImplementation(() => undefined); + mockEnsureOrganizationAccess.mockResolvedValue('member'); mockVerifyOrgOwnsSessionV2ByCloudAgentId.mockResolvedValue({ kiloSessionId: 'ses_12345678901234567890123456', }); @@ -533,7 +759,7 @@ describe('organizationCloudAgentNextRouter.cancelQueuedMessage', () => { describe('organizationCloudAgentNextRouter helper procedures', () => { beforeEach(() => { jest.clearAllMocks(); - mockEnsureOrganizationAccess.mockImplementation(() => undefined); + mockEnsureOrganizationAccess.mockResolvedValue('member'); mockOrderRepositoriesByUsage.mockImplementation(async ({ repositories }) => repositories); }); @@ -552,7 +778,10 @@ describe('organizationCloudAgentNextRouter helper procedures', () => { isEligible, accessLevel, }); - expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith('member-user', ORGANIZATION_ID); + expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith( + expect.objectContaining({ user: { id: 'member-user', is_admin: false } }), + ORGANIZATION_ID + ); expect(mockGetBalanceForOrganizationUser).toHaveBeenCalledWith( ORGANIZATION_ID, 'member-user' @@ -593,7 +822,10 @@ describe('organizationCloudAgentNextRouter helper procedures', () => { await expect( caller[method]({ organizationId: ORGANIZATION_ID, forceRefresh: true }) ).resolves.toEqual(repositories); - expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith('member-user', ORGANIZATION_ID); + expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith( + expect.objectContaining({ user: { id: 'member-user', is_admin: false } }), + ORGANIZATION_ID + ); if (platform === 'GitLab') { expect(mockFetchGitLabRepositoriesForOrganization).toHaveBeenCalledWith( ORGANIZATION_ID, @@ -688,7 +920,7 @@ describe('organizationCloudAgentNextRouter terminal ownership', () => { beforeEach(() => { jest.clearAllMocks(); - mockEnsureOrganizationAccess.mockImplementation(() => undefined); + mockEnsureOrganizationAccess.mockResolvedValue('member'); }); it('issues a terminal ticket for a session owned by the organization', async () => { @@ -1024,7 +1256,7 @@ describe('organizationCloudAgentNextRouter.createWorktreeChat', () => { beforeEach(() => { jest.clearAllMocks(); - mockEnsureOrganizationAccess.mockImplementation(() => undefined); + mockEnsureOrganizationAccess.mockResolvedValue('member'); mockCreateWorktreeChat.mockResolvedValue(result); }); @@ -1041,7 +1273,10 @@ describe('organizationCloudAgentNextRouter.createWorktreeChat', () => { }) ).resolves.toEqual(result); - expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith(user.id, ORGANIZATION_ID); + expect(mockEnsureOrganizationAccess).toHaveBeenCalledWith( + expect.objectContaining({ user }), + ORGANIZATION_ID + ); expect(mockCreateWorktreeChat).toHaveBeenCalledWith({ user, headersList, @@ -1093,7 +1328,7 @@ describe('organizationCloudAgentNextRouter.createWorktreeChat', () => { describe('organizationCloudAgentNextRouter Bitbucket repository listing', () => { beforeEach(() => { jest.clearAllMocks(); - mockEnsureOrganizationAccess.mockImplementation(() => undefined); + mockEnsureOrganizationAccess.mockResolvedValue('member'); mockOrderRepositoriesByUsage.mockImplementation(async ({ repositories }) => repositories); }); diff --git a/apps/web/src/routers/organizations/organization-cloud-agent-next-router.ts b/apps/web/src/routers/organizations/organization-cloud-agent-next-router.ts index 3b68e46d33..f8fe523c64 100644 --- a/apps/web/src/routers/organizations/organization-cloud-agent-next-router.ts +++ b/apps/web/src/routers/organizations/organization-cloud-agent-next-router.ts @@ -1,5 +1,5 @@ import 'server-only'; -import { createTRPCRouter } from '@/lib/trpc/init'; +import { baseProcedure, createTRPCRouter } from '@/lib/trpc/init'; import { createCloudAgentNextClient, createCloudAgentNextClientForModel, @@ -11,6 +11,7 @@ import { createWorktreeChat } from '@/lib/cloud-agent-next/worktree-chat'; import { generateCloudAgentToken } from '@/lib/tokens'; import { isFeatureFlagEnabledOrDevelopment } from '@/lib/posthog-feature-flags'; import { + ensureOrganizationAccess, organizationMemberProcedure, organizationMemberMutationProcedure, } from '@/routers/organizations/utils'; @@ -37,6 +38,8 @@ import { baseCancelQueuedMessageNextSchema, baseGetSessionNextSchema, baseGetSessionNextOutputSchema, + baseGetSandboxStatusNextSchema, + baseGetSandboxStatusNextOutputSchema, baseAnswerQuestionNextSchema, baseRejectQuestionNextSchema, baseAnswerPermissionNextSchema, @@ -174,6 +177,10 @@ const GetSessionInput = baseGetSessionNextSchema.extend({ organizationId: z.uuid(), }); +const GetSandboxStatusInput = baseGetSandboxStatusNextSchema.extend({ + organizationId: z.uuid(), +}); + const CreateTerminalInput = baseCreateTerminalNextSchema.extend({ organizationId: z.uuid(), }); @@ -725,6 +732,31 @@ export const organizationCloudAgentNextRouter = createTRPCRouter({ return await client.getSession(input.cloudAgentSessionId); }), + getSandboxStatus: baseProcedure + .input(GetSandboxStatusInput) + .output(baseGetSandboxStatusNextOutputSchema) + .query(async ({ ctx, input }) => { + try { + await ensureOrganizationAccess(ctx, input.organizationId); + await assertOrganizationOwnsSession({ + organizationId: input.organizationId, + userId: ctx.user.id, + cloudAgentSessionId: input.cloudAgentSessionId, + }); + } catch (error) { + throw new TRPCError({ + code: + error instanceof TRPCError && error.code === 'UNAUTHORIZED' + ? 'UNAUTHORIZED' + : 'FORBIDDEN', + message: 'Session not found or access denied', + }); + } + return await createCloudAgentNextClient(generateCloudAgentToken(ctx.user)).getSandboxStatus( + input.cloudAgentSessionId + ); + }), + getComputeBillingStatus: organizationMemberProcedure .input(GetSessionInput) .query(async ({ ctx, input }) => { diff --git a/apps/web/tests/e2e/cloud-agent-sandbox-status.spec.ts b/apps/web/tests/e2e/cloud-agent-sandbox-status.spec.ts new file mode 100644 index 0000000000..50018ae7c6 --- /dev/null +++ b/apps/web/tests/e2e/cloud-agent-sandbox-status.spec.ts @@ -0,0 +1,1483 @@ +import { test, expect, type Locator, type Page, type WebSocketRoute } from '@playwright/test'; +import { randomUUID } from 'node:crypto'; +import { createDrizzleClient } from '@kilocode/db/client'; +import { organization_memberships, organizations } from '@kilocode/db/schema'; +import { eq } from 'drizzle-orm'; +import { z } from 'zod'; +import type { SandboxStatusSnapshot } from '@/routers/cloud-agent-next-schemas'; + +const firstId = 'ses_sandbox_status_first'; +const secondId = 'ses_sandbox_status_second'; +const firstWorkspace = 'workspace_00000000-0000-4000-8000-000000000001'; +const secondWorkspace = 'workspace_00000000-0000-4000-8000-000000000002'; +const privateSentinel = 'PRIVATE_SANDBOX_DIAGNOSTIC_SENTINEL'; +const baseTime = Date.parse('2026-08-28T12:00:00Z'); +const runtimeMetadata = { + sandboxType: 'isolated-standard', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + startedAt: baseTime - 600_000, + stoppedAt: null, +} satisfies NonNullable; +const lifecycleDetails = { + active: 'sandbox_ready', + sleeping: 'sandbox_stopped', + starting: 'sandbox_starting', + stopping: 'sandbox_stopping', + error: 'sandbox_failed', + unreachable: 'connection_unavailable', + unknown: 'insufficient_evidence', +} as const; + +type SessionFixture = { + id: string; + cloudId: string | null; + title: string; + organizationId?: string; + kind?: 'remote' | 'read-only' | 'unresolved' | 'unrelated'; +}; +type StatusRequest = { cloudAgentSessionId: string; organizationId?: string; at: number }; +type RpcResult = + | { result: { data: unknown } } + | { error: { message: string; code: number; data: { code: string; httpStatus: number } } }; + +function success(data: unknown): RpcResult { + return { result: { data } }; +} + +function failure(code = 'INTERNAL_SERVER_ERROR'): RpcResult { + return { + error: { + message: privateSentinel, + code: -32603, + data: { code, httpStatus: code === 'FORBIDDEN' ? 403 : 500 }, + }, + }; +} + +function deferred() { + let resolve: (value: T) => void = () => {}; + const promise = new Promise(fulfill => { + resolve = fulfill; + }); + return { promise, resolve }; +} + +async function mountFixtures( + page: Page, + sessions: SessionFixture[] = [ + { id: firstId, cloudId: firstWorkspace, title: 'First sandbox chat' }, + { id: secondId, cloudId: secondWorkspace, title: 'Second sandbox chat' }, + ] +) { + let now = baseTime; + await page.clock.install({ time: new Date(baseTime) }); + const statusRequests: StatusRequest[] = []; + const procedures: string[] = []; + const sockets = new Map(); + let reply: (request: StatusRequest) => RpcResult | Promise = () => success(snapshot()); + let eventId = 0; + + function snapshot(overrides: Partial = {}): SandboxStatusSnapshot { + return { + status: 'active', + provider: 'Cloudflare', + observedAt: now, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: now + 180_000, + runtime: runtimeMetadata, + ...overrides, + }; + } + + function row(session: SessionFixture) { + return { + session_id: session.id, + cloud_agent_session_id: session.cloudId, + title: session.title, + organization_id: session.organizationId ?? null, + created_on_platform: 'cloud-agent-web', + created_at: new Date(baseTime).toISOString(), + updated_at: new Date(baseTime).toISOString(), + version: 2, + git_url: 'https://github.com/example/sandbox-status-regressions.git', + git_branch: 'feature/a-long-branch-name-for-responsive-header-regressions', + parent_session_id: null, + status: 'idle', + status_updated_at: new Date(baseTime).toISOString(), + total_cost_microdollars: 0, + associatedPr: null, + runtimeState: session.cloudId + ? { + sessionId: session.cloudId, + mode: 'code', + model: 'kilo-auto/efficient', + githubRepo: 'example/sandbox-status-regressions', + initiatedAt: baseTime - 1000, + preparedAt: baseTime - 1000, + } + : null, + }; + } + + function send(cloudId: string, streamEventType: string, data: unknown) { + sockets.get(cloudId)?.send( + JSON.stringify({ + eventId: ++eventId, + sessionId: cloudId, + streamEventType, + timestamp: new Date(now).toISOString(), + data, + }) + ); + } + + await page.routeWebSocket('**', socket => { + const url = new URL(socket.url()); + if (url.pathname !== '/stream') { + if (url.pathname !== '/api/user/web') socket.connectToServer(); + return; + } + const cloudId = url.searchParams.get('cloudAgentSessionId'); + if (!cloudId) return; + sockets.set(cloudId, socket); + send(cloudId, 'connected', { sessionStatus: { type: 'idle' }, cloudStatus: { type: 'ready' } }); + }); + await page.route('**/api/cloud-agent-next/sessions/stream-ticket', route => + route.fulfill({ + json: { ticket: 'controlled-browser-ticket', expiresAt: baseTime / 1000 + 3600 }, + }) + ); + await page.route('**/api/trpc/**', async route => { + const request = route.request(); + const url = new URL(request.url()); + const paths = url.pathname.split('/api/trpc/')[1].split(','); + const batch = url.searchParams.get('batch') === '1'; + const input = + request.method() === 'GET' + ? JSON.parse(url.searchParams.get('input') ?? '{}') + : request.postDataJSON(); + const results = await Promise.all( + paths.map(async (procedure, index): Promise => { + procedures.push(procedure); + const args = (batch ? input?.[index] : input) ?? {}; + if (procedure.endsWith('.getSandboxStatus')) { + now = await page.evaluate(() => Date.now()); + const statusRequest = { ...args, at: now } as StatusRequest; + statusRequests.push(statusRequest); + return reply(statusRequest); + } + if (procedure === 'cliSessionsV2.list' || procedure === 'cliSessionsV2.search') + return success({ cliSessions: sessions.map(row), total: sessions.length }); + if (procedure === 'cliSessionsV2.recentRepositories') return success({ repositories: [] }); + if (procedure === 'kiloPass.getSidebarPromoEligibility') + return success({ showPromoBanner: false }); + if (procedure === 'activeSessions.list') + return success({ + sessions: sessions + .filter(s => s.kind === 'remote') + .map(s => ({ + id: s.id, + status: 'idle', + title: s.title, + connectionId: 'controlled-remote', + })), + }); + if (procedure === 'activeSessions.createWebTicket') + return success({ token: 'controlled-web-ticket' }); + if ( + procedure === 'cliSessionsV2.getWithRuntimeState' || + procedure === 'cliSessionsV2.get' + ) { + const session = sessions.find(s => s.id === args.session_id); + if (!session || session.kind === 'unrelated') return failure('NOT_FOUND'); + if (session.kind === 'unresolved') return new Promise(() => {}); + if (session.kind === 'read-only' && procedure === 'cliSessionsV2.get') + return success({ ...row(session), cloud_agent_session_id: null }); + return success(row(session)); + } + if (procedure === 'cliSessionsV2.getSessionMessagesPage') + return success({ + kiloSessionId: args.session_id, + history: { messages: [], nextCursor: null, omittedItemCount: 0 }, + }); + if (procedure === 'cliSessionsV2.getSessionMessages') + return success({ info: { id: args.session_id }, messages: [] }); + if (procedure.endsWith('.getComputeBillingStatus')) + return success({ phase: 'unavailable' }); + if (procedure.endsWith('.sendMessage')) + return success({ messageId: 'msg_controlled', delivery: 'sent' }); + const upstreamUrl = new URL(`/api/trpc/${procedure}`, url.origin); + if (request.method() === 'GET') upstreamUrl.searchParams.set('input', JSON.stringify(args)); + try { + const response = await route.fetch({ + url: upstreamUrl.toString(), + ...(request.method() === 'POST' ? { postData: JSON.stringify(args) } : {}), + }); + return response.json(); + } catch { + throw new Error(`Fixture prerequisite failed: ${procedure}`); + } + }) + ); + await route.fulfill({ json: batch ? results : results[0] }); + }); + + return { + statusRequests, + procedures, + snapshot, + setReply(handler: typeof reply) { + reply = handler; + }, + async open(id = firstId, organizationId?: string) { + await page.goto( + `${organizationId ? `/organizations/${organizationId}` : ''}/cloud/chat?sessionId=${id}` + ); + await expect(page.getByRole('button', { name: 'More options' })).toBeVisible(); + }, + async navigate(id: string) { + await page.evaluate(sessionId => { + const url = new URL(location.href); + url.searchParams.set('sessionId', sessionId); + window.history.pushState(null, '', url); + }, id); + }, + async advance(ms: number) { + await page.clock.fastForward(ms); + now = await page.evaluate(() => Date.now()); + }, + async currentSnapshot(overrides: Partial = {}) { + now = await page.evaluate(() => Date.now()); + return snapshot(overrides); + }, + async refresh() { + const count = statusRequests.length; + await page.evaluate(() => window.dispatchEvent(new Event('focus'))); + await expect.poll(() => statusRequests.length).toBeGreaterThan(count); + }, + async activity(cloudId: string, type: 'busy' | 'idle') { + await expect.poll(() => sockets.has(cloudId)).toBe(true); + const session = sessions.find(s => s.cloudId === cloudId); + send(cloudId, 'kilocode', { + type: 'session.status', + properties: { sessionID: session?.id, status: { type } }, + }); + if (type === 'idle') send(cloudId, 'complete', {}); + }, + }; +} + +async function withOrganization( + page: Pick, + run: (organizationId: string) => Promise +) { + const connectionString = process.env.POSTGRES_URL; + if (!connectionString) + throw new Error('POSTGRES_URL is required for the organization page fixture'); + const response = await page.request.get('/api/auth/session'); + const { kiloUserId } = z.object({ kiloUserId: z.string() }).parse(await response.json()); + const organizationId = randomUUID(); + const { db, pool } = createDrizzleClient({ connectionString, poolConfig: { max: 1 } }); + try { + await db.insert(organizations).values({ + id: organizationId, + name: 'Sandbox status browser regression organization', + created_by_kilo_user_id: kiloUserId, + }); + await db.insert(organization_memberships).values({ + organization_id: organizationId, + kilo_user_id: kiloUserId, + role: 'member', + }); + await run(organizationId); + } finally { + try { + await page.unrouteAll({ behavior: 'ignoreErrors' }); + } finally { + try { + await db + .delete(organization_memberships) + .where(eq(organization_memberships.organization_id, organizationId)); + await db.delete(organizations).where(eq(organizations.id, organizationId)); + } finally { + await pool.end(); + } + } + } +} + +const indicator = (page: Page) => page.getByRole('button', { name: /^Sandbox status:/ }); +const details = (page: Page) => page.getByRole('dialog', { name: 'Sandbox status details' }); +const detailValue = (container: Locator, label: string) => + container + .locator('dt') + .filter({ hasText: new RegExp(`^${label}$`) }) + .locator('+ dd'); +const sleepTime = (page: Page) => detailValue(details(page), 'Sleeps in').locator('time'); +const debugDisclosure = (page: Page) => details(page).locator('details'); +const debugSummary = (page: Page) => debugDisclosure(page).locator('summary'); + +async function expectDebugCollapsed(page: Page) { + await expect(debugSummary(page)).toBeVisible(); + await expect(debugSummary(page)).toHaveText('Debug'); + await expect(debugDisclosure(page)).toHaveJSProperty('open', false); + await expect(debugDisclosure(page).locator('dl')).toBeHidden(); + await expect(details(page).getByText('Runtime code', { exact: true })).toHaveCount(0); + await expect(details(page).locator('code')).toHaveCount(0); +} + +async function expectDebugValues(page: Page, kiloCliVersion: string, wrapperVersion: string) { + await expect(debugDisclosure(page).locator('dt')).toHaveText([ + 'Execution', + 'Kilo CLI', + 'Wrapper', + ]); + await expect(debugDisclosure(page).locator('dd')).toHaveText([ + 'Control plane', + kiloCliVersion, + wrapperVersion, + ]); +} + +async function settlePausedStatus(page: Page, label: string) { + await expect + .poll(async () => { + await page.clock.runFor(1); + return indicator(page).getAttribute('aria-label'); + }) + .toBe(`Sandbox status: ${label}`); +} + +async function expectStaticIndicator(page: Page) { + const button = indicator(page); + await expect(button).toHaveText(''); + await expect(button).toHaveCSS('width', '44px'); + await expect(button).toHaveCSS('height', '44px'); + await expect(button.locator('svg')).toHaveCount(2); + await expect(button.locator('svg').first()).toHaveClass(/lucide-box/); + for (const icon of await button.locator('svg').all()) { + await expect(icon).toBeVisible(); + await expect(icon).toHaveAttribute('aria-hidden', 'true'); + } + await expect( + button.locator('.animate-spin, [role="progressbar"], animate, animateTransform') + ).toHaveCount(0); + expect( + await button.evaluate(element => + [element, ...element.querySelectorAll('*')].every( + node => getComputedStyle(node).animationName === 'none' + ) + ) + ).toBe(true); +} + +async function expectUnavailableRuntime(page: Page) { + await expect(detailValue(details(page), 'Provider')).toHaveText('Unknown'); + await expect(detailValue(details(page), 'Sandbox type')).toHaveText('Unknown'); + await expectDebugValues(page, 'Unknown', 'Unknown'); + await expect(details(page).getByText('Runtime code', { exact: true })).toHaveCount(0); + await expect(details(page).locator('code')).toHaveCount(0); + await expect(details(page).getByRole('heading', { name: 'Timing', exact: true })).toHaveCount(0); + await expect(details(page).getByText('Local time', { exact: true })).toHaveCount(0); + await expect(details(page).locator('time')).toHaveCount(0); +} + +async function expectSafe(page: Page) { + await expect(page.locator('body')).not.toContainText(privateSentinel); + await expect(page.locator('body')).not.toContainText(/sandbox[_-]instance|runtime[_-]id/); +} + +test.describe('control-plane sandbox header', () => { + test.afterEach(async ({ page }) => { + await page.unrouteAll({ behavior: 'ignoreErrors' }); + }); + + test('removes organization fixtures when route teardown rejects without hiding the failure', async ({ + page, + }) => { + const connectionString = process.env.POSTGRES_URL; + if (!connectionString) + throw new Error('POSTGRES_URL is required for the organization page fixture'); + const { db, pool } = createDrizzleClient({ connectionString, poolConfig: { max: 1 } }); + const cleanupError = new Error('Route teardown rejected'); + let fixtureId: string | undefined; + try { + await expect( + withOrganization( + { + request: page.request, + unrouteAll: async () => { + throw cleanupError; + }, + }, + async organizationId => { + fixtureId = organizationId; + } + ) + ).rejects.toBe(cleanupError); + if (!fixtureId) throw new Error('Organization fixture was not created'); + expect( + await db + .select({ id: organizations.id }) + .from(organizations) + .where(eq(organizations.id, fixtureId)) + ).toEqual([]); + expect( + await db + .select({ organizationId: organization_memberships.organization_id }) + .from(organization_memberships) + .where(eq(organization_memberships.organization_id, fixtureId)) + ).toEqual([]); + } finally { + try { + if (fixtureId) { + await db + .delete(organization_memberships) + .where(eq(organization_memberships.organization_id, fixtureId)); + await db.delete(organizations).where(eq(organizations.id, fixtureId)); + } + } finally { + await pool.end(); + } + } + }); + + test('keeps the sandbox icon fixed with distinct static lifecycle badges independently of agent progress', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect + .poll(() => fixture.procedures, { timeout: 15000 }) + .toContain('cliSessionsV2.getWithRuntimeState'); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(details(page).getByText('Sandbox', { exact: true })).toBeVisible(); + await expect(detailValue(details(page), 'Provider')).toHaveText('Cloudflare'); + await expect(sleepTime(page)).toBeVisible(); + await expect(details(page)).not.toContainText( + /Sleeps after|Inactivity timing is unknown|Sandbox lifecycle, not agent progress or compute billing/ + ); + const sandboxShape = await indicator(page).locator('svg').first().innerHTML(); + const badgeShapes = new Set(); + for (const [status, detailCode] of Object.entries(lifecycleDetails)) { + fixture.setReply(() => + success({ + ...fixture.snapshot({ + status: status as SandboxStatusSnapshot['status'], + detailCode, + estimatedSleepAt: null, + }), + internal: privateSentinel, + }) + ); + await fixture.refresh(); + const label = `${status[0].toUpperCase()}${status.slice(1)}`; + await expect(indicator(page)).toHaveAccessibleName(`Sandbox status: ${label}`); + await expect( + details(page).getByText(label, { exact: true }).filter({ visible: true }) + ).toBeVisible(); + await expectStaticIndicator(page); + expect(await indicator(page).locator('svg').first().innerHTML()).toBe(sandboxShape); + const badgeShape = await indicator(page).locator('svg').last().innerHTML(); + expect(badgeShapes.has(badgeShape)).toBe(false); + badgeShapes.add(badgeShape); + await expect(sleepTime(page)).toHaveCount(0); + await expect(detailValue(details(page), 'Started').locator('time')).toBeVisible(); + await expect(page.getByRole('combobox', { name: 'Ask anything…' })).toBeEnabled(); + await expectSafe(page); + } + fixture.setReply(request => + success(fixture.snapshot({ estimatedSleepAt: request.at + 60_000 })) + ); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping soon'); + await expect(details(page).getByText('Sleeping soon', { exact: true })).toBeVisible(); + await expectStaticIndicator(page); + expect(await indicator(page).locator('svg').first().innerHTML()).toBe(sandboxShape); + expect(badgeShapes.has(await indicator(page).locator('svg').last().innerHTML())).toBe(false); + await expect(sleepTime(page)).toBeVisible(); + for (const provider of ['Vercel', 'Unknown'] as const) { + fixture.setReply(() => + success(fixture.snapshot({ provider, inactivityTimeoutMs: null, estimatedSleepAt: null })) + ); + await fixture.refresh(); + await expect(detailValue(details(page), 'Provider')).toHaveText(provider); + await expect(sleepTime(page)).toHaveCount(0); + } + await expect(details(page)).not.toContainText( + /Sleeps after|Inactivity timing is unknown|Sandbox lifecycle, not agent progress or compute billing/ + ); + }); + + test('shows reported versions only in collapsed Debug and local lifecycle times without raw identifiers', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success({ + ...fixture.snapshot({ estimatedSleepAt: fixture.statusRequests[0].at + 180_000 }), + sandboxInstanceId: 'sandbox_instance_private', + runtimeId: 'runtime_id_private', + runtime: { + ...runtimeMetadata, + wrapperRunId: privateSentinel, + cloudAgentSessionId: firstWorkspace, + }, + }) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect( + details(page).getByRole('heading', { name: 'Runtime', exact: true }) + ).toBeVisible(); + for (const [label, value] of [ + ['Provider', 'Cloudflare'], + ['Sandbox type', 'Standard'], + ]) { + await expect(detailValue(details(page), label)).toHaveText(value); + } + await expectDebugCollapsed(page); + await debugSummary(page).click(); + await expect(debugDisclosure(page)).toHaveJSProperty('open', true); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues(page, runtimeMetadata.kiloCliVersion, runtimeMetadata.wrapperVersion); + await expect(details(page)).not.toContainText( + /workspace_|ses_|sandbox[_-]instance|runtime[_-]id/ + ); + await expectSafe(page); + await expect(details(page).getByRole('heading', { name: 'Timing', exact: true })).toBeVisible(); + await expect(details(page).getByText('Local time', { exact: true })).toBeVisible(); + const started = detailValue(details(page), 'Started').locator('time'); + await expect(started).toHaveAttribute( + 'datetime', + new Date(runtimeMetadata.startedAt).toISOString() + ); + await expect(started).toHaveText( + await page.evaluate( + timestamp => + new Date(timestamp).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' }), + runtimeMetadata.startedAt + ) + ); + await expect(detailValue(details(page), 'Stopped')).toHaveCount(0); + await expect(sleepTime(page)).toHaveAttribute( + 'datetime', + new Date(fixture.statusRequests[0].at + 180_000).toISOString() + ); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + const stoppedAt = await page.evaluate(() => Date.now()); + fixture.setReply(() => + success( + fixture.snapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + estimatedSleepAt: null, + runtime: { ...runtimeMetadata, stoppedAt }, + }) + ) + ); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await expect(started).toBeVisible(); + const stopped = detailValue(details(page), 'Stopped').locator('time'); + await expect(stopped).toHaveAttribute('datetime', new Date(stoppedAt).toISOString()); + await expect(stopped).toHaveText( + await page.evaluate( + timestamp => + new Date(timestamp).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' }), + stoppedAt + ) + ); + await expect(sleepTime(page)).toHaveCount(0); + }); + + for (const missingRuntime of ['omitted', 'null fields'] as const) { + test(`does not invent runtime versions or dates for ${missingRuntime}`, async ({ page }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success( + fixture.snapshot({ + estimatedSleepAt: null, + runtime: + missingRuntime === 'omitted' + ? undefined + : { + sandboxType: null, + kiloCliVersion: null, + wrapperVersion: null, + startedAt: null, + stoppedAt: null, + }, + }) + ) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(detailValue(details(page), 'Provider')).toHaveText('Cloudflare'); + await expect(detailValue(details(page), 'Sandbox type')).toHaveText('Unknown'); + await expectDebugCollapsed(page); + await debugSummary(page).click(); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues(page, 'Unknown', 'Unknown'); + await expect(details(page).getByRole('heading', { name: 'Timing', exact: true })).toHaveCount( + 0 + ); + await expect(details(page).getByText('Local time', { exact: true })).toHaveCount(0); + await expect(detailValue(details(page), 'Started')).toHaveCount(0); + await expect(detailValue(details(page), 'Stopped')).toHaveCount(0); + await expect(sleepTime(page)).toHaveCount(0); + await expect(details(page).locator('time')).toHaveCount(0); + await expectSafe(page); + }); + } + + test('shows relative sleep timing without a Local time caption until a lifecycle date is known', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success( + fixture.snapshot({ + estimatedSleepAt: fixture.statusRequests[0].at + 180_000, + runtime: { ...runtimeMetadata, startedAt: null, stoppedAt: null }, + }) + ) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(details(page).getByRole('heading', { name: 'Timing', exact: true })).toBeVisible(); + await expect(details(page).getByText('Local time', { exact: true })).toHaveCount(0); + await expect(detailValue(details(page), 'Started')).toHaveCount(0); + await expect(detailValue(details(page), 'Stopped')).toHaveCount(0); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + await expect(sleepTime(page)).toHaveAttribute( + 'datetime', + new Date(fixture.statusRequests[0].at + 180_000).toISOString() + ); + const stoppedAt = await page.evaluate(() => Date.now()); + fixture.setReply(() => + success( + fixture.snapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + estimatedSleepAt: null, + runtime: { ...runtimeMetadata, startedAt: null, stoppedAt }, + }) + ) + ); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await expect(details(page).getByText('Local time', { exact: true })).toBeVisible(); + await expect(detailValue(details(page), 'Started')).toHaveCount(0); + await expect(detailValue(details(page), 'Stopped').locator('time')).toHaveAttribute( + 'datetime', + new Date(stoppedAt).toISOString() + ); + await expect(sleepTime(page)).toHaveCount(0); + }); + + test('shares only the status description across tooltip and popup, excluding Debug and versions', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).hover(); + await fixture.advance(250); + const tooltip = page.getByRole('tooltip'); + await expect(tooltip).toBeVisible(); + const tooltipSurface = page.locator('[data-slot="tooltip-content"]').filter({ has: tooltip }); + await expect(tooltipSurface).toHaveCSS('width', '320px'); + await expect(tooltipSurface).toHaveCSS('padding', '16px'); + await expect(tooltipSurface.locator('details, summary, code')).toHaveCount(0); + await expect(tooltipSurface).not.toContainText( + /Debug|Execution|Control plane|Kilo CLI|Wrapper|Runtime code|7\.4\.20|2\.4\.0/ + ); + await expect(detailValue(tooltip, 'Provider')).toHaveText('Cloudflare'); + await expect(detailValue(tooltip, 'Sandbox type')).toHaveText('Standard'); + await expect(detailValue(tooltip, 'Sleeps in').locator('time')).toHaveText( + 'About 3 min if inactive' + ); + const hoverDetails = await tooltip.innerText(); + await expect(indicator(page)).toHaveAttribute('aria-describedby', /.+/); + await page.keyboard.press('Escape'); + await fixture.advance(250); + await expect(tooltip).toHaveCount(0); + await page.mouse.move(0, 0); + await page.getByRole('button', { name: 'New terminal' }).focus(); + await page.keyboard.press('Tab'); + await expect(indicator(page)).toBeFocused(); + await expect(indicator(page)).not.toHaveCSS('box-shadow', 'none'); + await expect(tooltip).toBeVisible(); + await expect(tooltip).toHaveText(hoverDetails, { useInnerText: true }); + await page.keyboard.press('Enter'); + await fixture.advance(250); + await expect(details(page)).toBeVisible(); + await expect(tooltip).toHaveCount(0); + await expectDebugCollapsed(page); + await expect(debugSummary(page)).toBeFocused(); + await expect(indicator(page)).toHaveAttribute('aria-describedby', /.+/); + const descriptionId = await indicator(page).getAttribute('aria-describedby'); + if (!descriptionId) throw new Error('Sandbox status description is missing'); + const sharedDetails = details(page).locator(`[id="${descriptionId}"]`); + await expect(sharedDetails).toHaveText(hoverDetails, { useInnerText: true }); + await expect(sharedDetails.locator('details, summary, code')).toHaveCount(0); + await expect(sharedDetails).not.toContainText( + /Debug|Execution|Control plane|Kilo CLI|Wrapper|Runtime code|7\.4\.20|2\.4\.0/ + ); + await expect(details(page)).toHaveCSS('width', '320px'); + await expect(details(page)).toHaveCSS('padding', '16px'); + await expect(indicator(page)).toHaveAttribute('aria-expanded', 'true'); + await expect(indicator(page)).toHaveAccessibleDescription(hoverDetails); + await debugSummary(page).click(); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues(page, runtimeMetadata.kiloCliVersion, runtimeMetadata.wrapperVersion); + await expect(indicator(page)).toHaveAccessibleDescription(hoverDetails); + await expect(sharedDetails).toHaveText(hoverDetails, { useInnerText: true }); + await expectSafe(page); + await page.keyboard.press('Escape'); + await fixture.advance(250); + await expect(details(page)).toHaveCount(0); + await expect(tooltip).toHaveCount(0); + await expect(indicator(page)).toBeFocused(); + await page.keyboard.press('Space'); + await fixture.advance(250); + await expect(details(page)).toBeVisible(); + await expectDebugCollapsed(page); + await expect(debugSummary(page)).toBeFocused(); + await page.keyboard.press('Tab'); + await expect(page.getByRole('button', { name: 'Mute completion sounds' })).toBeFocused(); + await expect(details(page)).toHaveCount(0); + await indicator(page).click(); + await expect(details(page)).toBeVisible(); + await page.getByRole('combobox', { name: 'Ask anything…' }).click(); + await expect(details(page)).toHaveCount(0); + const box = await indicator(page).boundingBox(); + expect(box?.width).toBeGreaterThanOrEqual(44); + expect(box?.height).toBeGreaterThanOrEqual(44); + }); + + for (const key of ['Enter', 'Space']) { + test(`toggles Debug with ${key} and collapses it on reopen`, async ({ page }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).focus(); + await page.keyboard.press(key); + await fixture.advance(250); + await expectDebugCollapsed(page); + await expect(debugSummary(page)).toBeFocused(); + await page.keyboard.press(key); + await expect(debugDisclosure(page)).toHaveJSProperty('open', true); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues(page, runtimeMetadata.kiloCliVersion, runtimeMetadata.wrapperVersion); + await expect(debugSummary(page)).toBeFocused(); + await page.keyboard.press(key); + await expectDebugCollapsed(page); + await page.keyboard.press(key); + await expect(debugDisclosure(page)).toHaveJSProperty('open', true); + await page.keyboard.press('Escape'); + await fixture.advance(250); + await expect(details(page)).toHaveCount(0); + await expect(indicator(page)).toBeFocused(); + await expect(indicator(page)).toHaveAttribute('aria-expanded', 'false'); + await page.keyboard.press(key); + await fixture.advance(250); + await expectDebugCollapsed(page); + await expect(debugSummary(page)).toBeFocused(); + }); + } + + for (const focusTarget of ['Debug summary', 'dialog container']) { + for (const key of ['Tab', 'Shift+Tab', 'Escape']) { + test(`handles ${key} from the ${focusTarget} without trapping focus`, async ({ page }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expectDebugCollapsed(page); + await expect(debugSummary(page)).toBeFocused(); + if (focusTarget === 'dialog container') { + await detailValue(details(page), 'Provider').click(); + await expect(details(page)).toBeFocused(); + } else { + await page.keyboard.press('Space'); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expect(debugSummary(page)).toBeFocused(); + } + await page.keyboard.press(key); + if (focusTarget === 'dialog container' && key === 'Tab') { + await expect(details(page)).toBeVisible(); + await expect(debugSummary(page)).toBeFocused(); + await expectDebugCollapsed(page); + await page.keyboard.press('Tab'); + } + await fixture.advance(250); + await expect(details(page)).toHaveCount(0); + await expect(page.getByRole('tooltip')).toHaveCount(0); + const destination = + key === 'Tab' + ? page.getByRole('button', { name: 'Mute completion sounds' }) + : key === 'Shift+Tab' + ? page.getByRole('button', { name: 'New terminal' }) + : indicator(page); + await expect(destination).toBeFocused(); + }); + } + } + + test('shows a static Unknown badge for initial loading and never guesses Starting', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + const pending = deferred(); + fixture.setReply(() => pending.promise); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expectStaticIndicator(page); + await indicator(page).click(); + await expectUnavailableRuntime(page); + await expect(details(page)).not.toContainText(/Checking|Starting/); + await fixture.advance(20_000); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expectStaticIndicator(page); + pending.resolve( + success( + fixture.snapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + estimatedSleepAt: null, + }) + ) + ); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + }); + + test('polls every five seconds while idle or sleeping, but not hidden or offline', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success( + fixture.snapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + estimatedSleepAt: null, + }) + ) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + const initial = fixture.statusRequests.length; + await fixture.advance(3_000); + expect(fixture.statusRequests).toHaveLength(initial); + await fixture.advance(2_000); + await expect.poll(() => fixture.statusRequests.length).toBe(initial + 1); + const pollDelay = fixture.statusRequests[initial].at - fixture.statusRequests[initial - 1].at; + expect(pollDelay).toBeGreaterThanOrEqual(5_000); + expect(pollDelay).toBeLessThan(7_000); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await fixture.advance(5_000); + await expect.poll(() => fixture.statusRequests.length).toBe(initial + 2); + await page.evaluate(() => { + Object.defineProperty(document, 'visibilityState', { + configurable: true, + get: () => 'hidden', + }); + document.dispatchEvent(new Event('visibilitychange')); + }); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + const hiddenCount = fixture.statusRequests.length; + await fixture.advance(20_000); + await page.evaluate(() => window.dispatchEvent(new Event('focus'))); + expect(fixture.statusRequests).toHaveLength(hiddenCount); + const resumed = deferred(); + fixture.setReply(() => resumed.promise); + await page.evaluate(() => { + Object.defineProperty(document, 'visibilityState', { + configurable: true, + get: () => 'visible', + }); + document.dispatchEvent(new Event('visibilitychange')); + }); + await expect.poll(() => fixture.statusRequests.length).toBeGreaterThan(hiddenCount); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + resumed.resolve(success(fixture.snapshot())); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await page.context().setOffline(true); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + const offlineCount = fixture.statusRequests.length; + await fixture.advance(20_000); + await page.evaluate(() => window.dispatchEvent(new Event('focus'))); + expect(fixture.statusRequests).toHaveLength(offlineCount); + fixture.setReply(() => success(fixture.snapshot())); + await page.context().setOffline(false); + await expect.poll(() => fixture.statusRequests.length).toBeGreaterThan(offlineCount); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await fixture.refresh(); + }); + + for (const code of ['INTERNAL_SERVER_ERROR', 'UNAUTHORIZED', 'FORBIDDEN']) { + test(`suppresses cached status and estimates immediately on ${code}`, async ({ page }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + await debugSummary(page).click(); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues(page, runtimeMetadata.kiloCliVersion, runtimeMetadata.wrapperVersion); + fixture.setReply(() => failure(code)); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expect(details(page)).toContainText('does not mean the sandbox failed'); + await expect(sleepTime(page)).toHaveCount(0); + await expectUnavailableRuntime(page); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectSafe(page); + fixture.setReply(() => success(fixture.snapshot())); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + await expectDebugValues(page, runtimeMetadata.kiloCliVersion, runtimeMetadata.wrapperVersion); + }); + } + + test('waits two minutes of confirmed idle time before showing the sleep estimate', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success(fixture.snapshot({ estimatedSleepAt: fixture.statusRequests[0].at + 300_000 })) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(sleepTime(page)).toHaveCount(0); + await expect(detailValue(details(page), 'Started').locator('time')).toBeVisible(); + const deadline = fixture.statusRequests[0].at + 300_000; + const visibleAt = deadline - 300_000 + 120_000; + const pending = deferred(); + fixture.setReply(() => pending.promise); + await fixture.refresh(); + await page.clock.pauseAt((await page.evaluate(() => Date.now())) + 1_000); + await fixture.advance(visibleAt - 1_000 - (await page.evaluate(() => Date.now()))); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + pending.resolve(success(await fixture.currentSnapshot({ estimatedSleepAt: deadline }))); + await settlePausedStatus(page, 'Active'); + await fixture.advance(visibleAt - 1 - (await page.evaluate(() => Date.now()))); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await expect(sleepTime(page)).toHaveCount(0); + const beforeEstimate = fixture.statusRequests.length; + await fixture.advance(1); + await expect(sleepTime(page)).toHaveAttribute('datetime', new Date(deadline).toISOString()); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + expect(fixture.statusRequests).toHaveLength(beforeEstimate); + }); + + test('updates the relative countdown at a minute boundary without another status response', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success(fixture.snapshot({ estimatedSleepAt: fixture.statusRequests[0].at + 130_000 })) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + const deadline = fixture.statusRequests[0].at + 130_000; + const pending = deferred(); + fixture.setReply(() => pending.promise); + await fixture.refresh(); + await page.clock.pauseAt((await page.evaluate(() => Date.now())) + 1_000); + await fixture.advance(deadline - 120_001 - (await page.evaluate(() => Date.now()))); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + await expect(sleepTime(page)).toHaveAttribute('datetime', new Date(deadline).toISOString()); + const beforeBoundary = fixture.statusRequests.length; + await fixture.advance(1); + await expect(sleepTime(page)).toHaveText('About 2 min if inactive'); + await expect(sleepTime(page)).toHaveAttribute('datetime', new Date(deadline).toISOString()); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + expect(fixture.statusRequests).toHaveLength(beforeBoundary); + }); + + test('warns in the final minute and remains Active at expiry until authoritative sleep', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(() => + success(fixture.snapshot({ estimatedSleepAt: fixture.statusRequests[0].at + 70_000 })) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(sleepTime(page)).toBeVisible(); + const deadline = fixture.statusRequests[0].at + 70_000; + const pending = deferred(); + fixture.setReply(() => pending.promise); + await fixture.refresh(); + await page.clock.pauseAt((await page.evaluate(() => Date.now())) + 1_000); + await fixture.advance(deadline - 60_001 - (await page.evaluate(() => Date.now()))); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await expect(sleepTime(page)).toHaveText('About 2 min if inactive'); + const beforeWarning = fixture.statusRequests.length; + await fixture.advance(1); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping soon'); + await expect(details(page).getByText('Sleeping soon', { exact: true })).toBeVisible(); + await expect(sleepTime(page)).toHaveText('About 1 min if inactive'); + await expect(sleepTime(page)).toHaveAttribute('datetime', new Date(deadline).toISOString()); + await expectStaticIndicator(page); + expect(fixture.statusRequests).toHaveLength(beforeWarning); + await fixture.advance(deadline - 1_000 - (await page.evaluate(() => Date.now()))); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + pending.resolve(success(await fixture.currentSnapshot({ estimatedSleepAt: deadline }))); + await settlePausedStatus(page, 'Sleeping soon'); + await fixture.advance(deadline - 1 - (await page.evaluate(() => Date.now()))); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping soon'); + await expect(sleepTime(page)).toHaveText('About 1 min if inactive'); + const beforeExpiry = fixture.statusRequests.length; + await fixture.advance(1); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await expect(sleepTime(page)).toHaveCount(0); + expect(fixture.statusRequests).toHaveLength(beforeExpiry); + await page.clock.resume(); + fixture.setReply(() => + success( + fixture.snapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + estimatedSleepAt: null, + runtime: { ...runtimeMetadata, stoppedAt: deadline }, + }) + ) + ); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await expect(detailValue(details(page), 'Stopped').locator('time')).toHaveAttribute( + 'datetime', + new Date(deadline).toISOString() + ); + }); + + test('expires hung and stale observations within fifteen seconds without inventing sleep', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + fixture.setReply(request => + success(fixture.snapshot({ estimatedSleepAt: request.at + 2_000 })) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping soon'); + await indicator(page).click(); + await expect(sleepTime(page)).toBeVisible(); + const pending = deferred(); + fixture.setReply(() => pending.promise); + await fixture.advance(2_000); + await expect(sleepTime(page)).toHaveCount(0); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + const freshUntil = fixture.statusRequests[0].at + 15_000; + const beforeExpiry = freshUntil - 2_000 - (await page.evaluate(() => Date.now())); + await fixture.advance(Math.max(0, beforeExpiry)); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + const remaining = freshUntil - (await page.evaluate(() => Date.now())); + await fixture.advance(Math.max(0, remaining)); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + pending.resolve(success(fixture.snapshot({ observedAt: baseTime }))); + await expect(details(page)).toContainText('out of date'); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expectUnavailableRuntime(page); + fixture.setReply(() => success(fixture.snapshot())); + await fixture.refresh(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + }); + + test('expires a hung observation when its deadline callback sees an earlier wall clock', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(sleepTime(page)).toBeVisible(); + const pending = deferred(); + fixture.setReply(() => pending.promise); + await fixture.refresh(); + const pendingRequestCount = fixture.statusRequests.length; + const observedAt = fixture.statusRequests[0].at; + await page.clock.setFixedTime(observedAt + 14_999); + await fixture.advance(20_000); + expect(await page.evaluate(() => Date.now())).toBe(observedAt + 14_999); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await page.clock.setFixedTime(observedAt + 20_000); + await fixture.advance(10_000); + expect(fixture.statusRequests).toHaveLength(pendingRequestCount); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expect(details(page)).toContainText('out of date'); + await expectUnavailableRuntime(page); + await expect(sleepTime(page)).toHaveCount(0); + pending.resolve(success(await fixture.currentSnapshot())); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await expect(sleepTime(page)).toBeVisible(); + }); + + test('invalidates idle estimates during activity until fresh idle evidence arrives', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(sleepTime(page)).toBeVisible(); + await fixture.advance(1_000); + await fixture.activity(firstWorkspace, 'busy'); + await expect(sleepTime(page)).toHaveCount(0); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await fixture.advance(1_000); + await fixture.activity(firstWorkspace, 'idle'); + await expect(sleepTime(page)).toHaveCount(0); + await fixture.refresh(); + await expect(sleepTime(page)).toBeVisible(); + fixture.setReply(() => success(fixture.snapshot({ estimatedSleepAt: null }))); + await fixture.refresh(); + await expect(sleepTime(page)).toHaveCount(0); + }); + + test('isolates sessions when delayed requests resolve out of order and on return navigation', async ({ + page, + }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + const pending = deferred(); + fixture.setReply(request => + request.cloudAgentSessionId === firstWorkspace + ? pending.promise + : success( + fixture.snapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + provider: 'Vercel', + estimatedSleepAt: null, + }) + ) + ); + await fixture.refresh(); + await fixture.navigate(secondId); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + pending.resolve( + success( + fixture.snapshot({ status: 'error', detailCode: 'sandbox_failed', estimatedSleepAt: null }) + ) + ); + await fixture.advance(1); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await indicator(page).click(); + await expect(detailValue(details(page), 'Provider')).toHaveText('Vercel'); + const returning = deferred(); + fixture.setReply(() => returning.promise); + await fixture.navigate(firstId); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + returning.resolve(success(await fixture.currentSnapshot())); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + expect(fixture.statusRequests.some(r => r.cloudAgentSessionId === secondWorkspace)).toBe(true); + }); + + test('isolates personal and organization status in one QueryClient across pending requests and return navigation', async ({ + page, + }) => { + await withOrganization(page, async organizationId => { + const organizationName = 'Sandbox status browser regression organization'; + const session: SessionFixture = { + id: firstId, + cloudId: firstWorkspace, + title: 'Scoped sandbox chat', + }; + const fixture = await mountFixtures(page, [session]); + await fixture.open(); + const originalDocument = await page.evaluateHandle(() => document); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).click(); + await expect(detailValue(details(page), 'Provider')).toHaveText('Cloudflare'); + await expect(sleepTime(page)).toBeVisible(); + await page.keyboard.press('Escape'); + + async function switchScope(toOrganization: boolean) { + session.organizationId = toOrganization ? organizationId : undefined; + await page + .getByRole('button', { + name: toOrganization ? 'Personal Personal Workspace' : `${organizationName} Member`, + exact: true, + }) + .click(); + await page + .getByRole('menuitem', { + name: toOrganization ? `${organizationName} Member` : 'Personal Personal Workspace', + exact: true, + }) + .click(); + const basePath = toOrganization ? `/organizations/${organizationId}` : ''; + await page.locator(`a[href="${basePath}/cloud"]`).click(); + await page.getByText(session.title, { exact: true }).click(); + await expect(page).toHaveURL(`${basePath}/cloud/chat?sessionId=${firstId}`); + expect(await originalDocument.evaluate(node => node === document)).toBe(true); + } + + async function expectPendingUnknown() { + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await indicator(page).click(); + await expectUnavailableRuntime(page); + await expect(sleepTime(page)).toHaveCount(0); + } + + const personalPending = deferred(); + const organizationInitial = deferred(); + fixture.setReply(request => + request.organizationId ? organizationInitial.promise : personalPending.promise + ); + await fixture.refresh(); + await switchScope(true); + await expectPendingUnknown(); + personalPending.resolve(success(await fixture.currentSnapshot())); + await fixture.advance(250); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expectUnavailableRuntime(page); + await expect(sleepTime(page)).toHaveCount(0); + organizationInitial.resolve( + success( + await fixture.currentSnapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + provider: 'Vercel', + estimatedSleepAt: null, + }) + ) + ); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await expect(detailValue(details(page), 'Provider')).toHaveText('Vercel'); + await expect(sleepTime(page)).toHaveCount(0); + await page.keyboard.press('Escape'); + + const organizationPending = deferred(); + const personalReturning = deferred(); + fixture.setReply(request => + request.organizationId ? organizationPending.promise : personalReturning.promise + ); + await fixture.refresh(); + await switchScope(false); + await expectPendingUnknown(); + organizationPending.resolve(success(await fixture.currentSnapshot({ provider: 'Vercel' }))); + await fixture.advance(250); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unknown'); + await expectUnavailableRuntime(page); + await expect(sleepTime(page)).toHaveCount(0); + personalReturning.resolve(success(await fixture.currentSnapshot())); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await expect(detailValue(details(page), 'Provider')).toHaveText('Cloudflare'); + await expect(sleepTime(page)).toBeVisible(); + await page.keyboard.press('Escape'); + + const organizationReturning = deferred(); + fixture.setReply(() => organizationReturning.promise); + await switchScope(true); + await expectPendingUnknown(); + organizationReturning.resolve( + success( + await fixture.currentSnapshot({ + status: 'sleeping', + detailCode: 'sandbox_stopped', + provider: 'Vercel', + estimatedSleepAt: null, + }) + ) + ); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Sleeping'); + await expect(detailValue(details(page), 'Provider')).toHaveText('Vercel'); + await expect(sleepTime(page)).toHaveCount(0); + expect( + fixture.statusRequests.some(request => request.organizationId === organizationId) + ).toBe(true); + expect( + fixture.statusRequests.every(request => request.cloudAgentSessionId === firstWorkspace) + ).toBe(true); + expect(fixture.procedures).toContain('organizations.cloudAgentNext.getSandboxStatus'); + expect(fixture.procedures).toContain('cloudAgentNext.getSandboxStatus'); + await originalDocument.dispose(); + }); + }); + + test('never requests status for excluded contexts, including activity, focus, reconnect and navigation', async ({ + page, + }) => { + const sessions: SessionFixture[] = [ + { id: firstId, cloudId: firstWorkspace, title: 'Eligible' }, + { id: 'ses_legacy', cloudId: 'agent_00000000-0000-4000-8000-000000000003', title: 'Legacy' }, + { id: 'ses_placeholder', cloudId: 'Starting session…', title: 'Placeholder' }, + { id: 'ses_remote', cloudId: secondWorkspace, title: 'Remote', kind: 'remote' }, + { id: 'ses_readonly', cloudId: secondWorkspace, title: 'Read only', kind: 'read-only' }, + { + id: 'ses_wrongscope', + cloudId: secondWorkspace, + title: 'Different organization', + organizationId: '00000000-0000-4000-8000-000000000099', + }, + { id: 'ses_unrelated', cloudId: null, title: 'Unrelated', kind: 'unrelated' }, + { id: 'ses_unresolved', cloudId: secondWorkspace, title: 'Unresolved', kind: 'unresolved' }, + ]; + const fixture = await mountFixtures(page, sessions); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + for (const session of sessions.slice(1)) { + await fixture.navigate(session.id); + await expect(indicator(page)).toHaveCount(0); + const count = fixture.statusRequests.length; + await page.evaluate(() => { + window.dispatchEvent(new Event('focus')); + window.dispatchEvent(new Event('offline')); + window.dispatchEvent(new Event('online')); + }); + await fixture.advance(10_000); + expect(fixture.statusRequests).toHaveLength(count); + await expect(indicator(page)).toHaveCount(0); + if (session.id === 'ses_legacy') { + await fixture.activity(session.cloudId ?? '', 'busy'); + await fixture.activity(session.cloudId ?? '', 'idle'); + await fixture.advance(5_000); + expect(fixture.statusRequests).toHaveLength(count); + } + } + await page.goto('/cloud'); + await fixture.advance(10_000); + await expect(indicator(page)).toHaveCount(0); + expect(fixture.statusRequests.every(r => r.cloudAgentSessionId === firstWorkspace)).toBe(true); + }); + + for (const width of [375, 820, 1440]) { + test(`keeps a touch-safe indicator and existing controls usable at ${width}px`, async ({ + page, + }, testInfo) => { + await page.setViewportSize({ width, height: width === 375 ? 812 : 1000 }); + const fixture = await mountFixtures(page, [ + { + id: firstId, + cloudId: firstWorkspace, + title: + 'A very long workspace title that must not obstruct sandbox status or other chat controls', + }, + ]); + fixture.setReply(() => + success( + fixture.snapshot({ + status: 'unreachable', + detailCode: 'connection_unavailable', + estimatedSleepAt: null, + }) + ) + ); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Unreachable'); + await page.getByRole('button', { name: 'New terminal' }).focus(); + await indicator(page).focus(); + await page.keyboard.press('Enter'); + await fixture.advance(250); + await expect(details(page)).toBeVisible(); + const target = await indicator(page).boundingBox(); + expect(target?.width).toBeGreaterThanOrEqual(44); + expect(target?.height).toBeGreaterThanOrEqual(44); + await expectStaticIndicator(page); + await expectDebugCollapsed(page); + for (const expanded of [false, true]) { + if (expanded) { + await debugSummary(page).click(); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues( + page, + runtimeMetadata.kiloCliVersion, + runtimeMetadata.wrapperVersion + ); + } + await expect(details(page)).toHaveCSS('padding', '16px'); + await expect + .poll(async () => (await details(page).boundingBox())?.width) + .toBe(Math.min(320, width - 32)); + const popup = await details(page).boundingBox(); + expect(popup?.x).toBeGreaterThanOrEqual(0); + expect((popup?.x ?? 0) + (popup?.width ?? 0)).toBeLessThanOrEqual(width); + expect( + await details(page).evaluate(element => element.scrollWidth <= element.clientWidth) + ).toBe(true); + await expectSafe(page); + await page.screenshot({ + path: testInfo.outputPath( + `controlled-header-${width}-${expanded ? 'expanded' : 'collapsed'}.png` + ), + animations: 'disabled', + }); + } + for (const name of [ + 'New terminal', + 'Mute completion sounds', + 'More options', + 'Send feedback', + ]) { + const control = page.getByRole('button', { name, exact: true }); + await expect(control).toBeVisible(); + const box = await control.boundingBox(); + expect(box?.x).toBeGreaterThanOrEqual(0); + expect((box?.x ?? 0) + (box?.width ?? 0)).toBeLessThanOrEqual(width); + } + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe( + true + ); + await expectSafe(page); + await page.screenshot({ path: testInfo.outputPath(`controlled-header-${width}.png`) }); + }); + } + + test.describe('touch', () => { + test.use({ hasTouch: true, viewport: { width: 375, height: 812 } }); + test('expands Debug with taps and reopens safe details collapsed', async ({ page }) => { + const fixture = await mountFixtures(page); + await fixture.open(); + await expect(indicator(page)).toHaveAccessibleName('Sandbox status: Active'); + await indicator(page).tap(); + await fixture.advance(250); + await expect(details(page)).toBeVisible(); + await expect(page.getByRole('tooltip')).toHaveCount(0); + await expect(detailValue(details(page), 'Provider')).toHaveText('Cloudflare'); + await expect(sleepTime(page)).toHaveText('About 3 min if inactive'); + await expectDebugCollapsed(page); + await debugSummary(page).tap(); + await expect(debugDisclosure(page)).toHaveJSProperty('open', true); + await expect(debugDisclosure(page).locator('dl')).toBeVisible(); + await expectDebugValues(page, runtimeMetadata.kiloCliVersion, runtimeMetadata.wrapperVersion); + await expect(page.getByRole('tooltip')).toHaveCount(0); + await expect(details(page)).toHaveCSS('width', '320px'); + expect( + await details(page).evaluate(element => element.scrollWidth <= element.clientWidth) + ).toBe(true); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe( + true + ); + await expectSafe(page); + await debugSummary(page).tap(); + await expectDebugCollapsed(page); + await debugSummary(page).tap(); + await expect(debugDisclosure(page)).toHaveJSProperty('open', true); + await page.getByRole('combobox', { name: 'Ask anything…' }).tap(); + await expect(details(page)).toHaveCount(0); + await fixture.advance(250); + await indicator(page).tap(); + await fixture.advance(250); + await expectDebugCollapsed(page); + await expect(page.getByRole('tooltip')).toHaveCount(0); + }); + }); +}); diff --git a/services/cloud-agent-next/src/middleware/balance.test.ts b/services/cloud-agent-next/src/middleware/balance.test.ts index 474fb89af9..9102595f3e 100644 --- a/services/cloud-agent-next/src/middleware/balance.test.ts +++ b/services/cloud-agent-next/src/middleware/balance.test.ts @@ -1,30 +1,30 @@ import { Hono } from 'hono'; +import { trpcServer } from '@hono/trpc-server'; +import { TRPCError } from '@trpc/server'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { HonoContext } from '../hono-context.js'; import type { Env } from '../types.js'; +import type * as BalanceValidationModule from '../balance-validation.js'; +import type * as LoggerModule from '../logger.js'; const { requireCurrentSessionAccessMock } = vi.hoisted(() => ({ requireCurrentSessionAccessMock: vi.fn(), })); -vi.mock('../balance-validation.js', () => ({ - BALANCE_REQUIRED_MUTATIONS: new Set([ - 'initiateFromKilocodeSessionV2', - 'sendMessageV2', - 'start', - 'send', - ]), - extractProcedureName: (pathname: string) => { - const match = pathname.match(/^\/trpc\/([^?/]+)/); - return match ? match[1] : null; - }, +vi.mock('../balance-validation.js', async importOriginal => ({ + ...(await importOriginal()), fetchOrgIdForSession: vi.fn(), validateBalanceOnly: vi.fn(), })); -vi.mock('../logger.js', () => ({ +vi.mock('@cloudflare/sandbox', () => ({ getSandbox: vi.fn() })); + +vi.mock('../logger.js', async importOriginal => ({ + ...(await importOriginal()), logger: { withFields: () => ({ info: vi.fn(), warn: vi.fn() }), + warn: vi.fn(), + error: vi.fn(), }, })); @@ -35,6 +35,9 @@ vi.mock('../session-access.js', () => ({ const { balanceMiddleware } = await import('./balance.js'); const { fetchOrgIdForSession, validateBalanceOnly } = await import('../balance-validation.js'); +const { createSessionManagementHandlers } = + await import('../router/handlers/session-management.js'); +const { router } = await import('../router/auth.js'); describe('balanceMiddleware', () => { const env = {} as Env; @@ -73,6 +76,98 @@ describe('balanceMiddleware', () => { ); } + it('serves the real passive status query without credits or a balance bypass', async () => { + vi.mocked(validateBalanceOnly).mockResolvedValue({ + success: false, + status: 402, + message: 'Insufficient credits', + }); + const snapshot = { + status: 'sleeping', + provider: 'Cloudflare', + observedAt: Date.now(), + detailCode: 'sandbox_stopped', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: null, + }; + const getSandboxStatus = vi.fn().mockResolvedValue(snapshot); + const legacyLookup = vi.fn(); + const sandboxLookup = vi.fn(() => ({ getSandboxStatus })); + const statusEnv = { + SANDBOX_SESSION: { idFromName: (name: string) => name, get: sandboxLookup }, + CLOUD_AGENT_SESSION: { idFromName: legacyLookup, get: legacyLookup }, + } as unknown as Env; + const app = new Hono(); + app.use('/trpc/*', async (c, next) => { + c.set('userId', 'user-123'); + c.set('authToken', 'token-123'); + await next(); + }); + app.use('/trpc/*', balanceMiddleware); + app.use( + '/trpc/*', + trpcServer({ + router: router({ getSandboxStatus: createSessionManagementHandlers().getSandboxStatus }), + createContext: (_opts, c) => ({ + env: c.env, + userId: c.get('userId'), + authToken: c.get('authToken'), + request: c.req.raw, + }), + }) + ); + const input = encodeURIComponent( + JSON.stringify({ + cloudAgentSessionId: 'workspace_12345678-1234-4234-9234-123456789abc', + }) + ); + const request = () => new Request(`https://worker.test/trpc/getSandboxStatus?input=${input}`); + const response = await app.fetch(request(), statusEnv); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ result: { data: snapshot } }); + expect(getSandboxStatus).toHaveBeenCalledTimes(1); + expect(requireCurrentSessionAccessMock).toHaveBeenCalledTimes(1); + expect(validateBalanceOnly).not.toHaveBeenCalled(); + expect(legacyLookup).not.toHaveBeenCalled(); + + requireCurrentSessionAccessMock.mockRejectedValueOnce( + new TRPCError({ code: 'FORBIDDEN', message: 'Session access denied' }) + ); + const denied = await app.fetch(request(), statusEnv); + expect(denied.status).toBe(403); + expect(getSandboxStatus).toHaveBeenCalledTimes(1); + expect(sandboxLookup).toHaveBeenCalledTimes(1); + expect(validateBalanceOnly).not.toHaveBeenCalled(); + }); + + it.each(['start', 'send', 'sendMessageV2', 'initiateFromKilocodeSessionV2'])( + 'still rejects paid %s execution without credits', + async procedure => { + vi.mocked(validateBalanceOnly).mockResolvedValue({ + success: false, + status: 402, + message: 'Insufficient credits', + }); + const response = await postTrpc(procedure, { + cloudAgentSessionId: 'workspace_12345678-1234-4234-9234-123456789abc', + }); + expect(response.status).toBe(402); + if (procedure !== 'start') expect(requireCurrentSessionAccessMock).toHaveBeenCalledTimes(1); + expect(validateBalanceOnly).toHaveBeenCalledTimes(1); + } + ); + + it('still checks execution ownership before the paid send balance', async () => { + requireCurrentSessionAccessMock.mockRejectedValueOnce( + new TRPCError({ code: 'FORBIDDEN', message: 'Session access denied' }) + ); + const response = await postTrpc('send', { + cloudAgentSessionId: 'workspace_12345678-1234-4234-9234-123456789abc', + }); + expect(response.status).toBe(403); + expect(validateBalanceOnly).not.toHaveBeenCalled(); + }); + it('returns non-retryable clientError for insufficient credits', async () => { vi.mocked(validateBalanceOnly).mockResolvedValue({ success: false, diff --git a/services/cloud-agent-next/src/persistence/SandboxControl.ts b/services/cloud-agent-next/src/persistence/SandboxControl.ts index 5b5a7aa2bd..faa34690ed 100644 --- a/services/cloud-agent-next/src/persistence/SandboxControl.ts +++ b/services/cloud-agent-next/src/persistence/SandboxControl.ts @@ -30,6 +30,7 @@ import { getSandboxSessionStub } from '../sandbox-session/session-stub.js'; import { createSandboxControlSocketHandler, type SandboxControlConnectionIdentity, + readSandboxControlConnection, type SandboxControlOutboundRequest, type SandboxControlSocketHandler, } from '../sandbox-control/socket.js'; @@ -90,6 +91,7 @@ import { } from '../sandbox-control/session-routes.js'; import { projectReportedStatus, + projectSandboxStatus, type ConnectionState, type ReportedSandboxStatus, type WorkState, @@ -108,8 +110,12 @@ import { eraseSandboxRecord, loadDeadlines, loadPhysicalRecord, + initialRuntimeMetadata, + loadRuntimeMetadata, + saveRuntimeMetadata, loadRouteTable, loadTransitionLog, + readSandboxControlState, saveDeadlines, savePhysicalRecord, saveRouteTable, @@ -177,6 +183,11 @@ import { type SandboxTerminalAccessResult, } from '../sandbox-control/terminal-billing.js'; import type { AgentSandboxProvider } from '../types.js'; +import { + safeSandboxRuntimeVersion, + type SandboxRuntimeMetadata, + type SandboxStatusSnapshot, +} from '../shared/sandbox-status.js'; const CREDENTIAL_HASH_KEY = 'wrapper_credential_hash'; const OWNER_ID_KEY = 'owner_id'; @@ -285,6 +296,7 @@ export class SandboxControl extends DurableObject { private readonly readinessOperations = new Set>(); private readonly lifecycleOperations = new Set>(); private worktreeDeletionChain: Promise = Promise.resolve(); + private operationalInitialization: Promise | null = null; constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); @@ -295,7 +307,7 @@ export class SandboxControl extends DurableObject { ); this.socketHandler = createSandboxControlSocketHandler(ctx, this.sandboxId, undefined, { validateHandshake: providerInstanceId => this.validateHandshake(providerInstanceId), - onHandshakeComplete: identity => this.onHandshakeComplete(identity), + onHandshakeComplete: (identity, runtime) => this.onHandshakeComplete(identity, runtime), onReady: identity => this.onWrapperReady(identity), onHeartbeat: (payload, identity) => this.onHeartbeat(payload, identity), onSessionEvent: (sessionIdentity, payload, identity) => @@ -305,7 +317,11 @@ export class SandboxControl extends DurableObject { onSocketClosed: (handshakeComplete, identity) => this.onSocketClosed(handshakeComplete, identity), }); - void ctx.blockConcurrencyWhile(async () => { + } + + private ensureOperationalInitialized(): Promise { + return (this.operationalInitialization ??= this.ctx.blockConcurrencyWhile(async () => { + const ctx = this.ctx; const [readyAt, runtime, kind, physical] = await Promise.all([ ctx.storage.get(WRAPPER_READY_AT_KEY), ctx.storage.get(ACTIVE_WRAPPER_RUNTIME_KEY), @@ -348,10 +364,11 @@ export class SandboxControl extends DurableObject { } else { this.kiloReady = !runtime && current !== null && readyAt !== undefined; } - }); + })); } async fetch(request: Request): Promise { + await this.ensureOperationalInitialized(); const upgrade = request.headers.get('Upgrade'); if (upgrade?.toLowerCase() !== 'websocket') { return new Response('Expected WebSocket upgrade', { status: 426 }); @@ -369,6 +386,7 @@ export class SandboxControl extends DurableObject { } async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise { + await this.ensureOperationalInitialized(); if (this.runtimeDeleted) return; await this.trackLifecycleOperation( Promise.resolve(this.socketHandler.handleMessage(ws, message)) @@ -376,6 +394,7 @@ export class SandboxControl extends DurableObject { } async webSocketClose(ws: WebSocket): Promise { + await this.ensureOperationalInitialized(); if (this.runtimeDeleted) return; await this.trackLifecycleOperation(Promise.resolve(this.socketHandler.handleClose(ws))); } @@ -385,6 +404,7 @@ export class SandboxControl extends DurableObject { } async alarm(): Promise { + await this.ensureOperationalInitialized(); if (this.runtimeDeleted) return; await this.trackLifecycleOperation(this.runAlarm()); } @@ -430,6 +450,7 @@ export class SandboxControl extends DurableObject { } async setWrapperCredentialHash(hash: string): Promise { + await this.ensureOperationalInitialized(); if (!/^[0-9a-f]{64}$/.test(hash)) { throw new Error('Invalid wrapper credential hash'); } @@ -462,6 +483,7 @@ export class SandboxControl extends DurableObject { } async initializeOwner(ownerId: string): Promise<{ ownerId: string }> { + await this.ensureOperationalInitialized(); const normalized = typeof ownerId === 'string' ? ownerId.trim() : ''; if (normalized.length === 0) { throw new Error('ownerId must be a non-empty string'); @@ -480,10 +502,12 @@ export class SandboxControl extends DurableObject { } async getOwner(): Promise { + await this.ensureOperationalInitialized(); return this.readOwner(); } async request(input: SandboxControlOutboundRequest): Promise { + await this.ensureOperationalInitialized(); await this.assertRequestWorktreeAdmission(input); if (input.operation === 'worktree.delete' || input.operation === 'worktree.prepareDeletion') { throw new Error('Worktree cleanup requires the deletion coordinator'); @@ -580,6 +604,7 @@ export class SandboxControl extends DurableObject { wrapperInstanceId: string; reason: string; }): Promise<{ quarantined: boolean }> { + await this.ensureOperationalInitialized(); if ( typeof input.ownerId !== 'string' || !input.ownerId || @@ -730,6 +755,7 @@ export class SandboxControl extends DurableObject { url: string; method: string; }): Promise<{ credential: string; organizationId?: string } | null> { + await this.ensureOperationalInitialized(); return this.withCredentialUpdate(async () => { try { const alias = parseControlPlaneCredential(input.credential); @@ -810,6 +836,7 @@ export class SandboxControl extends DurableObject { private async runEnsureReady( input: Parameters[0] ): Promise { + await this.ensureOperationalInitialized(); this.assertWorktreeAdmission(input.worktreeId); const acquisition = input.acquisition === undefined @@ -1255,6 +1282,7 @@ export class SandboxControl extends DurableObject { } async detachSession(sessionId: string): Promise<{ existed: boolean }> { + await this.ensureOperationalInitialized(); const route = (await loadRouteTable(this.ctx.storage)).get(sessionId); let runtimeDetached = false; let existed = false; @@ -1585,6 +1613,7 @@ export class SandboxControl extends DurableObject { } async listRoutes(): Promise { + await this.ensureOperationalInitialized(); const table = await loadRouteTable(this.ctx.storage); return [...table.values()]; } @@ -1720,6 +1749,7 @@ export class SandboxControl extends DurableObject { } async getStatus(): Promise { + await this.ensureOperationalInitialized(); return this.statusForPhysical(await loadPhysicalRecord(this.ctx.storage)); } @@ -1738,11 +1768,41 @@ export class SandboxControl extends DurableObject { }; } + async getSandboxStatus(input: { + ownerId: string; + provider: AgentSandboxProvider; + }): Promise { + const [stored, ownerId, provider, runtime] = await Promise.all([ + readSandboxControlState(this.ctx.storage), + this.readOwner(), + this.ctx.storage.get(PROVIDER_KIND_KEY), + this.ctx.storage.get(ACTIVE_WRAPPER_RUNTIME_KEY), + ]); + const matches = + ownerId !== null && + ownerId === input.ownerId && + (provider === 'cloudflare' || provider === 'vercel') && + provider === input.provider; + return projectSandboxStatus({ + stored: matches ? stored : { physical: null, deadlines: null, routes: null }, + connection: readSandboxControlConnection( + this.ctx, + stored.physical?.providerRef ?? null, + runtime + ), + ownerId, + provider: matches ? provider : undefined, + now: Date.now(), + }); + } + async getPhysicalRecord(): Promise { + await this.ensureOperationalInitialized(); return loadPhysicalRecord(this.ctx.storage); } async getTransitionLog(): Promise { + await this.ensureOperationalInitialized(); return loadTransitionLog(this.ctx.storage); } @@ -1752,6 +1812,7 @@ export class SandboxControl extends DurableObject { allocationName?: string, containment?: CredentialContainmentRequirements ): Promise { + await this.ensureOperationalInitialized(); return this.ctx.storage.transaction(async () => { const current = await loadPhysicalRecord(this.ctx.storage, resumable); this.assertWorktreeAdmission(); @@ -1780,6 +1841,7 @@ export class SandboxControl extends DurableObject { } async confirmInstance(providerRef: string): Promise { + await this.ensureOperationalInitialized(); const current = await loadPhysicalRecord(this.ctx.storage); const next = confirmRunning(current, providerRef, Date.now()); await this.persistPhysical(current, next, 'instance confirmed'); @@ -1787,6 +1849,7 @@ export class SandboxControl extends DurableObject { } async observeProvider(result: ObserveResult): Promise { + await this.ensureOperationalInitialized(); const current = await loadPhysicalRecord(this.ctx.storage); let next = observe(current, result); if ((next.state === 'failed' || next.state === 'unknown') && !next.stopTombstone) { @@ -1808,6 +1871,7 @@ export class SandboxControl extends DurableObject { } async beginStop(reason: string): Promise { + await this.ensureOperationalInitialized(); const current = await loadPhysicalRecord(this.ctx.storage); if (current.state === 'stopped' || current.stopTombstone) return current; const next = beginStop(current, reason, Date.now(), this.activeConnection?.wrapperInstanceId); @@ -1816,6 +1880,7 @@ export class SandboxControl extends DurableObject { } async recordStopAttempt(): Promise { + await this.ensureOperationalInitialized(); const current = await loadPhysicalRecord(this.ctx.storage); if (this.stopAttemptInFlight && sameAllocation(this.stopAttemptInFlight.physical, current)) { this.logDiagnostic('stop_coalesced', { @@ -1983,6 +2048,7 @@ export class SandboxControl extends DurableObject { } async confirmStopped(): Promise { + await this.ensureOperationalInitialized(); const current = await loadPhysicalRecord(this.ctx.storage); const next = confirmStopped(current); await this.persistPhysical(current, next, 'terminal'); @@ -1990,6 +2056,7 @@ export class SandboxControl extends DurableObject { } async markFailed(): Promise { + await this.ensureOperationalInitialized(); const current = await loadPhysicalRecord(this.ctx.storage); const next = fail(current, Date.now()); await this.persistPhysical(current, next, 'failed'); @@ -2004,6 +2071,7 @@ export class SandboxControl extends DurableObject { } async eraseRecord(options?: { preserveAcquisitionReceipts: true }): Promise { + await this.ensureOperationalInitialized(); await eraseSandboxRecord(this.ctx.storage); await this.ctx.storage.delete([ OWNER_ID_KEY, @@ -2369,7 +2437,10 @@ export class SandboxControl extends DurableObject { ); } - private async onHandshakeComplete(identity: SandboxControlConnectionIdentity): Promise { + private async onHandshakeComplete( + identity: SandboxControlConnectionIdentity, + runtime?: Pick + ): Promise { const socketConnection = this.socketHandler.getConnectionIdentity(); if (!socketConnection || !this.sameConnection(socketConnection, identity)) return; @@ -2390,15 +2461,34 @@ export class SandboxControl extends DurableObject { return; } const now = Date.now(); - await this.ctx.storage.transaction(async () => { + const activated = await this.ctx.storage.transaction(async () => { + const [current, storedRuntime] = await Promise.all([ + loadPhysicalRecord(this.ctx.storage), + loadRuntimeMetadata(this.ctx.storage), + ]); + const connection = this.socketHandler.getConnectionIdentity(); + if ( + !connection || + !this.sameConnection(connection, identity) || + !sameAllocation(current, physical) || + current.stopTombstone || + (current.state !== 'creating' && current.state !== 'running') || + !this.matchesProviderReference(current, identity.providerInstanceId) + ) + return false; + await saveRuntimeMetadata(this.ctx.storage, { + ...(storedRuntime ?? initialRuntimeMetadata(this.sandboxId)), + wrapperVersion: safeSandboxRuntimeVersion(runtime?.wrapperVersion), + kiloCliVersion: null, + }); await this.ctx.storage.put(ACTIVE_WRAPPER_RUNTIME_KEY, identity); await this.ctx.storage.delete(WRAPPER_READY_AT_KEY); - if (physical.state === 'creating') { + if (current.state === 'creating') { const providerRef = - physical.providerRef ?? + current.providerRef ?? (this.providerKind === 'cloudflare' ? identity.providerInstanceId : undefined); if (providerRef !== undefined) { - await savePhysicalRecord(this.ctx.storage, confirmRunning(physical, providerRef, now)); + await savePhysicalRecord(this.ctx.storage, confirmRunning(current, providerRef, now)); await this.appendLog( physicalTransition(now, physical.state, 'running', 'hello', providerRef) ); @@ -2414,7 +2504,9 @@ export class SandboxControl extends DurableObject { await saveDeadlines(this.ctx.storage, deadlines); await this.scheduleAlarm(deadlines); await this.appendLog(connectionTransition(now, 'disconnected', 'connected', 'hello')); + return true; }); + if (!activated) return; this.activeConnection = identity; this.readyConnectionId = null; this.kiloReady = false; @@ -2498,8 +2590,24 @@ export class SandboxControl extends DurableObject { const now = Date.now(); const applied = await this.ctx.storage .transaction(async () => { + const physical = await loadPhysicalRecord(this.ctx.storage); const table = await loadRouteTable(this.ctx.storage); - if (!this.isCurrentConnection(identity)) return; + if ( + !this.isCurrentConnection(identity) || + physical.state !== 'running' || + physical.stopTombstone || + physical.providerRef !== identity.providerInstanceId + ) + return; + if (payload.kilo.version !== undefined) { + const runtime = + (await loadRuntimeMetadata(this.ctx.storage)) ?? initialRuntimeMetadata(this.sandboxId); + const kiloCliVersion = safeSandboxRuntimeVersion(payload.kilo.version); + if (!this.isCurrentConnection(identity)) return; + if (runtime.kiloCliVersion !== kiloCliVersion) { + await saveRuntimeMetadata(this.ctx.storage, { ...runtime, kiloCliVersion }); + } + } const reported = new Map(payload.sessions.map(session => [session.kiloSessionId, session])); let missingRoutes = 0; let activeRoutes = 0; @@ -2988,6 +3096,7 @@ export class SandboxControl extends DurableObject { input: SandboxTerminalAccessInput, allowExpiredCredentials = false ): Promise { + await this.ensureOperationalInitialized(); if ( typeof input.sessionId !== 'string' || input.sessionId.length === 0 || @@ -3184,6 +3293,9 @@ export class SandboxControl extends DurableObject { const unavailable = to.stopTombstone !== null || (to.state !== 'creating' && to.state !== 'running'); await savePhysicalRecord(this.ctx.storage, to); + if (from.state === 'stopped' && to.state === 'creating') { + await saveRuntimeMetadata(this.ctx.storage, initialRuntimeMetadata(this.sandboxId)); + } if (to.state === 'stopped') { await saveSessionCredentialGrants(this.ctx.storage, []); await this.ctx.storage.delete(CREDENTIAL_POLICY_DIRTY_KEY); @@ -3357,6 +3469,7 @@ export class SandboxControl extends DurableObject { } private async requireOwner(): Promise { + await this.ensureOperationalInitialized(); const ownerId = await this.readOwner(); if (ownerId === null) throw new Error('Sandbox owner is not initialized'); return ownerId; diff --git a/services/cloud-agent-next/src/router.test.ts b/services/cloud-agent-next/src/router.test.ts index 870e5dc62b..5a0bbec55e 100644 --- a/services/cloud-agent-next/src/router.test.ts +++ b/services/cloud-agent-next/src/router.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { TRPCError } from '@trpc/server'; // Mock Cloudflare sandbox to prevent module resolution errors @@ -91,6 +91,8 @@ import type { Env, TRPCContext, SessionId } from './types.js'; import type { CloudAgentSessionState } from './persistence/types.js'; import { parseSessionMetadata } from './persistence/session-metadata.js'; import { WrapperClient } from './kilo/wrapper-client.js'; +import { logger } from './logger.js'; +import type { SandboxStatusSnapshot } from './shared/sandbox-status.js'; type MockSessionStub = { deleteSession?: ReturnType; @@ -1870,6 +1872,223 @@ describe('router sessionId validation', () => { }); }); +describe('getSandboxStatus procedure', () => { + const cloudAgentSessionId = 'workspace_12345678-1234-4234-9234-123456789abc'; + const snapshot = { + status: 'active', + provider: 'Cloudflare', + observedAt: 1_800_000_000_000, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: 1_800_000_060_000, + } satisfies SandboxStatusSnapshot; + const unavailable = { + status: 'unknown', + provider: 'Unknown', + observedAt: expect.any(Number), + detailCode: 'status_unavailable', + inactivityTimeoutMs: null, + estimatedSleepAt: null, + }; + const getSandboxStatus = vi.fn<() => Promise>(); + const sandboxSession = { + idFromName: vi.fn((name: string) => name), + get: vi.fn(() => ({ getSandboxStatus })), + }; + const legacySession = { idFromName: vi.fn(), get: vi.fn() }; + let context: TRPCContext; + let caller: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + getSandboxStatus.mockReset().mockResolvedValue(snapshot); + sandboxSession.get.mockReset().mockImplementation(() => ({ getSandboxStatus })); + sandboxSession.idFromName.mockReset().mockImplementation(name => name); + requireCurrentSessionAccessMock.mockResolvedValue({ + kiloSessionId: 'ses_12345678901234567890123456', + organizationId: null, + }); + context = { + userId: 'oauth/provider:status-owner', + authToken: 'test-token', + request: new Request('https://worker.test/trpc/getSandboxStatus'), + env: { + SANDBOX_SESSION: sandboxSession, + CLOUD_AGENT_SESSION: legacySession, + }, + } as unknown as TRPCContext; + caller = appRouter.createCaller(context); + }); + + afterEach(() => { + expect(legacySession.idFromName).not.toHaveBeenCalled(); + expect(legacySession.get).not.toHaveBeenCalled(); + expect(getSandbox).not.toHaveBeenCalled(); + expect(fetchSessionMetadata).not.toHaveBeenCalled(); + expect(getOrCreateSessionMock).not.toHaveBeenCalled(); + vi.restoreAllMocks(); + }); + + it.each([null, '11111111-2222-4333-8444-555555555555'])( + 'authorizes current scope %s before any passive lookup', + async organizationId => { + requireCurrentSessionAccessMock.mockImplementationOnce(async () => { + expect(sandboxSession.idFromName).not.toHaveBeenCalled(); + expect(getSandboxStatus).not.toHaveBeenCalled(); + return { kiloSessionId: 'ses_12345678901234567890123456', organizationId }; + }); + await expect(caller.getSandboxStatus({ cloudAgentSessionId })).resolves.toEqual(snapshot); + expect(requireCurrentSessionAccessMock).toHaveBeenCalledWith({ + env: context.env, + kiloUserId: context.userId, + cloudAgentSessionId, + }); + expect(sandboxSession.idFromName).toHaveBeenCalledWith( + `oauth/provider:status-owner:${cloudAgentSessionId}` + ); + expect(getSandboxStatus).toHaveBeenCalledWith(); + } + ); + + it('requires authentication before authorization or DO access', async () => { + const unauthenticated = appRouter.createCaller({ + ...context, + userId: '', + authToken: '', + }); + await expect(unauthenticated.getSandboxStatus({ cloudAgentSessionId })).rejects.toMatchObject({ + code: 'UNAUTHORIZED', + }); + expect(requireCurrentSessionAccessMock).not.toHaveBeenCalled(); + expect(sandboxSession.idFromName).not.toHaveBeenCalled(); + }); + + it.each([ + 'agent_12345678-1234-4234-9234-123456789abc', + 'ses_12345678901234567890123456', + 'sess_12345678-1234-4234-9234-123456789abc', + 'workspace_', + 'workspace_pending', + `${cloudAgentSessionId} `, + `${cloudAgentSessionId}\n`, + 'workspace_../../private', + '', + ])('rejects ineligible reference %s before any lookup', async invalidId => { + await expect(caller.getSandboxStatus({ cloudAgentSessionId: invalidId })).rejects.toMatchObject( + { code: 'BAD_REQUEST' } + ); + expect(requireCurrentSessionAccessMock).not.toHaveBeenCalled(); + expect(sandboxSession.idFromName).not.toHaveBeenCalled(); + }); + + it.each([ + 'provider', + 'ownerId', + 'userId', + 'orgId', + 'organizationId', + 'sandboxId', + 'providerInstanceId', + 'observedAt', + 'inactivityTimeoutMs', + 'estimatedSleepAt', + ])('rejects caller-supplied %s before any lookup', async field => { + await expect( + caller.getSandboxStatus({ cloudAgentSessionId, [field]: 'private-override' }) + ).rejects.toMatchObject({ code: 'BAD_REQUEST' }); + expect(requireCurrentSessionAccessMock).not.toHaveBeenCalled(); + expect(sandboxSession.idFromName).not.toHaveBeenCalled(); + }); + + it.each(['FORBIDDEN', 'NOT_FOUND', 'SERVICE_UNAVAILABLE'] as const)( + 'preserves %s authorization failures rather than returning unknown', + async code => { + requireCurrentSessionAccessMock.mockRejectedValueOnce( + new TRPCError({ code, message: 'Session access denied' }) + ); + await expect(caller.getSandboxStatus({ cloudAgentSessionId })).rejects.toMatchObject({ + code, + }); + expect(sandboxSession.idFromName).not.toHaveBeenCalled(); + expect(getSandboxStatus).not.toHaveBeenCalled(); + } + ); + + it('strips private upstream fields and repeatedly uses only the passive RPC', async () => { + getSandboxStatus.mockResolvedValue({ + ...snapshot, + sessionId: 'private-session', + sandboxId: 'private-sandbox', + ownerId: 'private-owner', + providerInstanceId: 'private-instance', + credentials: { token: 'private-token' }, + error: 'private-error', + }); + for (let read = 0; read < 3; read++) { + const result = await caller.getSandboxStatus({ cloudAgentSessionId }); + expect(result).toEqual(snapshot); + expect(JSON.stringify(result)).not.toContain('private'); + } + expect(getSandboxStatus).toHaveBeenCalledTimes(3); + expect(requireCurrentSessionAccessMock).toHaveBeenCalledTimes(3); + }); + + it.each([ + null, + {}, + { ...snapshot, status: 'private-status' }, + { ...snapshot, provider: 'private-provider' }, + { ...snapshot, detailCode: 'private-error' }, + { ...snapshot, detailCode: 'sandbox_failed' }, + { ...snapshot, observedAt: Infinity }, + { ...snapshot, inactivityTimeoutMs: -1 }, + { ...snapshot, estimatedSleepAt: snapshot.observedAt }, + ])('makes malformed upstream data safely unavailable: %j', async response => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + getSandboxStatus.mockResolvedValue(response); + await expect(caller.getSandboxStatus({ cloudAgentSessionId })).resolves.toEqual(unavailable); + expect(getSandboxStatus).toHaveBeenCalledTimes(1); + expect(JSON.stringify(warn.mock.calls)).not.toContain('private'); + }); + + it('sanitizes retryable DO errors before logging and uses fresh stubs', async () => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + getSandboxStatus.mockRejectedValueOnce( + Object.assign(new Error('private-token provider diagnostics'), { retryable: true }) + ); + await expect(caller.getSandboxStatus({ cloudAgentSessionId })).resolves.toEqual(snapshot); + expect(sandboxSession.get).toHaveBeenCalledTimes(2); + expect(getSandboxStatus).toHaveBeenCalledTimes(2); + expect(JSON.stringify(warn.mock.calls)).not.toContain('private'); + expect(warn).toHaveBeenCalledWith( + '[do-retry] Retrying', + expect.objectContaining({ error: 'Sandbox status unavailable' }) + ); + }); + + it.each(['stub', 'rpc', 'exhausted'] as const)( + 'bounds %s failures without leaking diagnostics or confirming sandbox failure', + async failure => { + const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined); + const logError = vi.spyOn(logger, 'error').mockImplementation(() => undefined); + const error = Object.assign(new Error('private-token provider diagnostics'), { + retryable: failure === 'exhausted', + }); + if (failure === 'stub') + sandboxSession.get.mockImplementation(() => { + throw error; + }); + else getSandboxStatus.mockRejectedValue(error); + const response = await caller.getSandboxStatus({ cloudAgentSessionId }); + expect(response).toEqual(unavailable); + expect(JSON.stringify([response, warn.mock.calls, logError.mock.calls])).not.toContain( + 'private' + ); + expect(sandboxSession.get).toHaveBeenCalledTimes(failure === 'exhausted' ? 3 : 1); + } + ); +}); + describe('getMessageResult procedure', () => { const sessionId: SessionId = 'agent_12345678-1234-1234-1234-123456789abc'; const messageId = 'msg_018f1e2d3c4bAbCdEfGhIjKlMn'; diff --git a/services/cloud-agent-next/src/router/handlers/session-management.ts b/services/cloud-agent-next/src/router/handlers/session-management.ts index 8773982d19..6b1ad150d1 100644 --- a/services/cloud-agent-next/src/router/handlers/session-management.ts +++ b/services/cloud-agent-next/src/router/handlers/session-management.ts @@ -20,6 +20,8 @@ import { MessageIdSchema, GetSessionInput, GetSessionOutput, + GetSandboxStatusInput, + GetSandboxStatusOutput, GetSessionHealthInput, GetSessionHealthOutput, GetMessageResultInput, @@ -438,6 +440,43 @@ export function createSessionManagementHandlers() { }); }), + getSandboxStatus: protectedProcedure + .input(GetSandboxStatusInput) + .output(GetSandboxStatusOutput) + .query(async ({ input, ctx }) => { + await requireCurrentSessionAccess({ + env: ctx.env, + kiloUserId: ctx.userId, + cloudAgentSessionId: input.cloudAgentSessionId, + }); + + try { + return await withDORetry( + () => () => getSandboxSessionStub(ctx.env, ctx.userId, input.cloudAgentSessionId), + async getStub => { + try { + return GetSandboxStatusOutput.parse(await getStub().getSandboxStatus()); + } catch (error) { + throw Object.assign(new Error('Sandbox status unavailable'), { + retryable: + error instanceof Error && 'retryable' in error && error.retryable === true, + }); + } + }, + 'getSandboxStatus' + ); + } catch { + return { + status: 'unknown', + provider: 'Unknown', + observedAt: Date.now(), + detailCode: 'status_unavailable', + inactivityTimeoutMs: null, + estimatedSleepAt: null, + }; + } + }), + getComputeBillingStatus: protectedProcedure .input(GetSessionInput) .output(GetComputeBillingStatusOutput) diff --git a/services/cloud-agent-next/src/router/schemas.ts b/services/cloud-agent-next/src/router/schemas.ts index dd8afe65a4..6684d465bd 100644 --- a/services/cloud-agent-next/src/router/schemas.ts +++ b/services/cloud-agent-next/src/router/schemas.ts @@ -28,6 +28,10 @@ import { AgentModeSchema, BUILTIN_AGENT_MODES, Limits } from '../schema.js'; import { MESSAGE_ID_FORMAT_DESCRIPTION, MESSAGE_ID_PATTERN } from '../session/message-id.js'; import { SessionMessageCompletionSourceSchema } from '../session/session-message-state.js'; import { SafeFailureProjectionSchema } from '../session/safe-failure-projection.js'; +import { SandboxStatusSessionIdSchema } from '../shared/sandbox-status.js'; + +export { SandboxStatusSnapshotSchema as GetSandboxStatusOutput } from '../shared/sandbox-status.js'; +export type { SandboxStatusSnapshot as GetSandboxStatusResponse } from '../shared/sandbox-status.js'; // Re-export schemas from types.ts and persistence/schemas.ts for convenience export { sessionIdSchema, githubRepoSchema, gitUrlSchema, envVarsSchema }; @@ -999,6 +1003,12 @@ export const GetSessionInput = z.object({ cloudAgentSessionId: sessionIdSchema.describe('Cloud-agent session ID to retrieve'), }); +export const GetSandboxStatusInput = z + .object({ + cloudAgentSessionId: SandboxStatusSessionIdSchema, + }) + .strict(); + /** Customer-safe, no-wake compute billing status for an existing session. */ export const GetComputeBillingStatusOutput = z.object({ payer: z.object({ type: z.enum(['user', 'org']), id: z.string() }), diff --git a/services/cloud-agent-next/src/sandbox-control/durable-state.test.ts b/services/cloud-agent-next/src/sandbox-control/durable-state.test.ts index 20d778bbef..903d05a759 100644 --- a/services/cloud-agent-next/src/sandbox-control/durable-state.test.ts +++ b/services/cloud-agent-next/src/sandbox-control/durable-state.test.ts @@ -3,6 +3,10 @@ import { eraseSandboxRecord, loadDeadlines, loadPhysicalRecord, + initialRuntimeMetadata, + loadRuntimeMetadata, + saveRuntimeMetadata, + readSandboxControlState, loadRouteTable, loadSessionCredentialGrants, loadTransitionLog, @@ -76,6 +80,60 @@ function credentialGrant(): SessionCredentialGrant { } describe('sandbox control durable state', () => { + it.each([ + ['usr-abcd', 'shared'], + ['org-abcd', 'shared'], + ['legacy__shared', 'shared'], + ['ses-abcd', 'isolated-small'], + ['istd-abcd', 'isolated-standard'], + ['crv-abcd', 'code-review'], + ['dind-abcd', 'devcontainer'], + ['private-invalid', 'unknown'], + ])( + 'derives only the authoritative allocation classification for %s', + (sandboxId, sandboxType) => { + expect(initialRuntimeMetadata(sandboxId)).toEqual({ + sandboxType, + kiloCliVersion: null, + wrapperVersion: null, + startedAt: null, + stoppedAt: null, + }); + } + ); + + it('round-trips metadata independently of physical lifecycle writes and erases it on deletion', async () => { + const storage = memoryStorage(); + expect(await loadRuntimeMetadata(storage)).toBeUndefined(); + const runtime = { + ...initialRuntimeMetadata('istd-abcd'), + wrapperVersion: '2.4.0', + kiloCliVersion: '7.4.20', + }; + await saveRuntimeMetadata(storage, runtime); + await savePhysicalRecord(storage, initialPhysicalRecord(false)); + expect(await loadRuntimeMetadata(storage)).toEqual(runtime); + expect((await readSandboxControlState(storage)).runtime).toEqual(runtime); + await eraseSandboxRecord(storage); + expect(await loadRuntimeMetadata(storage)).toBeUndefined(); + }); + + it('does not backfill or reflect malformed stored runtime metadata', async () => { + const storage = memoryStorage(); + await savePhysicalRecord(storage, initialPhysicalRecord(false)); + for (const runtime of [ + undefined, + {}, + { ...initialRuntimeMetadata('ses-abcd'), wrapperVersion: 'private-error' }, + ]) { + await storage.put('runtime_metadata', runtime); + expect(await loadRuntimeMetadata(storage)).toBeUndefined(); + expect((await readSandboxControlState(storage)).physical?.state).toBe('stopped'); + expect((await readSandboxControlState(storage)).runtime).toBeUndefined(); + expect(await storage.get('runtime_metadata')).toEqual(runtime); + } + }); + it('loads no credential grants from a pre-worktree record', async () => { expect(await loadSessionCredentialGrants(memoryStorage())).toEqual([]); }); diff --git a/services/cloud-agent-next/src/sandbox-control/durable-state.ts b/services/cloud-agent-next/src/sandbox-control/durable-state.ts index 033cff205d..57057b05cf 100644 --- a/services/cloud-agent-next/src/sandbox-control/durable-state.ts +++ b/services/cloud-agent-next/src/sandbox-control/durable-state.ts @@ -1,5 +1,11 @@ +import { z } from 'zod'; +import { classifySandboxId } from '../sandbox-id.js'; +import { + SandboxRuntimeMetadataSchema, + type SandboxRuntimeMetadata, +} from '../shared/sandbox-status.js'; import type { DeadlineTable } from './deadlines.js'; -import { emptyDeadlines } from './deadlines.js'; +import { DEADLINE_IDS, emptyDeadlines } from './deadlines.js'; import { initialPhysicalRecord, type PhysicalRecord } from './physical-lifecycle.js'; import type { SessionRoute } from './session-routes.js'; import { @@ -13,6 +19,7 @@ const ROUTES_KEY = 'session_routes'; const DEADLINES_KEY = 'deadlines'; const LOG_KEY = 'transition_log'; const CREDENTIAL_GRANTS_KEY = 'worktree_credential_grants'; +const RUNTIME_METADATA_KEY = 'runtime_metadata'; type ControlStorage = { get(key: string): Promise; @@ -20,6 +27,86 @@ type ControlStorage = { delete(keys: string[]): Promise; }; +const timestampSchema = z.number().int().nonnegative().max(8_640_000_000_000_000); +const physicalRecordSchema = z.object({ + state: z.enum(['stopped', 'creating', 'running', 'stopping', 'failed', 'unknown']), + providerRef: z.string().min(1).nullable(), + createIntent: z.object({ intentId: z.string().min(1), createdAt: timestampSchema }).nullable(), + stopTombstone: z + .object({ + reason: z.string(), + attempts: z.number().int().nonnegative(), + createdAt: timestampSchema, + }) + .nullable(), + resumable: z.boolean(), +}); +const deadlineTableSchema = z.partialRecord(z.enum(DEADLINE_IDS), timestampSchema); +const sessionRouteSchema = z.object({ + sessionId: z.string().min(1), + kiloSessionId: z.string().min(1), + directory: z.string().min(1), + worktreeId: z.string().min(1).optional(), + ownerId: z.string().min(1), + lastState: z.enum(['idle', 'active', 'finalizing']).nullable(), + lastStateAt: timestampSchema.nullable(), + idleForMs: z.number().int().nonnegative().nullable(), + waitingOn: z.enum(['model', 'tool', 'finalizing', 'preparation', 'input']).nullable(), +}); +const routeTableSchema = z.array(sessionRouteSchema); + +export type StoredSandboxControlState = { + physical: PhysicalRecord | null; + deadlines: DeadlineTable | null; + routes: SessionRoute[] | null; + runtime?: SandboxRuntimeMetadata; +}; + +export function initialRuntimeMetadata(sandboxId: string): SandboxRuntimeMetadata { + const classification = classifySandboxId(sandboxId); + return { + sandboxType: classification === 'legacy-shared' ? 'shared' : classification, + kiloCliVersion: null, + wrapperVersion: null, + startedAt: null, + stoppedAt: null, + }; +} + +export async function loadRuntimeMetadata(storage: { + get(key: string): Promise; +}): Promise { + const parsed = SandboxRuntimeMetadataSchema.safeParse(await storage.get(RUNTIME_METADATA_KEY)); + return parsed.success ? parsed.data : undefined; +} + +export async function saveRuntimeMetadata( + storage: ControlStorage, + runtime: SandboxRuntimeMetadata +): Promise { + await storage.put(RUNTIME_METADATA_KEY, SandboxRuntimeMetadataSchema.parse(runtime)); +} + +export async function readSandboxControlState(storage: { + get(key: string): Promise; +}): Promise { + const [physical, deadlines, routes, runtime] = await Promise.all([ + storage.get(PHYSICAL_KEY), + storage.get(DEADLINES_KEY), + storage.get(ROUTES_KEY), + loadRuntimeMetadata(storage), + ]); + const parsedPhysical = physicalRecordSchema.safeParse(physical); + const parsedDeadlines = deadlineTableSchema.safeParse(deadlines); + const parsedRoutes = routeTableSchema.safeParse(routes); + return { + physical: parsedPhysical.success ? parsedPhysical.data : null, + deadlines: parsedDeadlines.success ? parsedDeadlines.data : null, + routes: parsedRoutes.success ? parsedRoutes.data : null, + ...(runtime ? { runtime } : {}), + }; +} + export async function loadPhysicalRecord( storage: ControlStorage, resumable = false @@ -83,5 +170,12 @@ export async function saveSessionCredentialGrants( } export async function eraseSandboxRecord(storage: ControlStorage): Promise { - await storage.delete([PHYSICAL_KEY, ROUTES_KEY, DEADLINES_KEY, LOG_KEY, CREDENTIAL_GRANTS_KEY]); + await storage.delete([ + PHYSICAL_KEY, + ROUTES_KEY, + DEADLINES_KEY, + LOG_KEY, + CREDENTIAL_GRANTS_KEY, + RUNTIME_METADATA_KEY, + ]); } diff --git a/services/cloud-agent-next/src/sandbox-control/frames.test.ts b/services/cloud-agent-next/src/sandbox-control/frames.test.ts index d61a67797a..396ce8325c 100644 --- a/services/cloud-agent-next/src/sandbox-control/frames.test.ts +++ b/services/cloud-agent-next/src/sandbox-control/frames.test.ts @@ -3,6 +3,7 @@ import { MAX_SANDBOX_CONTROL_FRAME_BYTES, SANDBOX_CONTROL_PROTOCOL_VERSION, sandboxControlSocketAttachmentSchema, + sandboxHelloResultSchema, sessionTerminalCloseResultSchema, sessionTerminalConnectResultSchema, sessionTerminalCreateResultSchema, @@ -11,6 +12,7 @@ import { } from '../shared/sandbox-control-protocol.js'; import { errorResponse, + helloResult, isControlEvent, isControlOperation, isSessionOperation, @@ -37,6 +39,29 @@ const terminalPty = { }; describe('sandbox control frames', () => { + it('advertises heartbeat version support without requiring it from an old Worker', () => { + expect(sandboxHelloResultSchema.parse(helloResult())).toEqual({ + protocolVersion: 1, + handshakeComplete: true, + capabilities: { kiloVersionHeartbeat: true }, + }); + const previous = { protocolVersion: 1, handshakeComplete: true }; + expect(sandboxHelloResultSchema.parse(previous)).toEqual(previous); + for (const capabilities of [ + null, + { kiloVersionHeartbeat: 'true' }, + { kiloVersionHeartbeat: 1 }, + ]) { + expect(sandboxHelloResultSchema.safeParse({ ...previous, capabilities }).success).toBe(false); + } + expect( + sandboxHelloResultSchema.safeParse({ ...helloResult(), protocolVersion: 2 }).success + ).toBe(false); + expect( + sandboxHelloResultSchema.safeParse({ ...helloResult(), handshakeComplete: false }).success + ).toBe(false); + }); + it('accepts a valid request envelope', () => { const parsed = parseControlFrame( JSON.stringify({ diff --git a/services/cloud-agent-next/src/sandbox-control/frames.ts b/services/cloud-agent-next/src/sandbox-control/frames.ts index 94b766ebc9..1669698b48 100644 --- a/services/cloud-agent-next/src/sandbox-control/frames.ts +++ b/services/cloud-agent-next/src/sandbox-control/frames.ts @@ -31,6 +31,7 @@ import { type ControlOperation, type ResponseFrame, type SandboxHelloPayload, + type SandboxHelloResult, } from '../shared/sandbox-control-protocol.js'; const encoder = new TextEncoder(); @@ -170,9 +171,10 @@ export function errorResponse( return { type: 'response', requestId, ok: false, error }; } -export function helloResult(): { protocolVersion: 1; handshakeComplete: true } { +export function helloResult(): SandboxHelloResult { return { protocolVersion: SANDBOX_CONTROL_PROTOCOL_VERSION, handshakeComplete: true, + capabilities: { kiloVersionHeartbeat: true }, }; } diff --git a/services/cloud-agent-next/src/sandbox-control/lifecycle.test.ts b/services/cloud-agent-next/src/sandbox-control/lifecycle.test.ts index a9e86607e6..f1476d7cb4 100644 --- a/services/cloud-agent-next/src/sandbox-control/lifecycle.test.ts +++ b/services/cloud-agent-next/src/sandbox-control/lifecycle.test.ts @@ -19,11 +19,13 @@ import type { import { DEADLINE_MS } from './deadlines.js'; import { loadDeadlines, loadRouteTable, loadSessionCredentialGrants } from './durable-state.js'; import type * as cloudflareProvider from './cloudflare-provider.js'; +import type * as SocketModule from './socket.js'; import { decodeCloudflareProviderRef } from './cloudflare-provider.js'; import { WORKTREE_CREDENTIAL_CONTAINMENT } from './physical-lifecycle.js'; import { parseSessionMetadata } from '../persistence/session-metadata.js'; import { logger } from '../logger.js'; import { validateControlLogUploadGrant } from './log-upload-grant.js'; +import { summarizeHeartbeatIdle } from './status-projection.js'; const mocks = vi.hoisted(() => ({ getSandbox: vi.fn(), @@ -59,7 +61,10 @@ vi.mock('./cloudflare-provider.js', async importOriginal => { }, }; }); -vi.mock('./socket.js', () => ({ createSandboxControlSocketHandler: mocks.socket })); +vi.mock('./socket.js', async importOriginal => ({ + ...(await importOriginal()), + createSandboxControlSocketHandler: mocks.socket, +})); vi.mock('../sandbox-session/session-stub.js', () => ({ getSandboxSessionStub: mocks.session })); const SANDBOX_ID = 'ses-abcdef'; @@ -321,6 +326,7 @@ async function harness( }, storage, records, + ctx, env, namespace, namespaces, @@ -353,7 +359,7 @@ async function harness( billing: BILLING, }); }, - async ready() { + async ready(runtime?: { wrapperVersion: string | null }) { const physical = await control.getPhysicalRecord(); if (!physical.providerRef) throw new Error('No physical allocation'); connection = { @@ -362,7 +368,7 @@ async function harness( providerInstanceId: physical.providerRef, }; const identity = connection; - await hooks.onHandshakeComplete?.(identity); + await hooks.onHandshakeComplete?.(identity, runtime); await hooks.onReady?.(identity); await control.attachSession(ROUTE); return identity; @@ -374,9 +380,10 @@ async function harness( await control.alarm(); await flush(); }, - async evict() { + async evict(passive = false) { control = new SandboxControl(ctx, env); await Promise.all(initializing); + if (!passive) await control.getStatus(); }, }; } @@ -401,6 +408,264 @@ afterEach(() => { }); describe('SandboxControl lifecycle boundaries', () => { + it('retains observed versions through readiness, eviction and stop, then clears them on a new allocation', async () => { + const h = await harness(); + const status = () => h.control.getSandboxStatus({ ownerId: OWNER, provider: 'cloudflare' }); + await h.create(); + const empty = { + sandboxType: 'isolated-small', + wrapperVersion: null, + kiloCliVersion: null, + startedAt: null, + stoppedAt: null, + }; + expect((await status()).runtime).toEqual(empty); + const identity = await h.ready({ wrapperVersion: '2.4.0' }); + expect((await status()).runtime).toEqual({ ...empty, wrapperVersion: '2.4.0' }); + await h.hooks.onHeartbeat?.( + { ...activeHeartbeat, kilo: { ready: true, version: '7.4.20' } }, + identity + ); + const runtime = { ...empty, wrapperVersion: '2.4.0', kiloCliVersion: '7.4.20' }; + expect((await status()).runtime).toEqual(runtime); + await h.hooks.onHeartbeat?.(activeHeartbeat, identity); + await h.evict(); + expect((await status()).runtime).toEqual(runtime); + await h.control.beginStop('idle'); + await h.hooks.onHeartbeat?.( + { ...activeHeartbeat, kilo: { ready: true, version: '9.9.9' } }, + identity + ); + await h.control.confirmStopped(); + await h.evict(true); + expect(await status()).toMatchObject({ status: 'sleeping', runtime }); + await h.create(); + expect((await status()).runtime).toEqual(empty); + const replacement = await h.ready({ wrapperVersion: '2.5.0' }); + await h.hooks.onHeartbeat?.( + { ...activeHeartbeat, kilo: { ready: true, version: '7.5.0' } }, + replacement + ); + await h.hooks.onHandshakeComplete?.(identity, { wrapperVersion: '9.9.9' }); + await h.hooks.onHeartbeat?.( + { ...activeHeartbeat, kilo: { ready: true, version: '9.9.9' } }, + identity + ); + expect((await status()).runtime).toEqual({ + ...empty, + wrapperVersion: '2.5.0', + kiloCliVersion: '7.5.0', + }); + await h.control.eraseRecord(); + expect(h.records.has('runtime_metadata')).toBe(false); + }); + + it('fences a delayed hello against allocation replacement inside its transaction', async () => { + const h = await harness(); + await h.create(); + const identity = await h.ready({ wrapperVersion: '2.4.0' }); + const gate = deferred(); + const entered = deferred(); + const replacement = { + sandboxType: 'isolated-small', + wrapperVersion: null, + kiloCliVersion: null, + startedAt: null, + stoppedAt: null, + }; + const blocked = h.storage.transaction(async () => { + entered.resolve(); + await gate.promise; + h.records.set('physical_record', { + state: 'creating', + providerRef: null, + createIntent: { intentId: 'replacement', createdAt: Date.now() }, + stopTombstone: null, + resumable: false, + }); + h.records.set('runtime_metadata', replacement); + }); + await entered.promise; + const hello = h.hooks.onHandshakeComplete?.(identity, { wrapperVersion: '9.9.9' }); + await Promise.resolve(); + gate.resolve(); + await blocked; + await hello; + expect(h.records.get('runtime_metadata')).toEqual(replacement); + }); + + it('omits runtime metadata on owner or provider mismatch without any operational effects', async () => { + const h = await harness(); + await h.create(); + await h.ready({ wrapperVersion: '2.4.0' }); + await h.evict(true); + const writes = [ + vi.spyOn(h.storage, 'put'), + vi.spyOn(h.storage, 'delete'), + vi.spyOn(h.storage, 'setAlarm'), + vi.spyOn(h.storage, 'deleteAlarm'), + ]; + const before = structuredClone([...h.records]); + for (const input of [ + { ownerId: 'other-owner', provider: 'cloudflare' }, + { ownerId: OWNER, provider: 'vercel' }, + ] as const) { + expect(await h.control.getSandboxStatus(input)).not.toHaveProperty('runtime'); + } + expect([...h.records]).toEqual(before); + for (const write of writes) expect(write).not.toHaveBeenCalled(); + }); + + it.each(['missing', 'stopped', 'creating', 'stopping'] as const)( + 'keeps cold passive %s reads independent of lifecycle repair and credentials', + async state => { + const h = await harness(); + h.records.clear(); + h.records.set('owner_id', OWNER); + h.records.set('provider_kind', 'cloudflare'); + h.records.set('worktree_credential_grants', [{ untouched: true }]); + const runtime = { + sandboxType: 'isolated-small', + wrapperVersion: '2.4.0', + kiloCliVersion: '7.4.20', + startedAt: null, + stoppedAt: null, + }; + h.records.set('runtime_metadata', runtime); + if (state !== 'missing') { + h.records.set('physical_record', { + state, + providerRef: state === 'stopped' ? null : 'instance_pending', + resumable: false, + createIntent: + state === 'creating' ? { intentId: 'pending', createdAt: Date.now() - 10_000 } : null, + stopTombstone: + state === 'stopping' + ? { reason: 'retired', attempts: 5, createdAt: Date.now() - 10_000 } + : null, + }); + } + await h.storage.setAlarm(Date.now() + 12_345); + const before = structuredClone([...h.records]); + const alarm = h.alarmAt; + const writes = [ + vi.spyOn(h.storage, 'put'), + vi.spyOn(h.storage, 'delete'), + vi.spyOn(h.storage, 'setAlarm'), + vi.spyOn(h.storage, 'deleteAlarm'), + ]; + const close = vi.fn(); + h.socket.closeAll = close; + await h.evict(true); + for (let i = 0; i < 3; i++) { + expect( + await h.control.getSandboxStatus({ ownerId: OWNER, provider: 'cloudflare' }) + ).toMatchObject({ + status: { + missing: 'unknown', + stopped: 'sleeping', + creating: 'starting', + stopping: 'stopping', + }[state], + estimatedSleepAt: null, + ...(state === 'missing' ? {} : { runtime }), + }); + } + expect([...h.records]).toEqual(before); + expect(h.alarmAt).toBe(alarm); + for (const write of writes) expect(write).not.toHaveBeenCalled(); + expect(close).not.toHaveBeenCalled(); + expect(h.sendRequest).not.toHaveBeenCalled(); + expect(h.allocations.size).toBe(0); + } + ); + + it('reads two shared-directory roots after reconstruction without initializing operational state', async () => { + const h = await harness(); + const now = Date.now(); + const identity = { + connectionId: crypto.randomUUID(), + wrapperInstanceId: crypto.randomUUID(), + providerInstanceId: 'instance_shared', + }; + const first = { + ...ROUTE, + worktreeId: 'worktree_shared', + lastState: 'idle', + lastStateAt: now, + idleForMs: 0, + waitingOn: null, + }; + const sibling = { ...first, sessionId: 'workspace_sibling', kiloSessionId: 'ses_sibling' }; + const idle = await summarizeHeartbeatIdle({ + state: 'idle', + pendingMessages: 0, + kilo: { ready: true }, + sessions: [first, sibling].map(route => ({ + kiloSessionId: route.kiloSessionId, + state: 'idle', + idleForMs: 0, + })), + }); + const attachment = { + ...identity, + handshakeComplete: true, + protocolVersion: 1, + acceptedAt: now - 1_000, + observation: { ready: true, receivedAt: now, idle }, + }; + const socket = { + readyState: 1, + deserializeAttachment: vi.fn(() => attachment), + serializeAttachment: vi.fn(), + send: vi.fn(), + close: vi.fn(), + }; + vi.spyOn(h.ctx, 'getWebSockets').mockReturnValue([socket as unknown as WebSocket]); + h.records.set('provider_kind', 'cloudflare'); + h.records.set('physical_record', { + state: 'running', + providerRef: identity.providerInstanceId, + createIntent: null, + stopTombstone: null, + resumable: false, + }); + h.records.set('active_wrapper_runtime', { + ...identity, + readyConnectionId: identity.connectionId, + }); + h.records.set('session_routes', [first, sibling]); + h.records.set('deadlines', { + idleStop: now + DEADLINE_MS.idleStop, + heartbeatExpiry: now + DEADLINE_MS.heartbeatExpiry, + }); + const before = structuredClone([...h.records]); + const alarm = h.alarmAt; + await h.evict(true); + for (let i = 0; i < 3; i++) { + expect( + await h.control.getSandboxStatus({ ownerId: OWNER, provider: 'cloudflare' }) + ).toMatchObject({ status: 'active', estimatedSleepAt: now + DEADLINE_MS.idleStop }); + } + expect([...h.records]).toEqual(before); + expect(h.alarmAt).toBe(alarm); + expect(socket.serializeAttachment).not.toHaveBeenCalled(); + expect(socket.send).not.toHaveBeenCalled(); + expect(socket.close).not.toHaveBeenCalled(); + h.records.set('session_routes', [first, { ...sibling, waitingOn: 'input' }]); + expect( + await h.control.getSandboxStatus({ ownerId: OWNER, provider: 'cloudflare' }) + ).toMatchObject({ status: 'active', estimatedSleepAt: null }); + h.records.set('active_wrapper_runtime', { + ...identity, + connectionId: crypto.randomUUID(), + readyConnectionId: identity.connectionId, + }); + expect( + await h.control.getSandboxStatus({ ownerId: OWNER, provider: 'cloudflare' }) + ).toMatchObject({ status: 'unknown', estimatedSleepAt: null }); + }); + describe.each(['create', 'acquire'] as const)('%s credential policy', createPath => { it.each([false, undefined])( 'launches and attaches using persisted containment %s', @@ -1130,11 +1395,18 @@ describe('SandboxControl lifecycle boundaries', () => { it('repairs a missing create alarm using the original startup deadline', async () => { const h = await harness(); const claimed = await h.control.claimCreate('intent_repair'); + h.records.set('provider_kind', 'cloudflare'); const deadline = (claimed.createIntent?.createdAt ?? 0) + DEADLINE_MS.startup; h.records.delete('deadlines'); await h.storage.deleteAlarm(); vi.setSystemTime(Date.now() + 30_000); - await h.evict(); + await h.evict(true); + expect( + await h.control.getSandboxStatus({ ownerId: OWNER, provider: 'cloudflare' }) + ).toMatchObject({ status: 'starting' }); + expect(await loadDeadlines(h.storage)).toEqual({}); + expect(h.alarmAt).toBeNull(); + await h.control.getStatus(); expect(await loadDeadlines(h.storage)).toEqual({ startup: deadline }); expect(h.alarmAt).toBe(deadline); expect(h.allocations.size).toBe(0); diff --git a/services/cloud-agent-next/src/sandbox-control/socket.test.ts b/services/cloud-agent-next/src/sandbox-control/socket.test.ts index d5a98f9fcb..1b887565de 100644 --- a/services/cloud-agent-next/src/sandbox-control/socket.test.ts +++ b/services/cloud-agent-next/src/sandbox-control/socket.test.ts @@ -1,6 +1,11 @@ -import { describe, expect, it, vi } from 'vitest'; -import { SANDBOX_CONTROL_PROTOCOL_VERSION } from '../shared/sandbox-control-protocol.js'; -import { createSandboxControlSocketHandler } from './socket.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + SANDBOX_CONTROL_PROTOCOL_VERSION, + sandboxControlSocketAttachmentSchema, + sandboxHeartbeatPayloadSchema, + type SandboxHeartbeatPayload, +} from '../shared/sandbox-control-protocol.js'; +import { createSandboxControlSocketHandler, readSandboxControlConnection } from './socket.js'; import { createControlRequestWaiters } from './waiters.js'; vi.mock('../logger.js', () => { @@ -16,9 +21,9 @@ vi.mock('../logger.js', () => { type FakeWebSocket = { deserializeAttachment: () => unknown; - serializeAttachment: ReturnType; - send: ReturnType; - close: ReturnType; + serializeAttachment: ReturnType void>>; + send: ReturnType void>>; + close: ReturnType void>>; readyState: number; }; @@ -75,6 +80,70 @@ const WRAPPER_INSTANCE_ID = '11111111-1111-4111-8111-111111111111'; const REPLACEMENT_WRAPPER_INSTANCE_ID = '22222222-2222-4222-8222-222222222222'; describe('sandbox control socket handler', () => { + it('accepts old heartbeats and ignores invalid optional versions without rejecting readiness', () => { + const heartbeat = { state: 'idle', kilo: { ready: true }, sessions: [] }; + expect(sandboxHeartbeatPayloadSchema.parse(heartbeat)).toEqual(heartbeat); + for (const version of ['7.4.20', null]) { + expect( + sandboxHeartbeatPayloadSchema.parse({ ...heartbeat, kilo: { ready: true, version } }).kilo + ).toEqual({ ready: true, version }); + } + for (const version of ['private-error', '7.4.20\n', { token: 'private' }]) { + const parsed = sandboxHeartbeatPayloadSchema.parse({ + ...heartbeat, + kilo: { ready: true, version }, + }); + expect(parsed.kilo.ready).toBe(true); + expect(parsed.kilo.version).toBeUndefined(); + expect(JSON.stringify(parsed)).not.toContain('private'); + } + }); + it.each([ + ['2.4.0', '2.4.0'], + [undefined, null], + ['https://private.invalid/error', null], + ['2.4.0-private-instance', null], + ])('retains only safe hello wrapper metadata: %s', async (wrapperVersion, expected) => { + const incoming = createFakeWebSocket(); + const onHandshakeComplete = vi.fn(); + const handler = createSandboxControlSocketHandler( + createFakeState([incoming]), + 'sbx_test', + undefined, + { + validateHandshake: () => true, + onHandshakeComplete, + } + ); + await handler.handleMessage( + asWs(incoming), + JSON.stringify({ + type: 'request', + requestId: 'hello_metadata', + operation: 'sandbox.hello', + payload: { + protocolVersion: 1, + providerInstanceId: 'inst_1', + wrapperInstanceId: WRAPPER_INSTANCE_ID, + wrapperVersion, + }, + }) + ); + expect(onHandshakeComplete).toHaveBeenCalledWith(handler.getConnectionIdentity(), { + wrapperVersion: expected, + }); + await handler.handleMessage( + asWs(incoming), + JSON.stringify({ + type: 'request', + requestId: 'replayed_metadata', + operation: 'sandbox.hello', + payload: { protocolVersion: 1, providerInstanceId: 'inst_1', wrapperVersion: '9.9.9' }, + }) + ); + expect(onHandshakeComplete).toHaveBeenCalledTimes(1); + }); + it('does not replace the current socket until sandbox.hello succeeds', async () => { const current = createFakeWebSocket({ handshakeComplete: true, @@ -246,7 +315,11 @@ describe('sandbox control socket handler', () => { type: 'response', requestId: 'req_hello', ok: true, - result: { protocolVersion: 1, handshakeComplete: true }, + result: { + protocolVersion: 1, + handshakeComplete: true, + capabilities: { kiloVersionHeartbeat: true }, + }, }) ); const statusFrame = JSON.parse(incoming.send.mock.calls[1]?.[0] as string) as { @@ -278,7 +351,9 @@ describe('sandbox control socket handler', () => { handshakeComplete: true, ...identity, }); - expect(onHandshakeComplete).toHaveBeenCalledWith(handler.getConnectionIdentity()); + expect(onHandshakeComplete).toHaveBeenCalledWith(handler.getConnectionIdentity(), { + wrapperVersion: null, + }); }); it('rejects duplicate hellos without replacing the current connection', async () => { @@ -452,7 +527,9 @@ describe('sandbox control socket handler', () => { helloFrame('inst_1', WRAPPER_INSTANCE_ID) ); - expect(onHandshakeComplete).toHaveBeenCalledWith(handler.getConnectionIdentity()); + expect(onHandshakeComplete).toHaveBeenCalledWith(handler.getConnectionIdentity(), { + wrapperVersion: null, + }); expect(incoming.send).not.toHaveBeenCalled(); await handler.handleMessage( @@ -1025,7 +1102,7 @@ describe('sandbox control socket handler', () => { wrapperInstanceId: WRAPPER_INSTANCE_ID, }); expect(currentIdentity?.connectionId).not.toBe(previousIdentity?.connectionId); - expect(onHandshakeComplete).toHaveBeenCalledWith(currentIdentity); + expect(onHandshakeComplete).toHaveBeenCalledWith(currentIdentity, { wrapperVersion: null }); const pending = handler.sendRequest({ operation: 'sandbox.status', payload: {} }); const request = JSON.parse(incoming.send.mock.calls[2]?.[0] as string) as { requestId: string }; @@ -1185,3 +1262,417 @@ describe('sandbox control socket handler', () => { expect(ws.send).not.toHaveBeenCalled(); }); }); + +const now = 1_000_000; +const handshaken = { + handshakeComplete: true, + acceptedAt: now - 1_000, + protocolVersion: SANDBOX_CONTROL_PROTOCOL_VERSION, + providerInstanceId: 'inst_1', +}; +const idleHeartbeat: SandboxHeartbeatPayload = { + state: 'idle', + pendingMessages: 0, + kilo: { ready: true }, + sessions: [{ kiloSessionId: 'kilo_1', state: 'idle', idleForMs: 0 }], +}; +const readyFrame = JSON.stringify({ + type: 'event', + event: 'sandbox.ready', + payload: { kiloReady: true, globalFeedAttached: true }, +}); +const busyFrame = JSON.stringify({ + type: 'event', + event: 'session.event', + session: { directory: '/workspace/a', kiloSessionId: 'kilo_1' }, + payload: { + type: 'session.status', + properties: { sessionID: 'kilo_1', status: { type: 'busy' } }, + }, +}); +function heartbeatFrame(payload: SandboxHeartbeatPayload = idleHeartbeat) { + return JSON.stringify({ type: 'event', event: 'sandbox.heartbeat', payload }); +} +function statusHelloFrame() { + return JSON.stringify({ + type: 'request', + requestId: 'hello-observation', + operation: 'sandbox.hello', + payload: { protocolVersion: 1, providerInstanceId: 'inst_1' }, + }); +} + +function readTestConnection( + state: Pick, + providerRef: string | null +) { + const socket = state.getWebSockets().find(ws => { + const parsed = sandboxControlSocketAttachmentSchema.safeParse(ws.deserializeAttachment()); + return ws.readyState === 1 && parsed.success && parsed.data.handshakeComplete; + }); + const parsed = sandboxControlSocketAttachmentSchema.safeParse(socket?.deserializeAttachment()); + return readSandboxControlConnection( + state, + providerRef, + parsed.success ? { ...parsed.data, readyConnectionId: parsed.data.connectionId } : null + ); +} + +function attachmentOf(ws: FakeWebSocket) { + return sandboxControlSocketAttachmentSchema.parse(ws.deserializeAttachment()); +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe('connection-local sandbox observations', () => { + it('records ready and explicit false heartbeats across handler reconstruction without frames', async () => { + vi.useFakeTimers({ now }); + const ws = createFakeWebSocket(handshaken); + const state = createFakeState([ws]); + const onReady = vi.fn(); + const onHeartbeat = vi.fn(); + const handler = createSandboxControlSocketHandler(state, 'sbx_test', undefined, { + onReady, + onHeartbeat, + }); + await handler.handleMessage(asWs(ws), readyFrame); + expect(attachmentOf(ws).observation).toEqual({ ready: true, receivedAt: now, idle: null }); + await handler.handleMessage(asWs(ws), heartbeatFrame()); + expect(attachmentOf(ws).observation).toEqual({ + ready: true, + receivedAt: now, + idle: { sessionCount: 1, sessionIdsHash: expect.stringMatching(/^[0-9a-f]{64}$/) }, + }); + vi.advanceTimersByTime(5); + const notReady = { ...idleHeartbeat, kilo: { ready: false } }; + await handler.handleMessage(asWs(ws), heartbeatFrame(notReady)); + createSandboxControlSocketHandler(state, 'sbx_test'); + expect(readTestConnection(state, 'inst_1')).toEqual({ + state: 'connected', + acceptedAt: handshaken.acceptedAt, + observation: { ready: false, receivedAt: now + 5, idle: null }, + }); + expect(onReady).toHaveBeenCalledTimes(1); + expect(onHeartbeat.mock.calls.map(call => call[0])).toEqual([idleHeartbeat, notReady]); + expect(ws.send).not.toHaveBeenCalled(); + }); + + it('resets observation on every handshake and ignores late replaced-socket readiness', async () => { + vi.useFakeTimers({ now }); + const previous = createFakeWebSocket(handshaken); + const replacement = createFakeWebSocket({ handshakeComplete: false, acceptedAt: now }); + const sockets = [previous, replacement]; + const state = createFakeState(sockets); + const handler = createSandboxControlSocketHandler(state, 'sbx_test'); + await handler.handleMessage(asWs(previous), heartbeatFrame()); + await handler.handleMessage(asWs(replacement), statusHelloFrame()); + previous.readyState = 2; + expect(attachmentOf(replacement).observation).toBeUndefined(); + await handler.handleMessage(asWs(previous), readyFrame); + await handler.handleMessage(asWs(previous), heartbeatFrame()); + expect(readTestConnection(state, 'inst_1')).toEqual({ state: 'unknown' }); + await handler.handleMessage( + asWs(replacement), + heartbeatFrame({ ...idleHeartbeat, kilo: { ready: false } }) + ); + expect(readTestConnection(state, 'inst_1')).toMatchObject({ + state: 'connected', + observation: { ready: false, idle: null }, + }); + await handler.handleMessage(asWs(replacement), readyFrame); + expect(readTestConnection(state, 'inst_1')).toMatchObject({ + state: 'connected', + observation: { ready: true, idle: null }, + }); + const beforeDuplicate = attachmentOf(replacement); + await handler.handleMessage(asWs(replacement), statusHelloFrame()); + expect(attachmentOf(replacement)).toEqual(beforeDuplicate); + expect(replacement.send).toHaveBeenLastCalledWith(expect.stringContaining('protocol_error')); + }); + + it('does not overwrite newer false evidence when an earlier idle hook completes late', async () => { + vi.useFakeTimers({ now }); + const ws = createFakeWebSocket(handshaken); + const firstHook = Promise.withResolvers(); + const onHeartbeat = vi.fn().mockImplementationOnce(() => firstHook.promise); + const handler = createSandboxControlSocketHandler( + createFakeState([ws]), + 'sbx_test', + undefined, + { + onHeartbeat, + } + ); + const earlier = handler.handleMessage(asWs(ws), heartbeatFrame()); + await handler.handleMessage( + asWs(ws), + heartbeatFrame({ ...idleHeartbeat, kilo: { ready: false } }) + ); + firstHook.resolve(); + await earlier; + expect(attachmentOf(ws).observation).toEqual({ ready: false, receivedAt: now, idle: null }); + }); + + it.each([ + ['busy', busyFrame], + [ + 'message', + JSON.stringify({ + type: 'event', + event: 'session.event', + session: { directory: '/workspace/a', kiloSessionId: 'kilo_1' }, + payload: { type: 'message.updated', properties: { id: 'msg_1' } }, + }), + ], + [ + 'preparing', + JSON.stringify({ + type: 'event', + event: 'session.preparing', + session: { directory: '/workspace/a', kiloSessionId: 'kilo_1' }, + payload: { + version: 2, + attemptId: 'att_1', + triggerMessageId: 'msg_1', + revision: 1, + timestamp: now, + step: 'cloning', + message: 'Cloning repository', + action: 'step_started', + }, + }), + ], + ])( + 'invalidates persisted idle evidence on a newer %s event without refreshing readiness', + async (_name, frame) => { + vi.useFakeTimers({ now }); + const ws = createFakeWebSocket(handshaken); + const state = createFakeState([ws]); + await createSandboxControlSocketHandler(state, 'sbx_test').handleMessage( + asWs(ws), + heartbeatFrame() + ); + expect(attachmentOf(ws).observation?.idle).toMatchObject({ sessionCount: 1 }); + vi.advanceTimersByTime(5); + const handler = createSandboxControlSocketHandler(state, 'sbx_test'); + await handler.handleMessage(asWs(ws), frame); + expect(attachmentOf(ws).observation).toEqual({ ready: true, receivedAt: now, idle: null }); + expect(ws.send).not.toHaveBeenCalled(); + await handler.handleMessage(asWs(ws), heartbeatFrame()); + expect(attachmentOf(ws).observation).toMatchObject({ + ready: true, + receivedAt: now + 5, + idle: { sessionCount: 1 }, + }); + } + ); + + it('fences an in-flight idle heartbeat before forwarding a newer busy event in the same millisecond', async () => { + vi.useFakeTimers({ now }); + const ws = createFakeWebSocket(handshaken); + const heartbeatHook = Promise.withResolvers(); + const eventHook = Promise.withResolvers(); + const handler = createSandboxControlSocketHandler( + createFakeState([ws]), + 'sbx_test', + undefined, + { + onHeartbeat: () => heartbeatHook.promise, + onSessionEvent: () => eventHook.promise, + } + ); + const heartbeat = handler.handleMessage(asWs(ws), heartbeatFrame()); + const event = handler.handleMessage(asWs(ws), busyFrame); + heartbeatHook.resolve(); + try { + await heartbeat; + expect(attachmentOf(ws).observation).toEqual({ ready: true, receivedAt: now, idle: null }); + } finally { + eventHook.resolve(); + await event; + } + }); + + it('ignores replaced-socket activity without invalidating current idle evidence', async () => { + vi.useFakeTimers({ now }); + const previous = createFakeWebSocket(handshaken); + const replacement = createFakeWebSocket({ ...handshaken, acceptedAt: now }); + const handler = createSandboxControlSocketHandler( + createFakeState([previous, replacement]), + 'sbx_test' + ); + previous.readyState = 2; + await handler.handleMessage(asWs(replacement), heartbeatFrame()); + const attachment = attachmentOf(replacement); + await handler.handleMessage(asWs(previous), busyFrame); + expect(attachmentOf(replacement)).toEqual(attachment); + expect(attachment.observation?.idle).toMatchObject({ sessionCount: 1 }); + expect(previous.serializeAttachment).not.toHaveBeenCalled(); + }); + + it('does not expose readiness during activation or under another persisted runtime identity', async () => { + vi.useFakeTimers({ now }); + const previous = createFakeWebSocket({ ...handshaken, wrapperInstanceId: WRAPPER_INSTANCE_ID }); + const replacement = createFakeWebSocket({ handshakeComplete: false, acceptedAt: now }); + const state = createFakeState([previous, replacement]); + const activation = Promise.withResolvers(); + const onReady = vi.fn(); + const handler = createSandboxControlSocketHandler(state, 'sbx_test', undefined, { + onHandshakeComplete: () => activation.promise, + onReady, + }); + const oldRuntime = attachmentOf(previous); + const handshake = handler.handleMessage( + asWs(replacement), + helloFrame('inst_1', REPLACEMENT_WRAPPER_INSTANCE_ID) + ); + await handler.handleMessage(asWs(replacement), readyFrame); + expect(onReady).not.toHaveBeenCalled(); + expect(attachmentOf(replacement).observation).toBeUndefined(); + activation.resolve(); + await handshake; + await handler.handleMessage(asWs(replacement), readyFrame); + const current = attachmentOf(replacement); + expect(onReady).toHaveBeenCalledWith({ + connectionId: current.connectionId, + providerInstanceId: 'inst_1', + wrapperInstanceId: REPLACEMENT_WRAPPER_INSTANCE_ID, + }); + const runtime = { ...current, readyConnectionId: current.connectionId }; + for (const stale of [ + undefined, + oldRuntime, + { ...runtime, readyConnectionId: oldRuntime.connectionId }, + { ...runtime, wrapperInstanceId: WRAPPER_INSTANCE_ID }, + ]) { + expect(readSandboxControlConnection(state, 'inst_1', stale)).toEqual({ state: 'unknown' }); + } + const serialize = replacement.serializeAttachment.mock.calls.length; + expect(readSandboxControlConnection(state, 'inst_1', runtime)).toMatchObject({ + state: 'connected', + observation: { ready: true, receivedAt: now }, + }); + expect(replacement.serializeAttachment).toHaveBeenCalledTimes(serialize); + }); + + it('does not let an in-flight heartbeat populate a same-provider replacement', async () => { + vi.useFakeTimers({ now }); + const previous = createFakeWebSocket(handshaken); + const replacement = createFakeWebSocket({ handshakeComplete: false, acceptedAt: now }); + const state = createFakeState([previous, replacement]); + const hook = Promise.withResolvers(); + const handler = createSandboxControlSocketHandler(state, 'sbx_test', undefined, { + onHeartbeat: () => hook.promise, + }); + const heartbeat = handler.handleMessage(asWs(previous), heartbeatFrame()); + await handler.handleMessage(asWs(replacement), statusHelloFrame()); + previous.readyState = 2; + hook.resolve(); + await heartbeat; + expect(attachmentOf(replacement).observation).toBeUndefined(); + expect(readTestConnection(state, 'inst_1')).toEqual({ state: 'unknown' }); + }); + + it('retains no unbounded session payload in attachments', async () => { + vi.useFakeTimers({ now }); + const ws = createFakeWebSocket(handshaken); + const payload: SandboxHeartbeatPayload = { + ...idleHeartbeat, + sessions: Array.from({ length: 2_000 }, (_, index) => ({ + kiloSessionId: `kilo_${index}`, + state: 'idle', + idleForMs: 0, + })), + }; + const handler = createSandboxControlSocketHandler(createFakeState([ws]), 'sbx_test'); + await handler.handleMessage(asWs(ws), heartbeatFrame(payload)); + expect(attachmentOf(ws).observation?.idle?.sessionCount).toBe(2_000); + const serialized = JSON.stringify(ws.deserializeAttachment()); + expect(serialized.length).toBeLessThan(512); + expect(serialized).not.toContain('kilo_1999'); + }); + + it.each([ + { event: 'sandbox.ready', payload: { kiloReady: false, globalFeedAttached: true } }, + { event: 'sandbox.ready', payload: { kiloReady: true } }, + { event: 'sandbox.heartbeat', payload: { ...idleHeartbeat, kilo: { ready: 'true' } } }, + { event: 'sandbox.heartbeat', payload: { ...idleHeartbeat, sessions: undefined } }, + { event: 'unknown', payload: idleHeartbeat }, + ])('does not record invalid or unrelated $event evidence', async frame => { + const ws = createFakeWebSocket(handshaken); + const handler = createSandboxControlSocketHandler(createFakeState([ws]), 'sbx_test'); + await handler.handleMessage(asWs(ws), JSON.stringify({ type: 'event', ...frame })); + expect(ws.serializeAttachment).not.toHaveBeenCalled(); + }); + + it('does not derive readiness from an operational sandbox.status response', async () => { + const ws = createFakeWebSocket(handshaken); + const handler = createSandboxControlSocketHandler(createFakeState([ws]), 'sbx_test'); + await handler.handleMessage( + asWs(ws), + JSON.stringify({ + type: 'response', + requestId: 'probe', + ok: true, + result: { healthy: true, kiloReady: true, state: 'idle', version: 'test' }, + }) + ); + expect(ws.serializeAttachment).not.toHaveBeenCalled(); + }); + + it.each([ + null, + handshaken, + { ...handshaken, observation: { ready: true, receivedAt: NaN, idle: null } }, + { ...handshaken, observation: { ready: true, receivedAt: now, idle: { sessionCount: 1 } } }, + { ...handshaken, observation: { ready: true, receivedAt: now - 2_000, idle: null } }, + { + ...handshaken, + protocolVersion: undefined, + observation: { ready: true, receivedAt: now, idle: null }, + }, + ])('treats old or invalid attachments as unknown without rewriting them: %j', attachment => { + const ws = createFakeWebSocket(attachment); + const state = createFakeState([ws]); + expect(readTestConnection(state, 'inst_1')).toEqual({ state: 'unknown' }); + expect(ws.serializeAttachment).not.toHaveBeenCalled(); + expect(ws.send).not.toHaveBeenCalled(); + expect(ws.close).not.toHaveBeenCalled(); + }); + + it('requires one matching open handshaken socket, without choosing by acceptance time', () => { + const attachment = { ...handshaken, observation: { ready: true, receivedAt: now, idle: null } }; + const older = createFakeWebSocket(attachment); + const newer = createFakeWebSocket({ ...attachment, acceptedAt: now }); + expect(readTestConnection(createFakeState([older, newer]), 'inst_1')).toEqual({ + state: 'unknown', + }); + expect(readTestConnection(createFakeState([older]), 'different')).toEqual({ + state: 'unknown', + }); + older.readyState = 2; + expect(readTestConnection(createFakeState([older]), 'inst_1')).toEqual({ + state: 'disconnected', + }); + const provisional = createFakeWebSocket({ handshakeComplete: false, acceptedAt: now }); + expect(readTestConnection(createFakeState([provisional]), 'inst_1')).toEqual({ + state: 'disconnected', + }); + expect( + readTestConnection(createFakeState([older, newer, provisional]), 'inst_1') + ).toMatchObject({ state: 'connected' }); + }); + + it('keeps operational dispatch usable when only the optional observation is invalid', async () => { + const ws = createFakeWebSocket({ ...handshaken, observation: { ready: 'invalid' } }); + const state = createFakeState([ws]); + const handler = createSandboxControlSocketHandler(state, 'sbx_test'); + expect(handler.hasHandshakenSocket()).toBe(true); + expect(readTestConnection(state, 'inst_1')).toEqual({ state: 'unknown' }); + await handler.handleMessage(asWs(ws), readyFrame); + expect(attachmentOf(ws).observation?.ready).toBe(true); + }); +}); diff --git a/services/cloud-agent-next/src/sandbox-control/socket.ts b/services/cloud-agent-next/src/sandbox-control/socket.ts index 5d0495ea3a..4d7e2dfa62 100644 --- a/services/cloud-agent-next/src/sandbox-control/socket.ts +++ b/services/cloud-agent-next/src/sandbox-control/socket.ts @@ -4,11 +4,17 @@ import { logControlDiagnostic, type ControlDiagnosticFields, } from './diagnostics.js'; +import { + safeSandboxRuntimeVersion, + type SandboxRuntimeMetadata, +} from '../shared/sandbox-status.js'; import { SANDBOX_CONTROL_PROTOCOL_VERSION, SANDBOX_CONTROL_WS_TAG, SANDBOX_HELLO_DEADLINE_MS, sandboxControlSocketAttachmentSchema, + sandboxControlObservationSchema, + type SandboxControlObservation, sessionRequestIdentitySchema, type ControlOperation, type RequestFrame, @@ -37,6 +43,7 @@ import { createControlRequestWaiters, type ControlRequestWaiters, } from './waiters.js'; +import { summarizeHeartbeatIdle } from './status-projection.js'; export type SandboxControlSocketState = DurableObjectState; @@ -56,7 +63,10 @@ export type SandboxControlConnectionIdentity = { export type SandboxControlSocketHooks = { validateHandshake?(providerInstanceId: string): boolean | Promise; - onHandshakeComplete?(identity: SandboxControlConnectionIdentity): void | Promise; + onHandshakeComplete?( + identity: SandboxControlConnectionIdentity, + runtime?: Pick + ): void | Promise; onReady?(identity: SandboxControlConnectionIdentity): void | Promise; onHeartbeat?( payload: SandboxHeartbeatPayload, @@ -90,11 +100,62 @@ export type SandboxControlSocketHandler = { closeProvisionalSockets(): void; }; +const operationalAttachmentSchema = sandboxControlSocketAttachmentSchema.extend({ + observation: sandboxControlObservationSchema.optional().catch(undefined), +}); + function readAttachment(ws: WebSocket): SandboxControlSocketAttachment | null { - const parsed = sandboxControlSocketAttachmentSchema.safeParse(ws.deserializeAttachment()); + const parsed = operationalAttachmentSchema.safeParse(ws.deserializeAttachment()); return parsed.success ? parsed.data : null; } +export type SandboxControlConnectionObservation = + | { state: 'disconnected' } + | { state: 'unknown' } + | { state: 'connected'; acceptedAt: number; observation: SandboxControlObservation }; + +const observedRuntimeSchema = sandboxControlSocketAttachmentSchema + .pick({ connectionId: true, providerInstanceId: true, wrapperInstanceId: true }) + .required({ connectionId: true, providerInstanceId: true }) + .extend({ readyConnectionId: sandboxControlSocketAttachmentSchema.shape.connectionId }); + +export function readSandboxControlConnection( + state: Pick, + providerRef: string | null, + runtime: unknown +): SandboxControlConnectionObservation { + let current: SandboxControlSocketAttachment | undefined; + for (const ws of state.getWebSockets(SANDBOX_CONTROL_WS_TAG)) { + if (ws.readyState !== 1) continue; + const parsed = sandboxControlSocketAttachmentSchema.safeParse(ws.deserializeAttachment()); + if (!parsed.success) return { state: 'unknown' }; + if (!parsed.data.handshakeComplete) continue; + if (current) return { state: 'unknown' }; + current = parsed.data; + } + if (!current) return { state: 'disconnected' }; + const expected = observedRuntimeSchema.safeParse(runtime); + if ( + !expected.success || + current.connectionId !== expected.data.connectionId || + current.providerInstanceId !== expected.data.providerInstanceId || + current.wrapperInstanceId !== expected.data.wrapperInstanceId || + (current.observation?.ready && expected.data.readyConnectionId !== current.connectionId) || + current.protocolVersion !== SANDBOX_CONTROL_PROTOCOL_VERSION || + current.providerInstanceId !== providerRef || + !current.observation || + !Number.isSafeInteger(current.acceptedAt) || + current.observation.receivedAt < current.acceptedAt + ) { + return { state: 'unknown' }; + } + return { + state: 'connected', + acceptedAt: current.acceptedAt, + observation: current.observation, + }; +} + function sendJson(ws: WebSocket, value: unknown): void { if (ws.readyState !== 1) return; ws.send(JSON.stringify(value)); @@ -220,6 +281,32 @@ export function createSandboxControlSocketHandler( const activatingConnections = new Set(); const log = (event: string, fields: ControlDiagnosticFields) => logControlDiagnostic(event, { sandboxId, ...fields }); + const observations = new WeakMap(); + + function recordObservation( + ws: WebSocket, + attachment: SandboxControlSocketAttachment, + ready: boolean + ): SandboxControlObservation | undefined { + if (currentHandshakenSocket(state)?.socket !== ws) return undefined; + const observation: SandboxControlObservation = { ready, receivedAt: Date.now(), idle: null }; + observations.set(ws, observation); + ws.serializeAttachment({ ...attachment, observation }); + return observation; + } + + function invalidateIdleObservation( + ws: WebSocket, + attachment: SandboxControlSocketAttachment + ): void { + if (currentHandshakenSocket(state)?.socket !== ws) return; + observations.delete(ws); + if (!attachment.observation?.idle) return; + ws.serializeAttachment({ + ...attachment, + observation: { ...attachment.observation, idle: null }, + }); + } return { hasHandshakenSocket(): boolean { @@ -405,6 +492,7 @@ export function createSandboxControlSocketHandler( } } + observations.delete(ws); ws.serializeAttachment(completed); if (replaced) waiters.rejectAll('Wrapper socket replaced'); for (const existing of superseded) { @@ -416,7 +504,9 @@ export function createSandboxControlSocketHandler( activatingConnections.add(identity.connectionId); try { - const activation = hooks.onHandshakeComplete?.(identity); + const activation = hooks.onHandshakeComplete?.(identity, { + wrapperVersion: safeSandboxRuntimeVersion(payload.wrapperVersion), + }); if (activation) await activation; } finally { activatingConnections.delete(identity.connectionId); @@ -481,16 +571,27 @@ export function createSandboxControlSocketHandler( return; } if (frame.event === 'sandbox.ready') { + recordObservation(ws, attachment, true); await hooks.onReady?.(identity); } else if (frame.event === 'sandbox.heartbeat') { - await hooks.onHeartbeat?.(eventPayload.payload as SandboxHeartbeatPayload, identity); + const payload = eventPayload.payload as SandboxHeartbeatPayload; + const observation = recordObservation(ws, attachment, payload.kilo.ready); + await hooks.onHeartbeat?.(payload, identity); + if (observation) { + const idle = await summarizeHeartbeatIdle(payload); + if (isCurrentConnection(state, ws, identity) && observations.get(ws) === observation) { + ws.serializeAttachment({ ...attachment, observation: { ...observation, idle } }); + } + } } else if (frame.event === 'session.event') { + invalidateIdleObservation(ws, attachment); await hooks.onSessionEvent?.( frame.session, eventPayload.payload as SessionEventPayload, identity ); } else if (frame.event === 'session.preparing') { + invalidateIdleObservation(ws, attachment); await hooks.onSessionPreparing?.( frame.session, eventPayload.payload as SessionPreparingPayload, diff --git a/services/cloud-agent-next/src/sandbox-control/status-projection.test.ts b/services/cloud-agent-next/src/sandbox-control/status-projection.test.ts index aa54fb2969..652daa733f 100644 --- a/services/cloud-agent-next/src/sandbox-control/status-projection.test.ts +++ b/services/cloud-agent-next/src/sandbox-control/status-projection.test.ts @@ -1,5 +1,16 @@ -import { describe, expect, it } from 'vitest'; -import { projectReportedStatus } from './status-projection.js'; +import { describe, expect, it, vi } from 'vitest'; +import { + projectReportedStatus, + projectSandboxStatus, + summarizeHeartbeatIdle, +} from './status-projection.js'; +import { DEADLINE_MS } from './deadlines.js'; +import { readSandboxControlState, type StoredSandboxControlState } from './durable-state.js'; +import type { PhysicalRecord } from './physical-lifecycle.js'; +import type { SessionRoute } from './session-routes.js'; +import type { SandboxControlConnectionObservation } from './socket.js'; +import type { SandboxHeartbeatPayload } from '../shared/sandbox-control-protocol.js'; +import { SandboxStatusSnapshotSchema } from '../shared/sandbox-status.js'; describe('projectReportedStatus', () => { it('reports off when physical is stopped', () => { @@ -82,3 +93,417 @@ describe('projectReportedStatus', () => { ); }); }); + +const now = 1_000_000; +const receivedAt = now - 1_000; +const ownerId = 'owner_1'; +const physical: PhysicalRecord = { + state: 'running', + providerRef: 'instance_1', + createIntent: null, + stopTombstone: null, + resumable: false, +}; +const route: SessionRoute = { + sessionId: 'workspace_1', + kiloSessionId: 'kilo_1', + directory: '/workspace/one', + ownerId, + lastState: 'idle', + lastStateAt: receivedAt, + idleForMs: 0, + waitingOn: null, +}; +const idleHeartbeat: SandboxHeartbeatPayload = { + state: 'idle', + pendingMessages: 0, + kilo: { ready: true }, + sessions: [{ kiloSessionId: route.kiloSessionId, state: 'idle', idleForMs: 0 }], +}; + +async function fixture() { + const connection = { + state: 'connected', + acceptedAt: now - 10_000, + observation: { ready: true, receivedAt, idle: await summarizeHeartbeatIdle(idleHeartbeat) }, + } satisfies SandboxControlConnectionObservation; + return { + stored: { + physical: { ...physical }, + routes: [{ ...route }], + deadlines: { + idleStop: receivedAt + DEADLINE_MS.idleStop, + heartbeatExpiry: receivedAt + DEADLINE_MS.heartbeatExpiry, + }, + } satisfies StoredSandboxControlState, + connection, + ownerId, + provider: 'cloudflare', + now, + }; +} + +describe('passive sandbox status projection', () => { + it.each(['running', 'stopped', 'creating', 'stopping', 'unknown', 'failed'] as const)( + 'projects retained metadata for %s without changing lifecycle freshness', + async state => { + const input = await fixture(); + const runtime = { + sandboxType: 'isolated-standard' as const, + wrapperVersion: '2.4.0', + kiloCliVersion: '7.4.20', + startedAt: null, + stoppedAt: null, + }; + const stored = { ...input.stored, physical: { ...physical, state }, runtime }; + const before = structuredClone(stored); + const result = await projectSandboxStatus({ + ...input, + stored, + connection: { state: 'disconnected' }, + }); + expect(result.runtime).toEqual(runtime); + expect(result.estimatedSleepAt).toBeNull(); + if (state === 'running') expect(result.status).toBe('unreachable'); + expect(await projectSandboxStatus({ ...input, stored, ownerId: null })).not.toHaveProperty( + 'runtime' + ); + expect( + await projectSandboxStatus({ ...input, stored: { ...stored, physical: null } }) + ).not.toHaveProperty('runtime'); + expect(stored).toEqual(before); + } + ); + + it.each([ + ['stopped', 'sleeping', 'sandbox_stopped'], + ['creating', 'starting', 'sandbox_starting'], + ['stopping', 'stopping', 'sandbox_stopping'], + ['failed', 'error', 'sandbox_failed'], + ['unknown', 'unknown', 'insufficient_evidence'], + ] as const)( + 'maps confirmed %s to %s regardless of connection and work', + async (state, status, detailCode) => { + const input = await fixture(); + input.stored.physical.state = state; + input.stored.routes[0] = { ...route, lastState: 'active' }; + const result = await projectSandboxStatus(input); + expect(result).toEqual({ + status, + detailCode, + provider: 'Cloudflare', + observedAt: now, + inactivityTimeoutMs: DEADLINE_MS.idleStop, + estimatedSleepAt: null, + }); + expect(SandboxStatusSnapshotSchema.safeParse(result).success).toBe(true); + } + ); + + it('does not synthesize stopped state from missing records or owners', async () => { + const input = await fixture(); + expect( + await projectSandboxStatus({ ...input, stored: { ...input.stored, physical: null } }) + ).toMatchObject({ status: 'unknown', estimatedSleepAt: null }); + expect(await projectSandboxStatus({ ...input, ownerId: null })).toMatchObject({ + status: 'unknown', + estimatedSleepAt: null, + }); + expect( + await projectSandboxStatus({ + ...input, + stored: { ...input.stored, physical: { ...physical, providerRef: null } }, + }) + ).toMatchObject({ status: 'unknown', estimatedSleepAt: null }); + }); + + it('requires explicit fresh ready evidence, including false and exact expiry', async () => { + const input = await fixture(); + const project = (connection: SandboxControlConnectionObservation) => + projectSandboxStatus({ ...input, connection }); + expect(await project({ state: 'unknown' })).toMatchObject({ + status: 'unknown', + estimatedSleepAt: null, + }); + expect(await project({ state: 'disconnected' })).toMatchObject({ + status: 'unreachable', + detailCode: 'connection_unavailable', + estimatedSleepAt: null, + }); + expect( + await project({ + ...input.connection, + observation: { ...input.connection.observation, ready: false }, + }) + ).toMatchObject({ status: 'starting', detailCode: 'sandbox_starting', estimatedSleepAt: null }); + for (const ready of [true, false]) { + expect( + await project({ + ...input.connection, + observation: { ready, receivedAt: now - DEADLINE_MS.heartbeatExpiry, idle: null }, + }) + ).toMatchObject({ status: 'unreachable', estimatedSleepAt: null }); + } + expect( + await project({ + ...input.connection, + observation: { ready: true, receivedAt: now + 1, idle: null }, + }) + ).toMatchObject({ status: 'unknown', estimatedSleepAt: null }); + expect( + await project({ + ...input.connection, + observation: { ready: true, receivedAt: now - DEADLINE_MS.heartbeatExpiry + 1, idle: null }, + }) + ).toMatchObject({ status: 'active', estimatedSleepAt: null }); + }); + + it('uses only the stored future deadline with coherent current-connection sandbox-wide idle evidence', async () => { + const input = await fixture(); + const result = await projectSandboxStatus(input); + expect(result).toEqual({ + status: 'active', + detailCode: 'sandbox_ready', + provider: 'Cloudflare', + observedAt: now, + inactivityTimeoutMs: DEADLINE_MS.idleStop, + estimatedSleepAt: input.stored.deadlines.idleStop, + }); + expect(SandboxStatusSnapshotSchema.safeParse(result).success).toBe(true); + expect(await projectSandboxStatus({ ...input, provider: 'vercel' })).toMatchObject({ + provider: 'Vercel', + estimatedSleepAt: input.stored.deadlines.idleStop, + }); + expect( + await projectSandboxStatus({ ...input, provider: 'private-provider-sentinel' }) + ).toMatchObject({ provider: 'Unknown' }); + expect(JSON.stringify(result)).not.toMatch(/instance_1|owner_1|workspace_1|kilo_1/); + }); + + it.each([ + null, + {}, + { idleStop: now - 1 }, + { idleStop: now }, + { idleStop: now + DEADLINE_MS.idleStop + 1 }, + { idleStop: now - 20_000 + DEADLINE_MS.idleStop }, + ])('suppresses unsupported, expired, and inherited deadlines: %j', async deadlines => { + const input = await fixture(); + expect( + await projectSandboxStatus({ ...input, stored: { ...input.stored, deadlines } }) + ).toMatchObject({ status: 'active', estimatedSleepAt: null }); + }); + + it('suppresses estimates when only readiness, not a current idle heartbeat, was received', async () => { + const input = await fixture(); + input.connection.observation.idle = null; + expect(await projectSandboxStatus(input)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + }); + + it.each([ + { lastState: 'active' }, + { lastState: 'finalizing' }, + { waitingOn: 'preparation' }, + { waitingOn: 'input' }, + { lastState: null }, + { lastStateAt: null }, + { lastStateAt: receivedAt - 1 }, + { lastStateAt: now + 1 }, + { idleForMs: null }, + { waitingOn: 'model' }, + { waitingOn: 'tool' }, + { waitingOn: 'finalizing' }, + { ownerId: 'other-owner' }, + { kiloSessionId: 'other-kilo' }, + ] satisfies Partial[])( + 'suppresses estimates for incomplete or contradictory route evidence: %j', + async patch => { + const input = await fixture(); + input.stored.routes[0] = { ...route, ...patch }; + expect(await projectSandboxStatus(input)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + } + ); + + it('requires complete coverage of every shared-sandbox route', async () => { + const input = await fixture(); + const sibling = { + ...route, + sessionId: 'workspace_2', + kiloSessionId: 'kilo_2', + directory: '/workspace/two', + }; + input.stored.routes.push(sibling); + expect(await projectSandboxStatus(input)).toMatchObject({ estimatedSleepAt: null }); + input.connection.observation.idle = await summarizeHeartbeatIdle({ + ...idleHeartbeat, + sessions: [ + { kiloSessionId: 'kilo_2', state: 'idle', idleForMs: 0 }, + ...idleHeartbeat.sessions, + ], + }); + expect(await projectSandboxStatus(input)).toMatchObject({ + estimatedSleepAt: input.stored.deadlines.idleStop, + }); + input.stored.routes[1] = { ...sibling, lastState: 'active' }; + expect(await projectSandboxStatus(input)).toMatchObject({ estimatedSleepAt: null }); + input.stored.routes[1] = { ...route }; + expect(await projectSandboxStatus(input)).toMatchObject({ estimatedSleepAt: null }); + expect( + await projectSandboxStatus({ ...input, stored: { ...input.stored, routes: null } }) + ).toMatchObject({ estimatedSleepAt: null }); + }); + + it('projects same-directory sibling roots only with coherent worktree ownership and idle evidence', async () => { + const input = await fixture(); + const first = { ...route, worktreeId: 'worktree_shared' }; + const sibling = { ...first, sessionId: 'workspace_2', kiloSessionId: 'kilo_2' }; + const values = { + physical_record: physical, + deadlines: input.stored.deadlines, + session_routes: [first, sibling], + }; + const stored = await readSandboxControlState({ + get: async key => values[key as keyof typeof values], + }); + input.connection.observation.idle = await summarizeHeartbeatIdle({ + ...idleHeartbeat, + sessions: [ + ...idleHeartbeat.sessions, + { kiloSessionId: 'kilo_2', state: 'idle', idleForMs: 0 }, + ], + }); + expect(await projectSandboxStatus({ ...input, stored })).toMatchObject({ + status: 'active', + estimatedSleepAt: input.stored.deadlines.idleStop, + }); + for (const patch of [ + { worktreeId: 'worktree_other' }, + { worktreeId: undefined }, + { directory: '/workspace/other' }, + { ownerId: 'other-owner' }, + { waitingOn: 'input' }, + { waitingOn: 'preparation' }, + ] satisfies Partial[]) { + expect( + await projectSandboxStatus({ + ...input, + stored: { ...stored, routes: [first, { ...sibling, ...patch }] }, + }) + ).toMatchObject({ status: 'active', estimatedSleepAt: null }); + } + }); + + it('does not report a tombstoned running allocation as active', async () => { + const input = await fixture(); + input.stored.physical.stopTombstone = { reason: 'retired', attempts: 0, createdAt: now }; + expect(await projectSandboxStatus(input)).toMatchObject({ + status: 'stopping', + estimatedSleepAt: null, + }); + }); + + it('permits an observed empty sandbox but not a missing route table', async () => { + const input = await fixture(); + input.stored.routes = []; + input.connection.observation.idle = await summarizeHeartbeatIdle({ + ...idleHeartbeat, + sessions: [], + }); + expect(await projectSandboxStatus(input)).toMatchObject({ + estimatedSleepAt: input.stored.deadlines.idleStop, + }); + expect( + await projectSandboxStatus({ ...input, stored: { ...input.stored, routes: null } }) + ).toMatchObject({ estimatedSleepAt: null }); + }); +}); + +describe('bounded heartbeat idle summaries', () => { + it.each([ + { state: 'active' }, + { state: 'finalizing' }, + { pendingMessages: undefined }, + { pendingMessages: 1 }, + { activeKiloSessions: 1 }, + { kilo: { ready: false } }, + { sessions: [{ kiloSessionId: 'kilo_1', state: 'active', idleForMs: 180_000 }] }, + { sessions: [{ kiloSessionId: 'kilo_1', state: 'finalizing', idleForMs: 0 }] }, + { sessions: [{ kiloSessionId: 'kilo_1', state: 'idle', idleForMs: 0, waitingOn: 'tool' }] }, + { sessions: [...idleHeartbeat.sessions, ...idleHeartbeat.sessions] }, + ] satisfies Partial[])( + 'rejects busy or incomplete sandbox-wide evidence: %j', + async patch => { + expect(await summarizeHeartbeatIdle({ ...idleHeartbeat, ...patch })).toBeNull(); + } + ); +}); + +describe('validation-only control storage reads', () => { + function storage(values: Record) { + return { + get: vi.fn(async (key: string) => values[key]), + put: vi.fn(), + delete: vi.fn(), + }; + } + + it('returns null instead of synthetic sleeping or repaired values', async () => { + const target = storage({}); + expect(await readSandboxControlState(target)).toEqual({ + physical: null, + routes: null, + deadlines: null, + }); + expect(target.put).not.toHaveBeenCalled(); + expect(target.delete).not.toHaveBeenCalled(); + }); + + it.each([ + { physical_record: { state: 'stopped' } }, + { physical_record: { ...physical, state: 'unsupported' } }, + { physical_record: { ...physical, providerRef: 10 } }, + { physical_record: { ...physical, createIntent: { intentId: 'intent', createdAt: Infinity } } }, + { deadlines: { idleStop: NaN } }, + { deadlines: { idleStop: Infinity } }, + { deadlines: { idleStop: -1 } }, + { deadlines: { idleStop: 1.5 } }, + { deadlines: { idleStop: '1000000' } }, + { deadlines: { idleStop: 8_640_000_000_000_001 } }, + { session_routes: [{ ...route, lastStateAt: NaN }] }, + { session_routes: [{ ...route, worktreeId: 42 }] }, + { session_routes: {} }, + ])('does not accept or repair malformed stored evidence: %j', async values => { + const target = storage(values); + expect(await readSandboxControlState(target)).toEqual({ + physical: null, + routes: null, + deadlines: null, + }); + expect(target.put).not.toHaveBeenCalled(); + expect(target.delete).not.toHaveBeenCalled(); + }); + + it('preserves valid storage and strips unrelated fields from the projection inputs', async () => { + const values = { + physical_record: { ...physical, internal: 'private-value' }, + session_routes: [{ ...route, internal: 'private-value' }], + deadlines: { idleStop: now + DEADLINE_MS.idleStop }, + }; + const before = structuredClone(values); + const target = storage(values); + expect(await readSandboxControlState(target)).toEqual({ + physical, + routes: [route], + deadlines: values.deadlines, + }); + expect(values).toEqual(before); + expect(target.put).not.toHaveBeenCalled(); + expect(target.delete).not.toHaveBeenCalled(); + }); +}); diff --git a/services/cloud-agent-next/src/sandbox-control/status-projection.ts b/services/cloud-agent-next/src/sandbox-control/status-projection.ts index aef31d69e4..1e838607bd 100644 --- a/services/cloud-agent-next/src/sandbox-control/status-projection.ts +++ b/services/cloud-agent-next/src/sandbox-control/status-projection.ts @@ -1,3 +1,13 @@ +import { getSandboxProviderLabel, type SandboxStatusSnapshot } from '../shared/sandbox-status.js'; +import type { + SandboxControlObservation, + SandboxHeartbeatPayload, +} from '../shared/sandbox-control-protocol.js'; +import { sha256Hex } from '../utils/sha256.js'; +import { DEADLINE_MS } from './deadlines.js'; +import type { StoredSandboxControlState } from './durable-state.js'; +import type { SandboxControlConnectionObservation } from './socket.js'; + export type ReportedSandboxStatus = | 'off' | 'booting' @@ -42,3 +52,125 @@ export function projectReportedStatus(input: { } } } + +export async function summarizeHeartbeatIdle( + payload: SandboxHeartbeatPayload +): Promise { + if ( + !payload.kilo.ready || + payload.state !== 'idle' || + payload.pendingMessages !== 0 || + (payload.activeKiloSessions !== undefined && payload.activeKiloSessions !== 0) || + payload.sessions.some(session => session.state !== 'idle' || session.waitingOn !== undefined) + ) { + return null; + } + const sessionIds = payload.sessions.map(session => session.kiloSessionId); + if (new Set(sessionIds).size !== sessionIds.length) return null; + return { + sessionCount: sessionIds.length, + sessionIdsHash: await sha256Hex(JSON.stringify(sessionIds.sort())), + }; +} + +function hasConsistentWorktrees(routes: NonNullable): boolean { + const directories = new Map(); + const worktrees = new Map(); + for (const route of routes) { + if ( + directories.has(route.directory) && + (!route.worktreeId || directories.get(route.directory) !== route.worktreeId) + ) + return false; + if ( + route.worktreeId && + worktrees.has(route.worktreeId) && + worktrees.get(route.worktreeId) !== route.directory + ) + return false; + directories.set(route.directory, route.worktreeId); + if (route.worktreeId) worktrees.set(route.worktreeId, route.directory); + } + return true; +} + +export async function projectSandboxStatus(input: { + stored: StoredSandboxControlState; + connection: SandboxControlConnectionObservation; + ownerId: string | null; + provider: unknown; + now: number; +}): Promise { + const { stored, connection, ownerId, now } = input; + const snapshot: SandboxStatusSnapshot = { + status: 'unknown', + provider: getSandboxProviderLabel(input.provider), + observedAt: now, + detailCode: 'insufficient_evidence', + inactivityTimeoutMs: DEADLINE_MS.idleStop, + estimatedSleepAt: null, + }; + if (!ownerId || !stored.physical) return snapshot; + if (stored.runtime) snapshot.runtime = stored.runtime; + switch (stored.physical.state) { + case 'stopped': + return { ...snapshot, status: 'sleeping', detailCode: 'sandbox_stopped' }; + case 'creating': + return { ...snapshot, status: 'starting', detailCode: 'sandbox_starting' }; + case 'stopping': + return { ...snapshot, status: 'stopping', detailCode: 'sandbox_stopping' }; + case 'failed': + return { ...snapshot, status: 'error', detailCode: 'sandbox_failed' }; + case 'unknown': + return snapshot; + case 'running': + break; + } + if (stored.physical.stopTombstone) { + return { ...snapshot, status: 'stopping', detailCode: 'sandbox_stopping' }; + } + if (connection.state === 'unknown' || stored.physical.providerRef === null) return snapshot; + if (connection.state === 'disconnected') { + return { ...snapshot, status: 'unreachable', detailCode: 'connection_unavailable' }; + } + const { observation } = connection; + if (observation.receivedAt > now) return snapshot; + if (now - observation.receivedAt >= DEADLINE_MS.heartbeatExpiry) { + return { ...snapshot, status: 'unreachable', detailCode: 'connection_unavailable' }; + } + if (!observation.ready) { + return { ...snapshot, status: 'starting', detailCode: 'sandbox_starting' }; + } + + const { idle } = observation; + const { routes } = stored; + const idleStop = stored.deadlines?.idleStop; + let estimatedSleepAt: number | null = null; + if ( + idle && + idleStop !== undefined && + idleStop > now && + idleStop - DEADLINE_MS.idleStop >= connection.acceptedAt && + idleStop - DEADLINE_MS.idleStop <= observation.receivedAt && + routes && + routes.length === idle.sessionCount && + new Set(routes.map(route => route.sessionId)).size === routes.length && + new Set(routes.map(route => route.kiloSessionId)).size === routes.length && + hasConsistentWorktrees(routes) && + routes.every( + route => + route.ownerId === ownerId && + route.lastState === 'idle' && + route.lastStateAt !== null && + route.lastStateAt >= observation.receivedAt && + route.lastStateAt <= now && + route.idleForMs !== null && + route.waitingOn === null + ) && + (await sha256Hex(JSON.stringify(routes.map(route => route.kiloSessionId).sort()))) === + idle.sessionIdsHash + ) { + estimatedSleepAt = idleStop; + } + return { ...snapshot, status: 'active', detailCode: 'sandbox_ready', estimatedSleepAt }; +} diff --git a/services/cloud-agent-next/src/sandbox-session/SandboxSession.ts b/services/cloud-agent-next/src/sandbox-session/SandboxSession.ts index cfb191e24e..e87cd4d23b 100644 --- a/services/cloud-agent-next/src/sandbox-session/SandboxSession.ts +++ b/services/cloud-agent-next/src/sandbox-session/SandboxSession.ts @@ -71,6 +71,7 @@ import { import { applyControlPlanePreparingEvent } from './control-plane-preparing.js'; import { logger } from '../logger.js'; import { sandboxControlRpc } from './control-rpc.js'; +import { getSandboxControlStub } from '../sandbox-control/stub.js'; import { DEADLINE_MS } from '../sandbox-control/deadlines.js'; import { createMessageId } from '../session/message-id.js'; import { validateControlSessionOptions } from './attach-payload.js'; @@ -97,6 +98,11 @@ import { type SessionPreparingPayload, } from '../shared/sandbox-control-protocol.js'; import type { WrapperPty } from '../kilo/wrapper-client.js'; +import { + SandboxStatusSnapshotSchema, + getSandboxProviderLabel, + type SandboxStatusSnapshot, +} from '../shared/sandbox-status.js'; import { ControlRequestError, controlDispatchDisposition, @@ -114,6 +120,7 @@ import { createSandboxTerminalBridge, type SandboxTerminalRecord } from './termi import { createSandboxTerminalLifecycle, SANDBOX_SESSION_METADATA_KEY, + SANDBOX_SESSION_DELETED_WORKTREE_KEY, } from './terminal-lifecycle.js'; import { acceptQueuedMessage, @@ -138,7 +145,7 @@ import { const METADATA_KEY = SANDBOX_SESSION_METADATA_KEY; const MESSAGES_KEY = 'session_messages'; -const DELETED_WORKTREE_KEY = 'deleted_worktree'; +const DELETED_WORKTREE_KEY = SANDBOX_SESSION_DELETED_WORKTREE_KEY; const DELETION_COMPLETED_KEY = 'deletion_completed'; type SandboxControlEventInput = { @@ -222,9 +229,6 @@ export class SandboxSession extends DurableObject { this.deletedWorktreeId = cloudAgentWorktreeIdSchema .optional() .parse(ctx.storage.kv.get(DELETED_WORKTREE_KEY)); - if (this.deletedWorktreeId) { - this.terminalLifecycle.beginDeletion(this.terminalLifecycle.getStoredMetadata()); - } }); } @@ -547,6 +551,45 @@ export class SandboxSession extends DurableObject { return this.terminalLifecycle.isBlocked() ? null : this.terminalLifecycle.getStoredMetadata(); } + async getSandboxStatus(): Promise { + const metadata = + this.deletedWorktreeId || this.terminalLifecycle.isDeleted() + ? null + : this.terminalLifecycle.getStoredMetadata(); + const provider = metadata ? getSandboxProvider(metadata) : undefined; + const unknown: SandboxStatusSnapshot = { + status: 'unknown', + provider: getSandboxProviderLabel(provider), + observedAt: Date.now(), + detailCode: 'insufficient_evidence', + inactivityTimeoutMs: DEADLINE_MS.idleStop, + estimatedSleepAt: null, + }; + const sandboxId = metadata?.workspace?.sandboxId; + if (!metadata || !sandboxId || !provider || metadata.identity.sessionId !== this.sessionId) { + return unknown; + } + try { + return await withDORetry( + () => getSandboxControlStub(this.env, sandboxId), + async control => { + try { + return SandboxStatusSnapshotSchema.parse( + await control.getSandboxStatus({ ownerId: metadata.identity.userId, provider }) + ); + } catch (error) { + throw Object.assign(new Error('Sandbox status unavailable'), { + retryable: error instanceof Error && 'retryable' in error && error.retryable === true, + }); + } + }, + 'getSandboxStatus' + ); + } catch { + return { ...unknown, observedAt: Date.now(), detailCode: 'status_unavailable' }; + } + } + async validateKiloGlobalFeedProducer(_params: { kiloSessionId: string; wrapperRunId: string; diff --git a/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.test.ts b/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.test.ts index dda6e7fffa..9d77daf8b0 100644 --- a/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.test.ts +++ b/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.test.ts @@ -16,6 +16,7 @@ import { createSandboxTerminalLifecycle, SANDBOX_SESSION_LIFECYCLE_KEY, SANDBOX_SESSION_METADATA_KEY, + SANDBOX_SESSION_DELETED_WORKTREE_KEY, } from './terminal-lifecycle.js'; vi.mock('@cloudflare/sandbox', () => ({ getSandbox: vi.fn() })); @@ -192,6 +193,25 @@ function createFixture( } describe('SandboxSession terminal lifecycle', () => { + it('honors a retained worktree deletion marker without repairing storage during reads', async () => { + const fixture = createFixture(); + fixture.storage.kv.put(SANDBOX_SESSION_DELETED_WORKTREE_KEY, 'worktree_deleted'); + const before = structuredClone([...fixture.values]); + expect(fixture.lifecycle.isDeleted()).toBe(true); + expect(fixture.lifecycle.isBlocked()).toBe(true); + expect(fixture.lifecycle.captureEpoch()).toBeNull(); + expect( + await fixture.lifecycle.createTerminal({ operationId: FIRST_OPERATION_ID }) + ).toMatchObject({ success: false }); + expect([...fixture.values]).toEqual(before); + expect(fixture.control.request).not.toHaveBeenCalled(); + fixture.lifecycle.beginDeletion(fixture.metadata); + expect(fixture.storage.kv.get(SANDBOX_SESSION_LIFECYCLE_KEY)).toMatchObject({ + state: 'deleted', + epoch: 1, + }); + }); + it('requires successful runtime-bound session attachment without starting compute', async () => { const fixture = createFixture({ attached: false }); diff --git a/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.ts b/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.ts index 6a1d8ccff9..93149fae6b 100644 --- a/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.ts +++ b/services/cloud-agent-next/src/sandbox-session/terminal-lifecycle.ts @@ -24,6 +24,7 @@ import type { SandboxTerminalRecord } from './terminal-bridge.js'; export const SANDBOX_SESSION_METADATA_KEY = 'session_metadata'; export const SANDBOX_SESSION_LIFECYCLE_KEY = 'session_lifecycle_fence'; +export const SANDBOX_SESSION_DELETED_WORKTREE_KEY = 'deleted_worktree'; const ATTACHED_SESSION_KEY = 'terminal_attached_session'; const TERMINAL_PREFIX = 'terminal:'; @@ -148,7 +149,11 @@ export function createSandboxTerminalLifecycle(deps: TerminalLifecycleDeps) { function readFence(): LifecycleFence | undefined { const value = storage.kv.get(SANDBOX_SESSION_LIFECYCLE_KEY); - if (value === undefined) return undefined; + if (value === undefined) { + return storage.kv.get(SANDBOX_SESSION_DELETED_WORKTREE_KEY) === undefined + ? undefined + : { epoch: Number.MAX_SAFE_INTEGER, state: 'deleted' }; + } const parsed = lifecycleFenceSchema.safeParse(value); return parsed.success ? parsed.data : { epoch: Number.MAX_SAFE_INTEGER, state: 'deleted' }; } @@ -721,7 +726,8 @@ export function createSandboxTerminalLifecycle(deps: TerminalLifecycleDeps) { } function beginFence(state: LifecycleFence['state'], metadata: SessionMetadata | null) { - const previous = readFence(); + const previous = + storage.kv.get(SANDBOX_SESSION_LIFECYCLE_KEY) === undefined ? undefined : readFence(); if (previous?.state === 'deleted') return previous; const fence: LifecycleFence = { epoch: (previous?.epoch ?? 0) + 1, diff --git a/services/cloud-agent-next/src/shared/sandbox-control-protocol.ts b/services/cloud-agent-next/src/shared/sandbox-control-protocol.ts index 0e779c3b77..a8a7ae934e 100644 --- a/services/cloud-agent-next/src/shared/sandbox-control-protocol.ts +++ b/services/cloud-agent-next/src/shared/sandbox-control-protocol.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { SandboxRuntimeVersionSchema } from './sandbox-status.js'; export const SANDBOX_CONTROL_PROTOCOL_VERSION = 1; @@ -143,6 +144,7 @@ export const sandboxHelloPayloadSchema = z.object({ export const sandboxHelloResultSchema = z.object({ protocolVersion: z.literal(SANDBOX_CONTROL_PROTOCOL_VERSION), handshakeComplete: z.literal(true), + capabilities: z.object({ kiloVersionHeartbeat: z.boolean().optional() }).optional(), }); export type SandboxHelloPayload = z.infer; @@ -163,6 +165,7 @@ export const sandboxHeartbeatPayloadSchema = z kilo: z .object({ ready: z.boolean(), + version: SandboxRuntimeVersionSchema.nullable().optional().catch(undefined), reason: z .enum([ 'feed_stale', @@ -576,6 +579,24 @@ export type SessionTerminalConnectResult = z.infer; export type SessionPreparingPayload = z.infer; +const observationTimestampSchema = z.number().int().nonnegative().max(8_640_000_000_000_000); + +export const sandboxControlObservationSchema = z + .object({ + ready: z.boolean(), + receivedAt: observationTimestampSchema, + idle: z + .object({ + sessionCount: z.number().int().nonnegative(), + sessionIdsHash: z.string().regex(/^[0-9a-f]{64}$/), + }) + .strict() + .nullable(), + }) + .strict(); + +export type SandboxControlObservation = z.infer; + export const sandboxControlSocketAttachmentSchema = z.object({ handshakeComplete: z.boolean(), acceptedAt: z.number().int().nonnegative(), @@ -583,6 +604,7 @@ export const sandboxControlSocketAttachmentSchema = z.object({ protocolVersion: z.literal(SANDBOX_CONTROL_PROTOCOL_VERSION).optional(), providerInstanceId: z.string().min(1).max(256).optional(), wrapperInstanceId: wrapperInstanceIdSchema.optional(), + observation: sandboxControlObservationSchema.optional(), }); export type SandboxControlSocketAttachment = z.infer; diff --git a/services/cloud-agent-next/src/shared/sandbox-status.test.ts b/services/cloud-agent-next/src/shared/sandbox-status.test.ts new file mode 100644 index 0000000000..f2ec721bcd --- /dev/null +++ b/services/cloud-agent-next/src/shared/sandbox-status.test.ts @@ -0,0 +1,335 @@ +import { describe, expect, it } from 'vitest'; +import { + getSandboxProviderLabel, + safeSandboxRuntimeVersion, + SandboxRuntimeMetadataSchema, + SANDBOX_STATUS_DETAIL_MESSAGES, + SandboxStatusSnapshotSchema, + SandboxStatusSessionIdSchema, + type SandboxStatusSnapshot, +} from './sandbox-status.js'; + +const observedAt = 1_800_000_000_000; +const snapshot = { + status: 'active', + provider: 'Cloudflare', + observedAt, + detailCode: 'sandbox_ready', + inactivityTimeoutMs: 300_000, + estimatedSleepAt: null, +} satisfies SandboxStatusSnapshot; + +const lifecycleCases = [ + { status: 'active', detailCode: 'sandbox_ready' }, + { status: 'sleeping', detailCode: 'sandbox_stopped' }, + { status: 'starting', detailCode: 'sandbox_starting' }, + { status: 'stopping', detailCode: 'sandbox_stopping' }, + { status: 'error', detailCode: 'sandbox_failed' }, + { status: 'unreachable', detailCode: 'connection_unavailable' }, + { status: 'unknown', detailCode: 'insufficient_evidence' }, + { status: 'unknown', detailCode: 'status_unavailable' }, +] satisfies Pick[]; + +const invalidNumbers = [NaN, Infinity, -Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER + 1, '123']; + +const privateFields = { + sessionId: 'agent_private-session', + sandboxId: 'usr-private-sandbox', + providerInstanceId: 'private-instance', + wrapperRunId: 'private-wrapper', + userId: 'private-owner', + orgId: 'private-organization', + region: 'private-region', + url: 'https://private-runtime.invalid', + credentials: { token: 'private-credential' }, + headers: { Authorization: 'Bearer private-credential' }, + error: new Error('private-provider-failure'), + message: 'private-provider-failure', +}; + +describe('SandboxStatusSessionIdSchema', () => { + it.each([ + 'workspace_12345678-1234-4234-9234-123456789abc', + 'workspace_ABCDEF01-2345-6789-ABCD-EF0123456789', + ])('accepts a valid control-plane reference %s', sessionId => { + expect(SandboxStatusSessionIdSchema.parse(sessionId)).toBe(sessionId); + }); + + it.each([ + 'agent_12345678-1234-4234-9234-123456789abc', + 'sess_12345678-1234-4234-9234-123456789abc', + 'ses_12345678901234567890123456', + '12345678-1234-4234-9234-123456789abc', + 'workspace_', + 'workspace_pending', + 'workspace_../../private', + 'workspace_zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz', + 'workspace_12345678-1234-4234-9234-123456789ab', + 'workspace_12345678-1234-4234-9234-123456789abc\n', + ' workspace_12345678-1234-4234-9234-123456789abc', + '', + ])('rejects legacy, unrelated, and malformed reference %s', sessionId => { + expect(SandboxStatusSessionIdSchema.safeParse(sessionId).success).toBe(false); + }); +}); + +describe('sandbox status public contract', () => { + it.each(lifecycleCases)('accepts $status with $detailCode', lifecycle => { + const input = { ...snapshot, ...lifecycle }; + expect(SandboxStatusSnapshotSchema.parse(input)).toEqual(input); + }); + + it.each(['Cloudflare', 'Vercel', 'Unknown'])( + 'accepts only a bounded provider label: %s', + provider => { + expect(SandboxStatusSnapshotSchema.parse({ ...snapshot, provider }).provider).toBe(provider); + } + ); + + it('strips private fields from the serialized response', () => { + const parsed = SandboxStatusSnapshotSchema.parse({ ...snapshot, ...privateFields }); + expect(parsed).toStrictEqual(snapshot); + expect(JSON.stringify(parsed)).not.toContain('private'); + }); + + it.each(['status', 'provider', 'detailCode'] as const)('rejects arbitrary text in %s', field => { + expect( + SandboxStatusSnapshotSchema.safeParse({ ...snapshot, [field]: 'private-provider-failure' }) + .success + ).toBe(false); + }); + + it.each(['loading', 'healthy', 'destroyed', 'running', 'failed', 'idle', 'settling'])( + 'does not treat loading, agent progress, or billing phase %s as a lifecycle status', + status => { + expect(SandboxStatusSnapshotSchema.safeParse({ ...snapshot, status }).success).toBe(false); + } + ); + + it.each(lifecycleCases)('rejects contradictory details for $status/$detailCode', lifecycle => { + expect( + SandboxStatusSnapshotSchema.safeParse({ + ...snapshot, + ...lifecycle, + status: lifecycle.status === 'active' ? 'unknown' : 'active', + }).success + ).toBe(false); + }); + + it('distinguishes failed observation from a confirmed sandbox failure', () => { + expect( + SandboxStatusSnapshotSchema.safeParse({ + ...snapshot, + status: 'error', + detailCode: 'status_unavailable', + }).success + ).toBe(false); + expect(SANDBOX_STATUS_DETAIL_MESSAGES.status_unavailable).toBe( + 'Sandbox status is temporarily unavailable. This does not mean the sandbox failed.' + ); + expect(SANDBOX_STATUS_DETAIL_MESSAGES.sandbox_failed).toBe( + 'The sandbox encountered an error. Send a message to try again.' + ); + }); + + it.each(['observedAt', 'estimatedSleepAt'] as const)( + 'validates %s as a finite timestamp', + field => { + for (const invalid of [...invalidNumbers, 8_640_000_000_000_001]) { + expect( + SandboxStatusSnapshotSchema.safeParse({ ...snapshot, [field]: invalid }).success + ).toBe(false); + } + } + ); + + it('requires finite positive whole-millisecond inactivity durations', () => { + for (const invalid of [...invalidNumbers, 0]) { + expect( + SandboxStatusSnapshotSchema.safeParse({ ...snapshot, inactivityTimeoutMs: invalid }).success + ).toBe(false); + } + expect( + SandboxStatusSnapshotSchema.safeParse({ ...snapshot, inactivityTimeoutMs: 1 }).success + ).toBe(true); + }); + + it('allows unknown timing without fabricating a deadline', () => { + expect( + SandboxStatusSnapshotSchema.parse({ ...snapshot, inactivityTimeoutMs: null }) + ).toStrictEqual({ + ...snapshot, + inactivityTimeoutMs: null, + }); + }); + + it.each([ + 'status', + 'provider', + 'observedAt', + 'detailCode', + 'inactivityTimeoutMs', + 'estimatedSleepAt', + ])('requires the explicit %s field, including nullable timing', field => { + const input: Record = { ...snapshot }; + delete input[field]; + expect(SandboxStatusSnapshotSchema.safeParse(input).success).toBe(false); + }); + + it('accepts a future approximate deadline for an active sandbox with a known policy', () => { + const input = { ...snapshot, estimatedSleepAt: observedAt + 60_000 }; + expect(SandboxStatusSnapshotSchema.parse(input)).toEqual(input); + }); + + it.each([observedAt - 1, observedAt])('rejects an expired estimate: %s', estimatedSleepAt => { + expect(SandboxStatusSnapshotSchema.safeParse({ ...snapshot, estimatedSleepAt }).success).toBe( + false + ); + }); + + it('rejects an estimate without an inactivity policy', () => { + expect( + SandboxStatusSnapshotSchema.safeParse({ + ...snapshot, + inactivityTimeoutMs: null, + estimatedSleepAt: observedAt + 60_000, + }).success + ).toBe(false); + }); + + it.each(lifecycleCases.filter(lifecycle => lifecycle.status !== 'active'))( + 'rejects estimates for $status/$detailCode', + lifecycle => { + expect( + SandboxStatusSnapshotSchema.safeParse({ + ...snapshot, + ...lifecycle, + estimatedSleepAt: observedAt + 60_000, + }).success + ).toBe(false); + } + ); +}); + +describe('optional sandbox runtime metadata', () => { + const runtime = { + sandboxType: 'isolated-standard', + kiloCliVersion: '7.4.20', + wrapperVersion: '2.4.0', + startedAt: observedAt - 100_000, + stoppedAt: null, + }; + + it('retains the six-field contract and allows explicit unknown runtime metadata', () => { + expect(SandboxStatusSnapshotSchema.parse(snapshot)).toEqual(snapshot); + const unknown = Object.fromEntries(Object.keys(runtime).map(key => [key, null])); + expect(SandboxStatusSnapshotSchema.parse({ ...snapshot, runtime: unknown }).runtime).toEqual( + unknown + ); + expect(SandboxStatusSnapshotSchema.parse({ ...snapshot, runtime }).runtime).toEqual(runtime); + }); + + it.each(Object.keys(runtime))( + 'requires nullable runtime field %s when runtime is present', + field => { + const input: Record = { ...runtime }; + delete input[field]; + expect(SandboxRuntimeMetadataSchema.safeParse(input).success).toBe(false); + } + ); + + it.each([ + 'shared', + 'isolated-small', + 'isolated-standard', + 'code-review', + 'devcontainer', + 'unknown', + ])('accepts bounded sandbox type %s', sandboxType => { + expect(SandboxRuntimeMetadataSchema.parse({ ...runtime, sandboxType }).sandboxType).toBe( + sandboxType + ); + }); + + it.each(['legacy-shared', 'private-sandbox', 'https://runtime.invalid', 'standard'])( + 'rejects non-public sandbox type %s', + sandboxType => { + expect(SandboxRuntimeMetadataSchema.safeParse({ ...runtime, sandboxType }).success).toBe( + false + ); + } + ); + + it.each(['7.4.20', '2.4.0', '0.0.0-dev-20260902123456', '7.4.20-rc.1', '7.4.20-canary'])( + 'accepts a bounded version %s', + version => { + expect(safeSandboxRuntimeVersion(version)).toBe(version); + } + ); + + it.each([ + '', + '7.4', + 'private-instance', + 'https://runtime.invalid', + 'Bearer credential', + '7.4.20+private-instance', + '7.4.20-private-instance', + '7.4.20\n', + '7.4.20\r\n', + ' 7.4.20', + '1234567.1.1', + '1.2.3-rc.123456789012345', + '1'.repeat(128), + null, + undefined, + ])('does not reflect unsafe or unsupported version %j', version => { + expect(safeSandboxRuntimeVersion(version)).toBeNull(); + for (const field of ['kiloCliVersion', 'wrapperVersion']) { + if (version === null) continue; + expect(SandboxRuntimeMetadataSchema.safeParse({ ...runtime, [field]: version }).success).toBe( + false + ); + } + }); + + it.each(['startedAt', 'stoppedAt'])('validates lifecycle timestamp %s', field => { + for (const invalid of [...invalidNumbers, 8_640_000_000_000_001]) { + expect(SandboxRuntimeMetadataSchema.safeParse({ ...runtime, [field]: invalid }).success).toBe( + false + ); + } + }); + + it('strips private metadata fields without returning raw identifiers or errors', () => { + const result = SandboxStatusSnapshotSchema.parse({ + ...snapshot, + ...privateFields, + runtime: { ...runtime, ...privateFields }, + }); + expect(result).toEqual({ ...snapshot, runtime }); + expect(JSON.stringify(result)).not.toContain('private'); + }); +}); + +describe('getSandboxProviderLabel', () => { + it.each([ + ['cloudflare', 'Cloudflare'], + ['vercel', 'Vercel'], + ])('maps the stored provider %s to %s', (provider, label) => { + expect(getSandboxProviderLabel(provider)).toBe(label); + }); + + it.each([ + undefined, + null, + '', + 'private-provider', + 'Cloudflare', + 'vercel:private-instance', + {}, + 1, + ])('does not reflect unknown provider data', provider => { + expect(getSandboxProviderLabel(provider)).toBe('Unknown'); + }); +}); diff --git a/services/cloud-agent-next/src/shared/sandbox-status.ts b/services/cloud-agent-next/src/shared/sandbox-status.ts new file mode 100644 index 0000000000..2ffd80790c --- /dev/null +++ b/services/cloud-agent-next/src/shared/sandbox-status.ts @@ -0,0 +1,142 @@ +import * as z from 'zod'; +import { SESSION_ID_RE } from './protocol'; + +export const SandboxStatusSessionIdSchema = z + .string() + .regex(SESSION_ID_RE, 'Invalid session ID format') + .startsWith('workspace_', 'Sandbox status requires a control-plane session'); + +export const SandboxLifecycleStatusSchema = z.enum([ + 'active', + 'sleeping', + 'starting', + 'stopping', + 'error', + 'unreachable', + 'unknown', +]); + +export type SandboxLifecycleStatus = z.infer; + +export const SandboxProviderLabelSchema = z.enum(['Cloudflare', 'Vercel', 'Unknown']); + +export type SandboxProviderLabel = z.infer; + +export function getSandboxProviderLabel(provider: unknown): SandboxProviderLabel { + switch (provider) { + case 'cloudflare': + return 'Cloudflare'; + case 'vercel': + return 'Vercel'; + default: + return 'Unknown'; + } +} + +export const SandboxStatusDetailCodeSchema = z.enum([ + 'sandbox_ready', + 'sandbox_stopped', + 'sandbox_starting', + 'sandbox_stopping', + 'sandbox_failed', + 'connection_unavailable', + 'status_unavailable', + 'insufficient_evidence', +]); + +export type SandboxStatusDetailCode = z.infer; + +const STATUS_FOR_DETAIL_CODE = { + sandbox_ready: 'active', + sandbox_stopped: 'sleeping', + sandbox_starting: 'starting', + sandbox_stopping: 'stopping', + sandbox_failed: 'error', + connection_unavailable: 'unreachable', + status_unavailable: 'unknown', + insufficient_evidence: 'unknown', +} as const satisfies Record; + +export const SANDBOX_STATUS_DETAIL_MESSAGES = { + sandbox_ready: 'The sandbox is active.', + sandbox_stopped: 'The sandbox is sleeping. Send a message to resume.', + sandbox_starting: 'The sandbox is starting.', + sandbox_stopping: 'The sandbox is stopping.', + sandbox_failed: 'The sandbox encountered an error. Send a message to try again.', + connection_unavailable: + 'The sandbox connection is unavailable. Its current state cannot be confirmed.', + status_unavailable: + 'Sandbox status is temporarily unavailable. This does not mean the sandbox failed.', + insufficient_evidence: "There is not enough information to confirm the sandbox's current state.", +} as const satisfies Record; + +const timestampSchema = z.number().finite().int().nonnegative().max(8_640_000_000_000_000); + +export const SandboxRuntimeVersionSchema = z + .string() + .max(64) + .regex(/^\d{1,6}\.\d{1,6}\.\d{1,6}(?:-(?:alpha|beta|rc|canary|dev)(?:[.-]\d{1,14})?)?(?![\s\S])/); + +export function safeSandboxRuntimeVersion(value: unknown): string | null { + const parsed = SandboxRuntimeVersionSchema.safeParse(value); + return parsed.success ? parsed.data : null; +} + +export const SandboxRuntimeMetadataSchema = z.object({ + sandboxType: z + .enum([ + 'shared', + 'isolated-small', + 'isolated-standard', + 'code-review', + 'devcontainer', + 'unknown', + ]) + .nullable(), + kiloCliVersion: SandboxRuntimeVersionSchema.nullable(), + wrapperVersion: SandboxRuntimeVersionSchema.nullable(), + startedAt: timestampSchema.nullable(), + stoppedAt: timestampSchema.nullable(), +}); + +export type SandboxRuntimeMetadata = z.infer; + +export const SandboxStatusSnapshotSchema = z + .object({ + status: SandboxLifecycleStatusSchema, + provider: SandboxProviderLabelSchema, + observedAt: timestampSchema.describe( + 'Snapshot creation time in Unix epoch milliseconds, not a fresh sandbox probe' + ), + detailCode: SandboxStatusDetailCodeSchema, + runtime: SandboxRuntimeMetadataSchema.optional(), + inactivityTimeoutMs: z + .number() + .finite() + .int() + .positive() + .nullable() + .describe('Applicable inactivity timeout in milliseconds, or null when unknown'), + estimatedSleepAt: timestampSchema + .nullable() + .describe( + 'Approximate sleep time supported by authoritative sandbox-wide idle evidence, in Unix epoch milliseconds' + ), + }) + .refine(snapshot => STATUS_FOR_DETAIL_CODE[snapshot.detailCode] === snapshot.status, { + message: 'Sandbox status and detail code must agree', + path: ['detailCode'], + }) + .refine( + snapshot => + snapshot.estimatedSleepAt === null || + (snapshot.status === 'active' && + snapshot.inactivityTimeoutMs !== null && + snapshot.estimatedSleepAt > snapshot.observedAt), + { + message: 'Sleep estimates require an active sandbox, a known timeout, and a future timestamp', + path: ['estimatedSleepAt'], + } + ); + +export type SandboxStatusSnapshot = z.infer; diff --git a/services/cloud-agent-next/test/integration/sandbox-control.test.ts b/services/cloud-agent-next/test/integration/sandbox-control.test.ts index bd0d8cb444..94fdc06ccf 100644 --- a/services/cloud-agent-next/test/integration/sandbox-control.test.ts +++ b/services/cloud-agent-next/test/integration/sandbox-control.test.ts @@ -27,7 +27,7 @@ import type { AgentSelectionOverride, SubmittedSessionMessageRequest, } from '../../src/execution/types.js'; -import type { AttachSessionInput, SandboxControl } from '../../src/persistence/SandboxControl.js'; +import { type AttachSessionInput, SandboxControl } from '../../src/persistence/SandboxControl.js'; import { serializeSessionMetadata, type SessionMetadata, @@ -46,7 +46,7 @@ import { } from '../../src/sandbox-control/session-credentials.js'; import { findMatchingCredentialInjectionRule } from '../../src/sandbox-control/vercel-network-policy.js'; import { MANAGED_SCM_OUTBOUND_HANDLER } from '../../src/sandbox-id.js'; -import type { SandboxSession } from '../../src/sandbox-session/SandboxSession.js'; +import { SandboxSession } from '../../src/sandbox-session/SandboxSession.js'; import { SANDBOX_SESSION_LIFECYCLE_KEY, SANDBOX_SESSION_METADATA_KEY, @@ -106,7 +106,11 @@ import { type RequestFrame, type ResponseFrame, type SessionAttachPayload, + SANDBOX_CONTROL_WS_TAG, + sandboxControlSocketAttachmentSchema, + type SandboxHeartbeatPayload, } from '../../src/shared/sandbox-control-protocol.js'; +import { SandboxStatusSnapshotSchema } from '../../src/shared/sandbox-status.js'; vi.mock('../../src/session-access.js', () => ({ requireCurrentSessionAccess: vi.fn(), @@ -356,7 +360,11 @@ async function completeHello( type: 'response', requestId, ok: true, - result: { protocolVersion: 1, handshakeComplete: true }, + result: { + protocolVersion: 1, + handshakeComplete: true, + capabilities: { kiloVersionHeartbeat: true }, + }, }) ); const status = JSON.parse(await nextMessage(ws)) as { @@ -5954,6 +5962,688 @@ describe('SandboxControl durable remainder', () => { }); }); +const statusOwner = 'owner_status'; +const statusInput = { ownerId: statusOwner, provider: 'cloudflare' as const }; +const statusHeartbeat: SandboxHeartbeatPayload = { + state: 'idle', + pendingMessages: 0, + kilo: { ready: true }, + sessions: [{ kiloSessionId: 'kilo_status', state: 'idle', idleForMs: 0 }], +}; + +function statusSocket(state: DurableObjectState) { + const socket = state.getWebSockets(SANDBOX_CONTROL_WS_TAG).find(ws => ws.readyState === 1); + if (!socket) throw new Error('Expected an open control socket'); + return socket; +} + +async function receiveHeartbeat( + instance: SandboxControl, + state: DurableObjectState, + payload: SandboxHeartbeatPayload = statusHeartbeat +) { + await instance.webSocketMessage( + statusSocket(state), + JSON.stringify({ type: 'event', event: 'sandbox.heartbeat', payload }) + ); +} + +async function seedStatusControl(id: string) { + const stub = env.SANDBOX_CONTROL.getByName(id); + await runInDurableObject(stub, async (instance, state) => { + await instance.initializeOwner(statusOwner); + await state.storage.put('provider_kind', 'cloudflare'); + await seedRunningCloudflare(instance); + await saveRouteTable( + state.storage, + attachRoute( + new Map(), + { + sessionId: 'workspace_status', + kiloSessionId: 'kilo_status', + directory: '/workspace/status', + worktreeId: WORKTREE_ID, + ownerId: statusOwner, + }, + statusOwner + ).table + ); + instance['provider'].ensureLeaseAtLeast = vi.fn(async () => undefined); + }); + return stub; +} + +async function completeStatusHello(ws: WebSocket, requestId: string) { + await completeHello(ws, requestId, { wrapperInstanceId: crypto.randomUUID() }); + const id = socketSandboxIds.get(ws); + if (!id) throw new Error('Missing status sandbox identity'); + await runInDurableObject(env.SANDBOX_CONTROL.getByName(id), async (instance, state) => { + await instance.webSocketMessage( + statusSocket(state), + JSON.stringify({ + type: 'event', + event: 'sandbox.ready', + payload: { kiloReady: true, globalFeedAttached: true }, + }) + ); + }); +} + +async function reconstructControl(instance: SandboxControl, state: DurableObjectState) { + const callbacks: Promise[] = []; + const block = vi.spyOn(state, 'blockConcurrencyWhile').mockImplementation(callback => { + const promise = callback(); + callbacks.push(promise); + return promise; + }); + try { + const fresh = new SandboxControl(state, instance['env']); + await Promise.all(callbacks); + expect(callbacks).toHaveLength(0); + return fresh; + } finally { + block.mockRestore(); + } +} + +function forbidControlOperations(instance: SandboxControl) { + const forbidden = () => { + throw new Error('Passive status reached a control operation'); + }; + instance['provider'] = { + resumable: false, + ensureBillingAdmission: forbidden, + launch: forbidden, + create: forbidden, + stop: forbidden, + observe: forbidden, + ensureLeaseAtLeast: forbidden, + logs: forbidden, + }; + instance.ensureReady = forbidden; + instance.initializeOwner = forbidden; + instance.getStatus = forbidden; + instance.request = forbidden; + instance['pinProvider'] = forbidden; + instance['socketHandler'].sendRequest = forbidden; +} + +afterEach(() => vi.restoreAllMocks()); + +describe('SandboxControl passive status', () => { + it('returns unknown for a cold missing record without synthesizing or storing sleeping', async () => { + const stub = env.SANDBOX_CONTROL.getByName('sbx_status_empty'); + expect(await stub.getSandboxStatus(statusInput)).toMatchObject({ + status: 'unknown', + provider: 'Unknown', + estimatedSleepAt: null, + inactivityTimeoutMs: DEADLINE_MS.idleStop, + }); + await runInDurableObject(stub, async (_instance, state) => { + expect(await state.storage.list()).toEqual(new Map()); + expect(await state.storage.getAlarm()).toBeNull(); + }); + }); + + it.each([null, 10_000, 86_400_000])( + 'preserves records and exact alarm %j on repeated reconstructed reads', + async offset => { + const stub = await seedStatusControl(`sbx_status_passive_${offset}`); + await runInDurableObject(stub, async (instance, state) => { + await state.storage.put('wrapper_ready_at', Date.now()); + await state.storage.put('deadlines', { + heartbeatExpiry: Date.now() + DEADLINE_MS.heartbeatExpiry, + idleStop: Date.now() + DEADLINE_MS.idleStop, + }); + if (offset === null) await state.storage.deleteAlarm(); + else await state.storage.setAlarm(Date.now() + offset); + const alarm = await state.storage.getAlarm(); + const records = await state.storage.list(); + const fresh = await reconstructControl(instance, state); + forbidControlOperations(fresh); + const put = vi.spyOn(state.storage, 'put'); + const remove = vi.spyOn(state.storage, 'delete'); + const setAlarm = vi.spyOn(state.storage, 'setAlarm'); + const deleteAlarm = vi.spyOn(state.storage, 'deleteAlarm'); + for (let read = 0; read < 3; read++) { + expect(await fresh.getSandboxStatus(statusInput)).toMatchObject({ + status: 'unreachable', + estimatedSleepAt: null, + }); + } + expect(await state.storage.list()).toEqual(records); + expect(await state.storage.getAlarm()).toBe(alarm); + expect(put).not.toHaveBeenCalled(); + expect(remove).not.toHaveBeenCalled(); + expect(setAlarm).not.toHaveBeenCalled(); + expect(deleteAlarm).not.toHaveBeenCalled(); + }); + } + ); + + it('preserves unhealthy-runtime quarantine through passive reconstruction', async () => { + const id = 'sbx_status_false_reconstruction'; + const stub = await seedStatusControl(id); + const credential = generateSandboxCredential(); + await seedCredential(credential, id); + const ws = await connect(credential, id); + await completeStatusHello(ws, 'hello-status-false'); + await runInDurableObject(stub, async (instance, state) => { + const socket = statusSocket(state); + await instance.webSocketMessage( + socket, + JSON.stringify({ + type: 'event', + event: 'sandbox.ready', + payload: { kiloReady: true, globalFeedAttached: true }, + }) + ); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + instance['provider'].stop = vi.fn(async () => 'retryable'); + await receiveHeartbeat(instance, state, { ...statusHeartbeat, kilo: { ready: false } }); + await vi.waitFor(async () => { + expect((await instance.getPhysicalRecord()).stopTombstone).toMatchObject({ + reason: 'kilo_unhealthy', + attempts: 1, + }); + }); + expect(await state.storage.get('wrapper_ready_at')).toBeUndefined(); + expect((await loadDeadlines(state.storage)).heartbeatExpiry).toBeUndefined(); + const fresh = await reconstructControl(instance, state); + expect(fresh['kiloReady']).toBe(false); + const records = await state.storage.list(); + const alarm = await state.storage.getAlarm(); + const attachment = socket.deserializeAttachment(); + forbidControlOperations(fresh); + const serialize = vi.spyOn(socket, 'serializeAttachment'); + const send = vi.spyOn(socket, 'send'); + expect(await fresh.getSandboxStatus(statusInput)).toMatchObject({ + status: 'stopping', + detailCode: 'sandbox_stopping', + estimatedSleepAt: null, + }); + expect(await fresh.getSandboxStatus(statusInput)).toMatchObject({ + status: 'stopping', + estimatedSleepAt: null, + }); + expect(await state.storage.list()).toEqual(records); + expect(await state.storage.getAlarm()).toBe(alarm); + expect(socket.deserializeAttachment()).toEqual(attachment); + expect(serialize).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + ws.close(); + }); + + it('does not inherit readiness when a same-provider replacement quarantines the runtime', async () => { + const id = 'sbx_status_replacement'; + const stub = await seedStatusControl(id); + const credential = generateSandboxCredential(); + await seedCredential(credential, id); + const first = await connect(credential, id); + await completeStatusHello(first, 'hello-status-old'); + await runInDurableObject(stub, async (instance, state) => { + instance['provider'].stop = vi.fn(async () => 'retryable'); + await receiveHeartbeat(instance, state); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ status: 'active' }); + }); + const second = await connect(credential, id); + const closed = new Promise(resolve => + second.addEventListener('close', event => resolve(event.code), { once: true }) + ); + sendHello(second, 'hello-status-new', { wrapperInstanceId: crypto.randomUUID() }); + await expect(closed).resolves.toBe(4001); + await vi.waitFor(async () => { + expect((await stub.getPhysicalRecord()).stopTombstone).toMatchObject({ + reason: 'control_replaced', + attempts: 1, + }); + }); + await runInDurableObject(stub, async (instance, state) => { + const fresh = await reconstructControl(instance, state); + forbidControlOperations(fresh); + const before = await state.storage.list(); + const alarm = await state.storage.getAlarm(); + expect(await fresh.getSandboxStatus(statusInput)).toMatchObject({ + status: 'stopping', + estimatedSleepAt: null, + }); + expect(await state.storage.list()).toEqual(before); + expect(await state.storage.getAlarm()).toBe(alarm); + }); + }); + + it('requires fresh coherent evidence for all shared routes and leaves ordinary heartbeat scheduling unchanged', async () => { + const id = 'sbx_status_shared_idle'; + const stub = await seedStatusControl(id); + const credential = generateSandboxCredential(); + await seedCredential(credential, id); + const ws = await connect(credential, id); + await completeStatusHello(ws, 'hello-status-idle'); + await runInDurableObject(stub, async (instance, state) => { + const renew = vi.fn(async () => undefined); + instance['provider'].ensureLeaseAtLeast = renew; + await saveRouteTable( + state.storage, + attachRoute( + await loadRouteTable(state.storage), + { + sessionId: 'workspace_sibling', + kiloSessionId: 'kilo_sibling', + directory: '/workspace/status', + worktreeId: WORKTREE_ID, + ownerId: statusOwner, + }, + statusOwner + ).table + ); + const socket = statusSocket(state); + await instance.webSocketMessage( + socket, + JSON.stringify({ + type: 'event', + event: 'sandbox.ready', + payload: { kiloReady: true, globalFeedAttached: true }, + }) + ); + const initialDeadline = (await loadDeadlines(state.storage)).idleStop; + expect(initialDeadline).toEqual(expect.any(Number)); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + await receiveHeartbeat(instance, state); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + estimatedSleepAt: null, + }); + const sharedIdle: SandboxHeartbeatPayload = { + ...statusHeartbeat, + sessions: [ + ...statusHeartbeat.sessions, + { kiloSessionId: 'kilo_sibling', state: 'idle', idleForMs: 0 }, + ], + }; + await receiveHeartbeat(instance, state, sharedIdle); + expect((await loadDeadlines(state.storage)).idleStop).toBe(initialDeadline); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: initialDeadline, + }); + const records = await state.storage.list(); + const alarm = await state.storage.getAlarm(); + const beforeAttachment = socket.deserializeAttachment(); + const send = vi.spyOn(socket, 'send'); + const serialize = vi.spyOn(socket, 'serializeAttachment'); + const reconstructed = await reconstructControl(instance, state); + forbidControlOperations(reconstructed); + for (let read = 0; read < 3; read++) { + const snapshot = await reconstructed.getSandboxStatus(statusInput); + expect(snapshot.estimatedSleepAt).toBe(initialDeadline); + expect(SandboxStatusSnapshotSchema.safeParse(snapshot).success).toBe(true); + } + expect(await state.storage.list()).toEqual(records); + expect(await state.storage.getAlarm()).toBe(alarm); + expect(socket.deserializeAttachment()).toEqual(beforeAttachment); + expect(send).not.toHaveBeenCalled(); + expect(serialize).not.toHaveBeenCalled(); + expect(renew).toHaveBeenCalledTimes(2); + const active: SandboxHeartbeatPayload = { + ...sharedIdle, + state: 'active', + sessions: [ + ...statusHeartbeat.sessions, + { kiloSessionId: 'kilo_sibling', state: 'active', idleForMs: 0 }, + ], + }; + await receiveHeartbeat(instance, state, active); + expect((await loadDeadlines(state.storage)).idleStop).toBeUndefined(); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + let heartbeatTime = Date.now(); + vi.spyOn(Date, 'now').mockImplementation(() => heartbeatTime++); + await receiveHeartbeat(instance, state, sharedIdle); + const nextDeadline = (await loadDeadlines(state.storage)).idleStop; + expect(nextDeadline).toBeGreaterThanOrEqual(initialDeadline ?? 0); + const rearmedObservation = sandboxControlSocketAttachmentSchema.parse( + socket.deserializeAttachment() + ).observation; + expect(rearmedObservation?.receivedAt).toBeLessThan( + (nextDeadline ?? 0) - DEADLINE_MS.idleStop + ); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + expect(renew).toHaveBeenCalledTimes(4); + await receiveHeartbeat(instance, state, sharedIdle); + expect((await loadDeadlines(state.storage)).idleStop).toBe(nextDeadline); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + estimatedSleepAt: nextDeadline, + }); + expect(renew).toHaveBeenCalledTimes(5); + expect(renew).toHaveBeenLastCalledWith( + cloudflareRef(id), + DEADLINE_MS.idleStop + DEADLINE_MS.idleStopLeaseMargin + ); + const attachment = sandboxControlSocketAttachmentSchema.parse(socket.deserializeAttachment()); + const expiredAt = Date.now() - DEADLINE_MS.heartbeatExpiry; + socket.serializeAttachment({ + ...attachment, + acceptedAt: expiredAt - 1, + observation: { ...attachment.observation, receivedAt: expiredAt }, + }); + await state.storage.put('deadlines', { + heartbeatExpiry: Date.now() + DEADLINE_MS.heartbeatExpiry, + idleStop: Date.now() + DEADLINE_MS.idleStop, + }); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'unreachable', + estimatedSleepAt: null, + }); + }); + ws.close(); + }); + + it('suppresses an idle estimate after a busy event without changing operational scheduling', async () => { + const id = 'usr-b05a1d1e1'; + const stub = await seedStatusControl(id); + const session = env.SANDBOX_SESSION.getByName(`${statusOwner}:workspace_status`); + await runInDurableObject(session, async instance => { + await instance.registerSession({ + identity: { sessionId: 'workspace_status', userId: statusOwner }, + auth: { kiloSessionId: 'kilo_status' }, + agent: { mode: 'code', model: 'test' }, + workspace: { + sandboxId: id, + sandboxProvider: 'cloudflare', + workspacePath: '/workspace/status', + }, + }); + }); + const credential = generateSandboxCredential(); + await seedCredential(credential, id); + const ws = await connect(credential, id); + await completeStatusHello(ws, 'hello-status-busy'); + const runtime = await stub.getStatus(); + await runInDurableObject(session, (_instance, state) => { + state.storage.kv.put('session_messages', [ + { + messageId: 'msg_status_busy', + state: 'accepted', + acceptedAt: Date.now(), + wrapperInstanceId: runtime.wrapperInstanceId, + } satisfies SessionMessageRecord, + ]); + }); + try { + await runInDurableObject(stub, async (instance, state) => { + await receiveHeartbeat(instance, state); + const deadline = (await loadDeadlines(state.storage)).idleStop; + expect(deadline).toEqual(expect.any(Number)); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: deadline, + }); + const fresh = await reconstructControl(instance, state); + await fresh.getStatus(); + const renew = vi.fn(async () => undefined); + fresh['provider'].ensureLeaseAtLeast = renew; + const socket = statusSocket(state); + const before = sandboxControlSocketAttachmentSchema.parse(socket.deserializeAttachment()); + const records = await state.storage.list(); + const alarm = await state.storage.getAlarm(); + const send = vi.spyOn(socket, 'send'); + await fresh.webSocketMessage( + socket, + JSON.stringify({ + type: 'event', + event: 'session.event', + session: { directory: '/workspace/status', kiloSessionId: 'kilo_status' }, + payload: { + type: 'session.status', + properties: { sessionID: 'kilo_status', status: { type: 'busy' } }, + }, + }) + ); + await Promise.all(fresh['sessionForwardChains'].values()); + expect(await fresh.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + expect(socket.deserializeAttachment()).toEqual({ + ...before, + observation: { ...before.observation, idle: null }, + }); + const reconstructed = await reconstructControl(fresh, state); + forbidControlOperations(reconstructed); + const serialize = vi.spyOn(socket, 'serializeAttachment'); + expect(await reconstructed.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: null, + }); + expect(await state.storage.list()).toEqual(records); + expect(await state.storage.getAlarm()).toBe(alarm); + expect(renew).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + expect(serialize).not.toHaveBeenCalled(); + await receiveHeartbeat(fresh, state); + expect((await loadDeadlines(state.storage)).idleStop).toBe(deadline); + expect(await fresh.getSandboxStatus(statusInput)).toMatchObject({ + status: 'active', + estimatedSleepAt: deadline, + }); + expect(renew).toHaveBeenCalledTimes(1); + }); + await runInDurableObject(session, async (_instance, state) => { + const events = createEventQueries( + drizzle(state.storage, { logger: false }), + state.storage.sql + ).findByFilters({ eventTypes: ['kilocode'] }); + expect(events.map(event => JSON.parse(event.payload))).toContainEqual({ + type: 'session.status', + event: 'session.status', + properties: { sessionID: 'kilo_status', status: { type: 'busy' } }, + }); + }); + } finally { + ws.close(); + } + }); + + it.each([ + { owner: undefined, provider: 'cloudflare' }, + { owner: 'other-owner', provider: 'cloudflare' }, + { owner: statusOwner, provider: undefined }, + { owner: statusOwner, provider: 'vercel' }, + { owner: statusOwner, provider: 'private-provider' }, + ])( + 'does not use missing or disagreeing owner/provider records: %j', + async ({ owner, provider }) => { + const stub = env.SANDBOX_CONTROL.getByName(`sbx_status_mismatch_${owner}_${provider}`); + await runInDurableObject(stub, async (instance, state) => { + if (owner !== undefined) await state.storage.put('owner_id', owner); + if (provider !== undefined) await state.storage.put('provider_kind', provider); + await state.storage.put('physical_record', { + state: 'stopped', + providerRef: null, + createIntent: null, + stopTombstone: null, + resumable: false, + }); + const before = await state.storage.list(); + expect(await instance.getSandboxStatus(statusInput)).toMatchObject({ + status: 'unknown', + provider: 'Unknown', + estimatedSleepAt: null, + }); + expect(await state.storage.list()).toEqual(before); + }); + } + ); +}); + +describe('SandboxSession passive delegation', () => { + it.each(['fence', 'worktree-marker'] as const)( + 'does not expose or repair retained deleted metadata through %s', + async mode => { + const sessionId = `workspace_status_deleted_${mode}`; + const session = env.SANDBOX_SESSION.getByName(`${statusOwner}:${sessionId}`); + await runInDurableObject(session, async (instance, state) => { + await instance.registerSession({ + identity: { sessionId, userId: statusOwner }, + auth: { kiloSessionId: 'kilo_status' }, + agent: { mode: 'code', model: 'test' }, + workspace: { sandboxId: 'usr-abcdef123', sandboxProvider: 'cloudflare' }, + }); + if (mode === 'fence') + state.storage.kv.put(SANDBOX_SESSION_LIFECYCLE_KEY, { state: 'deleted', epoch: 1 }); + else state.storage.kv.put('deleted_worktree', WORKTREE_ID); + const before = await state.storage.list(); + const callbacks: Promise[] = []; + const block = vi.spyOn(state, 'blockConcurrencyWhile').mockImplementation(callback => { + const promise = callback(); + callbacks.push(promise); + return promise; + }); + const fresh = new SandboxSession(state, instance['env']); + await Promise.all(callbacks); + block.mockRestore(); + const lookup = vi.spyOn(instance['env'].SANDBOX_CONTROL, 'getByName'); + expect(await fresh.getMetadata()).toBeNull(); + expect(await fresh.getSandboxStatus()).toMatchObject({ + status: 'unknown', + estimatedSleepAt: null, + }); + expect(lookup).not.toHaveBeenCalled(); + expect(await state.storage.list()).toEqual(before); + }); + } + ); + + it('uses stored assignment and owner without metadata recovery or control initialization', async () => { + const id = 'usr-abcdef123'; + const control = await seedStatusControl(id); + await runInDurableObject(control, async (instance, state) => { + await instance.beginStop('test'); + await instance.confirmStopped(); + await state.storage.deleteAlarm(); + forbidControlOperations(instance); + }); + const sessionId = 'workspace_status_delegation'; + const session = env.SANDBOX_SESSION.getByName(`${statusOwner}:${sessionId}`); + await runInDurableObject(session, async (instance, state) => { + await instance.registerSession({ + identity: { sessionId, userId: statusOwner }, + auth: { kiloSessionId: 'kilo_status' }, + agent: { mode: 'code', model: 'test' }, + workspace: { sandboxId: id, sandboxProvider: 'cloudflare' }, + }); + await state.storage.put('session_messages', [{ messageId: 'msg_status', state: 'queued' }]); + await state.storage.setAlarm(Date.now() + 86_400_000); + const before = await state.storage.list(); + const alarm = await state.storage.getAlarm(); + const callbacks: Promise[] = []; + const block = vi.spyOn(state, 'blockConcurrencyWhile').mockImplementation(callback => { + const promise = callback(); + callbacks.push(promise); + return promise; + }); + const fresh = new SandboxSession(state, instance['env']); + await Promise.all(callbacks); + block.mockRestore(); + fresh.getMetadata = () => { + throw new Error('Passive delegation called operational metadata'); + }; + fresh['dispatchQueued'] = () => { + throw new Error('Passive delegation dispatched'); + }; + for (let read = 0; read < 3; read++) { + expect(await fresh.getSandboxStatus()).toMatchObject({ + status: 'sleeping', + provider: 'Cloudflare', + inactivityTimeoutMs: DEADLINE_MS.idleStop, + estimatedSleepAt: null, + }); + } + expect(await state.storage.list()).toEqual(before); + expect(await state.storage.getAlarm()).toBe(alarm); + }); + await runInDurableObject(control, async (_instance, state) => { + expect(await state.storage.getAlarm()).toBeNull(); + }); + }); + + it('sanitizes and bounds failed passive delegation before retry logging', async () => { + const sessionId = 'workspace_status_retry'; + const stub = env.SANDBOX_SESSION.getByName(`${statusOwner}:${sessionId}`); + await runInDurableObject(stub, async (instance, state) => { + await instance.registerSession({ + identity: { sessionId, userId: statusOwner }, + auth: {}, + agent: { mode: 'code', model: 'test' }, + workspace: { sandboxId: 'usr-abcde987', sandboxProvider: 'cloudflare' }, + }); + const control = instance['env'].SANDBOX_CONTROL.getByName('usr-abcde987'); + const failure = vi + .spyOn(control, 'getSandboxStatus') + .mockRejectedValue( + Object.assign(new Error('private-runtime-error-sentinel'), { retryable: true }) + ); + const getStub = vi + .spyOn(instance['env'].SANDBOX_CONTROL, 'getByName') + .mockReturnValue(control); + const before = await state.storage.list(); + const result = await instance.getSandboxStatus(); + expect(result).toMatchObject({ + status: 'unknown', + detailCode: 'status_unavailable', + estimatedSleepAt: null, + }); + expect(getStub).toHaveBeenCalledTimes(3); + expect(failure).toHaveBeenCalledTimes(3); + expect(JSON.stringify(result)).not.toContain('private-runtime-error-sentinel'); + expect(await state.storage.list()).toEqual(before); + expect(await state.storage.getAlarm()).toBeNull(); + }); + }); + + it.each(['empty', 'missing-assignment', 'invalid', 'wrong-session'] as const)( + 'does not generate an assignment for %s metadata', + async mode => { + const sessionId = `workspace_status_${mode}`; + const stub = env.SANDBOX_SESSION.getByName(`${statusOwner}:${sessionId}`); + await runInDurableObject(stub, async (instance, state) => { + if (mode === 'invalid') await state.storage.put('session_metadata', { invalid: true }); + if (mode === 'missing-assignment' || mode === 'wrong-session') { + await instance.registerSession({ + identity: { + sessionId: mode === 'wrong-session' ? 'workspace_other' : sessionId, + userId: statusOwner, + }, + auth: {}, + agent: { mode: 'code', model: 'test' }, + ...(mode === 'wrong-session' ? { workspace: { sandboxId: 'usr-abcdef123' } } : {}), + }); + } + const before = await state.storage.list(); + const getStub = vi.spyOn(instance['env'].SANDBOX_CONTROL, 'getByName'); + expect(await instance.getSandboxStatus()).toMatchObject({ + status: 'unknown', + estimatedSleepAt: null, + }); + expect(getStub).not.toHaveBeenCalled(); + expect(await state.storage.list()).toEqual(before); + expect(await state.storage.getAlarm()).toBeNull(); + }); + } + ); +}); + describe('SandboxSession control-plane regressions', () => { beforeEach(() => { vi.spyOn(globalThis, 'fetch').mockImplementation(async () => Response.json({ valid: true })); diff --git a/services/cloud-agent-next/test/integration/worktree-deletion.test.ts b/services/cloud-agent-next/test/integration/worktree-deletion.test.ts index d0f4d027c2..dfa58e9c1e 100644 --- a/services/cloud-agent-next/test/integration/worktree-deletion.test.ts +++ b/services/cloud-agent-next/test/integration/worktree-deletion.test.ts @@ -155,7 +155,8 @@ function createDeletionProvider(sandboxId: string) { } satisfies ProviderAdapter; } -function installDeletionProvider(instance: SandboxControl, provider: ProviderAdapter) { +async function installDeletionProvider(instance: SandboxControl, provider: ProviderAdapter) { + await instance.getStatus(); Object.assign(instance, { env: { ...instance['env'], @@ -601,7 +602,7 @@ describe('worktree deletion in Durable Objects', () => { }), }, }); - installDeletionProvider(instance, provider); + await installDeletionProvider(instance, provider); Object.assign(instance, { socketHandler: { hasHandshakenSocket: () => true, @@ -783,7 +784,7 @@ describe('worktree deletion in Durable Objects', () => { }), }, }); - installDeletionProvider(instance, provider); + await installDeletionProvider(instance, provider); Object.assign(instance, { socketHandler: { hasHandshakenSocket: () => true, @@ -858,7 +859,7 @@ describe('worktree deletion in Durable Objects', () => { if (!('providerRef' in created)) throw new Error('Missing native fixture allocation'); const stop = vi.fn(async () => 'retryable'); const observe = vi.fn(native.observe.bind(native)); - installDeletionProvider(instance, { ...native, stop, observe }); + await installDeletionProvider(instance, { ...native, stop, observe }); const original = instance['env'].SESSION_INGEST; Object.assign(instance['env'], { SESSION_INGEST: { @@ -998,7 +999,7 @@ describe('worktree deletion in Durable Objects', () => { await runInDurableObject(control, async (instance, state) => { const provider = createDeletionProvider(sandboxId); const create = vi.fn(provider.create.bind(provider)); - installDeletionProvider(instance, { ...provider, create }); + await installDeletionProvider(instance, { ...provider, create }); const original = instance['env'].SESSION_INGEST; Object.assign(instance['env'], { SESSION_INGEST: { @@ -1527,7 +1528,7 @@ describe('worktree deletion in Durable Objects', () => { ...memory, stop: ref => (mayStop ? memory.stop(ref) : Promise.resolve('retryable')), }; - installDeletionProvider(instance, provider); + await installDeletionProvider(instance, provider); const original = instance['env'].SESSION_INGEST; Object.assign(instance['env'], { SESSION_INGEST: { @@ -1667,7 +1668,7 @@ describe('worktree deletion in Durable Objects', () => { return memory.stop(ref); }, }; - installDeletionProvider(instance, provider); + await installDeletionProvider(instance, provider); const original = instance['env'].SESSION_INGEST; const workerUrl = instance['env'].WORKER_URL; Object.assign(instance['env'], { diff --git a/services/cloud-agent-next/test/unit/wrapper/worktree-credential-refresh.test.ts b/services/cloud-agent-next/test/unit/wrapper/worktree-credential-refresh.test.ts index 70a07612d8..b06a281637 100644 --- a/services/cloud-agent-next/test/unit/wrapper/worktree-credential-refresh.test.ts +++ b/services/cloud-agent-next/test/unit/wrapper/worktree-credential-refresh.test.ts @@ -495,17 +495,18 @@ describe('direct worktree credential refresh', () => { it.each(['connection', 'http'] as const)( 'classifies real SDK SSE startup %s errors without retaining private data', async failure => { + vi.mocked(fetch).mockImplementation(async request => { + const url = request instanceof Request ? request.url : String(request); + if (new URL(url).pathname === '/global/health') { + return Response.json({ healthy: true, version: '7.4.20' }); + } + if (failure === 'connection') throw new Error('private-connection-credential'); + return new Response(null, { status: 503, statusText: 'private-http-credential' }); + }); const f = fixture(); const runtime = await f.attach(); const feed = vi.mocked(startSandboxControlEventFeed).mock.calls[0]?.[0]; if (!feed) throw new Error('Missing worktree event feed'); - if (failure === 'connection') { - vi.mocked(fetch).mockRejectedValueOnce(new Error('private-connection-credential')); - } else { - vi.mocked(fetch).mockResolvedValueOnce( - new Response(null, { status: 503, statusText: 'private-http-credential' }) - ); - } const { stream } = await feed.open(feed.signal); if (!stream) throw new Error('Missing SDK SSE stream'); const error = await stream[Symbol.asyncIterator]() @@ -518,7 +519,12 @@ describe('direct worktree credential refresh', () => { expect(error).not.toHaveProperty('cause'); expect(String(error)).not.toContain('private-'); expect(JSON.stringify(error)).not.toContain('private-'); - expect(fetch).toHaveBeenCalledTimes(1); + expect( + vi.mocked(fetch).mock.calls.filter(([request]) => { + const url = request instanceof Request ? request.url : String(request); + return new URL(url).pathname === '/global/event'; + }) + ).toHaveLength(1); expect(runtime.signal.aborted).toBe(false); feed.onUnexpectedClose(error); expect(f.onUnexpectedClose.mock.calls).toEqual([ diff --git a/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.test.ts b/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.test.ts index 5308d30f83..95e73fa717 100644 --- a/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.test.ts +++ b/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it, mock, spyOn } from 'bun:test'; +import { z } from 'zod'; +import { helloResult } from '../../../src/sandbox-control/frames'; +import { buildHeartbeatPayload } from './sandbox-control-handlers'; import { MAX_SANDBOX_CONTROL_FRAME_BYTES, SANDBOX_CONTROL_REQUEST_TIMEOUT_MS, + sandboxHeartbeatPayloadSchema, sessionEventPayloadSchema, } from '../../../src/shared/sandbox-control-protocol'; import { unfilteredKiloEvents } from './feed'; @@ -74,6 +78,204 @@ function createClientFixture(options: Partial = {}) return { client, sockets, openWebSocket, onDisconnected }; } +const previousHelloResultSchema = z.object({ + protocolVersion: z.literal(1), + handshakeComplete: z.literal(true), +}); + +const previousHeartbeatSchema = z + .object({ + state: z.enum(['idle', 'active', 'finalizing']), + activeKiloSessions: z.number().int().nonnegative().optional(), + pendingMessages: z.number().int().nonnegative().optional(), + kilo: z + .object({ + ready: z.boolean(), + reason: z + .enum([ + 'feed_stale', + 'feed_reconnected', + 'feed_ended', + 'feed_failed', + 'process_exited', + 'credential_refresh_failed', + 'control_disconnected', + 'shutdown', + ]) + .optional(), + }) + .strict(), + sessions: z.array( + z + .object({ + kiloSessionId: z.string().min(1), + state: z.enum(['idle', 'active', 'finalizing']), + idleForMs: z.number().int().nonnegative(), + waitingOn: z.enum(['model', 'tool', 'finalizing', 'preparation', 'input']).optional(), + }) + .strict() + ), + }) + .strict(); + +function versionHeartbeat(version: string | null) { + return buildHeartbeatPayload({ + kiloRuntimes: { + kiloCliVersion: version, + attach() { + throw new Error('Unexpected attach'); + }, + detach: () => false, + deleteDirectory: async () => {}, + get: () => undefined, + isHealthy: () => true, + shutdown() {}, + }, + version: '2.4.0', + kiloReady: true, + sessions: [], + tasks: new Map(), + emitSessionEvent() {}, + retireRuntime() {}, + }); +} + +describe('heartbeat version rollout compatibility', () => { + it.each(['7.4.20', null])( + 'reproduces the previous exact strict contract rejection of version %j', + version => { + expect(previousHeartbeatSchema.safeParse(versionHeartbeat(version)).success).toBe(false); + } + ); + + it.each(['7.4.20', null])( + 'new wrapper omits unnegotiated version %j for an old Worker', + async version => { + const fake = new FakeWebSocket(); + const { client } = createClientFixture({ openWebSocket: () => fake as unknown as WebSocket }); + try { + const connecting = client.connect(); + await handshake(fake); + await connecting; + const heartbeat = versionHeartbeat(version); + const before = structuredClone(heartbeat); + for (const kilo of [ + heartbeat.kilo, + { ...heartbeat.kilo, ready: false, reason: 'shutdown' as const }, + ]) { + expect(client.sendEvent?.('sandbox.heartbeat', { ...heartbeat, kilo })).toBe(true); + const frame = JSON.parse(fake.sent.at(-1) ?? '{}'); + expect(previousHeartbeatSchema.safeParse(frame.payload).success).toBe(true); + expect(frame.payload.kilo).not.toHaveProperty('version'); + expect(frame.payload.kilo.ready).toBe(kilo.ready); + expect(frame.payload.kilo.reason).toBe(kilo.reason); + } + expect(heartbeat).toEqual(before); + } finally { + client.close(); + } + } + ); + + it.each([{}, { kiloVersionHeartbeat: false }])( + 'does not infer support from capabilities %j', + async capabilities => { + const fake = new FakeWebSocket(); + const { client } = createClientFixture({ openWebSocket: () => fake as unknown as WebSocket }); + try { + const connecting = client.connect(); + await handshake(fake, { protocolVersion: 1, handshakeComplete: true, capabilities }); + await connecting; + expect(client.sendEvent?.('sandbox.heartbeat', versionHeartbeat('7.4.20'))).toBe(true); + const frame = JSON.parse(fake.sent.at(-1) ?? '{}'); + expect(previousHeartbeatSchema.safeParse(frame.payload).success).toBe(true); + expect(frame.payload.kilo).not.toHaveProperty('version'); + } finally { + client.close(); + } + } + ); + + it('does not retain capability support from a failed connection attempt', async () => { + const { client, sockets } = createClientFixture(); + try { + const connecting = client.connect(); + await Promise.resolve(); + const first = sockets[0]; + if (!first) throw new Error('Missing first socket'); + first.open(); + const hello = JSON.parse(first.sent[0] ?? '{}'); + first.respond( + JSON.stringify({ + type: 'response', + requestId: hello.requestId, + ok: true, + result: helloResult(), + }) + ); + first.close(); + await waitForReconnect(); + const second = sockets[1]; + await handshake(second); + await connecting; + if (!second) throw new Error('Missing second socket'); + const nextHello = JSON.parse(second.sent[0] ?? '{}'); + second.respond( + JSON.stringify({ + type: 'response', + requestId: nextHello.requestId, + ok: true, + result: helloResult(), + }) + ); + expect(client.sendEvent?.('sandbox.heartbeat', versionHeartbeat('7.4.20'))).toBe(true); + const frame = JSON.parse(second.sent.at(-1) ?? '{}'); + expect(previousHeartbeatSchema.safeParse(frame.payload).success).toBe(true); + expect(frame.payload.kilo).not.toHaveProperty('version'); + } finally { + client.close(); + } + }); + + it('old wrappers accept new Worker hello acknowledgements and their old heartbeats remain valid', () => { + expect(previousHelloResultSchema.parse(helloResult())).toEqual({ + protocolVersion: 1, + handshakeComplete: true, + }); + const heartbeat = previousHeartbeatSchema.parse({ + state: 'idle', + kilo: { ready: true }, + sessions: [], + }); + expect(sandboxHeartbeatPayloadSchema.parse(heartbeat)).toEqual(heartbeat); + }); + + it.each(['7.4.20', null])( + 'new wrapper retains negotiated version %j for a new Worker', + async version => { + const fake = new FakeWebSocket(); + const { client } = createClientFixture({ openWebSocket: () => fake as unknown as WebSocket }); + try { + const connecting = client.connect(); + await handshake(fake, helloResult()); + await connecting; + const heartbeat = versionHeartbeat(version); + expect(client.sendEvent?.('sandbox.heartbeat', heartbeat)).toBe(true); + const frame = JSON.parse(fake.sent.at(-1) ?? '{}'); + expect(sandboxHeartbeatPayloadSchema.parse(frame.payload)).toEqual(heartbeat); + expect(frame.payload.kilo.version).toBe(version); + const withoutVersion = { state: 'idle' as const, kilo: { ready: true }, sessions: [] }; + expect(client.sendEvent?.('sandbox.heartbeat', withoutVersion)).toBe(true); + expect( + sandboxHeartbeatPayloadSchema.parse(JSON.parse(fake.sent.at(-1) ?? '{}').payload) + ).toEqual(withoutVersion); + } finally { + client.close(); + } + } + ); +}); + describe('createSandboxControlClient', () => { it('opens with an Authorization header and completes sandbox.hello plus status probe', async () => { const fake = new FakeWebSocket(); @@ -964,7 +1166,10 @@ describe('createSandboxControlClient', () => { }); }); -async function handshake(fake: FakeWebSocket | undefined): Promise { +async function handshake( + fake: FakeWebSocket | undefined, + result: unknown = { protocolVersion: 1, handshakeComplete: true } +): Promise { if (!fake) throw new Error('missing socket'); await Promise.resolve(); fake.open(); @@ -975,7 +1180,7 @@ async function handshake(fake: FakeWebSocket | undefined): Promise { type: 'response', requestId: hello.requestId, ok: true, - result: { protocolVersion: 1, handshakeComplete: true }, + result, }) ); fake.respond( diff --git a/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.ts b/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.ts index 0ead0a3316..381f768572 100644 --- a/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.ts +++ b/services/cloud-agent-next/wrapper/src/control/sandbox-control-client.ts @@ -15,6 +15,7 @@ import { controlFrameSchema, controlErrorCodes, sandboxHelloResultSchema, + sandboxHeartbeatPayloadSchema, sessionEventPayloadSchema, type ControlError, type EventFrame, @@ -62,7 +63,7 @@ export type SandboxControlClient = { type ClientState = | { kind: 'idle' } | { kind: 'starting'; promise: Promise; abort: AbortController } - | { kind: 'ready'; socket: WebSocket; dispose: () => void } + | { kind: 'ready'; socket: WebSocket; dispose: () => void; kiloVersionHeartbeat: boolean } | { kind: 'closed' }; const CONNECT_TIMEOUT_MS = 10_000; @@ -237,6 +238,7 @@ export function createSandboxControlClient( const signal = starting.abort.signal; const requestId = crypto.randomUUID(); let phase: 'opening' | 'hello' | 'status' | 'finished' = 'opening'; + let kiloVersionHeartbeat = false; let timeout = setTimeout(fail, Math.min(CONNECT_TIMEOUT_MS, deadlineAt - Date.now())); function dispose(): void { @@ -326,10 +328,13 @@ export function createSandboxControlClient( } if (phase === 'opening') return; if (frame.type === 'response' && frame.requestId === requestId) { - if (!frame.ok || !sandboxHelloResultSchema.safeParse(frame.result).success) { + if (phase !== 'hello') return; + const hello = sandboxHelloResultSchema.safeParse(frame.result); + if (!frame.ok || !hello.success) { fail(); return; } + kiloVersionHeartbeat = hello.data.capabilities?.kiloVersionHeartbeat === true; phase = 'status'; diagnostic('hello_accepted', ws); return; @@ -371,6 +376,7 @@ export function createSandboxControlClient( state = { kind: 'ready', socket: ws, + kiloVersionHeartbeat, dispose: () => { clearInterval(keepalive); dispose(); @@ -499,7 +505,10 @@ export function createSandboxControlClient( return false; } try { - const serialized = serializeEvent(event, payload, session); + const heartbeat = + event === 'sandbox.heartbeat' ? sandboxHeartbeatPayloadSchema.parse(payload) : null; + if (heartbeat && !state.kiloVersionHeartbeat) delete heartbeat.kilo.version; + const serialized = serializeEvent(event, heartbeat ?? payload, session); socket.send(serialized); eventDiagnostic('sent', Buffer.byteLength(serialized)); return true; diff --git a/services/cloud-agent-next/wrapper/src/control/sandbox-control-handlers.ts b/services/cloud-agent-next/wrapper/src/control/sandbox-control-handlers.ts index 2f4bc44581..38bf22a387 100644 --- a/services/cloud-agent-next/wrapper/src/control/sandbox-control-handlers.ts +++ b/services/cloud-agent-next/wrapper/src/control/sandbox-control-handlers.ts @@ -286,7 +286,12 @@ export function buildHeartbeatPayload(deps: HandlerDeps): SandboxHeartbeatPayloa : 'active', activeKiloSessions: active.length, pendingMessages: deps.tasks.size, - kilo: { ready: deps.kiloReady && !deps.signal?.aborted }, + kilo: { + ready: deps.kiloReady && !deps.signal?.aborted, + ...(deps.kiloRuntimes?.kiloCliVersion !== undefined + ? { version: deps.kiloRuntimes.kiloCliVersion } + : {}), + }, sessions, }; } diff --git a/services/cloud-agent-next/wrapper/src/control/worktree-runtime.test.ts b/services/cloud-agent-next/wrapper/src/control/worktree-runtime.test.ts index 5159576d64..c7a0eea475 100644 --- a/services/cloud-agent-next/wrapper/src/control/worktree-runtime.test.ts +++ b/services/cloud-agent-next/wrapper/src/control/worktree-runtime.test.ts @@ -14,7 +14,11 @@ import { type SessionEventIdentity, type SessionRequestIdentity, } from '../../../src/shared/sandbox-control-protocol'; -import { handleControlRequest, type HandlerDeps } from './sandbox-control-handlers'; +import { + buildHeartbeatPayload, + handleControlRequest, + type HandlerDeps, +} from './sandbox-control-handlers'; import type { WrapperKiloClient } from '../kilo-api'; import { ControlTerminalRuntimeError, @@ -88,7 +92,13 @@ async function rejected(operation: Promise): Promise { throw new Error('Expected runtime operation to fail'); } -function createKiloStub() { +function asFetch( + fn: (...args: Parameters) => ReturnType +): typeof fetch { + return Object.assign(fn, { preconnect: fetch.preconnect }); +} + +function createKiloStub(health: unknown = { healthy: true, version: '7.4.20' }) { const requests: Array<{ pathname: string; directory: string | null; body?: unknown }> = []; const permissions: Awaited> = []; const feeds = new Set>(); @@ -127,6 +137,8 @@ function createKiloStub() { decodeURIComponent(request.headers.get('x-kilo-directory') ?? ''), ...(body ? { body } : {}), }); + if (request.method === 'GET' && url.pathname === '/global/health') + return Response.json(health); if (request.method === 'GET' && url.pathname === '/permission') { return Response.json(permissions); } @@ -229,6 +241,9 @@ function createRegistry(overrides: Partial) { const attachment = registry.attach(rootIdentity(directory), kilo, env); try { @@ -282,6 +297,86 @@ afterEach(async () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); +describe('observed Kilo runtime version', () => { + it('ignores a delayed health response from a retired Kilo process', async () => { + const response = Promise.withResolvers(); + const requested = Promise.withResolvers(); + const originalFetch = globalThis.fetch; + let delayHealth = true; + const fetchSpy = spyOn(globalThis, 'fetch').mockImplementation( + asFetch((request, init) => { + const url = request instanceof Request ? request.url : String(request); + if (delayHealth && new URL(url).pathname === '/global/health') { + delayHealth = false; + requested.resolve(); + return response.promise; + } + return originalFetch(request, init); + }) + ); + const { registry } = createRegistry(); + try { + const firstDirectory = path.join(tmpDir, 'first'); + await registry.ensure(firstDirectory, auth); + await requested.promise; + registry.detach(rootIdentity(firstDirectory)); + await registry.ensure(path.join(tmpDir, 'second'), { ...auth, scopeId: 'second' }); + await waitUntil(() => registry.kiloCliVersion === '7.4.20'); + response.resolve(Response.json({ healthy: true, version: '9.9.9' })); + await new Promise(resolve => setImmediate(resolve)); + expect(registry.kiloCliVersion).toBe('7.4.20'); + } finally { + response.resolve(Response.json({})); + registry.shutdown(); + fetchSpy.mockRestore(); + } + }); + it('reports health version in heartbeats without status-time probes and retains it after detach', async () => { + const { registry } = createRegistry(); + const directory = path.join(tmpDir, 'runtime'); + const deps = createHandlerDeps(registry); + expect(buildHeartbeatPayload(deps).kilo.version).toBeNull(); + await registry.ensure(directory, auth); + await waitUntil(() => registry.kiloCliVersion === '7.4.20'); + for (let i = 0; i < 3; i++) { + expect(buildHeartbeatPayload(deps).kilo.version).toBe('7.4.20'); + await handleControlRequest('sandbox.status', undefined, {}, deps); + } + expect( + servers[0].requests.filter(request => request.pathname === '/global/health') + ).toHaveLength(1); + registry.detach(rootIdentity(directory)); + expect(buildHeartbeatPayload(deps).kilo.version).toBe('7.4.20'); + }); + + it.each([ + { healthy: true, version: '7.5.0' }, + { healthy: true, version: 'https://private.invalid/credential' }, + { healthy: false, version: '7.4.20' }, + {}, + ])( + 'does not report one version for inconsistent or unavailable health: %j', + async secondHealth => { + let count = 0; + const { registry } = createRegistry({ + startServer: async () => { + const server = createKiloStub( + count++ === 0 ? { healthy: true, version: '7.4.20' } : secondHealth + ); + servers.push(server); + return { url: server.url, close() {} }; + }, + }); + await registry.ensure(path.join(tmpDir, 'one'), auth); + await waitUntil(() => registry.kiloCliVersion === '7.4.20'); + await registry.ensure(path.join(tmpDir, 'two'), { ...auth, scopeId: 'worktree_two' }); + await waitUntil(() => registry.kiloCliVersion === null); + expect(buildHeartbeatPayload(createHandlerDeps(registry)).kilo.version).toBeNull(); + expect(registry.isHealthy()).toBe(true); + } + ); +}); + describe('worktree Kilo environments', () => { it.each([undefined, 'org-trusted'])( 'uses only trusted organization attribution: %s', @@ -442,6 +537,7 @@ describe('worktree Kilo runtime registry', () => { ) ); expect(servers[0]?.requests.map(request => request.pathname).sort()).toEqual([ + '/global/health', '/session/root_one/prompt_async', '/session/root_two/prompt_async', ]); @@ -497,7 +593,14 @@ describe('worktree Kilo runtime registry', () => { }), { headers: { 'Content-Type': 'text/event-stream' } } ); - const fetchSpy = spyOn(globalThis, 'fetch').mockResolvedValue(response); + const fetchSpy = spyOn(globalThis, 'fetch').mockImplementation( + asFetch(async request => { + const url = request instanceof Request ? request.url : String(request); + return new URL(url).pathname === '/global/health' + ? Response.json({ healthy: true, version: '7.4.20' }) + : response; + }) + ); const failures: unknown[] = []; const harness = createRegistry({ onUnexpectedClose: error => failures.push(error) }); try { @@ -521,7 +624,12 @@ describe('worktree Kilo runtime registry', () => { expect(runtime.signal.aborted).toBe(true); expect(harness.registry.isHealthy()).toBe(false); expect(harness.registry.get(runtime.directory)).toBeUndefined(); - expect(fetchSpy).toHaveBeenCalledTimes(1); + expect( + fetchSpy.mock.calls.filter( + ([request]) => + request instanceof Request && new URL(request.url).pathname === '/global/event' + ) + ).toHaveLength(1); } finally { harness.registry.shutdown(); fetchSpy.mockRestore(); diff --git a/services/cloud-agent-next/wrapper/src/control/worktree-runtime.ts b/services/cloud-agent-next/wrapper/src/control/worktree-runtime.ts index bcf4a51ef6..6c73fb767b 100644 --- a/services/cloud-agent-next/wrapper/src/control/worktree-runtime.ts +++ b/services/cloud-agent-next/wrapper/src/control/worktree-runtime.ts @@ -10,6 +10,7 @@ import { emitControlDiagnostic, type ControlDiagnosticReporter, } from '../../../src/shared/control-diagnostics.js'; +import { safeSandboxRuntimeVersion } from '../../../src/shared/sandbox-status.js'; import type { ControlErrorCode, SessionAttachPayload, @@ -53,6 +54,7 @@ export type WorktreeKiloAttachment = { }; export type WorktreeKiloRuntimes = { + readonly kiloCliVersion?: string | null; attach( identity: SessionRequestIdentity, kilo: WorktreeKiloAuth, @@ -284,8 +286,14 @@ export function createWorktreeKiloRuntimes(options: { const roots = new Map(); const homesByDirectory = new Map>(); const deletedDirectories = new Set(); + let observedVersion: string | null | undefined; let closed = false; + function recordVersion(value: unknown): void { + const version = safeSandboxRuntimeVersion(value); + observedVersion = observedVersion === undefined || observedVersion === version ? version : null; + } + function findRoot(identity: SessionRequestIdentity): RootAttachment | undefined { for (const root of roots.values()) { if ( @@ -459,6 +467,20 @@ export function createWorktreeKiloRuntimes(options: { ); abort.signal.throwIfAborted(); entry.runtime = runtime; + void withKiloRequestDeadline( + signal => eventClient.global.health({ signal }), + abort.signal + ).then( + health => { + if (closed || abort.signal.aborted || entry.processAbort !== abort) return; + recordVersion(health.data?.healthy === true ? health.data.version : null); + }, + () => { + if (!closed && !abort.signal.aborted && entry.processAbort === abort) { + recordVersion(null); + } + } + ); return runtime; } catch { if (entry.runtime) failRuntime(entry, 'credential_refresh_failed'); @@ -526,6 +548,9 @@ export function createWorktreeKiloRuntimes(options: { } return { + get kiloCliVersion() { + return observedVersion ?? null; + }, attach(identity, kilo, environment, canRefreshCredentials) { if (closed) { throw new WorktreeKiloRuntimeError('not_ready', 'Kilo worktrees are closed', false); From a68bfa801f88fa62205cb0571d707e513a6c73f9 Mon Sep 17 00:00:00 2001 From: Evgeny Shurakov Date: Thu, 3 Sep 2026 11:44:57 +0200 Subject: [PATCH 2/3] fix(cloud-agent): unify sandbox popup and toolbar controls --- .specs/cloud-agent-session.md | 10 +- .../cloud-agent-next/ChatHeader.tsx | 6 +- .../cloud-agent-next/FeedbackDialog.tsx | 8 +- .../SandboxStatusIndicator.tsx | 155 ++++++--- .../cloud-agent-next/sandbox-status.test.ts | 1 + .../cloud-agent-next/sandbox-status.ts | 4 +- .../components/shared/SoundToggleButton.tsx | 11 +- apps/web/src/components/ui/button-variants.ts | 1 + .../e2e/cloud-agent-sandbox-status.spec.ts | 319 +++++++++++++++--- 9 files changed, 401 insertions(+), 114 deletions(-) diff --git a/.specs/cloud-agent-session.md b/.specs/cloud-agent-session.md index d0863b6f43..09c69d1754 100644 --- a/.specs/cloud-agent-session.md +++ b/.specs/cloud-agent-session.md @@ -274,10 +274,12 @@ repository. locally invent `sleeping`; new authoritative evidence is required. 9. Details MUST group the current text status, runtime information, and known lifecycle timing. Runtime information MUST identify the provider and bounded - sandbox type using plain-language labels. The click/touch popup MUST include - a Debug section, collapsed by default, with labeled control-plane mode and - reported CLI/wrapper versions. The hover tooltip MUST remain noninteractive - and omit Debug. Compact runtime codes MUST NOT be shown. Missing versions + sandbox type using plain-language labels. Hover, click, and touch MUST show + the same popup, including a Debug section collapsed by default with labeled + control-plane mode and reported CLI/wrapper versions. Hover MUST NOT move + keyboard focus. Moving into the popup MUST keep it open; clicking the trigger + or interacting inside MUST keep the same view open until dismissed. Compact + runtime codes MUST NOT be shown. Missing versions MUST display Unknown rather than guessed values. Debug MUST NOT contain a unique sandbox or provider identifier. Known physical start/stop dates MUST use local display times; unknown dates diff --git a/apps/web/src/components/cloud-agent-next/ChatHeader.tsx b/apps/web/src/components/cloud-agent-next/ChatHeader.tsx index 5e02772a67..e7774b3d2e 100644 --- a/apps/web/src/components/cloud-agent-next/ChatHeader.tsx +++ b/apps/web/src/components/cloud-agent-next/ChatHeader.tsx @@ -136,15 +136,15 @@ export function ChatHeader({ /> )} {onToggleSound && ( - + )} diff --git a/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx b/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx index fdd341b066..d0c682b8d5 100644 --- a/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx +++ b/apps/web/src/components/cloud-agent-next/SandboxStatusIndicator.tsx @@ -17,7 +17,6 @@ import { } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; -import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; import { useTRPC } from '@/lib/trpc/utils'; import { cn } from '@/lib/utils'; import { @@ -126,11 +125,43 @@ export function SandboxStatusIndicator({ }); const [clock, setClock] = useState(0); const [activity, setActivity] = useState({ active: sessionActive, changedAt: 0 }); - const [popoverOpen, setPopoverOpen] = useState(false); - const [tooltipOpen, setTooltipOpen] = useState(false); + const [openMode, setOpenMode] = useState<'preview' | 'pinned' | null>(null); const triggerRef = useRef(null); + const debugSummaryRef = useRef(null); + const closeTimerRef = useRef | null>(null); const detailsId = useId(); + function cancelPreviewClose() { + if (closeTimerRef.current !== null) { + clearTimeout(closeTimerRef.current); + closeTimerRef.current = null; + } + } + + function closePopover() { + cancelPreviewClose(); + setOpenMode(null); + } + + function schedulePreviewClose() { + cancelPreviewClose(); + closeTimerRef.current = setTimeout(() => { + closeTimerRef.current = null; + setOpenMode(current => (current === 'preview' ? null : current)); + }, 150); + } + + function pinPopover() { + cancelPreviewClose(); + setOpenMode('pinned'); + } + + useEffect(() => { + return () => { + if (closeTimerRef.current !== null) clearTimeout(closeTimerRef.current); + }; + }, []); + if (activity.active !== sessionActive) { setActivity({ active: sessionActive, changedAt: Date.now() }); } @@ -217,62 +248,77 @@ export function SandboxStatusIndicator({ return ( { - setTooltipOpen(false); - setPopoverOpen(open); + if (open) pinPopover(); + else closePopover(); }} > - - - - - - - {!popoverOpen && ( - - - - )} - - {popoverOpen && ( + + + + {openMode !== null && ( triggerRef.current?.focus()} + onPointerEnter={cancelPreviewClose} + onPointerLeave={schedulePreviewClose} + onPointerDownCapture={pinPopover} + onFocusCapture={pinPopover} + onOpenAutoFocus={event => { + if (openMode === 'preview') event.preventDefault(); + }} + onCloseAutoFocus={event => { + if (openMode === 'preview') event.preventDefault(); + }} + onEscapeKeyDown={() => { + if (openMode === 'pinned') triggerRef.current?.focus(); + }} onKeyDown={event => { if ( event.key === 'Tab' && @@ -282,7 +328,7 @@ export function SandboxStatusIndicator({ (event.shiftKey || event.target !== event.currentTarget) ) { triggerRef.current?.focus(); - setPopoverOpen(false); + closePopover(); } }} > @@ -290,7 +336,10 @@ export function SandboxStatusIndicator({
- + diff --git a/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts b/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts index b2094dce47..e901f03f5e 100644 --- a/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts +++ b/apps/web/src/components/cloud-agent-next/sandbox-status.test.ts @@ -417,6 +417,7 @@ describe('sandbox status presentation', () => { }) ).toMatchObject({ status: 'sleeping', + detail: 'Send a message to resume.', provider: 'Vercel', sandboxType: 'Small', kiloCliVersion: '7.4.20', diff --git a/apps/web/src/components/cloud-agent-next/sandbox-status.ts b/apps/web/src/components/cloud-agent-next/sandbox-status.ts index 537caf943b..7c30cd94ed 100644 --- a/apps/web/src/components/cloud-agent-next/sandbox-status.ts +++ b/apps/web/src/components/cloud-agent-next/sandbox-status.ts @@ -167,7 +167,9 @@ export function sandboxStatusPresentation({ label: sleepingSoon ? 'Sleeping soon' : statusLabels[snapshot.status], detail: sleepingSoon ? 'The sandbox will sleep soon if inactivity continues.' - : SANDBOX_STATUS_DETAIL_MESSAGES[snapshot.detailCode], + : snapshot.status === 'sleeping' + ? 'Send a message to resume.' + : SANDBOX_STATUS_DETAIL_MESSAGES[snapshot.detailCode], provider: snapshot.provider, sandboxType: sandboxTypes[runtime?.sandboxType ?? 'unknown'], kiloCliVersion: runtime?.kiloCliVersion ?? null, diff --git a/apps/web/src/components/shared/SoundToggleButton.tsx b/apps/web/src/components/shared/SoundToggleButton.tsx index 06bcdbd81e..d5459d3657 100644 --- a/apps/web/src/components/shared/SoundToggleButton.tsx +++ b/apps/web/src/components/shared/SoundToggleButton.tsx @@ -10,7 +10,7 @@ type SoundToggleButtonProps = { /** Callback when toggle is clicked */ onToggle: () => void; /** Size variant - affects icon and button dimensions */ - size?: 'sm' | 'default'; + size?: 'sm' | 'default' | 'toolbar'; /** Additional CSS classes */ className?: string; }; @@ -27,14 +27,17 @@ export function SoundToggleButton({ className, }: SoundToggleButtonProps) { const iconSize = size === 'sm' ? 'h-3 w-3' : 'h-4 w-4'; - const buttonSize = size === 'sm' ? 'h-6 w-6' : 'h-8 w-8'; + const buttonClassName = + size === 'toolbar' + ? 'text-muted-foreground' + : cn(size === 'sm' ? 'h-6 w-6' : 'h-8 w-8', 'hover:text-blue-400'); return (