Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
330 changes: 330 additions & 0 deletions tst/manual/remote_computer_clock_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,330 @@
"""Run explicitly with ``pytest -s tst/manual/remote_computer_clock_check.py``: a computer's universe clock on a remote
sandbox provider, through the steps ``tst/integration/computer/computer_clock_test.py`` takes on the local one. Armed,
a day and a week on, a month back, a rate of 10, frozen, running again, a minute and a half with no change, the
follower killed, and a reset: after each, the session is within ``MAX_SKEW_S`` of the gateway's universe, give or take
how far the reading itself may be off, and it runs at the clock's rate. A browser computer's pages show it too.

A provider whose sandbox is the computer's container itself, as ``modal``'s is, can't reset it in place, which core
refuses; the check takes that refusal as the reset's outcome. On a provider that boots the desktop from its disk form,
whose guest follows no clock yet, arming the clock skips it, as tolerated by default, and a strict arm refuses it.

It needs a config whose image and object stores the provider reaches, the provider's credentials, and two multi envs
written there. ``AGENTENV_CLOCK_CHECK_OFFICE`` names one whose computer ``desktop`` is the ubuntu-24.04 desktop, and
``AGENTENV_CLOCK_CHECK_BROWSING`` one whose computer ``browser`` is a browser computer; either left out skips its check.
Each multi env deploys on ``AGENTENV_CLOCK_CHECK_GATEWAY_SANDBOX`` (``modal_vm``), its computer on each of
``AGENTENV_CLOCK_CHECK_SANDBOXES`` (``modal_vm,modal``).

To write them, register a bundle into those stores with ``agent-env bundle register <folder> --sandbox <provider>``,
which composes its computers on the provider. Its envs:
- ``desktop``: ``type = "computer"``, ``computer = "ubuntu-24.04"``, ``driver = "reference"``.
- ``browser``: ``type = "computer"``, ``computer = "ubuntu-24.04-browser"``, ``driver = "page"``,
``apps = ["chromium"]``.
- ``office``: ``type = "multi"``, ``computer_envs = ["desktop"]``.
- ``browsing``: ``type = "multi"``, ``computer_envs = ["browser"]``.
- Copies of those bases, drivers and app, since registration refuses entities of installed bundles.
Registration builds each image it writes on this machine, for amd64. The stores need the gateway's and the service-db's
infra envs too (``agent-env env gateway put --id default``, ``agent-env env service-db put --id default-db``); a VM
sandbox builds an image from its context when it isn't published or was built for another platform.

Every reading is taken inside the computer: the gateway's universe, read at the address the computer reaches the gateway
at, as its follower does, and the session's own, each between two readings of real time on the computer's clock. So
neither this machine's distance from the provider nor its clock is in them."""

import asyncio
import json
import os
import re
import time
import urllib.parse
from datetime import datetime, timedelta, timezone

import pytest
from agentenv_protocol.computer.clock import FOLLOWER, FOLLOWER_USER

from agent_env.env.env import DeployedEnv
from agent_env.env.envs.computer.utils import inside
from agent_env.env.envs.computer.utils.sites import RootChannel
from agent_env.env.gateway.constants import EXT_CLOCK_URI
from agent_env.providers import build_sandbox_provider
from agent_env.providers.sandbox_providers.sandbox import VmSandbox
from agent_env.task.teardown import teardown_run
from agent_env.task_step.context import StepContextError, TaskStepContext
from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep
from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep
from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep
from tst.integration.computer.browser_computer_test import _browser_op

OFFICE = os.environ.get("AGENTENV_CLOCK_CHECK_OFFICE")
BROWSING = os.environ.get("AGENTENV_CLOCK_CHECK_BROWSING")
GATEWAY_SANDBOX = os.environ.get("AGENTENV_CLOCK_CHECK_GATEWAY_SANDBOX", "modal_vm")
SANDBOXES = os.environ.get("AGENTENV_CLOCK_CHECK_SANDBOXES", "modal_vm,modal").split(",")
RATE = 10
#: How far the session's clock may be from the gateway's: room for reading the two, as on the local provider.
MAX_SKEW_S = 1.0
#: How long a minute and a half with no clock change is: past the gateway's 25 s hold on a watch more than three times,
#: so a provider's tunnel that drops an idle long poll shows as a follower that misses the next change.
IDLE_S = 90
#: How soon a killed follower may be back: supervisord's second, and a remote sandbox's slower start.
FOLLOWER_BACK_S = 10
TTL_S = 2400
#: How far back of now the universe is armed: half a year, as the local clock test arms it.
ARMED_BACK = timedelta(days=182)
#: How long a clock write waits for every computer to show it, and how long a call to the gateway may take.
FOLLOWERS_WAIT_S = 15
CALL_TIMEOUT_S = 30
#: How long the reading script inside the computer may run, how many times it reads, keeping the narrowest, and how
#: long its read of the gateway may take.
READ_TIMEOUT_S = 120
READ_ATTEMPTS = 10
GATEWAY_READ_TIMEOUT_S = 10
#: How far apart the two readings a rate is measured from are, and how far that rate may be from the clock's: a
#: fraction of it, or outright when frozen.
RATE_OVER_S = 3.0
RATE_TOLERANCE = 0.05
FROZEN_TOLERANCE = 0.01
#: How long a killed follower, and a page's time, are waited for, and how often each is looked for.
FOLLOWER_WAIT_S = 60
FOLLOWER_POLL_S = 0.5
PAGE_WAIT_S = 60
PAGE_POLL_S = 0.25
#: How far a page's Date.now() may be from the gateway's universe: room for its title's refresh and the reads from here.
MAX_PAGE_SKEW_S = 3
#: The reset's outcome on a provider whose sandbox is the computer's container itself, which core can't replace.
REFUSED_IN_PLACE = "refused: its sandbox is the container itself"
#: Inside the computer, on real time: the gateway's universe as the computer reaches it at argv[1], read within
#: argv[3] seconds, between two readings of real time, then the session's own (``date``, without the real-time marker),
#: between two more, taken argv[2] times, keeping the narrowest; argv[1] "-" reads the session alone. Each bracket's
#: width is printed, so the caller can say how far the reading may be from the truth.
READ = """
import json, os, subprocess, sys, time, urllib.request
url, attempts, timeout = sys.argv[1], int(sys.argv[2]), float(sys.argv[3])
session = {k: v for k, v in os.environ.items() if k != "AGENTENV_REAL_TIME"}
best = None
for _ in range(attempts):
a = time.time()
gateway = json.load(urllib.request.urlopen(url, timeout=timeout))["virtual_time"] if url != "-" else None
b = time.time()
universe = float(subprocess.run(["date", "+%s.%N"], env=session, capture_output=True, text=True, check=True).stdout)
c = time.time()
read = {"gateway_at": (a + b) / 2, "gateway": gateway, "session_at": (b + c) / 2, "session": universe,
"widths": [round(b - a, 4), round(c - b, 4)]}
if best is None or max(read["widths"]) < max(best["widths"]):
best = read
print(json.dumps(best))
"""
#: A page that names its own Date.now(), in milliseconds, in its title, five times a second.
CLOCK_PAGE = "data:text/html," + urllib.parse.quote(
"<script>setInterval(() => document.title = 'now ' + Date.now(), 200)</script>")


def _iso(moment: datetime) -> str:
return moment.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")


def _at(text: str) -> float:
return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp()


async def _write(deployed, method: str, **params) -> float:
"""A clock write that waits up to ``FOLLOWERS_WAIT_S`` for every computer to show it; the seconds it took."""
started = time.monotonic()
await deployed.invoke(EXT_CLOCK_URI, method, {**params, "wait_followers_s": FOLLOWERS_WAIT_S},
timeout=CALL_TIMEOUT_S)
return round(time.monotonic() - started, 2)


async def _now(deployed) -> datetime:
state = await deployed.invoke(EXT_CLOCK_URI, "state", timeout=CALL_TIMEOUT_S)
return datetime.fromtimestamp(_at(state["virtual_time"]), timezone.utc)


async def _reading(computer, time_url: str) -> dict:
argv = ["env", "AGENTENV_REAL_TIME=1", "python3", "-c", READ, time_url, str(READ_ATTEMPTS),
str(GATEWAY_READ_TIMEOUT_S)]
ran = await inside.run(computer, argv, timeout_s=READ_TIMEOUT_S)
assert ran["exit_code"] == 0, ran
return json.loads(ran["stdout"])


async def _skew(computer, time_url: str, rate: float) -> tuple[float, float]:
"""How far the session's clock is from the gateway's, in universe seconds, both read inside the computer, and how
far that may be from the truth: half of each reading's bracket of real time, at the clock's rate."""
read = await _reading(computer, time_url)
skew = read["session"] - (_at(read["gateway"]) + rate * (read["session_at"] - read["gateway_at"]))
return round(skew, 3), round(rate * sum(read["widths"]) / 2, 3)


async def _rate(computer, over_s: float = RATE_OVER_S) -> float:
"""The session clock's rate, from two of its own readings ``over_s`` apart: no gateway read is in it."""
first = await _reading(computer, "-")
await asyncio.sleep(over_s)
then = await _reading(computer, "-")
return round((then["session"] - first["session"]) / (then["session_at"] - first["session_at"]), 3)


async def _followers(computer) -> list[str]:
channel = RootChannel(await build_sandbox_provider(computer.sandbox_type).get_sandbox(computer.sandbox_id))
return (await channel.run(["sh", "-c", f"pgrep -u {FOLLOWER_USER} -f {FOLLOWER} || true"])).split()


async def _kill_follower(computer) -> tuple[list[str], list[str], float]:
"""The computer's follower killed with SIGKILL through the root channel: its pids, those back, and how long that
took, timed from here."""
channel = RootChannel(await build_sandbox_provider(computer.sandbox_type).get_sandbox(computer.sandbox_id))
killed = await _followers(computer)
await channel.run(["kill", "-9", *killed])
started = time.monotonic()
while time.monotonic() - started < FOLLOWER_WAIT_S:
back = await _followers(computer)
if back and set(back) != set(killed):
return killed, back, round(time.monotonic() - started, 1)
await asyncio.sleep(FOLLOWER_POLL_S)
return killed, [], float(FOLLOWER_WAIT_S)


async def _page_now(browser: DeployedEnv) -> float:
"""What a fresh page in ``browser`` takes for now, in seconds, from its Date.now()."""
opened = await _browser_op(browser, "tabs/open", {"url": CLOCK_PAGE})
deadline, tabs = time.monotonic() + PAGE_WAIT_S, []
while time.monotonic() < deadline:
tabs = (await _browser_op(browser, "tabs/list", {}))["tabs"]
tab = next((tab for tab in tabs if tab["id"] == opened["tab"]), None)
if tab and (tab.get("title") or "").startswith("now "):
await _browser_op(browser, "tabs/close", {"tab": tab["id"]})
return int(tab["title"].split()[1]) / 1000
await asyncio.sleep(PAGE_POLL_S)
raise AssertionError(f"the clock page named no time within {PAGE_WAIT_S} s: {tabs}")


async def _through_the_steps(env_id: str, child: str, sandbox: str, browser: bool = False) -> dict:
"""The clock test's steps on ``env_id``'s computer ``child`` deployed on ``sandbox``: what each showed."""
results: dict = {"sandbox": sandbox}
context = TaskStepContext()
try:
started = time.monotonic()
[deployed] = (await DeployEnvTaskStep(
id="office", version=None, env_id=env_id, sandbox_type=GATEWAY_SANDBOX, ttl_seconds=TTL_S,
child_envs={child: {"sandbox_type": sandbox, "env_params": {"form": "container"}}}).execute(context)
).deployed_envs
results["deployed in s"] = round(time.monotonic() - started)
own = DeployedEnv.from_dict(deployed.child_envs[child])
kept = own.metadata["computer"]["clock"]
results["follows"] = {"name": kept["name"], "watch_url": kept["watch_url"].split("://")[0] + "://…",
"sandbox_type": own.sandbox_type}
time_url = kept["watch_url"].removesuffix("/clock/watch") + "/clock/time"

universe = datetime.now(timezone.utc) - ARMED_BACK
await SyncEnvClockTaskStep(id="clock", version=None, env_id=env_id,
virtual_time=_iso(universe)).execute(context)
results["synced"] = [(entry["service"], entry["result"]["follower"]["armed"])
for entry in context.metadata["clock_configurations"][-1]["synced"]
if "follower" in (entry.get("result") or {})]
skews, writes, rates = {}, {}, {}
results.update(skews=skews, rates=rates, writes=writes)
skews["armed"] = await _skew(own, time_url, 1)
if browser:
results["page minus universe s"] = round(await _page_now(own) - _at(
(await deployed.invoke(EXT_CLOCK_URI, "state", timeout=CALL_TIMEOUT_S))["virtual_time"]), 1)
for name, by in (("a day on", timedelta(days=1)), ("a week on", timedelta(weeks=1))):
writes[name] = await _write(deployed, "advance", virtual_time=_iso(await _now(deployed) + by))
skews[name] = await _skew(own, time_url, 1)
writes["a month back"] = await _write(deployed, "set_time", virtual_time=_iso(
await _now(deployed) - timedelta(days=30)), virtual_seconds_per_real_second=1)
skews["a month back"] = await _skew(own, time_url, 1)
writes[f"rate {RATE}"] = await _write(deployed, "set_rate", virtual_seconds_per_real_second=RATE)
rates[RATE] = await _rate(own)
skews[f"at a rate of {RATE}"] = await _skew(own, time_url, RATE)
writes["frozen"] = await _write(deployed, "set_rate", virtual_seconds_per_real_second=0)
rates["frozen"], skews["frozen"] = await _rate(own), await _skew(own, time_url, 0)
writes["running again"] = await _write(deployed, "set_rate", virtual_seconds_per_real_second=1)
skews["running again"] = await _skew(own, time_url, 1)

await asyncio.sleep(IDLE_S)
writes[f"after {IDLE_S} s idle"] = await _write(deployed, "advance", virtual_time=_iso(
await _now(deployed) + timedelta(hours=1)))
skews[f"after {IDLE_S} s idle"] = await _skew(own, time_url, 1)

killed, back, results["follower back in s"] = await _kill_follower(own)
results["follower killed, back"] = (bool(killed), bool(back))
writes["after the follower's kill"] = await _write(deployed, "advance", virtual_time=_iso(
await _now(deployed) + timedelta(hours=8)))
skews["after the follower's kill"] = await _skew(own, time_url, 1)

provider = build_sandbox_provider(own.sandbox_type)
if not isinstance(await provider.get_sandbox(own.sandbox_id), VmSandbox):
# Only a VM's Docker can replace the computer's container in place (SandboxProvider.reset_container), and
# core's refusal names the provider; any other failure of the reset fails the check.
with pytest.raises(RuntimeError, match=re.escape(f"can't be reset on {type(provider).__name__}")):
await ResetEnvTaskStep(id="reset", version=None, env_id=env_id,
fail_task_on_error=True).execute(context)
results["reset"] = REFUSED_IN_PLACE
return results
await ResetEnvTaskStep(id="reset", version=None, env_id=env_id, fail_task_on_error=True).execute(context)
own = DeployedEnv.from_dict(context.deployed_envs[0].child_envs[child])
skews["after a reset"] = await _skew(own, time_url, 1)
if browser:
results["page minus universe s after a reset"] = round(await _page_now(own) - _at(
(await deployed.invoke(EXT_CLOCK_URI, "state", timeout=CALL_TIMEOUT_S))["virtual_time"]), 1)
state = await deployed.invoke(EXT_CLOCK_URI, "state", timeout=CALL_TIMEOUT_S)
results["shown after a reset"] = state["followers"][kept["name"]]["version"] == state["version"]
results["reset"] = "done"
finally:
report = await teardown_run(context)
results["torn down"] = not report.failed
print("CLOCK CHECK", json.dumps(results, indent=1, default=str), flush=True)
return results


def _assert(results: dict) -> None:
assert results["synced"] and all(armed for _, armed in results["synced"]), results
assert all(abs(skew) - error < MAX_SKEW_S for skew, error in results["skews"].values()), results
assert abs(results["rates"][RATE] / RATE - 1) < RATE_TOLERANCE, results
assert abs(results["rates"]["frozen"]) < FROZEN_TOLERANCE, results
assert results["follower killed, back"] == (True, True) and results["follower back in s"] < FOLLOWER_BACK_S, results
assert results["reset"] == REFUSED_IN_PLACE or results["shown after a reset"], results
assert results["torn down"], results


@pytest.mark.skipif(not OFFICE, reason="AGENTENV_CLOCK_CHECK_OFFICE names no multi env")
@pytest.mark.parametrize("sandbox", SANDBOXES)
@pytest.mark.asyncio
async def test_a_remote_desktop_keeps_to_its_gateways_clock_through_jumps_rates_an_idle_spell_a_kill_and_a_reset(
sandbox):
_assert(await _through_the_steps(OFFICE, "desktop", sandbox))


@pytest.mark.skipif(not BROWSING, reason="AGENTENV_CLOCK_CHECK_BROWSING names no multi env")
@pytest.mark.parametrize("sandbox", SANDBOXES)
@pytest.mark.asyncio
async def test_a_remote_browser_computer_and_its_pages_keep_to_its_gateways_clock(sandbox):
results = await _through_the_steps(BROWSING, "browser", sandbox, browser=True)
_assert(results)
assert abs(results["page minus universe s"]) < MAX_PAGE_SKEW_S, results
assert results["reset"] != "done" or abs(results["page minus universe s after a reset"]) < MAX_PAGE_SKEW_S, results


@pytest.mark.skipif(not OFFICE, reason="AGENTENV_CLOCK_CHECK_OFFICE names no multi env")
@pytest.mark.parametrize("sandbox", [sandbox for sandbox in SANDBOXES
if build_sandbox_provider(sandbox).disk_image_platforms()])
@pytest.mark.asyncio
async def test_a_remote_desktop_booted_from_its_disk_is_skipped_when_tolerated_and_refused_by_a_strict_arm(sandbox):
context, results, refused = TaskStepContext(), {"sandbox": sandbox}, None
try:
await DeployEnvTaskStep(id="office", version=None, env_id=OFFICE, sandbox_type=GATEWAY_SANDBOX,
ttl_seconds=TTL_S, child_envs={"desktop": {"sandbox_type": sandbox,
"env_params": {"form": "disk"}}}
).execute(context)
universe = _iso(datetime.now(timezone.utc) - ARMED_BACK)
await SyncEnvClockTaskStep(id="clock", version=None, env_id=OFFICE, virtual_time=universe).execute(context)
results["tolerated"] = [(entry["service"], entry.get("reason"))
for entry in context.metadata["clock_configurations"][-1]["skipped"]]
with pytest.raises(StepContextError) as refused:
await SyncEnvClockTaskStep(id="strict", version=None, env_id=OFFICE, virtual_time=universe,
tolerate_missing_sync_time=False).execute(context)
results["refused"] = str(refused.value)
finally:
report = await teardown_run(context)
print("CLOCK CHECK", json.dumps(results), flush=True)

assert ("desktop", "not_advertised") in results["tolerated"], results
assert f"desktop does not advertise {EXT_CLOCK_URI}" in results["refused"] and not report.failed
Loading