From cbc7569da7c04243d92cb3e8e94cf2fc49c21fc6 Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Wed, 26 Aug 2026 12:40:06 +0100 Subject: [PATCH 1/2] Poll jobs in short intervals: long blocks are unreliable Found by running the real thing: a headless Claude Code session against the deployed server, scoring 1p on the UK basic rate through start_job. It got the right answer -- costings 6.463..7.378, cumulative GDP -10.713bn, matching a local run to the pound -- but needed NINE polls, and reported that "several polls hit transport timeouts and one Modal InternalFailure; wait_seconds: 30 polled reliably where 60-120 did not". So staying under the 150s ceiling is necessary but not sufficient. The flakiness tracks how long the connection is held open, not how close it gets to the ceiling -- the same intermittent `InternalFailure: Server has lost track of input` that shows up on long direct calls. Default wait drops 60 -> 30 and the cap 120 -> 60, so a caller asking for a long block is capped well below the ceiling rather than at it. Both descriptions now say a poll that ERRORS is not a job that failed: the job runs on regardless of the connection watching it, so the right response is to call again with the same job_id. An agent that reads a transport error as a failed score would throw away a running job and start another. 293 integration tests pass (16 in test_jobs). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Lcj9DDqam9KmVCfhEdnJcJ --- integration/src/policyengine_macro/jobs.py | 30 +++++++++++++------ .../src/policyengine_macro/mcp_server.py | 11 +++++-- integration/tests/test_jobs.py | 22 ++++++++++++-- integration/tests/test_remote_mcp.py | 2 +- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/integration/src/policyengine_macro/jobs.py b/integration/src/policyengine_macro/jobs.py index 68bcc6d..4d257dc 100644 --- a/integration/src/policyengine_macro/jobs.py +++ b/integration/src/policyengine_macro/jobs.py @@ -53,10 +53,17 @@ "pe_population_impact", ) -# The longest a get_job_result call may block. The transport ceiling is 150s; -# this leaves room for request overhead so the poll itself never becomes the -# thing that times out. -MAX_WAIT_SECONDS = 120 +# How long a get_job_result call blocks. +# +# The hard ceiling is 150s, but staying under it is not sufficient: measured +# end-to-end through a real Claude Code session, polls at 60-120s repeatedly +# hit transport timeouts and one Modal `InternalFailure: Server has lost track +# of input`, while 30s polls came back reliably. The flakiness scales with how +# long the connection is held open, not with how close it gets to 150s -- so +# the default is short, and a caller asking for a long block gets capped well +# below the ceiling rather than at it. +DEFAULT_WAIT_SECONDS = 30 +MAX_WAIT_SECONDS = 60 _spawn: Callable[[str, dict], str] | None = None _poll: Callable[[str, int], Any] | None = None @@ -112,13 +119,15 @@ def start(tool: str, arguments: dict | None = None) -> dict: "status": "running", "next_step": ( f"Call get_job_result(job_id={job_id!r}). It blocks until the job " - f"finishes or up to wait_seconds (max {MAX_WAIT_SECONDS}); if it " - "returns status 'running', call it again with the same job_id." + f"finishes or up to wait_seconds (default {DEFAULT_WAIT_SECONDS}, " + f"max {MAX_WAIT_SECONDS}); if it returns status 'running', call it " + "again with the same job_id. Short polls are more reliable than " + "long ones -- prefer the default over asking for a long block." ), } -def result(job_id: str, wait_seconds: int = 60) -> dict: +def result(job_id: str, wait_seconds: int = DEFAULT_WAIT_SECONDS) -> dict: """Poll a job. Blocks up to wait_seconds, then reports back either way.""" if not backend_available(): raise NoBackend(_NO_BACKEND_MESSAGE.format(tools=", ".join(JOB_TOOLS))) @@ -132,7 +141,10 @@ def result(job_id: str, wait_seconds: int = 60) -> dict: "waited_seconds": wait, "next_step": ( "Not finished yet. Call get_job_result again with the same " - "job_id; a score_reform over the default five-year window " - "typically needs two or three polls." + "job_id -- polling repeatedly is expected and cheap. A " + "score_reform over the default five-year window typically needs " + "several polls. If a poll errors rather than returning, retry it: " + "the job keeps running regardless of what happens to the " + "connection watching it." ), } diff --git a/integration/src/policyengine_macro/mcp_server.py b/integration/src/policyengine_macro/mcp_server.py index 533fda2..0237ee5 100644 --- a/integration/src/policyengine_macro/mcp_server.py +++ b/integration/src/policyengine_macro/mcp_server.py @@ -49,14 +49,19 @@ def start_job( def get_job_result( job_id: Annotated[str, Field(description="Job id from start_job")], wait_seconds: Annotated[ - int, Field(description="Seconds to block waiting, max 120") - ] = 60, + int, Field(description="Seconds to block waiting, max 60") + ] = 30, ) -> dict: """Fetch a started job's result, waiting up to wait_seconds for it. Returns status 'done' with the result, or status 'running' -- in which case call again with the same job_id. A score_reform over the default - five-year window typically needs two or three polls. + five-year window typically needs several polls; that is expected. + + Keep wait_seconds short. Long blocking polls hit transport timeouts far + more often than short ones, and a poll that errors is not a job that + failed -- the job runs on regardless of the connection watching it, so + just call again with the same job_id. """ return jobs.result(job_id, wait_seconds) diff --git a/integration/tests/test_jobs.py b/integration/tests/test_jobs.py index 80dd0eb..5add74d 100644 --- a/integration/tests/test_jobs.py +++ b/integration/tests/test_jobs.py @@ -128,13 +128,29 @@ def test_unfinished_job_tells_the_caller_to_poll_again(): assert "again" in out["next_step"] -def test_wait_is_capped_below_the_transport_ceiling(): - """A poll that outlived the 150s ceiling would be the bug it works around.""" +def test_wait_is_capped_well_below_the_transport_ceiling(): + """A poll that outlived the 150s ceiling would be the bug it works around. + + The cap sits far below it, not just inside it. Measured end-to-end through + a real Claude Code session: polls at 60-120s repeatedly hit transport + timeouts and one Modal `InternalFailure: Server has lost track of input`, + while 30s polls came back reliably. The flakiness tracks how long the + connection is held open rather than how close it gets to 150s. + """ seen = [] jobs.set_backend(lambda t, a: "fc-1", lambda j, w: (seen.append(w), (False, None))[1]) jobs.result("fc-1", wait_seconds=10_000) assert seen == [jobs.MAX_WAIT_SECONDS] - assert jobs.MAX_WAIT_SECONDS < 150, "the cap must sit inside the ceiling" + assert jobs.MAX_WAIT_SECONDS <= 60, "long blocking polls are unreliable" + assert jobs.DEFAULT_WAIT_SECONDS <= 30, "the default poll must be short" + + +def test_default_wait_is_the_short_one(): + """Callers who pass nothing get the interval that actually works.""" + seen = [] + jobs.set_backend(lambda t, a: "fc-1", lambda j, w: (seen.append(w), (False, None))[1]) + jobs.result("fc-1") + assert seen == [jobs.DEFAULT_WAIT_SECONDS] def test_negative_wait_is_clamped_not_passed_through(): diff --git a/integration/tests/test_remote_mcp.py b/integration/tests/test_remote_mcp.py index a2e809f..c7edf51 100644 --- a/integration/tests/test_remote_mcp.py +++ b/integration/tests/test_remote_mcp.py @@ -422,7 +422,7 @@ async def test_score_reform_default_window_works_through_a_job(): out = None while waited < deadline: out = await asyncio.wait_for( - _call("get_job_result", {"job_id": job_id, "wait_seconds": 120}), + _call("get_job_result", {"job_id": job_id, "wait_seconds": 30}), timeout=180, ) if out["status"] == "done": From 765ccb0b33d883ab054a0b00a3ed9037b8a6ad7a Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Wed, 26 Aug 2026 12:42:13 +0100 Subject: [PATCH 2/2] Regenerate navigation on the new release-note pages Repairs main. The vintage refresh (#177) generated twelve release-note pages under notes/releases/2026-08-26-* without running site_nav.py, so the Site provenance job has been failing on main since that merge with "FAIL stale navigation" on each new page. My miss: I checked #177 against the site test suite, site_contract.py and its own no-edited-vintages invariant, all of which passed, and merged it while only the Vercel checks had reported. site_nav.py is a separate gate and I did not run it. Mechanical regeneration -- `python3 site_nav.py --write`, now clean on 110 pages. --- notes/releases/2026-08-26-uk_average_weekly_earnings/index.html | 1 - notes/releases/2026-08-26-uk_bank_rate/index.html | 1 - notes/releases/2026-08-26-uk_core_cpi_yoy/index.html | 1 - notes/releases/2026-08-26-uk_cpi_yoy/index.html | 1 - notes/releases/2026-08-26-uk_gilt_10y/index.html | 1 - notes/releases/2026-08-26-uk_gilt_20y/index.html | 1 - notes/releases/2026-08-26-uk_gilt_5y/index.html | 1 - notes/releases/2026-08-26-uk_monthly_gva/index.html | 1 - .../2026-08-26-uk_public_sector_net_borrowing/index.html | 1 - .../releases/2026-08-26-uk_public_sector_net_debt_gdp/index.html | 1 - notes/releases/2026-08-26-uk_unemployment_rate/index.html | 1 - notes/releases/2026-08-26-uk_vacancies/index.html | 1 - notes/releases/2026-08-26-us_cpi/index.html | 1 - notes/releases/2026-08-26-us_treasury_10y/index.html | 1 - 14 files changed, 14 deletions(-) diff --git a/notes/releases/2026-08-26-uk_average_weekly_earnings/index.html b/notes/releases/2026-08-26-uk_average_weekly_earnings/index.html index 00c95d6..f684c78 100644 --- a/notes/releases/2026-08-26-uk_average_weekly_earnings/index.html +++ b/notes/releases/2026-08-26-uk_average_weekly_earnings/index.html @@ -21,7 +21,6 @@