From 357096e6d372c784a773699ccbb397392d699e97 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 11 Nov 2026 09:12:33 -0600 Subject: [PATCH 1/2] server: the host prompt cache gives its bytes back on request (POST /prompt-cache/limit) A cache is never need (Joel, 2026-11-11: 'Why would you grab what you aren't actively using'). On the M5 a 4 MB learner waited forever ('0 of 4096000 governed Vram bytes are free') while the host prompt cache (--cache-ram 13056) held ~13 GB counted as serving's need (continuum card 494c99f1). The cache already trims to a mutable limit (server_prompt_cache::update evicts the oldest states while size() > limit_size), but only --cache-ram set it, at launch. POST /prompt-cache/limit {limit_mib >= 1} runs on the serving loop (it owns the cache), sets the limit, trims, and replies {refusal, limit_mib, held_bytes, freed_bytes, states}, so the core frees exactly what was measured. 0 is refused (to the cache it means 'no limit'). Router mode proxies it. Fixture: tests/prompt-cache-limit.py (fill, refuse 0, lower to 1 MiB and assert bytes freed and the cache fits, raise again). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 40 ++++++++++++++++++++++++ tools/server/server-context.h | 1 + tools/server/server-task.h | 4 +++ tools/server/server.cpp | 2 ++ tools/server/tests/prompt-cache-limit.py | 35 +++++++++++++++++++++ 5 files changed, 82 insertions(+) create mode 100755 tools/server/tests/prompt-cache-limit.py diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index b03adacd3d33..ef81facb027f 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3323,6 +3323,26 @@ struct server_context_impl { case SERVER_TASK_TYPE_LEARN_GRANTS: process_learn_grants(task); break; + case SERVER_TASK_TYPE_PROMPT_CACHE_LIMIT: + { + // The host prompt cache gives its bytes back: a cache is never need (card 494c99f1). + // The limit is lowered or raised, and update() evicts the oldest states until the + // cache fits it. The reply is what the cache holds now, so the core frees exactly + // what it measured, never what it hoped. + auto result = std::make_unique(); + result->id = task.id; + if (!prompt_cache) { + result->receipt = {{"refusal", "prompt_cache_disabled"}}; + } else { + const size_t before = prompt_cache->size(); + prompt_cache->limit_size = 1024ull * 1024ull * (size_t) task.prompt_cache_limit_mib; + prompt_cache->update(); + result->receipt = {{"refusal", "none"}, {"limit_mib", task.prompt_cache_limit_mib}, + {"held_bytes", prompt_cache->size()}, {"freed_bytes", before - std::min(before, prompt_cache->size())}, + {"states", prompt_cache->states.size()}}; + } + queue_results.send(std::move(result)); + } break; case SERVER_TASK_TYPE_LEARN: { if (!task.restore_checkpoint.is_null()) { @@ -6578,6 +6598,26 @@ void server_routes::init_routes() { this->release_learn_residency = [grant_route](const server_http_req & req) { return grant_route(req, "residency_release"); }; this->revoke_learn_residency = [grant_route](const server_http_req & req) { return grant_route(req, "residency_revoke"); }; this->get_learn_grants = [grant_route](const server_http_req & req) { return grant_route(req, "list"); }; + // A cache is never need: the core lowers the host prompt cache's limit to give its bytes to an + // active consumer (a learner's step), and raises it again after (card 494c99f1). + this->post_prompt_cache_limit = [this](const server_http_req & req) { + auto res = create_response(); + const json body = req.body.empty() ? json::object() : json::parse(req.body, nullptr, false); + if (!body.is_object() || !body.contains("limit_mib") || !body.at("limit_mib").is_number_integer() + || body.at("limit_mib").get() < 1) { + // 0 would mean "no limit" to the cache itself, so a release to nothing is 1 MiB, never 0. + res->error(format_error_response("limit_mib must be an integer of at least 1", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + server_task task(SERVER_TASK_TYPE_PROMPT_CACHE_LIMIT); + task.prompt_cache_limit_mib = body.at("limit_mib").get(); + auto & rd = res->rd; + task.id = rd.get_new_id(); + rd.post_task(std::move(task)); + auto result = rd.next(req.should_stop); + if (result) { res->ok(result->to_json()); } + return res; + }; this->post_learn_grants = [grant_route](const server_http_req & req) { return grant_route(req, "register"); }; this->revoke_learn_grant = [grant_route](const server_http_req & req) { return grant_route(req, "revoke"); }; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index a810c3e3a1b9..d6acce4840ea 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -177,6 +177,7 @@ struct server_routes { server_http_context::handler_t post_learn; server_http_context::handler_t post_learn_residency, release_learn_residency, revoke_learn_residency; server_http_context::handler_t get_learn_grants, post_learn_grants, revoke_learn_grant, post_learn_preflight; + server_http_context::handler_t post_prompt_cache_limit; server_http_context::handler_t get_train; server_http_context::handler_t post_train; server_http_context::handler_t post_train_cancel; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 652697ba4d2b..681ee333c184 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -63,6 +63,7 @@ enum server_task_type { SERVER_TASK_TYPE_UNLOAD_LORA, SERVER_TASK_TYPE_LEARN, SERVER_TASK_TYPE_LEARN_GRANTS, + SERVER_TASK_TYPE_PROMPT_CACHE_LIMIT, }; // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common @@ -288,6 +289,9 @@ struct server_task { std::string unload_engine; json learn_grants; // authority operation, processed on the serving queue + // Continuum fork: the host prompt cache's new limit in MiB (POST /prompt-cache/limit). A cache is + // never need: the core gives its bytes back when an active consumer needs them (card 494c99f1). + int64_t prompt_cache_limit_mib = -1; // used by SERVER_TASK_TYPE_LEARN learn_request learn; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 4a4ec63ab39f..dfecafd73892 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -226,6 +226,7 @@ int llama_server(common_params & params, int argc, char ** argv) { routes.get_learn_grants = models_routes->proxy_get; routes.post_learn_grants = models_routes->proxy_post; routes.post_learn_preflight = models_routes->proxy_post; + routes.post_prompt_cache_limit = models_routes->proxy_post; routes.post_learn_residency = models_routes->proxy_post; routes.release_learn_residency = models_routes->proxy_post; routes.revoke_learn_residency = models_routes->proxy_post; @@ -295,6 +296,7 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.post("/learn/residency/revoke", ex_wrapper(routes.revoke_learn_residency)); ctx_http.post("/learn/preflight", ex_wrapper(routes.post_learn_preflight)); ctx_http.post("/learn/grants", ex_wrapper(routes.post_learn_grants)); + ctx_http.post("/prompt-cache/limit", ex_wrapper(routes.post_prompt_cache_limit)); ctx_http.post("/learn/grants/revoke", ex_wrapper(routes.revoke_learn_grant)); ctx_http.get ("/train", ex_wrapper(routes.get_train)); ctx_http.post("/train", ex_wrapper(routes.post_train)); diff --git a/tools/server/tests/prompt-cache-limit.py b/tools/server/tests/prompt-cache-limit.py new file mode 100755 index 000000000000..9856d7586da6 --- /dev/null +++ b/tools/server/tests/prompt-cache-limit.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Real server test; requires an isolated server launched with a host prompt cache (--cache-ram N, +one slot). Usage: prompt-cache-limit.py URL + +What this catches (card 494c99f1): the host prompt cache counted as serving's need, so a 4 MB learner +waited forever on the M5 ('0 of 4096000 governed Vram bytes are free') while ~13 GB of cache held +states nothing predicted reusing. The cache gives its bytes back: POST /prompt-cache/limit lowers its +limit and evicts the oldest states until it fits, and replies what it now holds and what it freed, so +the core frees exactly what was measured. Raising the limit again lets it refill. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from learn_fixture import Engine + +engine = Engine(sys.argv[1], "cache-limit", timeout=120) + +# Two different prompts on one slot: the first is saved to the host cache when the second replaces it. +for prompt in ("Describe a lighthouse at dawn in one sentence:", "List three uses for a paperclip:"): + engine.ok("/completion", {"prompt": prompt, "n_predict": 8, "temperature": 0}) + +status, refused = engine.post("/prompt-cache/limit", {"limit_mib": 0}) +assert status == 400, ("0 means 'no limit' to the cache, so it is refused, never taken as unlimited", status, refused) + +lowered = engine.ok("/prompt-cache/limit", {"limit_mib": 1}) +assert lowered["refusal"] == "none", lowered +assert lowered["freed_bytes"] > 0, ("the filled cache gave bytes back", lowered) +assert lowered["held_bytes"] <= 1024 * 1024, ("the cache fits its new limit", lowered) + +raised = engine.ok("/prompt-cache/limit", {"limit_mib": 512}) +assert raised["refusal"] == "none" and raised["limit_mib"] == 512, raised + +engine.close() +print("ok: the host prompt cache gives its bytes back on request and reports what it freed") From 9b04459298ce4e65fe35073689053ad4e1168d2a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 11 Nov 2026 09:33:09 -0600 Subject: [PATCH 2/2] server: /prompt-cache/limit refuses a limit whose bytes overflow size_t (Cormac on #110) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 8 +++++--- tools/server/tests/prompt-cache-limit.py | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index ef81facb027f..1005d386f9dc 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6603,10 +6603,12 @@ void server_routes::init_routes() { this->post_prompt_cache_limit = [this](const server_http_req & req) { auto res = create_response(); const json body = req.body.empty() ? json::object() : json::parse(req.body, nullptr, false); + // 0 would mean "no limit" to the cache itself, so a release to nothing is 1 MiB, never 0; and the + // limit in bytes must fit a size_t, so a value past it is refused, never wrapped (Cormac on #110). + constexpr int64_t max_mib = (int64_t) (SIZE_MAX >> 20) < INT64_MAX ? (int64_t) (SIZE_MAX >> 20) : INT64_MAX; if (!body.is_object() || !body.contains("limit_mib") || !body.at("limit_mib").is_number_integer() - || body.at("limit_mib").get() < 1) { - // 0 would mean "no limit" to the cache itself, so a release to nothing is 1 MiB, never 0. - res->error(format_error_response("limit_mib must be an integer of at least 1", ERROR_TYPE_INVALID_REQUEST)); + || body.at("limit_mib").get() < 1 || body.at("limit_mib").get() > max_mib) { + res->error(format_error_response("limit_mib must be an integer from 1 to the bytes a size_t holds, in MiB", ERROR_TYPE_INVALID_REQUEST)); return res; } server_task task(SERVER_TASK_TYPE_PROMPT_CACHE_LIMIT); diff --git a/tools/server/tests/prompt-cache-limit.py b/tools/server/tests/prompt-cache-limit.py index 9856d7586da6..243ee1d5bfef 100755 --- a/tools/server/tests/prompt-cache-limit.py +++ b/tools/server/tests/prompt-cache-limit.py @@ -23,6 +23,9 @@ status, refused = engine.post("/prompt-cache/limit", {"limit_mib": 0}) assert status == 400, ("0 means 'no limit' to the cache, so it is refused, never taken as unlimited", status, refused) +status, overflow = engine.post("/prompt-cache/limit", {"limit_mib": 2**62}) +assert status == 400, ("a limit whose bytes overflow a size_t is refused, never wrapped", status, overflow) + lowered = engine.ok("/prompt-cache/limit", {"limit_mib": 1}) assert lowered["refusal"] == "none", lowered assert lowered["freed_bytes"] > 0, ("the filled cache gave bytes back", lowered)