From e183785cee767ea670b4f52b9183b32694d94efc Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 10 Nov 2026 10:39:16 -0600 Subject: [PATCH 1/7] server: explicitly initialize training admission callbacks --- tools/server/server-train.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/server/server-train.cpp b/tools/server/server-train.cpp index 098cc26c2004..4c6d4b2f42e0 100644 --- a/tools/server/server-train.cpp +++ b/tools/server/server-train.cpp @@ -1055,6 +1055,10 @@ void server_trainer::run(json req, examples_data ex) { // the single-context gradient): one step per window on the window's whole loss /*walk_exact =*/ req.value("exact", false), /*walk_horizon =*/ (uint32_t) req.value("walk_horizon", (int64_t) 0), + /*static_admission =*/ nullptr, + /*static_admission_data =*/ nullptr, + /*workspace_admission =*/ nullptr, + /*workspace_admission_data =*/ nullptr, }; // THE BUDGET COVERS THE WHOLE RUN (continuum, the 5090, 2026-10-11): the training context's own // memory (its Q8_0 K/V for the whole window, and any recurrent state) is allocated when it is From fcce169456169bc5bc005375a0ea14831d3f13b2 Mon Sep 17 00:00:00 2001 From: joelteply Date: Tue, 10 Nov 2026 10:51:54 -0600 Subject: [PATCH 2/7] server: initialize every pending learn field, close checkpoint files by type, and give LoRA tests real residency custody The integration line's CI failed on paths fork #103 does not touch (Codex's audit, logs 42832757120 and 42832757300): - server-context-learn.inc:396 aggregate-initialized pending_learn with only its first nine fields, so grant/finished/failure tripped -Wmissing-field-initializers on the ARM -Werror build. pending_learn now has a constructor for those nine; every other field keeps its stated default. - server-context-learn-checkpoint.inc:45-46 used decltype(&fclose) as a unique_ptr deleter, and fclose's attributes are dropped in a template argument (-Wignored-attributes). A file_closer type replaces it. - test_lora.py and dream-adapter-routes.py loaded or freshly created adapters by name alone; the engine now seats an adapter only inside a residency registered for its exact subject, so they failed unknown_residency. Each now measures (dry_run), registers the rows and leases through /learn/residency, and loads under that residency, as the core does. Admission is not relaxed. With e183785ce (training admission initializers, Codex) this repairs the gate. llama-server builds with LLAMA_FATAL_WARNINGS=ON (macOS x86_64 clang, CPU); both tests compile under python. Assisted-by: Claude (Cormac) --- .../server/server-context-learn-checkpoint.inc | 6 ++++-- tools/server/server-context-learn.inc | 4 ++-- tools/server/server-context.cpp | 7 +++++++ tools/server/tests/dream-adapter-routes.py | 18 +++++++++++++++--- tools/server/tests/unit/test_lora.py | 13 ++++++++++++- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/tools/server/server-context-learn-checkpoint.inc b/tools/server/server-context-learn-checkpoint.inc index d7bfa4e3f480..eb990627a7ce 100644 --- a/tools/server/server-context-learn-checkpoint.inc +++ b/tools/server/server-context-learn-checkpoint.inc @@ -42,8 +42,10 @@ gguf_get_val_i64(metadata.get(), iteration) <= 0 || gguf_get_val_i64(metadata.get(), iteration) - 1 != steps || std::string(gguf_get_val_str(metadata.get(), bound)) != binding) { return refuse("optimizer generation mismatch"); } - std::unique_ptr weights(fopen((path + "/adapter.gguf").c_str(), "rb"), fclose); - std::unique_ptr optimizer(fopen((path + "/optimizer.gguf").c_str(), "rb"), fclose); + // a closer type, not decltype(&fclose): fclose's attributes are dropped in a template argument (-Wignored-attributes) + struct file_closer { void operator()(FILE * file) const { if (file) { fclose(file); } } }; + std::unique_ptr weights(fopen((path + "/adapter.gguf").c_str(), "rb")); + std::unique_ptr optimizer(fopen((path + "/optimizer.gguf").c_str(), "rb")); if (!weights || !optimizer) { return refuse("checkpoint components unreadable"); } const auto started = ggml_time_us(); auto held = learn_ledger.hold_restoration(request.grant, started); diff --git a/tools/server/server-context-learn.inc b/tools/server/server-context-learn.inc index 2b2a0aa08099..925adcfcf01b 100644 --- a/tools/server/server-context-learn.inc +++ b/tools/server/server-context-learn.inc @@ -392,8 +392,8 @@ const bool begun = llama_opt_learn_begin(ctx_tgt, slot.id, n0, toks.data(), labels.data(), (int32_t) toks.size(), served_tokens.data() + n0, (int32_t) served_tokens.size() - n0, result); common_set_adapter_lora(ctx_tgt, slot.lora); - learning = std::unique_ptr(new pending_learn{ &slot, cp, std::move(set), result, - toks[0], L.n_prefix, (int32_t) toks.size() - 1, loras[L.lora_id].ptr, t0 }); + learning = std::make_unique(&slot, cp, std::move(set), result, + toks[0], L.n_prefix, (int32_t) toks.size() - 1, loras[L.lora_id].ptr, t0); learning->began = begun; learning->grant = L.grant; learning->lora_id = L.lora_id; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c51277e13599..b03adacd3d33 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -996,6 +996,13 @@ struct server_context_impl { return p; } struct pending_learn { + // The step's identity; every other field starts at its stated default (an aggregate with + // only these set trips -Wmissing-field-initializers on the ARM CI build). + pending_learn(server_slot * slot, const common_prompt_checkpoint * checkpoint, std::vector adapters, + ggml_opt_result_t result, llama_token last_prompt, int32_t prefix, int32_t n_labels, + const llama_adapter_lora * shadow, int64_t started) + : slot(slot), checkpoint(checkpoint), adapters(std::move(adapters)), result(result), last_prompt(last_prompt), + prefix(prefix), n_labels(n_labels), shadow(shadow), started(started) {} server_slot * slot; const common_prompt_checkpoint * checkpoint; std::vector adapters; diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py index 2e954b190a9a..fb5eedfe795d 100644 --- a/tools/server/tests/dream-adapter-routes.py +++ b/tools/server/tests/dream-adapter-routes.py @@ -48,6 +48,16 @@ def get(route): ledger = get("/learn/grants") +def register(suffix, subject, rows): + # The engine seats an adapter only inside a residency the core registered for its exact + # subject, with the leases that charge its rows (server-learn-residency.h). + rid = prefix + suffix + leases = [{"kind": kind, "lease_id": rid + kind, + "bytes": sum(row["bytes"] for row in rows if row["host"] == host)} + for kind, host in (("vram", False), ("ram", True))] + assert ok("/learn/residency", {"id": rid, "engine": ledger["engine"], "owner": prefix, + "subject": subject, "rows": rows, "leases": leases})["refusal"] == "none" + return rid grant = prefix + "-grant" assert ok("/learn/grants", {"id": grant, "engine": ledger["engine"], "period_ms": 1000, "share_ppm": 1000000, "expires_in_ms": 120000})["refusal"] == "none" @@ -59,8 +69,8 @@ def get(route): assert not plan["reserved"] and plan["weights"] assert not (train_dir / fresh_name).exists(), "dry run published an adapter" assert complete() == baseline, "dry run changed serving" -fresh = ok("/lora-adapters/fresh", {"name": fresh_name, "rank": 8, - "alpha": 16, "targets": "attn_q,attn_v", "top_layers": 1, "seed": 42}) +fresh_weights = register("-fresh", {"fresh": fresh_spec}, [dict(row, kind="weights") for row in plan["weights"]]) +fresh = ok("/lora-adapters/fresh", dict(fresh_spec, residency=fresh_weights)) assert fresh["weights"] == plan["weights"], ("allocated weights differ from dry plan", fresh, plan) file_plan = ok("/lora-adapters/load", {"name": fresh_name, "dry_run": True}) assert file_plan["weights"] == plan["weights"] @@ -101,7 +111,9 @@ def get(route): assert snapshot["steps"] == 2, ("two whole replies learned are two committed steps", snapshot) after = hashlib.sha256((train_dir / snapshot_name).read_bytes()).hexdigest() assert after != before, "successful learn did not change saved adapter weights" -loaded = ok("/lora-adapters/load", {"name": snapshot_name}) +snapshot_plan = ok("/lora-adapters/load", {"name": snapshot_name, "dry_run": True}) +snapshot_weights = register("-snapshot", {"load": {"name": snapshot_name}}, [dict(row, kind="weights") for row in snapshot_plan["weights"]]) +loaded = ok("/lora-adapters/load", {"name": snapshot_name, "residency": snapshot_weights}) assert complete(loaded["id"]) == trained_text, "trained snapshot reload changed output" assert post("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name})[0] >= 400 assert post("/lora-adapters/fresh", {"name": fresh_name})[0] >= 400 diff --git a/tools/server/tests/unit/test_lora.py b/tools/server/tests/unit/test_lora.py index e84377745269..25cb252820e2 100644 --- a/tools/server/tests/unit/test_lora.py +++ b/tools/server/tests/unit/test_lora.py @@ -89,7 +89,18 @@ def report(): assert server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": 0.25}]).status_code == 200 assert report()[0]["scale"] == 0.25 - loaded = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf"}) + # A runtime load is seated only inside a residency registered for its exact subject, as the core + # registers one (server-learn-residency.h): measure, register its rows and leases, then load. + measured = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf", "dry_run": True}) + assert measured.status_code == 200, measured.body + engine = server.make_request("GET", "/learn/grants").body["engine"] + rows = [dict(row, kind="weights") for row in measured.body["weights"]] + leases = [{"kind": kind, "lease_id": "second-" + kind, "bytes": sum(row["bytes"] for row in rows if row["host"] == host)} + for kind, host in (("vram", False), ("ram", True))] + registered = server.make_request("POST", "/learn/residency", data={"id": "second-weights", "engine": engine, "owner": "test", + "subject": {"load": {"name": "second.gguf"}}, "rows": rows, "leases": leases}) + assert registered.status_code == 200 and registered.body["refusal"] == "none", registered.body + loaded = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf", "residency": "second-weights"}) assert loaded.status_code == 200, loaded.body assert sorted(report()) == [0, 1] From 06e78f08552b2708550acec31ae96d0563621802 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 10 Nov 2026 11:09:35 -0600 Subject: [PATCH 3/7] examples: initialize training admission callbacks explicitly --- examples/training/finetune-lora.cpp | 4 ++++ examples/training/finetune.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/examples/training/finetune-lora.cpp b/examples/training/finetune-lora.cpp index 1f97b2bcde03..ab5c844226b6 100644 --- a/examples/training/finetune-lora.cpp +++ b/examples/training/finetune-lora.cpp @@ -136,6 +136,10 @@ int main(int argc, char ** argv) { /*recompute =*/false, // keep every layer's intermediates, as before /*walk_exact =*/false, // the per-chunk walk, as before /*walk_horizon =*/0, + /*static_admission =*/nullptr, + /*static_admission_data =*/nullptr, + /*workspace_admission =*/nullptr, + /*workspace_admission_data =*/nullptr, }; llama_opt_init(ctx, model, lopt_params); diff --git a/examples/training/finetune.cpp b/examples/training/finetune.cpp index c19e5f82dd19..8c1ccb2efc40 100644 --- a/examples/training/finetune.cpp +++ b/examples/training/finetune.cpp @@ -77,6 +77,10 @@ int main(int argc, char ** argv) { /*recompute =*/false, // keep every layer's intermediates, as before /*walk_exact =*/false, // the per-chunk walk, as before /*walk_horizon =*/0, + /*static_admission =*/nullptr, + /*static_admission_data =*/nullptr, + /*workspace_admission =*/nullptr, + /*workspace_admission_data =*/nullptr, }; llama_opt_init(ctx, model, lopt_params); From 5ca01bdeed13446c8cfcff39d02d9468acf70a91 Mon Sep 17 00:00:00 2001 From: joelteply Date: Tue, 10 Nov 2026 11:10:47 -0600 Subject: [PATCH 4/7] test: the dream route test learns under an idle-only grant, the authority that may calibrate an unmeasured geometry Its full-share grant (share_ppm 1000000, no max_chunk_us) is refused pending_calibration by design (server-learn-ledger.h): only idle-only authority may learn before a geometry's chunk cost is measured, as learn-past-example.py does. Nothing else decodes while the test learns. The guard is unchanged. Assisted-by: Claude (Cormac) --- tools/server/tests/dream-adapter-routes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py index fb5eedfe795d..ad1e0e4b23ca 100644 --- a/tools/server/tests/dream-adapter-routes.py +++ b/tools/server/tests/dream-adapter-routes.py @@ -59,8 +59,12 @@ def register(suffix, subject, rows): "subject": subject, "rows": rows, "leases": leases})["refusal"] == "none" return rid grant = prefix + "-grant" +# An idle-only grant (no share, no period): the one authority the ledger lets learn before this +# geometry's chunk cost is measured (server-learn-ledger.h, pending_calibration). A shared grant +# needs a measured max_chunk_us first. Nothing else decodes while this test learns, so it is never +# contended. assert ok("/learn/grants", {"id": grant, "engine": ledger["engine"], - "period_ms": 1000, "share_ppm": 1000000, "expires_in_ms": 120000})["refusal"] == "none" + "period_ms": 0, "share_ppm": 0, "expires_in_ms": 120000})["refusal"] == "none" baseline = complete() fresh_name = prefix + "-fresh.gguf" # Dry-run uses the real loader's placement/alignment before any adapter is seated or file written. From 112675793a9b57b933f1318ab615787b6e3d134b Mon Sep 17 00:00:00 2001 From: joelteply Date: Tue, 10 Nov 2026 11:20:43 -0600 Subject: [PATCH 5/7] test: the dream route test learns inside the custody the core registers from the engine's preflight Every learn now needs optimizer_residency and workspace_residency (admit_learning_residency). The test preflights each example through /learn/preflight's completion form, registers her optimizer once for the shadow's exact loaded subject, and a workspace per example sized to that reply, then attaches both references to the completion's learn. Rows split exactly as admission splits them (shared_compute_held skipped, shared_compute_required to the workspace, the rest to the optimizer). Weights already came through registered residency. Same pattern as learn-past-example.py. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01FRQWzgSfo79JtnwZywHKrE --- tools/server/tests/dream-adapter-routes.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py index ad1e0e4b23ca..51381d78c985 100644 --- a/tools/server/tests/dream-adapter-routes.py +++ b/tools/server/tests/dream-adapter-routes.py @@ -88,16 +88,30 @@ def register(suffix, subject, rows): repeat_name = prefix + "-repeat.gguf" ok("/lora-adapters/save", {"id": fresh["id"], "name": repeat_name}) assert hashlib.sha256((train_dir / repeat_name).read_bytes()).hexdigest() == before, "save bytes are unstable without learning" +optimizer_rid = None +def learning(reply, suffix): + # A learn runs only inside custody the core registered from the engine's own preflight of this + # example: her optimizer (persistent per shadow, registered once) and the engine-wide step + # workspace sized to this reply (server-context-learn-residency.inc admit_learning_residency). + global optimizer_rid + plan = ok("/learn/preflight", {"engine": ledger["engine"], "lora_id": fresh["id"], "learn": {"reply": reply}}) + assert plan["refusal"] == "none", plan + rows = [row for row in plan["requirements"] if row["kind"] != "shared_compute_held"] + if optimizer_rid is None: + optimizer_rid = register("-optimizer", {"loaded": {"lora_id": fresh["id"], "path": fresh["path"]}}, + [row for row in rows if row["kind"] != "shared_compute_required"]) + workspace = register("-workspace" + suffix, {"workspace": True}, + [row for row in rows if row["kind"] == "shared_compute_required"]) + return {"reply": reply, "lora_id": fresh["id"], "grant": grant, "lr": 0.001, + "optimizer_residency": optimizer_rid, "workspace_residency": workspace} learned = ok("/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, "n_predict": 1, "temperature": 0, - "learn": {"reply": "Topeka is the capital of Kansas.", - "lora_id": fresh["id"], "grant": grant, "lr": 0.001}}) + "learn": learning("Topeka is the capital of Kansas.", "-short")}) assert learned["learn"]["success"], learned # A queued save must refuse the active shadow even between resumable chunks. with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: future = pool.submit(ok, "/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, - "n_predict": 1, "temperature": 0, "learn": {"grant": grant, "lora_id": fresh["id"], - "lr": 0.001, "reply": "A room contains a card and a window. " * 55}}) + "n_predict": 1, "temperature": 0, "learn": learning("A room contains a card and a window. " * 55, "-long")}) deadline = time.monotonic() + 120 while time.monotonic() < deadline: if get("/slots")[0]["is_learning"]: From e582c1fa920c9827c1352ef565912d957ac3c0a2 Mon Sep 17 00:00:00 2001 From: joelteply Date: Tue, 10 Nov 2026 11:30:24 -0600 Subject: [PATCH 6/7] test: the dream test sizes its long learn from the engine's own window and tokenizer CI serves stories260K at -c 512, and 55 repeats of the sentence exceed that window, so the example preflight correctly refused it (reply_rows past the served window). The route forbids a caller-computed reply_rows (server-context.cpp post_learn_preflight). So the test reads the slot's n_ctx, counts the served prompt (with BOS) and the reply's end of turn through /tokenize with the schema's reply semantics (add_special false, parse_special true), and takes the largest repeat count that fits. It is still a multi-chunk learn for the save race. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01FRQWzgSfo79JtnwZywHKrE --- tools/server/tests/dream-adapter-routes.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py index 51381d78c985..4b8d7a88126b 100644 --- a/tools/server/tests/dream-adapter-routes.py +++ b/tools/server/tests/dream-adapter-routes.py @@ -108,10 +108,22 @@ def learning(reply, suffix): "n_predict": 1, "temperature": 0, "learn": learning("Topeka is the capital of Kansas.", "-short")}) assert learned["learn"]["success"], learned +# The long reply spans as many resumable chunks as this engine's window holds: sized from its own +# slot window and tokenizer (the reply's tokens plus its end of turn, after the served prompt), +# never a fixed repeat count, since preflight refuses an example past the served window. +def rows(text, special): + return len(ok("/tokenize", {"content": text, "add_special": special})["tokens"]) +long_prompt = "The capital of Kansas is" +room = get("/slots")[0]["n_ctx"] - rows(long_prompt, True) - 1 +unit = "A room contains a card and a window. " +repeats = room // rows(unit, False) +while repeats > 1 and rows(unit * repeats, False) > room: + repeats -= 1 +assert repeats > 1, ("the served window holds no multi-sentence reply", room) # A queued save must refuse the active shadow even between resumable chunks. with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit(ok, "/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, - "n_predict": 1, "temperature": 0, "learn": learning("A room contains a card and a window. " * 55, "-long")}) + future = pool.submit(ok, "/completion", {"prompt": long_prompt, "id_slot": 0, + "n_predict": 1, "temperature": 0, "learn": learning(unit * repeats, "-long")}) deadline = time.monotonic() + 120 while time.monotonic() < deadline: if get("/slots")[0]["is_learning"]: From 3851243088393b180c7cfdc6aaad4c6c56c34801 Mon Sep 17 00:00:00 2001 From: joelteply Date: Tue, 10 Nov 2026 11:39:58 -0600 Subject: [PATCH 7/7] test: the adapter-report test retires its adapter against the bound engine incarnation /lora-adapters/unload requires the engine incarnation (server-context.cpp post_lora_adapters_unload). Audit of the pytest suite: test_lora.py is its only caller of a lifecycle route (load, unload, residency, grants). Every other call there already carries its residency or engine. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01FRQWzgSfo79JtnwZywHKrE --- tools/server/tests/unit/test_lora.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/server/tests/unit/test_lora.py b/tools/server/tests/unit/test_lora.py index 25cb252820e2..a7c437551cda 100644 --- a/tools/server/tests/unit/test_lora.py +++ b/tools/server/tests/unit/test_lora.py @@ -107,7 +107,8 @@ def report(): # an adapter active by default is retired only once its scale is 0 (the engine refuses otherwise) assert server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": 0.0}, {"id": 1, "scale": 0.0}]).status_code == 200 assert report()[0]["scale"] == 0.0 - unloaded = server.make_request("POST", "/lora-adapters/unload", data={"id": 0}) + # a retire names the engine incarnation it was decided against, like every lifecycle verb + unloaded = server.make_request("POST", "/lora-adapters/unload", data={"id": 0, "engine": engine}) assert unloaded.status_code == 200, unloaded.body assert list(report()) == [1], "a retired adapter leaves the report; the ids it left stay taken"