diff --git a/examples/training/finetune-lora.cpp b/examples/training/finetune-lora.cpp index 1f97b2bcde03..ab5c844226b6 100644 --- a/examples/training/finetune-lora.cpp +++ b/examples/training/finetune-lora.cpp @@ -136,6 +136,10 @@ int main(int argc, char ** argv) { /*recompute =*/false, // keep every layer's intermediates, as before /*walk_exact =*/false, // the per-chunk walk, as before /*walk_horizon =*/0, + /*static_admission =*/nullptr, + /*static_admission_data =*/nullptr, + /*workspace_admission =*/nullptr, + /*workspace_admission_data =*/nullptr, }; llama_opt_init(ctx, model, lopt_params); diff --git a/examples/training/finetune.cpp b/examples/training/finetune.cpp index c19e5f82dd19..8c1ccb2efc40 100644 --- a/examples/training/finetune.cpp +++ b/examples/training/finetune.cpp @@ -77,6 +77,10 @@ int main(int argc, char ** argv) { /*recompute =*/false, // keep every layer's intermediates, as before /*walk_exact =*/false, // the per-chunk walk, as before /*walk_horizon =*/0, + /*static_admission =*/nullptr, + /*static_admission_data =*/nullptr, + /*workspace_admission =*/nullptr, + /*workspace_admission_data =*/nullptr, }; llama_opt_init(ctx, model, lopt_params); diff --git a/tools/server/server-context-learn-checkpoint.inc b/tools/server/server-context-learn-checkpoint.inc index d7bfa4e3f480..eb990627a7ce 100644 --- a/tools/server/server-context-learn-checkpoint.inc +++ b/tools/server/server-context-learn-checkpoint.inc @@ -42,8 +42,10 @@ gguf_get_val_i64(metadata.get(), iteration) <= 0 || gguf_get_val_i64(metadata.get(), iteration) - 1 != steps || std::string(gguf_get_val_str(metadata.get(), bound)) != binding) { return refuse("optimizer generation mismatch"); } - std::unique_ptr weights(fopen((path + "/adapter.gguf").c_str(), "rb"), fclose); - std::unique_ptr optimizer(fopen((path + "/optimizer.gguf").c_str(), "rb"), fclose); + // a closer type, not decltype(&fclose): fclose's attributes are dropped in a template argument (-Wignored-attributes) + struct file_closer { void operator()(FILE * file) const { if (file) { fclose(file); } } }; + std::unique_ptr weights(fopen((path + "/adapter.gguf").c_str(), "rb")); + std::unique_ptr optimizer(fopen((path + "/optimizer.gguf").c_str(), "rb")); if (!weights || !optimizer) { return refuse("checkpoint components unreadable"); } const auto started = ggml_time_us(); auto held = learn_ledger.hold_restoration(request.grant, started); diff --git a/tools/server/server-context-learn.inc b/tools/server/server-context-learn.inc index 2b2a0aa08099..925adcfcf01b 100644 --- a/tools/server/server-context-learn.inc +++ b/tools/server/server-context-learn.inc @@ -392,8 +392,8 @@ const bool begun = llama_opt_learn_begin(ctx_tgt, slot.id, n0, toks.data(), labels.data(), (int32_t) toks.size(), served_tokens.data() + n0, (int32_t) served_tokens.size() - n0, result); common_set_adapter_lora(ctx_tgt, slot.lora); - learning = std::unique_ptr(new pending_learn{ &slot, cp, std::move(set), result, - toks[0], L.n_prefix, (int32_t) toks.size() - 1, loras[L.lora_id].ptr, t0 }); + learning = std::make_unique(&slot, cp, std::move(set), result, + toks[0], L.n_prefix, (int32_t) toks.size() - 1, loras[L.lora_id].ptr, t0); learning->began = begun; learning->grant = L.grant; learning->lora_id = L.lora_id; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c51277e13599..b03adacd3d33 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -996,6 +996,13 @@ struct server_context_impl { return p; } struct pending_learn { + // The step's identity; every other field starts at its stated default (an aggregate with + // only these set trips -Wmissing-field-initializers on the ARM CI build). + pending_learn(server_slot * slot, const common_prompt_checkpoint * checkpoint, std::vector adapters, + ggml_opt_result_t result, llama_token last_prompt, int32_t prefix, int32_t n_labels, + const llama_adapter_lora * shadow, int64_t started) + : slot(slot), checkpoint(checkpoint), adapters(std::move(adapters)), result(result), last_prompt(last_prompt), + prefix(prefix), n_labels(n_labels), shadow(shadow), started(started) {} server_slot * slot; const common_prompt_checkpoint * checkpoint; std::vector adapters; diff --git a/tools/server/server-train.cpp b/tools/server/server-train.cpp index 098cc26c2004..4c6d4b2f42e0 100644 --- a/tools/server/server-train.cpp +++ b/tools/server/server-train.cpp @@ -1055,6 +1055,10 @@ void server_trainer::run(json req, examples_data ex) { // the single-context gradient): one step per window on the window's whole loss /*walk_exact =*/ req.value("exact", false), /*walk_horizon =*/ (uint32_t) req.value("walk_horizon", (int64_t) 0), + /*static_admission =*/ nullptr, + /*static_admission_data =*/ nullptr, + /*workspace_admission =*/ nullptr, + /*workspace_admission_data =*/ nullptr, }; // THE BUDGET COVERS THE WHOLE RUN (continuum, the 5090, 2026-10-11): the training context's own // memory (its Q8_0 K/V for the whole window, and any recurrent state) is allocated when it is diff --git a/tools/server/tests/dream-adapter-routes.py b/tools/server/tests/dream-adapter-routes.py index 2e954b190a9a..4b8d7a88126b 100644 --- a/tools/server/tests/dream-adapter-routes.py +++ b/tools/server/tests/dream-adapter-routes.py @@ -48,9 +48,23 @@ def get(route): ledger = get("/learn/grants") +def register(suffix, subject, rows): + # The engine seats an adapter only inside a residency the core registered for its exact + # subject, with the leases that charge its rows (server-learn-residency.h). + rid = prefix + suffix + leases = [{"kind": kind, "lease_id": rid + kind, + "bytes": sum(row["bytes"] for row in rows if row["host"] == host)} + for kind, host in (("vram", False), ("ram", True))] + assert ok("/learn/residency", {"id": rid, "engine": ledger["engine"], "owner": prefix, + "subject": subject, "rows": rows, "leases": leases})["refusal"] == "none" + return rid grant = prefix + "-grant" +# An idle-only grant (no share, no period): the one authority the ledger lets learn before this +# geometry's chunk cost is measured (server-learn-ledger.h, pending_calibration). A shared grant +# needs a measured max_chunk_us first. Nothing else decodes while this test learns, so it is never +# contended. assert ok("/learn/grants", {"id": grant, "engine": ledger["engine"], - "period_ms": 1000, "share_ppm": 1000000, "expires_in_ms": 120000})["refusal"] == "none" + "period_ms": 0, "share_ppm": 0, "expires_in_ms": 120000})["refusal"] == "none" baseline = complete() fresh_name = prefix + "-fresh.gguf" # Dry-run uses the real loader's placement/alignment before any adapter is seated or file written. @@ -59,8 +73,8 @@ def get(route): assert not plan["reserved"] and plan["weights"] assert not (train_dir / fresh_name).exists(), "dry run published an adapter" assert complete() == baseline, "dry run changed serving" -fresh = ok("/lora-adapters/fresh", {"name": fresh_name, "rank": 8, - "alpha": 16, "targets": "attn_q,attn_v", "top_layers": 1, "seed": 42}) +fresh_weights = register("-fresh", {"fresh": fresh_spec}, [dict(row, kind="weights") for row in plan["weights"]]) +fresh = ok("/lora-adapters/fresh", dict(fresh_spec, residency=fresh_weights)) assert fresh["weights"] == plan["weights"], ("allocated weights differ from dry plan", fresh, plan) file_plan = ok("/lora-adapters/load", {"name": fresh_name, "dry_run": True}) assert file_plan["weights"] == plan["weights"] @@ -74,16 +88,42 @@ def get(route): repeat_name = prefix + "-repeat.gguf" ok("/lora-adapters/save", {"id": fresh["id"], "name": repeat_name}) assert hashlib.sha256((train_dir / repeat_name).read_bytes()).hexdigest() == before, "save bytes are unstable without learning" +optimizer_rid = None +def learning(reply, suffix): + # A learn runs only inside custody the core registered from the engine's own preflight of this + # example: her optimizer (persistent per shadow, registered once) and the engine-wide step + # workspace sized to this reply (server-context-learn-residency.inc admit_learning_residency). + global optimizer_rid + plan = ok("/learn/preflight", {"engine": ledger["engine"], "lora_id": fresh["id"], "learn": {"reply": reply}}) + assert plan["refusal"] == "none", plan + rows = [row for row in plan["requirements"] if row["kind"] != "shared_compute_held"] + if optimizer_rid is None: + optimizer_rid = register("-optimizer", {"loaded": {"lora_id": fresh["id"], "path": fresh["path"]}}, + [row for row in rows if row["kind"] != "shared_compute_required"]) + workspace = register("-workspace" + suffix, {"workspace": True}, + [row for row in rows if row["kind"] == "shared_compute_required"]) + return {"reply": reply, "lora_id": fresh["id"], "grant": grant, "lr": 0.001, + "optimizer_residency": optimizer_rid, "workspace_residency": workspace} learned = ok("/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, "n_predict": 1, "temperature": 0, - "learn": {"reply": "Topeka is the capital of Kansas.", - "lora_id": fresh["id"], "grant": grant, "lr": 0.001}}) + "learn": learning("Topeka is the capital of Kansas.", "-short")}) assert learned["learn"]["success"], learned +# The long reply spans as many resumable chunks as this engine's window holds: sized from its own +# slot window and tokenizer (the reply's tokens plus its end of turn, after the served prompt), +# never a fixed repeat count, since preflight refuses an example past the served window. +def rows(text, special): + return len(ok("/tokenize", {"content": text, "add_special": special})["tokens"]) +long_prompt = "The capital of Kansas is" +room = get("/slots")[0]["n_ctx"] - rows(long_prompt, True) - 1 +unit = "A room contains a card and a window. " +repeats = room // rows(unit, False) +while repeats > 1 and rows(unit * repeats, False) > room: + repeats -= 1 +assert repeats > 1, ("the served window holds no multi-sentence reply", room) # A queued save must refuse the active shadow even between resumable chunks. with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit(ok, "/completion", {"prompt": "The capital of Kansas is", "id_slot": 0, - "n_predict": 1, "temperature": 0, "learn": {"grant": grant, "lora_id": fresh["id"], - "lr": 0.001, "reply": "A room contains a card and a window. " * 55}}) + future = pool.submit(ok, "/completion", {"prompt": long_prompt, "id_slot": 0, + "n_predict": 1, "temperature": 0, "learn": learning(unit * repeats, "-long")}) deadline = time.monotonic() + 120 while time.monotonic() < deadline: if get("/slots")[0]["is_learning"]: @@ -101,7 +141,9 @@ def get(route): assert snapshot["steps"] == 2, ("two whole replies learned are two committed steps", snapshot) after = hashlib.sha256((train_dir / snapshot_name).read_bytes()).hexdigest() assert after != before, "successful learn did not change saved adapter weights" -loaded = ok("/lora-adapters/load", {"name": snapshot_name}) +snapshot_plan = ok("/lora-adapters/load", {"name": snapshot_name, "dry_run": True}) +snapshot_weights = register("-snapshot", {"load": {"name": snapshot_name}}, [dict(row, kind="weights") for row in snapshot_plan["weights"]]) +loaded = ok("/lora-adapters/load", {"name": snapshot_name, "residency": snapshot_weights}) assert complete(loaded["id"]) == trained_text, "trained snapshot reload changed output" assert post("/lora-adapters/save", {"id": fresh["id"], "name": snapshot_name})[0] >= 400 assert post("/lora-adapters/fresh", {"name": fresh_name})[0] >= 400 diff --git a/tools/server/tests/unit/test_lora.py b/tools/server/tests/unit/test_lora.py index e84377745269..a7c437551cda 100644 --- a/tools/server/tests/unit/test_lora.py +++ b/tools/server/tests/unit/test_lora.py @@ -89,14 +89,26 @@ def report(): assert server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": 0.25}]).status_code == 200 assert report()[0]["scale"] == 0.25 - loaded = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf"}) + # A runtime load is seated only inside a residency registered for its exact subject, as the core + # registers one (server-learn-residency.h): measure, register its rows and leases, then load. + measured = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf", "dry_run": True}) + assert measured.status_code == 200, measured.body + engine = server.make_request("GET", "/learn/grants").body["engine"] + rows = [dict(row, kind="weights") for row in measured.body["weights"]] + leases = [{"kind": kind, "lease_id": "second-" + kind, "bytes": sum(row["bytes"] for row in rows if row["host"] == host)} + for kind, host in (("vram", False), ("ram", True))] + registered = server.make_request("POST", "/learn/residency", data={"id": "second-weights", "engine": engine, "owner": "test", + "subject": {"load": {"name": "second.gguf"}}, "rows": rows, "leases": leases}) + assert registered.status_code == 200 and registered.body["refusal"] == "none", registered.body + loaded = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf", "residency": "second-weights"}) assert loaded.status_code == 200, loaded.body assert sorted(report()) == [0, 1] # an adapter active by default is retired only once its scale is 0 (the engine refuses otherwise) assert server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": 0.0}, {"id": 1, "scale": 0.0}]).status_code == 200 assert report()[0]["scale"] == 0.0 - unloaded = server.make_request("POST", "/lora-adapters/unload", data={"id": 0}) + # a retire names the engine incarnation it was decided against, like every lifecycle verb + unloaded = server.make_request("POST", "/lora-adapters/unload", data={"id": 0, "engine": engine}) assert unloaded.status_code == 200, unloaded.body assert list(report()) == [1], "a retired adapter leaves the report; the ids it left stay taken"