diff --git a/.gitignore b/.gitignore index 806e297..7e23010 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,8 @@ env/ # Pi agent files .pi/ _task.md +_pi_tasks/ + +# Generated test artifacts (harness output; regenerated per run) +tests/_assets/ +tests/acceptance-results/ diff --git a/chat_template_sharp.jinja b/chat_template_sharp.jinja new file mode 100644 index 0000000..5debfc7 --- /dev/null +++ b/chat_template_sharp.jinja @@ -0,0 +1,340 @@ +{%- set template_version = "qwen3.6-froggeric-v21.3" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- set _preserve_thinking = preserve_thinking if preserve_thinking is defined else true %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set ns_state = namespace(thinking=enable_thinking) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set _first_role = messages[0].role %} +{%- if _first_role == 'system' or _first_role == 'developer' %} + {%- set _sys_msg = messages[0] %} + {%- set _msgs = messages[1:] %} +{%- else %} + {%- set _sys_msg = none %} + {%- set _msgs = messages %} +{%- endif %} +{%- set _sc = '' %} +{%- if _sys_msg is not none %} + {%- set _sc = render_content(_sys_msg.content, false, true) | trim %} + {%- if '<|think_off|>' in _sc %} + {%- set ns_state.thinking = false %} + {%- set _sc = _sc.split('<|think_off|>') | join('') | trim %} + {%- elif '<|think_on|>' in _sc %} + {%- set ns_state.thinking = true %} + {%- set _sc = _sc.split('<|think_on|>') | join('') | trim %} + {%- endif %} +{%- endif %} +{%- set _terse %} +Answer directly, after thinking. Lead with the answer, then only what it needs to be correct and usable. +Never: open with preamble or pleasantries; restate the question; add filler transitions; hedge with niceties; or repeat a point you've already made. +Always: keep essential steps, caveats, uncertainties, and specifics — never drop correctness or a needed warning for brevity. Keep the final answer lean. Use the least structure that conveys it (plain prose when short; lists or code only when they earn their place). If genuinely uncertain, say so and explain why — never omit uncertainty for the sake of brevity. +If a user request is genuinely ambiguous, ask a sharp question, don't guess. +{%- endset %} +{%- if not _sc %} + {%- set _sc = _terse | trim %} +{%- else %} + {%- set _sc = (_sc | trim) ~ '\n\n' ~ (_terse | trim) %} +{%- endif %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- set tool_instructions %} +If you choose to call a function ONLY reply in the following format with NO suffix: + +{%- if _tool_format == 'json' %} + +Brief explanation of tool call + + +{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}} + +{%- else %} + +Brief explanation of tool call + + + + +value_1 + + +This is the value for the second parameter +that can span +multiple lines + + + +{%- endif %} + + +Reminder: +- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user. +- ALL explanation and reasoning MUST be placed strictly inside the block. +{%- if _tool_format == 'json' %} +- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags. +{%- else %} +- Function calls MUST follow the specified format: an inner block must be nested within XML tags. +{%- endif %} +- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it. +{%- if _tool_format == 'json' %} +- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it. +{%- else %} +- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them. +{%- endif %} +- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks. +- If you have all necessary data, provide your final answer directly to the user without any tool call. + + {%- endset %} + {{- '\n\n' ~ tool_instructions | trim }} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + _sc + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %} + {%- set ns_state.thinking = false %} + {%- set content = content.split('<|think_off|>') | join('') | trim %} + {%- elif '<|think_on|>' in content %} + {%- set ns_state.thinking = true %} + {%- set content = content.split('<|think_on|>') | join('') | trim %} + {%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set reasoning_content = message.thinking %} + {%- else %} + {%- set reasoning_content = message.thinking | string %} + {%- endif %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) and reasoning_content %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- if _tool_format == 'json' %} + {%- if not loop.first or content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string and tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc.name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is mapping or (args_value is sequence and args_value is not string) %} + {%- set _av = args_value | tojson %} + {%- else %} + {%- set _av = args_value | string %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED — original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- elif tc.arguments is string and tc.arguments %} + {{- tc.arguments }} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:80] %} + {%- if content | length < 500 and '$ ' not in content and 'took ' not in _content_lower and ('"error":' in _content_head or 'error:' in _content_head or 'err!' in _content_head or 'fatal:' in _content_head or 'exception:' in _content_head or 'traceback' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'failed to' in _content_head) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- if max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED — original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- elif ns2.consecutive_failures >= 2 %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/docs/dev/model-acceptance.md b/docs/dev/model-acceptance.md new file mode 100644 index 0000000..53cb927 --- /dev/null +++ b/docs/dev/model-acceptance.md @@ -0,0 +1,131 @@ +# New-Model Acceptance Tests + +Gate a newly-added router model **before** wiring it into day-to-day use. Run this +for every model added to `proxy_config.py` (chat) — Nail, future Qwen drops, any +re-quant. If a check fails, the model is not ready; fix the preset/binary/template +and re-run. + +The philosophy: **fits, loads, talks, thinks, calls tools, remembers long context, +sees images, and stays on the GPU** — proven empirically, not assumed. + +--- + +## 0. Preconditions + +- Weights + mmproj present under `models/` (and `models/_aux/`). +- Preset added to `MODELS` in `proxy_config.py`, proxy restarted so + `models-preset.ini` is regenerated (see `docs/dev/workflow.md` for the safe + restart). Confirm the new preset id shows in `/v1/models`. +- Chat proxy (`:8001`) and router (`:8002`) both up (`/health` → ok). +- `LLAMA_API_KEY` in env or `.env`. +- `.venv\Scripts\python.exe` available (system `python` fails — tzdata/ZoneInfo). + +The router is pinned to the 3090 Ti (`CUDA_VISIBLE_DEVICES=0`, +`router_manager.py`). VRAM ceiling is **24,564 MiB**; treat **> ~24,000 MiB used** +as "no safe headroom" and **any spill to shared/system RAM** (WDDM) as a hard fail — +it silently tanks throughput. + +--- + +## 1. Automated harness + +```powershell +.venv\Scripts\python.exe tests\model_acceptance.py --model +# e.g. +.venv\Scripts\python.exe tests\model_acceptance.py --model nail-35b-a3b-q4-256k +``` + +Useful flags: + +| Flag | Default | Meaning | +|------|---------|---------| +| `--model` | *(required)* | Preset id to test (must exist in `/v1/models`). | +| `--proxy` | `http://localhost:8001` | Client-facing proxy base (IPv6-friendly host). | +| `--router` | `http://127.0.0.1:8002` | Router base for load/unload/introspection. | +| `--vram-ceiling` | `24000` | MiB; fail if GPU0 `memory.used` exceeds this after load / during probe. | +| `--ctx-probe-tokens` | `32000` | Filler size for the needle-in-haystack long-context test. | +| `--full-ctx` | off | Push the needle probe to ~95% of the preset ctx (slow; real long-context proof). | +| `--image` | *(auto)* | Path to a test image; if omitted a known solid-color PNG is generated. | +| `--min-decode-tps` | `15` | Fail if decode throughput drops below this (regression floor). | +| `--skip` | *(none)* | Comma list of check ids to skip (e.g. `vision,long_context`). | + +The harness prints PASS/FAIL per check and writes a dated report to +`tests/acceptance-results/-.md`. Non-zero exit if any +non-skipped check fails. + +--- + +## 2. What each check proves + +| id | Check | Method | Pass criteria | +|----|-------|--------|---------------| +| `registered` | Preset is exposed | GET router `/v1/models` | id present; `n_ctx` meta == expected preset ctx | +| `load` | Cold load works | POST router `/models/load`, poll status | reaches `loaded` within timeout; record load seconds | +| `vram_fit` | Fits on 3090 Ti, no spill | `nvidia-smi` GPU0 after load | `memory.used` ≤ `--vram-ceiling`; no shared-RAM spill | +| `basic` | Coherent completion | POST proxy `/v1/chat/completions` (non-stream) | deterministic answer correct (e.g. "2+2" → contains `4`) | +| `streaming` | SSE path healthy | POST proxy `stream=true` | ≥2 events + `[DONE]` | +| `think_integrity` | Reasoning clean (issue #8) | inspect streamed content | reasoning present; **no leaked ``/`` in visible content** | +| `tools` | Tool calling works | POST proxy with a `tools` schema | returns a `tool_calls[0]` with **valid JSON** arguments | +| `long_context` | Long-ctx retrieval + no OOM | needle-in-haystack prompt | needle returned verbatim; no OOM/spill; records **pp** from server timings | +| `vision` | Multimodal + CPU-mmproj | POST proxy image turn, then text follow-up | image described correctly; **turn-2 (text) works and is fast** (no re-encode); GPU stays ≤ ceiling | +| `perf` | Throughput | authoritative llama-server timings | records **pp** and **tg** tok/s; `tg` ≥ `--min-decode-tps` | + +Speed is logged as **pp** (prompt processing) and **tg** (token generation) taken +from llama-server's own `timings` (`prompt_per_second` / `predicted_per_second`), +not wall-clock estimates. Both land in the report header and the `perf` row. + +Notes on the tricky ones: + +- **`think_integrity`** is the issue-#8 guard. The model must emit properly-closed + `` blocks and the proxy must keep them out of the user-visible `content`. + A leak here means the SSEChunkLogger reroute regressed or the template's + `preserve_thinking` handling is wrong for this model. +- **`vision`** proves the earlier design decision: with `no-mmproj-offload` the + projector runs on CPU. The image is encoded **once**; the follow-up text turn must + reuse cached KV (fast) and must **not** re-run the encoder. The check times both + turns — turn-2 should be in the normal text-latency range, not image-encode range. + It also asserts GPU `memory.used` never exceeds the ceiling during the image turn + (the encode spike lives in RAM, not VRAM). +- **`long_context`** default is a moderate 32k filler for a fast gate. Run + `--full-ctx` at least once per model to prove the preset's headline context + (e.g. 256k) actually retrieves and doesn't OOM. + +--- + +## 3. Manual / judgment checks (not automated) + +Automation proves plumbing; a human confirms quality. Spend 5 minutes: + +- [ ] **Terseness / system-prompt behavior** — froggeric models should answer + without preamble. Ask something open-ended; confirm no "Certainly! Here's…". +- [ ] **Thinking length sane** — reasoning shouldn't run away (Nail/Dagger are + tuned for compressed thinking). Watch a couple of hard questions. +- [ ] **Multi-turn coherence** — a 3–4 turn exchange; confirm it tracks context and + tools across turns. +- [ ] **Tool-call ergonomics in the real client** — run one real task through pi + (or your daily agent) and confirm tool calls parse and execute end-to-end. +- [ ] **Vision accuracy on a real screenshot** — describe an actual UI screenshot, + not just the synthetic color swatch. + +--- + +## 4. Per-model sign-off + +Copy into the dated results file and fill in: + +``` +Model: +Binary (b###): +Weights: +mmproj: (offload: cpu|gpu) +Preset ctx: +VRAM used @load: / 24564 (headroom: ) +pp tok/s: tg tok/s: +Automated: +Manual review: +Verdict: ACCEPTED / REJECTED for daily use +``` + +Keep every run under `tests/acceptance-results/` so regressions across binary +upgrades and re-quants are visible over time. + diff --git a/proxy_config.py b/proxy_config.py index 4f2b901..e260bbf 100644 --- a/proxy_config.py +++ b/proxy_config.py @@ -18,6 +18,7 @@ ROOT = Path(__file__).resolve().parent SERVER_EXE = ROOT / "llama.cpp_latest" / "llama-server.exe" PRESET_PATH = ROOT / "models-preset.ini" +DEFAULT_CHAT_TEMPLATE = ROOT / "chat_template.jinja" # used by any model without its own # Each (model, ctx) pair becomes its own preset section so the router # exposes them as distinct models that the pi-llama-cpp extension can @@ -30,29 +31,31 @@ class ModelChoice: model_file: Path # GGUF weights mmproj_file: Path # multimodal projector spec_mtp: bool = False # enable built-in MTP speculative decoding (draft-mtp) + mmproj_offload: bool = True # False → run projector on CPU/RAM (frees ~1.1 GB VRAM; + # image encode moves to CPU, only on image turns) + chat_template_file: Path | None = None # per-model template; overrides the + # router-global --chat-template-file + ctx_choices: tuple[int, ...] | None = None # per-model ctx list; None → global CTX_CHOICES def preset_id(self, ctx: int) -> str: return f"{self.base_id}-{ctx // 1024}k" + def contexts(self, default: list[int]) -> tuple[int, ...] | list[int]: + return self.ctx_choices if self.ctx_choices is not None else default + +# Two production models, each exposed at a single 262k preset. Both fit the +# 3090 Ti alone at full context with the projector on CPU (mmproj_offload=False); +# see docs/dev/model-acceptance.md. Retired presets (base 35B A3B Q3/Q4, 27B MTP, +# and all sub-262k ctx variants) — weights remain on disk, just no longer served. MODELS: list[ModelChoice] = [ ModelChoice( - "Qwen3.6-35B-A3B Q3", - "qwen3.6-35b-q3", - ROOT / "models" / "Qwen3.6-35B-A3B-UD-Q3_K_XL.gguf", - ROOT / "models" / "_aux" / "mmproj-F16.gguf", - ), - ModelChoice( - "Qwen3.6-35B-A3B Q4", - "qwen3.6-35b-q4", - ROOT / "models" / "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + "Nail-Qwen3.6-35B-A3B Q4", + "nail-35b-a3b-q4", + ROOT / "models" / "Nail-Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf", ROOT / "models" / "_aux" / "mmproj-F16.gguf", - ), - ModelChoice( - "Qwen3.6-27B Q4", - "qwen3.6-27b-q4", - ROOT / "models" / "Qwen3.6-27B-UD-Q4_K_XL.gguf", - ROOT / "models" / "_aux" / "mmproj-27b-BF16.gguf", + mmproj_offload=False, + chat_template_file=ROOT / "chat_template_sharp.jinja", ), ModelChoice( "Qwen3.6-27B Q4 MTP", @@ -60,10 +63,14 @@ def preset_id(self, ctx: int) -> str: ROOT / "models" / "Qwen3.6-27B-UD-Q4_K_XL.mtp.gguf", ROOT / "models" / "_aux" / "mmproj-27b-BF16.gguf", spec_mtp=True, + mmproj_offload=False, + # MTP draft buffers add ~1.7 GB, so full 262k leaves only ~300 MiB — + # too tight for the prefill spike. 224k is the safe max (~1 GB headroom). + ctx_choices=(229376,), ), ] -CTX_CHOICES: list[int] = [32768, 65536, 98304, 131072] +CTX_CHOICES: list[int] = [262144] PROXY_HOST = "0.0.0.0" PROXY_PORT = 8001 @@ -90,12 +97,11 @@ def server_command(self) -> list[str]: "--models-preset", str(PRESET_PATH), "--models-max", "1", "--no-models-autoload", - # --- perf A/B test: chat-template flags (see chat_template_perf_test.md) --- - # Variant D: full new config (jinja + custom template + preserve_thinking kwarg) + # Chat template is set PER-PRESET (chat-template-file in each section), + # not globally: a router-global --chat-template-file overrides every + # preset's own template, which would deny Nail its froggeric template. + # Both templates default preserve_thinking=true, so no global kwarg needed. "--jinja", - "--chat-template-file", str(ROOT / "chat_template.jinja"), - "--chat-template-kwargs", '{"preserve_thinking":true}', - # ------------------------------------------------------------------------ "--host", self.server_host, "--port", str(self.server_port), "--api-key", self.api_key, @@ -142,6 +148,9 @@ def _model_preset_section(model: ModelChoice, ctx: int) -> str: if model.spec_mtp else "" ) + mmproj_off = "" if model.mmproj_offload else "mmproj-offload = off\n" + tmpl_path = model.chat_template_file or DEFAULT_CHAT_TEMPLATE + template = f"chat-template-file = {tmpl_path.as_posix()}\n" return ( f"[{model.preset_id(ctx)}]\n" f"model = {model.model_file.as_posix()}\n" @@ -157,6 +166,8 @@ def _model_preset_section(model: ModelChoice, ctx: int) -> str: f"temp = 0.6\n" f"top-p = 0.95\n" f"top-k = 20\n" + + mmproj_off + + template + spec ) @@ -171,7 +182,7 @@ def write_preset(models: list[ModelChoice], ctx_choices: list[int]) -> None: sections = [ _model_preset_section(m, ctx) for m in models - for ctx in ctx_choices + for ctx in m.contexts(ctx_choices) ] content = "\n".join(sections) + "\n" PRESET_PATH.write_text(content, encoding="utf-8") @@ -208,8 +219,9 @@ def build_config() -> ProxyConfig: model, ctx = pick_setup(args.model, args.ctx_size) write_preset(MODELS, CTX_CHOICES) default_id = model.preset_id(ctx) + n_presets = sum(len(m.contexts(CTX_CHOICES)) for m in MODELS) print(f"Default: {model.label} @ {ctx // 1024}k ctx (id: {default_id})") - print(f"Exposed presets: {len(MODELS) * len(CTX_CHOICES)} (one per model×ctx combo)") + print(f"Exposed presets: {n_presets} (one per model×ctx combo)") return ChatProxyConfig( proxy_host=args.proxy_host, diff --git a/tests/model_acceptance.py b/tests/model_acceptance.py new file mode 100644 index 0000000..38675b2 --- /dev/null +++ b/tests/model_acceptance.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python +"""Live acceptance harness for a newly-added router model. + +Run BEFORE putting a model into daily use. Proves the model fits on the GPU, +loads, talks, thinks cleanly (issue #8), calls tools, retrieves long context, +and handles vision with the projector on CPU — empirically, not by assumption. + +Usage: + .venv\\Scripts\\python.exe tests\\model_acceptance.py --model + +Companion plan: docs/dev/model-acceptance.md + +Dependency-free (stdlib urllib only). Drives the client-facing proxy (:8001) +for functional checks and the router (:8002) for load/introspection/VRAM. +""" +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import struct +import subprocess +import sys +import time +import urllib.error +import urllib.request +import zlib +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +# ── config / secrets ───────────────────────────────────────────────────────── + +def resolve_api_key() -> str: + key = os.environ.get("LLAMA_API_KEY") + if key: + return key + envf = ROOT / ".env" + if envf.is_file(): + for line in envf.read_text(encoding="utf-8").splitlines(): + m = re.match(r"\s*LLAMA_API_KEY\s*=\s*(.+)\s*$", line) + if m: + return m.group(1).strip().strip("\"'") + raise SystemExit("No LLAMA_API_KEY in env or .env") + + +# ── tiny HTTP layer ────────────────────────────────────────────────────────── + +class Http: + def __init__(self, key: str) -> None: + self.key = key + + def _req(self, url: str, method: str, body: dict | None, timeout: float): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f"Bearer {self.key}") + if data is not None: + req.add_header("Content-Type", "application/json") + return urllib.request.urlopen(req, timeout=timeout) + + def get(self, url: str, timeout: float = 15) -> dict: + with self._req(url, "GET", None, timeout) as r: + return json.loads(r.read().decode()) + + def post(self, url: str, body: dict, timeout: float = 240) -> dict: + with self._req(url, "POST", body, timeout) as r: + return json.loads(r.read().decode()) + + def post_stream(self, url: str, body: dict, timeout: float = 240): + """Yield parsed SSE JSON payloads (skips [DONE]).""" + body = {**body, "stream": True} + with self._req(url, "POST", body, timeout) as r: + for raw in r: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if not payload or payload == "[DONE]": + if payload == "[DONE]": + yield "__DONE__" + continue + try: + yield json.loads(payload) + except json.JSONDecodeError: + pass + + +def gpu_used_mib(index: int = 0) -> int | None: + try: + out = subprocess.run( + ["nvidia-smi", "--query-gpu=memory.used", + "--format=csv,noheader,nounits", "-i", str(index)], + capture_output=True, text=True, timeout=15, + ) + return int(out.stdout.strip().splitlines()[0]) + except Exception: + return None + + +# ── result plumbing ────────────────────────────────────────────────────────── + +@dataclass +class Result: + id: str + passed: bool + detail: str + metrics: dict = field(default_factory=dict) + + +class Runner: + def __init__(self, args) -> None: + self.a = args + self.http = Http(resolve_api_key()) + self.results: list[Result] = [] + self.skip = {s.strip() for s in (args.skip or "").split(",") if s.strip()} + self.pp: float | None = None # prompt-processing tok/s (llama-server timings) + self.tg: float | None = None # token-generation tok/s (llama-server timings) + + def record(self, r: Result) -> None: + self.results.append(r) + tag = "PASS" if r.passed else "FAIL" + extra = f" {r.metrics}" if r.metrics else "" + print(f" [{tag}] {r.id}: {r.detail}{extra}") + + def should(self, cid: str) -> bool: + if cid in self.skip: + print(f" [SKIP] {cid}") + return False + return True + + # ── individual checks ──────────────────────────────────────────────────── + + def check_registered(self) -> None: + data = self.http.get(f"{self.a.router}/v1/models") + entry = next((e for e in data.get("data", []) if e.get("id") == self.a.model), None) + if not entry: + self.record(Result("registered", False, f"{self.a.model} not in /v1/models")) + return + n_ctx = (entry.get("meta") or {}).get("n_ctx") + self.record(Result("registered", True, f"found; meta n_ctx={n_ctx}", + {"n_ctx": n_ctx})) + + def _status(self, model: str) -> str: + data = self.http.get(f"{self.a.router}/v1/models") + for e in data.get("data", []): + if e.get("id") == model: + st = e.get("status") + return st.get("value") if isinstance(st, dict) else str(st) + return "unknown" + + def check_load(self) -> None: + t0 = time.monotonic() + try: + self.http.post(f"{self.a.router}/models/load", {"model": self.a.model}, timeout=20) + except urllib.error.HTTPError as e: + if b"already running" not in e.read(): + self.record(Result("load", False, f"load POST failed: {e}")) + return + deadline = t0 + 300 + while time.monotonic() < deadline: + st = self._status(self.a.model) + if st == "loaded": + secs = round(time.monotonic() - t0, 1) + self.record(Result("load", True, "reached loaded", {"load_s": secs})) + return + if st == "failed": + self.record(Result("load", False, "status=failed")) + return + time.sleep(1) + self.record(Result("load", False, "did not load in 300s")) + + def check_vram_fit(self) -> None: + used = gpu_used_mib(0) + if used is None: + self.record(Result("vram_fit", False, "nvidia-smi unavailable")) + return + ok = used <= self.a.vram_ceiling + self.record(Result("vram_fit", ok, + f"GPU0 used={used} MiB (ceiling {self.a.vram_ceiling})", + {"vram_used_mib": used, "headroom_mib": 24564 - used})) + + def check_basic(self) -> None: + body = {"model": self.a.model, "temperature": 0, + "messages": [{"role": "user", + "content": "What is 2+2? Answer with only the number."}]} + r = self.http.post(f"{self.a.proxy}/v1/chat/completions", body) + content = r["choices"][0]["message"]["content"] or "" + ok = "4" in content + self.record(Result("basic", ok, f"answer={content.strip()[:40]!r}")) + + def check_streaming_and_think(self) -> None: + # deterministic, multi-token output so the stream actually flows and + # decode throughput is measurable + body = {"model": self.a.model, "temperature": 0, + "stream_options": {"include_usage": True}, + "messages": [{"role": "user", + "content": "Count from 1 to 40, comma-separated, on one line."}]} + events = 0 # any streamed delta (content or reasoning) + got_done = False + visible = [] + reasoning_seen = False + usage = {} + t_first = t_last = None + t0 = time.monotonic() + for ev in self.http.post_stream(f"{self.a.proxy}/v1/chat/completions", body): + if ev == "__DONE__": + got_done = True + continue + if ev.get("usage"): + usage = ev["usage"] + for ch in ev.get("choices", []): + delta = ch.get("delta", {}) + content = delta.get("content") + reasoning = delta.get("reasoning_content") + if content or reasoning: + events += 1 + now = time.monotonic() + if t_first is None: + t_first = now + t_last = now + if content: + visible.append(content) + if reasoning: + reasoning_seen = True + t_done = time.monotonic() + text = "".join(visible) + # issue #8 guard: no raw think tags leaked into visible content + leaked = "" in text or "" in text + # decode window: prefer first->last token; fall back to whole request if too small + window = (t_last - t_first) if (t_first and t_last and t_last - t_first > 0.1) else (t_done - t0) + comp_tok = usage.get("completion_tokens") + tps = round(comp_tok / window, 1) if comp_tok and window > 0.05 else None + stream_ok = got_done and events >= 2 and len(text) > 5 + detail = f"events={events} done={got_done} chars={len(text)} think_leak={leaked} reasoning={reasoning_seen}" + self.record(Result("streaming", stream_ok, detail, + {"wall_decode_tps": tps})) # rough; authoritative tg is in perf + self.record(Result("think_integrity", not leaked, + "no leaked think tags" if not leaked + else "RAW LEAKED INTO CONTENT")) + + def check_tools(self) -> None: + tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string", + "description": "City name"}}, + "required": ["location"], + }, + }, + }] + body = {"model": self.a.model, "temperature": 0, "tools": tools, + "tool_choice": "auto", + "messages": [{"role": "user", + "content": "Use the get_weather tool for Paris."}]} + r = self.http.post(f"{self.a.proxy}/v1/chat/completions", body) + msg = r["choices"][0]["message"] + calls = msg.get("tool_calls") or [] + if not calls: + self.record(Result("tools", False, f"no tool_calls; content={str(msg.get('content'))[:60]!r}")) + return + try: + args = json.loads(calls[0]["function"]["arguments"]) + ok = "paris" in json.dumps(args).lower() + self.record(Result("tools", ok, f"call={calls[0]['function']['name']} args={args}")) + except (json.JSONDecodeError, KeyError, TypeError) as e: + self.record(Result("tools", False, f"unparseable arguments: {e}")) + + def check_long_context(self) -> None: + target_tok = self.a.ctx_probe_tokens + if self.a.full_ctx: + n_ctx = next((r.metrics.get("n_ctx") for r in self.results + if r.id == "registered"), None) or 262144 + target_tok = int(n_ctx * 0.95) + needle = "The vault override code is TANGERINE-9417." + # ~1.3 tokens/word; build distinct filler lines so it can't be trivially compressed + n_words = int(target_tok / 1.3) + filler_lines = [] + wc = 0 + i = 0 + while wc < n_words: + line = f"Log entry {i}: routine telemetry sample alpha bravo charlie delta echo foxtrot." + filler_lines.append(line) + wc += 11 + i += 1 + mid = len(filler_lines) // 2 + filler_lines.insert(mid, needle) + haystack = "\n".join(filler_lines) + body = {"model": self.a.model, "temperature": 0, + "messages": [ + {"role": "user", + "content": haystack + "\n\nQuestion: What is the vault override code? " + "Reply with only the code."}]} + t0 = time.monotonic() + try: + r = self.http.post(f"{self.a.proxy}/v1/chat/completions", body, timeout=600) + except Exception as e: + self.record(Result("long_context", False, f"request failed (possible OOM): {e}")) + return + dt = round(time.monotonic() - t0, 1) + content = r["choices"][0]["message"]["content"] or "" + usage = r.get("usage", {}) + timings = r.get("timings") or {} + # authoritative pp (prompt processing) at real context scale + if timings.get("prompt_per_second"): + self.pp = round(timings["prompt_per_second"], 1) + ptok = usage.get("prompt_tokens") or timings.get("prompt_n") + ok = "TANGERINE-9417" in content + used = gpu_used_mib(0) + vram_ok = used is None or used <= self.a.vram_ceiling + self.record(Result("long_context", ok and vram_ok, + f"needle_found={ok} prompt_tok={ptok} vram={used}", + {"pp_tok_s": self.pp, "wall_s": dt})) + + def check_vision(self) -> None: + img_path = self.a.image + if not img_path: + img_path = str(ROOT / "tests" / "_assets" / "acceptance-red.png") + Path(img_path).parent.mkdir(parents=True, exist_ok=True) + make_solid_png(img_path, (220, 30, 30)) + b64 = base64.b64encode(Path(img_path).read_bytes()).decode() + data_uri = f"data:image/png;base64,{b64}" + img_msg = {"role": "user", "content": [ + {"type": "text", "text": "What is the dominant color of this image? Answer with one word."}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ]} + t0 = time.monotonic() + try: + r1 = self.http.post(f"{self.a.proxy}/v1/chat/completions", + {"model": self.a.model, "temperature": 0, "messages": [img_msg]}, + timeout=300) + except Exception as e: + self.record(Result("vision", False, f"image turn failed: {e}")) + return + t_img = round(time.monotonic() - t0, 1) + reply1 = r1["choices"][0]["message"]["content"] or "" + color_ok = "red" in reply1.lower() + used_after_img = gpu_used_mib(0) + vram_ok = used_after_img is None or used_after_img <= self.a.vram_ceiling + # turn 2: text follow-up in the same conversation — must work and be fast + # (proves cached KV reuse, no image re-encode) + t1 = time.monotonic() + r2 = self.http.post(f"{self.a.proxy}/v1/chat/completions", + {"model": self.a.model, "temperature": 0, "messages": [ + img_msg, + {"role": "assistant", "content": reply1}, + {"role": "user", "content": "What is 3+3? Only the number."}, + ]}, timeout=120) + t_turn2 = round(time.monotonic() - t1, 1) + reply2 = r2["choices"][0]["message"]["content"] or "" + turn2_ok = "6" in reply2 + ok = color_ok and turn2_ok and vram_ok + self.record(Result("vision", ok, + f"color={reply1.strip()[:20]!r} turn2_ok={turn2_ok} vram_after_img={used_after_img}", + {"img_turn_s": t_img, "turn2_s": t_turn2})) + + def check_perf(self) -> None: + # authoritative tg (token generation) from llama-server timings on a + # clean, decent-length generation; pp comes from long_context (real + # scale) or falls back to this request's small-prompt pp + body = {"model": self.a.model, "temperature": 0, "max_tokens": 200, + "messages": [{"role": "user", + "content": "Count from 1 to 200, comma-separated."}]} + try: + r = self.http.post(f"{self.a.proxy}/v1/chat/completions", body, timeout=180) + except Exception as e: + self.record(Result("perf", False, f"tg probe failed: {e}")) + return + t = r.get("timings") or {} + if t.get("predicted_per_second"): + self.tg = round(t["predicted_per_second"], 1) + if self.pp is None and t.get("prompt_per_second"): + self.pp = round(t["prompt_per_second"], 1) + ok = self.tg is not None and self.tg >= self.a.min_decode_tps + self.record(Result("perf", ok, + f"pp={self.pp} tg={self.tg} tok/s (tg floor {self.a.min_decode_tps})", + {"pp_tok_s": self.pp, "tg_tok_s": self.tg})) + + # ── orchestration ──────────────────────────────────────────────────────── + + ORDER = [ + ("registered", "check_registered"), + ("load", "check_load"), + ("vram_fit", "check_vram_fit"), + ("basic", "check_basic"), + ("streaming", "check_streaming_and_think"), + ("tools", "check_tools"), + ("long_context", "check_long_context"), + ("vision", "check_vision"), + ("perf", "check_perf"), + ] + + def run(self) -> bool: + print(f"\n=== acceptance: {self.a.model} ===") + for cid, method in self.ORDER: + # streaming method emits two result ids; gate on the primary id + if cid in self.skip and cid not in ("streaming",): + print(f" [SKIP] {cid}") + continue + if cid == "streaming" and "streaming" in self.skip and "think_integrity" in self.skip: + print(" [SKIP] streaming/think_integrity") + continue + try: + getattr(self, method)() + except Exception as e: + self.record(Result(cid, False, f"exception: {type(e).__name__}: {e}")) + return self.write_report() + + def write_report(self) -> bool: + failed = [r for r in self.results if not r.passed] + outdir = ROOT / "tests" / "acceptance-results" + outdir.mkdir(parents=True, exist_ok=True) + stamp = datetime.now().strftime("%Y-%m-%d-%H%M") + path = outdir / f"{self.a.model}-{stamp}.md" + lines = [f"# Acceptance report — {self.a.model}", "", + f"- When: {datetime.now().isoformat(timespec='seconds')}", + f"- Proxy: {self.a.proxy} Router: {self.a.router}", + f"- Verdict: {'ACCEPTED' if not failed else 'REJECTED'} " + f"({len(self.results) - len(failed)}/{len(self.results)} passed)", + f"- Speed (llama-server timings): pp={self.pp} tok/s tg={self.tg} tok/s", + "", "| id | result | detail | metrics |", "|----|--------|--------|---------|"] + for r in self.results: + lines.append(f"| {r.id} | {'PASS' if r.passed else 'FAIL'} | {r.detail} | " + f"{r.metrics or ''} |") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"\nReport: {path}") + print(f"Verdict: {'ACCEPTED' if not failed else 'REJECTED -- ' + ', '.join(r.id for r in failed)}") + return not failed + + +# ── known test image (stdlib PNG, no PIL) ──────────────────────────────────── + +def make_solid_png(path: str, rgb=(220, 30, 30), size: int = 64) -> None: + raw = bytearray() + for _ in range(size): + raw.append(0) # filter type 0 per scanline + raw += bytes(rgb) * size + comp = zlib.compress(bytes(raw), 9) + + def chunk(typ: bytes, data: bytes) -> bytes: + return (struct.pack(">I", len(data)) + typ + data + + struct.pack(">I", zlib.crc32(typ + data) & 0xffffffff)) + + ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0) # RGB, 8-bit + png = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", comp) + chunk(b"IEND", b"")) + Path(path).write_bytes(png) + + +def main() -> int: + p = argparse.ArgumentParser(description="Live new-model acceptance harness") + p.add_argument("--model", required=True, help="preset id (must exist in /v1/models)") + p.add_argument("--proxy", default="http://localhost:8001") + p.add_argument("--router", default="http://127.0.0.1:8002") + p.add_argument("--vram-ceiling", type=int, default=24000) + p.add_argument("--ctx-probe-tokens", type=int, default=32000) + p.add_argument("--full-ctx", action="store_true") + p.add_argument("--image", default=None) + p.add_argument("--min-decode-tps", type=float, default=15.0) + p.add_argument("--skip", default="") + args = p.parse_args() + return 0 if Runner(args).run() else 1 + + +if __name__ == "__main__": + sys.exit(main())