[TRTLLM-14764][feat] trtllm-serve: Kimi K3 API compliance for the Kimi Vendor Verifier (KVV)

docs/source/deployment-guide/deployment-guide-for-kimi-k3-on-trtllm.md

@@ -210,6 +210,16 @@ These options are set within the YAML file passed to `trtllm-serve` via the `--c
 
 * **Description:** Required to load the Kimi K3 configuration and tokenizer code shipped with the checkpoint.
 
+### Kimi-Specific API Behavior and Environment Variables
+
+When the served model is Kimi K3, `trtllm-serve` applies Kimi/Moonshot API semantics on `/v1/chat/completions` (all of these are exercised by Moonshot's [Kimi Vendor Verifier](https://www.kimi.com/blog/kimi-vendor-verifier.html)):
+
+* **Request extensions:** the `thinking` object (`{"type": "enabled"|"disabled", "keep": "all", "effort": "low"|"high"|"max"}`), `reasoning_effort` values `"low"`, `"high"`, `"max"`, and `"none"` (an explicit `thinking` object takes precedence), `tool_choice: "required"`, message-level (dynamic) tools declared on system messages, and `response_format` `json_object`/`json_schema` (the `json_schema` wrapper must carry a non-empty `name` and a `schema` object). These map onto the checkpoint chat template's native control messages; explicit `chat_template_kwargs` always win.
+* **Streaming usage:** `usage` is reported in the final streaming chunk even when the client does not send `stream_options` (Kimi API parity).
+* **Prompt-token accounting:** reported `usage.prompt_tokens` excludes the trailing 3-token generation channel opener, matching Kimi's reference accounting; the model still consumes the full rendered prompt.
+* **`TRTLLM_KIMI_PARAM_POLICY`** (default `0`, off): when set to `1`, enforces Kimi's immutable sampling parameters — `top_p` pinned to 0.95 (unset or the OpenAI default `1.0` are coerced to 0.95; other values are rejected with HTTP 400), `presence_penalty`/`frequency_penalty` 0, `n` 1, and `temperature` bounded to [0, 1]. Off by default so existing deployments keep accepting the requests they accept today; a Kimi-Vendor-Verifier certification run must set it to `1` (the KVV params suite requires the rejections).
+* **`TRTLLM_KIMI_K3_STRICT_TOOL_GRAMMAR`** (default `0`): opt-in constrained decoding for tools with `strict: true` (requires `guided_decoding_backend: xgrammar`). Disabled by default pending the investigation of a device-side assert observed under sustained concurrent guided load; strict tools otherwise fall back to warn-and-continue.
+
 ## Testing API Endpoint
 
 The server (the OpenAI-compatible REST endpoint) runs on the rank-0 node, listening on port `8000`. Send requests to that node's hostname or IP; `localhost` only works from the rank-0 node itself.

tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py

@@ -443,6 +443,9 @@ class KimiK3InputProcessor(KimiK25InputProcessor):
             "image": "<|kimi_image_placeholder|>",
         },
         placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
+        # K3's reference renderer concatenates content parts with no
+        # separator; the default "\n" join skews prompt-token parity.
+        placeholders_separator="",
     ),
 )
 class KimiK3ForConditionalGeneration(KimiK25ForConditionalGeneration):

tensorrt_llm/executor/postproc_worker.py

@@ -32,6 +32,12 @@
 class PostprocArgs:
     first_iteration: bool = True
     num_prompt_tokens: Optional[int] = None
+    # Subtracted from num_prompt_tokens when reporting usage. Lets servers
+    # exclude generation-prompt stub tokens the vendor accounting treats as
+    # pending output (e.g. Kimi K3's 3-token channel opener) without changing
+    # what the model sees. num_prompt_tokens itself is overwritten by the
+    # executor on the postproc-worker path, so the offset must be separate.
+    num_prompt_tokens_offset: int = 0
     tokenizer: Optional[TransformersTokenizer] = None
     ctx_usage: Optional[Any] = None
 

tensorrt_llm/inputs/utils.py

@@ -351,11 +351,15 @@ class ConversationMessage(TypedDict, total=False):
             This is used by `interleave_mm_placeholders` to insert multimodal placeholders at the
             correct positions, and to reconstruct the OpenAI-style content list for templates that
             handle media natively.
+        tools: Message-level (dynamic) tool declarations carried on system messages. Only
+            populated for models whose python-renderer chat template consumes them (kimi_k3);
+            absent for other models.
     """
     role: str
     content: str
     media: List[MultimodalData]
     content_parts: List[Union[str, dict]]
+    tools: List[Dict[str, Any]]
 
 
 class MultimodalDataTracker:

tensorrt_llm/serve/chat_utils.py

@@ -247,7 +247,9 @@ def parse_chat_message_content_parts(
 
 def parse_chat_message_content(
         message: ChatCompletionMessageParam,
-        mm_data_tracker: MultimodalDataTracker) -> ConversationMessage:
+        mm_data_tracker: MultimodalDataTracker,
+        lenient_tool_call_arguments: bool = False,
+        keep_message_tools: bool = False) -> ConversationMessage:
     """Parse the content of a chat message."""
     role = message["role"]
     content = message.get("content")
@@ -265,9 +267,17 @@ def parse_chat_message_content(
         mm_data_tracker,
     )
     if role == "assistant":
-        result.update(_parse_assistant_message_content(message))
+        result.update(
+            _parse_assistant_message_content(message,
+                                             lenient_tool_call_arguments))
     elif role == "tool":
         result.update(_parse_tool_message_content(message))
+    elif keep_message_tools and role == "system" and message.get("tools"):
+        # Message-level (dynamic) tool declarations: python-renderer chat
+        # templates (kimi_k3) render these as an in-conversation tool
+        # declare block at this message's position. Other models keep the
+        # pre-existing behavior of silently ignoring the key.
+        result["tools"] = message["tools"]
     return result
 
 
@@ -329,7 +339,10 @@ def _validate_fallback_tool_calls(
     return [tc.model_dump(exclude_unset=True) for tc in validated]
 
 
-def _normalize_tool_call_arguments(index: int, item: Any) -> dict[str, Any]:
+def _normalize_tool_call_arguments(index: int,
+                                   item: Any,
+                                   lenient_json: bool = False
+                                   ) -> dict[str, Any]:
     """Normalize `function.arguments` to the internal dict form."""
     item = dict(item)
     item["function"] = dict(item["function"])
@@ -341,6 +354,11 @@ def _normalize_tool_call_arguments(index: int, item: Any) -> dict[str, Any]:
         try:
             arguments = json.loads(arguments)
         except json.JSONDecodeError as e:
+            if lenient_json:
+                # Keep the raw string: python-renderer templates (kimi_k3)
+                # normalize unparsable arguments themselves and render them
+                # verbatim as a JSON block, matching the reference tokenizer.
+                return item
             raise ValueError(
                 f"tool_calls[{index}].function.arguments must be valid JSON."
             ) from e
@@ -357,7 +375,9 @@ def _normalize_tool_call_arguments(index: int, item: Any) -> dict[str, Any]:
     return item
 
 
-def _parse_fallback_tool_calls(tool_calls: list[Any]) -> list[dict[str, Any]]:
+def _parse_fallback_tool_calls(
+        tool_calls: list[Any],
+        lenient_json: bool = False) -> list[dict[str, Any]]:
     """Parse raw tool-call lists accepted only by the tau2-bench fallback path.
 
     `openai_server.py` first attempts strict OpenAI request validation. Some tau2-bench requests
@@ -369,13 +389,15 @@ def _parse_fallback_tool_calls(tool_calls: list[Any]) -> list[dict[str, Any]]:
     """
     tool_calls = _validate_fallback_tool_calls(tool_calls)
     return [
-        _normalize_tool_call_arguments(index, item)
+        _normalize_tool_call_arguments(index, item, lenient_json)
         for index, item in enumerate(tool_calls)
     ]
 
 
 # Adapted from: https://github.com/vllm-project/vllm/blob/4574d48bab9c4e38b7c0a830eeefc8f0980e8c58/vllm/entrypoints/chat_utils.py#L1406
-def _parse_assistant_message_content(message: Dict[str, Any]) -> Dict[str, Any]:
+def _parse_assistant_message_content(message: Dict[str, Any],
+                                     lenient_json: bool = False
+                                     ) -> Dict[str, Any]:
     result = {}
     # Include reasoning if present for interleaved thinking.
     reasoning_content = message.get("reasoning")
@@ -387,13 +409,14 @@ def _parse_assistant_message_content(message: Dict[str, Any]) -> Dict[str, Any]:
     tool_calls = message.get("tool_calls")
     if tool_calls is not None:
         if isinstance(tool_calls, list):
-            result["tool_calls"] = _parse_fallback_tool_calls(tool_calls)
+            result["tool_calls"] = _parse_fallback_tool_calls(
+                tool_calls, lenient_json)
         else:
             # The strict parse path delivers tool_calls as a single-use Pydantic `ValidatorIterator`
             # of already-validated OpenAI tool calls, so only materialize and normalize arguments.
             tool_calls = list(tool_calls)
             result["tool_calls"] = [
-                _normalize_tool_call_arguments(index, item)
+                _normalize_tool_call_arguments(index, item, lenient_json)
                 for index, item in enumerate(tool_calls)
             ]
 
@@ -478,7 +501,13 @@ def parse_chat_messages_coroutines(
         content_format = ContentFormat.STRING
 
     for msg in messages:
-        parsed_msg = parse_chat_message_content(msg, mm_data_tracker)
+        # kimi_k3's reference renderer keeps unparsable tool-call argument
+        # strings verbatim; other templates expect the strict dict contract.
+        parsed_msg = parse_chat_message_content(
+            msg,
+            mm_data_tracker,
+            lenient_tool_call_arguments=(model_type == "kimi_k3"),
+            keep_message_tools=(model_type == "kimi_k3"))
         conversation.append(parsed_msg)
 
         # Track placeholders added for this message only.

tensorrt_llm/serve/harmony_adapter.py

@@ -1977,12 +1977,17 @@ def _create_usage_info(num_prompt_tokens,
 
 
 def maybe_transform_reasoning_effort(
-    reasoning_effort: ReasoningEffort | Literal["low", "medium", "high"] | None
+    reasoning_effort: ReasoningEffort
+    | Literal["low", "medium", "high", "max", "none"] | None
 ) -> ReasoningEffort | None:
     str_to_effort = {
         "low": ReasoningEffort.LOW,
         "medium": ReasoningEffort.MEDIUM,
-        "high": ReasoningEffort.HIGH
+        "high": ReasoningEffort.HIGH,
+        # Kimi-style efforts accepted by the shared request schema; map to
+        # the nearest harmony level ("none" means no explicit effort).
+        "max": ReasoningEffort.HIGH,
+        "none": None,
     }
     if reasoning_effort and not isinstance(reasoning_effort, ReasoningEffort):
         return str_to_effort[reasoning_effort]

tensorrt_llm/serve/openai_protocol.py

@@ -409,6 +409,7 @@ class EmbeddingResponse(OpenAIBaseModel):
 def _response_format_to_guided_decoding_params(
     response_format: Optional[ResponseFormat],
     reasoning_parser: Optional[str] = None,
+    chat_template_kwargs: Optional[Dict[str, Any]] = None,
 ) -> Optional[GuidedDecodingParams]:
     if response_format is None:
         guided_decoding_params = None
@@ -490,6 +491,32 @@ def _response_format_to_guided_decoding_params(
             "stop_after_first":
             True,
         }
+    elif reasoning_parser == "kimi_k3":
+        # K3 XTML: the generation prompt already ends inside the channel the
+        # model starts in. In thinking mode (the default) the response channel
+        # opens mid-generation, so trigger the user constraint on it
+        # (mirrors the gpt_oss final-channel handling). In non-thinking mode
+        # the prompt ends inside <|open|>response<|sep|>, the trigger would
+        # never be generated, and the raw grammar applies from the first
+        # generated token instead.
+        thinking = (chat_template_kwargs or {}).get("thinking",
+                                                    True) is not False
+        if not thinking:
+            return guided_decoding_params
+        stag_format = {
+            "type":
+            "triggered_tags",
+            "triggers": ["<|open|>response<|sep|>"],
+            "tags": [
+                {
+                    "begin": "<|open|>response<|sep|>",
+                    "content": content,
+                    "end": "<|close|>response<|sep|>",
+                },
+            ],
+            "stop_after_first":
+            True,
+        }
     else:
         # Force thinking and then trigger user constraint
         parser = ReasoningParserFactory.create_reasoning_parser(
@@ -793,7 +820,24 @@ class ReasoningAssistantMessage(ChatCompletionAssistantMessageParam):
     reasoning_content: Optional[str]
 
 
-ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
+class DynamicToolsSystemMessageParam(TypedDict, total=False):
+    """System message carrying message-level (dynamic) tool declarations.
+
+    Kimi-style templates render such messages as an in-conversation tool
+    declare block. Must come first in `ChatCompletionMessageParam`: the
+    stock OpenAI system-message TypedDict otherwise wins smart-union scoring
+    and silently drops the `tools` key.
+    """
+    __pydantic_config__ = ConfigDict(extra="allow")  # type: ignore
+
+    role: Required[Literal["system"]]
+    tools: Required[List[dict]]
+    content: Union[str, List[ChatCompletionContentPartParam], None]
+    name: str
+
+
+ChatCompletionMessageParam = Union[DynamicToolsSystemMessageParam,
+                                   OpenAIChatCompletionMessageParam,
                                    CustomChatCompletionMessageParam,
                                    ReasoningAssistantMessage]
 
@@ -879,6 +923,19 @@ class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
     type: Literal["function"] = "function"
 
 
+class ChatCompletionThinkingParam(OpenAIBaseModel):
+    """Kimi/Moonshot `thinking` extension controlling reasoning output.
+
+    `keep` is fixed to `"all"` when thinking is enabled and ignored when
+    disabled; `effort` is only meaningful when enabled. An explicit
+    `effort` wins over a request-level `reasoning_effort`, which applies
+    only when `effort` is absent (and never for a disabled request).
+    """
+    type: Literal["enabled", "disabled"] = "enabled"
+    keep: Optional[Literal["all"]] = None
+    effort: Optional[Literal["low", "high", "max"]] = None
+
+
 class ChatCompletionRequest(OpenAIBaseModel):
     # Ordered by official OpenAI API documentation
     # https://platform.openai.com/docs/api-reference/chat/create
@@ -906,17 +963,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
     temperature: Optional[float] = None
     top_p: Optional[float] = None
     tools: Optional[List[ChatCompletionToolsParam]] = None
-    tool_choice: Optional[Union[Literal["none", "auto"],
+    tool_choice: Optional[Union[Literal["none", "auto", "required"],
                                 ChatCompletionNamedToolChoiceParam]] = "none"
     user: Optional[str] = None
     reasoning_effort: Optional[ReasoningEffort | Literal[
-        "low", "medium", "high"]] = Field(
+        "low", "medium", "high", "max", "none"]] = Field(
             default=ReasoningEffort.LOW,
             description=(
                 "The level of reasoning effort to use. Controls how much "
                 "reasoning is shown in the model's response. Options: "
-                "'low', 'medium', 'high'."),
+                "'low', 'medium', 'high' (harmony/gpt-oss), plus 'max' and "
+                "'none' for models with Kimi-style thinking control."),
         )
+    # Kimi/Moonshot extension: structured control of reasoning output. The
+    # serving layer maps it into the chat-template kwargs for models whose
+    # template understands it (e.g. kimi_k3); other models ignore it.
+    thinking: Optional[ChatCompletionThinkingParam] = None
     thinking_token_budget: Optional[int] = None
     prompt_ignore_length: Optional[int] = 0
 
@@ -1086,7 +1148,9 @@ def to_sampling_params(self,
             spaces_between_special_tokens=self.spaces_between_special_tokens,
             truncate_prompt_tokens=self.truncate_prompt_tokens,
             guided_decoding=_response_format_to_guided_decoding_params(
-                self.response_format, reasoning_parser=reasoning_parser),
+                self.response_format,
+                reasoning_parser=reasoning_parser,
+                chat_template_kwargs=self.chat_template_kwargs),
             thinking_token_budget=self.thinking_token_budget,
 
             # logits_bias
@@ -1112,14 +1176,47 @@ def validate_stream_options(cls, values):
     @model_validator(mode="before")
     @classmethod
     def check_tool_choice(cls, data):
-        if "tool_choice" not in data and data.get("tools"):
+        if not isinstance(data, dict):
+            return data
+        has_dynamic_tools = any(
+            isinstance(msg, dict) and msg.get("role") == "system"
+            and msg.get("tools") for msg in data.get("messages") or [])
+        if "tool_choice" not in data and (data.get("tools")
+                                          or has_dynamic_tools):
             data["tool_choice"] = "auto"
-        if "tool_choice" in data and data["tool_choice"] != "none":
-            if "tools" not in data or data["tools"] is None:
+        # "none" and "auto" are meaningful without tools. "required" needs a
+        # non-empty tool set — request-level or message-level (dynamic)
+        # tools both count. A named function must be a request-level tool.
+        if "tool_choice" in data and data["tool_choice"] not in ("none",
+                                                                 "auto"):
+            satisfied = data.get("tools") or (data["tool_choice"] == "required"
+                                              and has_dynamic_tools)
+            if not satisfied:
                 raise ValueError(
                     "When using `tool_choice`, `tools` must be set.")
         return data
 
+    @model_validator(mode="before")
+    @classmethod
+    def check_message_tools_role(cls, data):
+        """Message-level tool declarations ride on system messages only.
+
+        Union validation strips unknown keys from non-system messages, so
+        this raw-payload validator is the only layer that can reject the
+        misuse loudly instead of silently dropping the client's tools. The
+        rest of the dynamic-tools contract is model-specific and validated
+        in the serving layer (kimi_k3 only).
+        """
+        if not isinstance(data, dict):
+            return data
+        for message in data.get("messages") or []:
+            if (isinstance(message, dict) and message.get("tools") is not None
+                    and message.get("role") != "system"):
+                raise ValueError(
+                    "Message-level `tools` are only allowed on system "
+                    "messages.")
+        return data
+
     @model_validator(mode="before")
     @classmethod
     def check_logprobs(cls, data):