From b4fd077b70279324617f6c180dc11393f529ba06 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:34:29 -0400 Subject: [PATCH 1/9] feat: add Kimi-K3 MXFP4 B200 aggregated TP8xPP2 Dynamo-vLLM agentic recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (no expert parallelism) for the agentic-coding trace replay. Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, fastsafetensors load format, kimi_k3 tool-call/reasoning parsers. Model pre-staged at /lustre/fsw/models/Kimi-K3; launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay, and the agentic cache default_mounts used by the GB200/GB300 agentic paths. 中文:新增 Kimi-K3 MXFP4 B200 聚合式 TP8xPP2 Dynamo-vLLM 智能体编码基准测试配方 (2 节点 / 16 GPU,纯 TP,不启用专家并行(EP))。使用专用 bring-up 镜像 vllm/vllm-openai:kimi-k3(VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1、 fastsafetensors 加载格式、kimi_k3 工具调用/推理解析器)。模型已预置于 /lustre/fsw/models/Kimi-K3;启动器 launch_b200-dgxc.sh 增加 kimik3/fp4 模型路径映射、智能体配方覆盖及智能体缓存挂载。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 109 ++++++++++++++++++ configs/nvidia-master.yaml | 43 +++++++ perf-changelog.yaml | 11 ++ runners/launch_b200-dgxc.sh | 28 +++++ 4 files changed, 191 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml new file mode 100644 index 0000000000..2ded17c2fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -0,0 +1,109 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0.dev20260618" + +dynamo: + install: true + wheel: "1.3.0.dev20260618" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.95 + no-enable-flashinfer-autotune: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # Full native 1M context: the AgentX trace carries multi-hundred-k-token + # requests, and K3's KDA layers keep per-token KV small (only the 24 + # gated-MLA layers hold cache). Cap prefill chunks so a single long + # request cannot OOM a pipeline stage; let vLLM pick max-num-seqs. + max-model-len: 1048576 + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36ae816809..bf1cb716d5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8325,3 +8325,46 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: tp: 16 ep: 16 dp-attn: true + +# Kimi-K3 MXFP4 B200 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes / 16 +# GPUs), agentic bring-up. The native MXFP4 checkpoint (2.8T total params, +# ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense +# and PP2 splits layers. Plain TP (NOT TEP): ep 1, no expert parallelism — +# the 896 routed experts are TP-sharded within each pipeline stage. Node +# count = tp*pp/gpus_per_node = 8*2/8 = 2. Aggregated (prefill num-worker 1 + +# decode num-worker 0, RECIPES.md section 5) — the single worker serves both +# phases, so no P/D KV transfer. Dedicated kimi-k3 vLLM bring-up image with +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 and the kimi_k3 tool-call/reasoning +# parsers. +# Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +kimik3-fp4-b200-dynamo-vllm-agentic: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [8, 16, 32] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts the 16 GPUs only once. + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c9..0a4e40426b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5116,3 +5116,14 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic + description: + - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" + - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" + - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..e166dbb595 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -72,6 +72,10 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then # NVFP4 checkpoint, pre-staged on the b200-dgxc scratch tree. export MODEL_PATH="/scratch/fsw/models/MiniMax-M3-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" +elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + # Native MXFP4 checkpoint, pre-staged on the SRE-managed Lustre tree. + export MODEL_PATH="/lustre/fsw/models/Kimi-K3" + export SRT_SLURM_MODEL_PREFIX="kimik3" else echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION" echo "Available models under /lustre/fsw/models:" @@ -107,6 +111,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 + # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so + # an upstream stub directory is merged rather than nested). + if [[ $MODEL_PREFIX == "kimik3" ]]; then + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic + fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -207,6 +218,22 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" + # Agentic runs bind-mount two persistent caches into every worker + # container (Lustre, shared across nodes): aiperf's content-addressed + # dataset mmap cache and the HF hub cache holding the trace dataset + # download. The container-side paths are referenced by the agentic + # recipes' benchmark.env (AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache, + # HF_HUB_CACHE=/hf_hub_cache). + DEFAULT_MOUNTS_BLOCK="" + if [[ "$IS_AGENTIC" == "1" ]]; then + HF_HUB_CACHE_HOST_PATH="/lustre/fsw/gharunners/hf-hub-cache" + mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" + chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true + DEFAULT_MOUNTS_BLOCK="default_mounts: + ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache + ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + fi + # Create srtslurm.yaml for srtctl (used by both frameworks) SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." @@ -234,6 +261,7 @@ containers: "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" use_exclusive_sbatch_directive: true +${DEFAULT_MOUNTS_BLOCK} EOF echo "Generated srtslurm.yaml:" From 4dbbdc8a5908a6d6177674be46a226b721be80f3 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:35:35 -0400 Subject: [PATCH 2/9] docs: link PR #2355 in changelog entry and MODELS rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:在更新日志条目与 MODELS 表格行中补充 PR #2355 链接。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 3fc822420e..31cb9df252 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index 177a4b4ba3..e71067c13b 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0a4e40426b..5a7800bd48 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,4 +5126,4 @@ - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From e8d42a73cdc7126fae0a3c1a30b8e54132dff1f1 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:48:01 -0400 Subject: [PATCH 3/9] fix: pin agentic srt-slurm to NVIDIA v1.0.36, dynamo 1.2.1, conc-8 smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork rejected the recipe (benchmark.aiperf_server_metrics: Unknown field). Switch the b200-dgxc agentic clone to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341), drop the aiperf_server_metrics field, pin dynamo wheel/router to 1.2.1 (the combination validated with v1.0.36), and reduce the bring-up to a single conc-8 smoke test. 中文:cquil11/srt-slurm-nv 分支的 srtctl 校验拒绝了配方字段 benchmark.aiperf_server_metrics(Unknown field)。将 b200-dgxc 智能体路径改用 上游 NVIDIA/srt-slurm v1.0.36(已在 #2302/#2341 验证),移除该字段,dynamo wheel/router 固定为 1.2.1,并将 bring-up 缩减为单并发(conc 8)冒烟测试。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 5 ++--- configs/nvidia-master.yaml | 6 ++++-- perf-changelog.yaml | 5 +++-- runners/launch_b200-dgxc.sh | 13 ++++++++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 2ded17c2fd..cbee00a2b5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,11 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.3.0.dev20260618" + dynamo: "1.2.1" dynamo: + wheel: "1.2.1" install: true - wheel: "1.3.0.dev20260618" slurm: time_limit: "8:00:00" @@ -94,7 +94,6 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bf1cb716d5..82c69638c8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,14 +8344,16 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } + router: { name: dynamo-router, version: "1.2.1" } multinode: true disagg: false scenarios: agentic-coding: - search-space: + # Single-concurrency smoke test for the bring-up; widen the conc curve + # once the topology is proven green. - spec-decoding: none - conc-list: [8, 16, 32] + conc-list: [8] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5a7800bd48..b8b11fa508 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,7 +5123,8 @@ - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" - - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" - - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index e166dbb595..1dd4e19d14 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -109,14 +109,21 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + # Agentic recipes use NVIDIA/srt-slurm v1.0.36, the upstream version + # validated in InferenceX PR #2302/#2341 for the vLLM agentic path + # (BenchmarkType.CUSTOM + benchmark.command/env, DynamoConfig.wheel, + # srun_options propagation, per-node DP, matching Dynamo health + # counts). Keep it pinned so sweeps are reproducible. Note the older + # cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork previously + # cloned here rejects newer recipe schema fields. + git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so # an upstream stub directory is merged rather than nested). if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic + recipes/vllm/kimi-k3/agentic || exit 1 fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" From c1e2a568101d37556da96f9fd80c262f30d2dabb Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:54:05 -0400 Subject: [PATCH 4/9] fix: drop OpenAI-frontend tool-choice flags from dynamo-vllm worker args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dynamo-vllm worker entrypoint rejected --enable-auto-tool-choice --tool-call-parser kimi_k3 (unrecognized arguments; different arg parser than vllm serve). Chat parsing happens at the dynamo frontend — same convention as the DSv4 GB300 agentic recipes. Keep --reasoning-parser kimi_k3 (accepted by the worker). Also drop the explicit max-model-len and let vLLM derive the native 1M window from the model config, mirroring the agentic recipe convention. 中文:dynamo-vllm worker 入口不接受 --enable-auto-tool-choice 与 --tool-call-parser kimi_k3(unrecognized arguments,与 vllm serve 的参数解析器 不同),聊天解析由 dynamo 前端处理,与 DSv4 GB300 智能体配方约定一致;保留 worker 可接受的 --reasoning-parser kimi_k3。同时移除显式 max-model-len, 由 vLLM 从模型配置推导原生 1M 上下文窗口。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 19 ++++++++++++------- perf-changelog.yaml | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index cbee00a2b5..dcb1ec7afa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -76,14 +76,19 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 + # Note: --enable-auto-tool-choice / --tool-call-parser are NOT set on + # the worker. The dynamo-vllm worker entrypoint doesn't accept them + # (different arg parser than `vllm serve`; first sweep attempt died with + # "unrecognized arguments"). Chat parsing happens at the dynamo + # frontend, not at the worker — same convention as the DSv4 GB300 + # agentic recipes. --reasoning-parser IS accepted by the worker. reasoning-parser: kimi_k3 - # Full native 1M context: the AgentX trace carries multi-hundred-k-token - # requests, and K3's KDA layers keep per-token KV small (only the 24 - # gated-MLA layers hold cache). Cap prefill chunks so a single long - # request cannot OOM a pipeline stage; let vLLM pick max-num-seqs. - max-model-len: 1048576 + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. max-num-batched-tokens: 8192 sbatch_directives: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b8b11fa508..61b86f3db3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,8 +5122,8 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" - - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" + - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From ef35fd1f030fd821fa31436d9a26f425e77aaeec Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:20:42 -0400 Subject: [PATCH 5/9] fix: try dynamo wheel 1.2.0.dev20260426 for Kimi-K3 frontend tokenizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third sweep attempt: the engine loaded and served (TP8xPP2 healthy in ~14 min), but dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type 'kimi_k3' (supported: kimi, kimi_k2, kimi_k25, deepseek_v3), so the model never registered and all chat completions returned 404, aborting the AgentX warmup. Switch to the 1.2.0.dev20260426 wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Upstream published v1.4.0-kimi-k3-dev.1 (2026-07-27) as the day-zero K3 build if this wheel also lacks support. 中文:第三次扫描中引擎已成功加载并提供服务(TP8xPP2 约 14 分钟就绪),但 dynamo 1.2.1 的 rust 前端分词器不支持 Kimi-K3 的 tiktoken model_type 'kimi_k3',模型未能注册,所有请求返回 404,AgentX 预热中止。改用 DSv4 GB300/B200 Dynamo-vLLM 配方所用的 1.2.0.dev20260426 wheel;如仍不支持, 上游已于 2026-07-27 发布 day-zero 构建 v1.4.0-kimi-k3-dev.1。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index dcb1ec7afa..d36158073f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,11 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.2.1" + dynamo: "1.2.0.dev20260426" dynamo: - wheel: "1.2.1" install: true + wheel: "1.2.0.dev20260426" slurm: time_limit: "8:00:00" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 82c69638c8..3dd5261b27 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,7 +8344,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "1.2.0.dev20260426" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 61b86f3db3..2001751894 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5124,7 +5124,7 @@ - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" + - "Dynamo 1.2.0.dev20260426 (ai-dynamo wheel + dynamo-router), the wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Note: dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registers and all chat completions 404 (third sweep attempt); upstream published v1.4.0-kimi-k3-dev.1 on 2026-07-27 as the day-zero K3 build if this wheel also lacks kimi_k3 tiktoken support" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From be6c56e6a1a6717d3fc018570e21471f496c7138 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:26:38 -0400 Subject: [PATCH 6/9] fix: pin dynamo to day-zero Kimi-K3 commit, restore kimi_k3 parser flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin dynamo to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ("feat: Added support for Kimi-K3", tag v1.4.0-kimi-k3-dev.1) via srt-slurm's hash-cached source install: it adds the kimi_k3 tiktoken tokenizer to the rust frontend (dynamo <=1.2.1 404s every request because the model never registers) and accepts the kimi_k3 tool-call/reasoning parser worker args, so restore --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3. 中文:将 dynamo 固定到 day-zero Kimi-K3 提交 ba83080("feat: Added support for Kimi-K3",标签 v1.4.0-kimi-k3-dev.1),通过 srt-slurm 的哈希缓存源码 安装:该提交为 rust 前端新增 kimi_k3 tiktoken 分词器(dynamo <=1.2.1 因模型 无法注册而全部返回 404),worker 亦支持 kimi_k3 解析器参数,故恢复 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 21 ++++++++++++------- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index d36158073f..22362d730c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,16 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.2.0.dev20260426" + dynamo: "1.4.0-kimi-k3-dev.1" dynamo: install: true - wheel: "1.2.0.dev20260426" + # Day-zero Kimi-K3 dynamo ("feat: Added support for Kimi-K3", tag + # v1.4.0-kimi-k3-dev.1 == this commit): adds the kimi_k3 tiktoken tokenizer + # to the rust frontend (dynamo <=1.2.1 only knows kimi/kimi_k2/kimi_k25/ + # deepseek_v3, so the model never registers and every request 404s) and the + # kimi_k3 tool-call/reasoning parser worker args. + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" slurm: time_limit: "8:00:00" @@ -76,12 +81,12 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # Note: --enable-auto-tool-choice / --tool-call-parser are NOT set on - # the worker. The dynamo-vllm worker entrypoint doesn't accept them - # (different arg parser than `vllm serve`; first sweep attempt died with - # "unrecognized arguments"). Chat parsing happens at the dynamo - # frontend, not at the worker — same convention as the DSv4 GB300 - # agentic recipes. --reasoning-parser IS accepted by the worker. + # kimi_k3 parsers: the day-zero K3 dynamo build pinned above accepts + # these on the worker. Dynamo <=1.2.1 rejected --enable-auto-tool-choice + # / --tool-call-parser as unrecognized arguments (different arg parser + # than `vllm serve`). + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3dd5261b27..89bf2fad2b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,7 +8344,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } + router: { name: dynamo-router, version: "1.4.0-kimi-k3-dev.1" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2001751894..c687f6da00 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,9 +5122,9 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code (the kimi_k3 parser worker args require the day-zero K3 dynamo below; dynamo <=1.2.1 rejects the tool-choice flags as unrecognized arguments)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo 1.2.0.dev20260426 (ai-dynamo wheel + dynamo-router), the wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Note: dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registers and all chat completions 404 (third sweep attempt); upstream published v1.4.0-kimi-k3-dev.1 on 2026-07-27 as the day-zero K3 build if this wheel also lacks kimi_k3 tiktoken support" + - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From c6917e640fb34327cd684189f5741ae475ada231 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:32:17 -0400 Subject: [PATCH 7/9] fix: use dynamo namespaced --dyn-* kimi_k3 parser args on the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the vLLM OpenAI-frontend spellings (--enable-auto-tool-choice / --tool-call-parser) with dynamo's namespaced worker args: --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3. 中文:将 vLLM OpenAI 前端风格参数(--enable-auto-tool-choice / --tool-call-parser)替换为 dynamo 命名空间的 worker 参数: --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 13 +++++++------ perf-changelog.yaml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 22362d730c..ab4c5314b0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -81,13 +81,14 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # kimi_k3 parsers: the day-zero K3 dynamo build pinned above accepts - # these on the worker. Dynamo <=1.2.1 rejected --enable-auto-tool-choice - # / --tool-call-parser as unrecognized arguments (different arg parser - # than `vllm serve`). - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 + # kimi_k3 parsers via dynamo's namespaced worker args (--dyn-*), which + # the day-zero K3 dynamo build pinned above provides. The vLLM + # OpenAI-frontend spellings (--enable-auto-tool-choice / + # --tool-call-parser) are rejected by the dynamo worker entrypoint as + # unrecognized arguments (different arg parser than `vllm serve`). + dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c687f6da00..43fd4df391 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,7 +5122,7 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code (the kimi_k3 parser worker args require the day-zero K3 dynamo below; dynamo <=1.2.1 rejects the tool-choice flags as unrecognized arguments)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via dynamo's namespaced worker args --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3 (the vLLM OpenAI-frontend spellings --enable-auto-tool-choice/--tool-call-parser are rejected by the dynamo worker entrypoint as unrecognized arguments)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" From 288349f0b49fbf72a7c2abc904c86c345a240cc8 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:44:16 -0400 Subject: [PATCH 8/9] =?UTF-8?q?feat:=20parser-flag=20experiment=20B=20?= =?UTF-8?q?=E2=80=94=20plain=20vLLM=20tool-call-parser=20spellings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variant of the Kimi-K3 B200 agg TP8xPP2 agentic bring-up (#2355) using --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 on the worker. Sibling experiments try dynamo's --dyn-* namespaced args and no parser flags; whichever goes green first merges. 中文:Kimi-K3 B200 聚合式 TP8xPP2 智能体 bring-up(#2355)的解析器参数实验 变体 B:worker 使用 --tool-call-parser kimi_k3 --reasoning-parser kimi_k3。 兄弟实验分别尝试 dynamo --dyn-* 命名空间参数与不带解析器参数;先通过者合并。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 12 +++++------- perf-changelog.yaml | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index ab4c5314b0..7109b14097 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -81,14 +81,12 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # kimi_k3 parsers via dynamo's namespaced worker args (--dyn-*), which - # the day-zero K3 dynamo build pinned above provides. The vLLM - # OpenAI-frontend spellings (--enable-auto-tool-choice / - # --tool-call-parser) are rejected by the dynamo worker entrypoint as - # unrecognized arguments (different arg parser than `vllm serve`). - dyn-tool-call-parser: kimi_k3 + # kimi_k3 parsers via the plain vLLM spellings (parser-flag experiment + # variant B — sibling PRs try the --dyn-* namespaced args and no parser + # flags). Note dynamo <=1.2.1's worker rejected --tool-call-parser as + # unrecognized; the day-zero K3 dynamo pinned above may accept it. + tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 43fd4df391..06341f4bf1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,7 +5122,7 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via dynamo's namespaced worker args --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3 (the vLLM OpenAI-frontend spellings --enable-auto-tool-choice/--tool-call-parser are rejected by the dynamo worker entrypoint as unrecognized arguments)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via the plain vLLM spellings --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 (parser-flag experiment variant B; sibling agentic-experiment PRs try dynamo's --dyn-* namespaced args and no parser flags)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" From 2fe84a8d149d09e5497978d316b5851980846d9e Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:46:15 -0400 Subject: [PATCH 9/9] docs: point changelog and MODELS rows at experiment PR #2357 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:将更新日志条目与 MODELS 表格行链接指向实验 PR #2357。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 31cb9df252..00936552c9 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2357](https://github.com/SemiAnalysisAI/InferenceX/pull/2357)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index e71067c13b..7be80603bd 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2357](https://github.com/SemiAnalysisAI/InferenceX/pull/2357)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 06341f4bf1..4980393654 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5127,4 +5127,4 @@ - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2357