From ced99b4699ea121c04f27c4a3f2d1d93ce6e9691 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 18 Aug 2026 21:57:30 -0400 Subject: [PATCH] Add the Kimi K3 FP4 B200 agentic sweep on Mooncake Two-node TP8 x DP2 x EP16 with DCP8, offloading every point to a Mooncake external tier. Concurrency 1 to 8 drafts with DSpark level 7, 10 to 16 with level 3, and 18 to 24 runs unspeculated, with the golden acceptance length injected only for throughput so eval verifies against the target model. Runs on the b200-nscale cluster, rendered by upstream srt-slurm. --- ... agg-b200-tp8dp2-mooncake-c1-agentic.yaml} | 45 +++--- ...agg-b200-tp8dp2-mooncake-c12-agentic.yaml} | 44 +++--- ...agg-b200-tp8dp2-mooncake-c32-agentic.yaml} | 42 ++++-- ... agg-b200-tp8dp2-mooncake-c4-agentic.yaml} | 47 ++++--- .../agg-b200-tp8pp2-mooncake-c72-agentic.yaml | 128 ++++++++++++++++++ configs/nvidia-master.yaml | 72 ++++++---- configs/runners.yaml | 13 ++ perf-changelog.yaml | 8 ++ runners/launch_b200-nscale-slurm.sh | 24 +++- 9 files changed, 332 insertions(+), 91 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml => agg-b200-tp8dp2-mooncake-c1-agentic.yaml} (64%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp8dp2-latency-dspark-agentic.yaml => agg-b200-tp8dp2-mooncake-c12-agentic.yaml} (65%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml => agg-b200-tp8dp2-mooncake-c32-agentic.yaml} (65%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml => agg-b200-tp8dp2-mooncake-c4-agentic.yaml} (62%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml similarity index 64% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml index a3ae8342b..66463a748 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml @@ -1,18 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-eval-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c1-agentic" -# Real-verification eval counterpart to the B200 DEP offload throughput recipe. -# It keeps TP8/DP2/EP16 and the CPU KV connector unchanged; only DSpark -# acceptance changes from synthetic golden AL to target-verified block. model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -42,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -68,23 +82,22 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 language-model-only: true - max-num-seqs: 8 + max-num-seqs: 2 max-num-batched-tokens: 8192 - # Eval must verify every draft against real target logits. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -94,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml similarity index 65% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml index 7dc3d4979..581992087 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml @@ -1,19 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c12-agentic" -# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates -# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. -# This keeps FlashInfer fused collectives node-local while using shipped vLLM. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -43,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -69,12 +82,13 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true @@ -83,9 +97,7 @@ backend: language-model-only: true max-num-seqs: 8 max-num-batched-tokens: 8192 - # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval - # selects the paired real-verification recipe instead. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -95,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml similarity index 65% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml index f56268252..8f8eb4884 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml @@ -1,18 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-eval-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c32-agentic" -# Real-verification eval counterpart to the B200 DEP throughput recipe. -# It keeps TP8/DP2/EP16 and target settings unchanged; only DSpark acceptance -# changes from synthetic golden AL to target-verified block. model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -42,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -68,12 +82,13 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true @@ -82,8 +97,7 @@ backend: language-model-only: true max-num-seqs: 8 max-num-batched-tokens: 8192 - # Eval must verify every draft against real target logits. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -93,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml similarity index 62% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml index 0119d7921..ed49a6046 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml @@ -1,19 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c4-agentic" -# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates -# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. -# This keeps FlashInfer fused collectives node-local while using shipped vLLM. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -43,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -69,24 +82,22 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 language-model-only: true - max-num-seqs: 8 + max-num-seqs: 16 max-num-batched-tokens: 8192 - # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval - # selects the paired real-verification recipe instead. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -96,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml new file mode 100644 index 000000000..de46b4584 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml @@ -0,0 +1,128 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c72-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 144 + max-num-batched-tokens: 8192 + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a99531023..7dc6834af 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8785,86 +8785,106 @@ kimik3-fp4-b200-dynamo-vllm-agentic: # Expert parallel spans all 16 ranks, so the 896 experts remain EP16 while # FlashInfer fused collectives stay inside each B200 NVSwitch domain. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9 model: moonshotai/Kimi-K3 model-prefix: kimik3 - runner: cluster:b200-dgxc + runner: cluster:b200-nscale precision: fp4 framework: dynamo-vllm multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.63 + - dram-utilization: 0.8 search-space: - # Low-latency and high-interactivity points. - spec-decoding: mtp - conc-list: [1, 2, 4] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [1] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - # Balanced medium-concurrency points. - spec-decoding: mtp - conc-list: [8, 12, 16] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [4] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - # GPU-resident throughput points around the prior c16-c32 KV cliff. - spec-decoding: mtp - conc-list: [20, 24, 28, 32] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [12] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - # CPU KV-offload crossover and capacity points. Keep the resident points - # above so the same concurrency can be compared with one variable changed. - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [8, 12, 16, 20, 24, 28, 32, 48, 64] + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [32] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [72] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/configs/runners.yaml b/configs/runners.yaml index 6f0b35e9c..6b6fdd0c7 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -232,6 +232,16 @@ labels: - h200-dgxc-slurm_13 cluster:h200-nb: - h200-nb_0 + cluster:b200-nscale: + - b200-nscale-slurm_0 + - b200-nscale-slurm_1 + - b200-nscale-slurm_2 + - b200-nscale-slurm_3 + - b200-nscale-slurm_4 + - b200-nscale-slurm_5 + - b200-nscale-slurm_6 + - b200-nscale-slurm_7 + - b200-nscale-slurm_8 cluster:b200-dgxc: - b200-dgxc_00 - b200-dgxc_01 @@ -334,6 +344,9 @@ hardware: b200-nscale: available-cpu-dram-mib: 2_063_920 gpus-per-node: 8 + cluster:b200-nscale: + available-cpu-dram-mib: 2_063_920 + gpus-per-node: 8 cluster:b300-nv: available-cpu-dram-mib: 2_964_436 gpus-per-node: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 538a800d9..357846241 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6161,3 +6161,11 @@ description: - "Update the B200 SGLang AgentX HiCache MTP DEP8 configuration." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2656 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add new B200 multi-node configs with DP2, EP16" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2618 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 9b9a1e3f9..4bf9531d0 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -43,6 +43,9 @@ if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/Kimi-K2.6-NVFP4}" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" +elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/Kimi-K3}" + export SRT_SLURM_MODEL_PREFIX="kimik3" else echo "Unsupported model prefix/precision for b200-nscale: $MODEL_PREFIX/$PRECISION" >&2 echo "Models staged under $NSCALE_MODEL_ROOT:" >&2 @@ -61,7 +64,16 @@ export SERVED_MODEL_NAME=$MODEL echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then +if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then + # Pin the tested renderer so branch movement cannot change generated rank + # commands between sweep points. + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout 217f9438 || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 +elif [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 # Pin the srt-slurm revision used by these checked-in recipes. @@ -193,6 +205,15 @@ export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" echo "Submitting job with srtctl..." echo "MODEL_PATH=$MODEL_PATH" +# An eval row may point at a committed real-verification recipe while its +# throughput row keeps synthetic golden acceptance. Only configs that set +# EVAL_CONFIG_FILE opt into this selection; all other configs keep using +# CONFIG_FILE unchanged. +if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + CONFIG_FILE="$EVAL_CONFIG_FILE" + echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" +fi + if [[ -z "$CONFIG_FILE" ]]; then echo "Error: CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings." >&2 echo "Config: MODEL_PREFIX=${MODEL_PREFIX} PRECISION=${PRECISION} FRAMEWORK=${FRAMEWORK}" >&2 @@ -213,6 +234,7 @@ inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 SRTCTL_PREFLIGHT_ARGS=() # These weights are staged on the Slurm compute nodes, not the login node. if [[ $MODEL_PREFIX == "kimik2.6" ]] || + [[ $MODEL_PREFIX == "kimik3" ]] || [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi