diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml similarity index 64% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml index a3ae8342b9..66463a748b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml @@ -1,18 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-eval-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c1-agentic" -# Real-verification eval counterpart to the B200 DEP offload throughput recipe. -# It keeps TP8/DP2/EP16 and the CPU KV connector unchanged; only DSpark -# acceptance changes from synthetic golden AL to target-verified block. model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -42,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -68,23 +82,22 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 language-model-only: true - max-num-seqs: 8 + max-num-seqs: 2 max-num-batched-tokens: 8192 - # Eval must verify every draft against real target logits. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -94,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml similarity index 65% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml index 7dc3d49793..5819920877 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml @@ -1,19 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c12-agentic" -# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates -# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. -# This keeps FlashInfer fused collectives node-local while using shipped vLLM. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -43,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -69,12 +82,13 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true @@ -83,9 +97,7 @@ backend: language-model-only: true max-num-seqs: 8 max-num-batched-tokens: 8192 - # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval - # selects the paired real-verification recipe instead. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -95,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml similarity index 65% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml index f562682522..8f8eb48845 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml @@ -1,18 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-eval-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c32-agentic" -# Real-verification eval counterpart to the B200 DEP throughput recipe. -# It keeps TP8/DP2/EP16 and target settings unchanged; only DSpark acceptance -# changes from synthetic golden AL to target-verified block. model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -42,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -68,12 +82,13 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true @@ -82,8 +97,7 @@ backend: language-model-only: true max-num-seqs: 8 max-num-batched-tokens: 8192 - # Eval must verify every draft against real target logits. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -93,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml similarity index 62% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml index 0119d79219..ed49a60468 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml @@ -1,19 +1,15 @@ -name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-mooncake-c4-agentic" -# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates -# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. -# This keeps FlashInfer fused collectives node-local while using shipped vLLM. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep model: path: "kimik3" - container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" dynamo: install: false @@ -43,16 +39,33 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: - # TP8 collectives stay within each B200 NVSwitch domain. VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" @@ -69,24 +82,22 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Keep headroom for the first FlashInfer MXFP4 MoE workspace. - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.9 no-enable-flashinfer-autotune: true max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' enable-prefix-caching: true enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 language-model-only: true - max-num-seqs: 8 + max-num-seqs: 16 max-num-batched-tokens: 8192 - # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval - # selects the paired real-verification recipe instead. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' sbatch_directives: segment: "1" @@ -96,12 +107,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml new file mode 100644 index 0000000000..de46b4584e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml @@ -0,0 +1,128 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c72-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 144 + max-num-batched-tokens: 8192 + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a995310234..7dc6834af2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8785,86 +8785,106 @@ kimik3-fp4-b200-dynamo-vllm-agentic: # Expert parallel spans all 16 ranks, so the 896 experts remain EP16 while # FlashInfer fused collectives stay inside each B200 NVSwitch domain. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-5ee73d9 model: moonshotai/Kimi-K3 model-prefix: kimik3 - runner: cluster:b200-dgxc + runner: cluster:b200-nscale precision: fp4 framework: dynamo-vllm multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.63 + - dram-utilization: 0.8 search-space: - # Low-latency and high-interactivity points. - spec-decoding: mtp - conc-list: [1, 2, 4] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [1] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c1-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - # Balanced medium-concurrency points. - spec-decoding: mtp - conc-list: [8, 12, 16] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [4] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - # GPU-resident throughput points around the prior c16-c32 KV cliff. - spec-decoding: mtp - conc-list: [20, 24, 28, 32] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [12] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c12-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - # CPU KV-offload crossover and capacity points. Keep the resident points - # above so the same concurrency can be compared with one variable changed. - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [8, 12, 16, 20, 24, 28, 32, 48, 64] + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [32] prefill: - num-worker: 1 + num-worker: 2 tp: 8 ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-mooncake-c32-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" decode: num-worker: 0 tp: 8 ep: 16 dp-attn: true - + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + conc-list: [72] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c72-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/configs/runners.yaml b/configs/runners.yaml index 6f0b35e9c0..6b6fdd0c71 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -232,6 +232,16 @@ labels: - h200-dgxc-slurm_13 cluster:h200-nb: - h200-nb_0 + cluster:b200-nscale: + - b200-nscale-slurm_0 + - b200-nscale-slurm_1 + - b200-nscale-slurm_2 + - b200-nscale-slurm_3 + - b200-nscale-slurm_4 + - b200-nscale-slurm_5 + - b200-nscale-slurm_6 + - b200-nscale-slurm_7 + - b200-nscale-slurm_8 cluster:b200-dgxc: - b200-dgxc_00 - b200-dgxc_01 @@ -334,6 +344,9 @@ hardware: b200-nscale: available-cpu-dram-mib: 2_063_920 gpus-per-node: 8 + cluster:b200-nscale: + available-cpu-dram-mib: 2_063_920 + gpus-per-node: 8 cluster:b300-nv: available-cpu-dram-mib: 2_964_436 gpus-per-node: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 538a800d95..3578462410 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6161,3 +6161,11 @@ description: - "Update the B200 SGLang AgentX HiCache MTP DEP8 configuration." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2656 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add new B200 multi-node configs with DP2, EP16" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2618 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 9b9a1e3f96..4bf9531d0f 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -43,6 +43,9 @@ if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/Kimi-K2.6-NVFP4}" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" +elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/Kimi-K3}" + export SRT_SLURM_MODEL_PREFIX="kimik3" else echo "Unsupported model prefix/precision for b200-nscale: $MODEL_PREFIX/$PRECISION" >&2 echo "Models staged under $NSCALE_MODEL_ROOT:" >&2 @@ -61,7 +64,16 @@ export SERVED_MODEL_NAME=$MODEL echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then +if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then + # Pin the tested renderer so branch movement cannot change generated rank + # commands between sweep points. + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout 217f9438 || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 +elif [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 # Pin the srt-slurm revision used by these checked-in recipes. @@ -193,6 +205,15 @@ export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" echo "Submitting job with srtctl..." echo "MODEL_PATH=$MODEL_PATH" +# An eval row may point at a committed real-verification recipe while its +# throughput row keeps synthetic golden acceptance. Only configs that set +# EVAL_CONFIG_FILE opt into this selection; all other configs keep using +# CONFIG_FILE unchanged. +if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + CONFIG_FILE="$EVAL_CONFIG_FILE" + echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" +fi + if [[ -z "$CONFIG_FILE" ]]; then echo "Error: CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings." >&2 echo "Config: MODEL_PREFIX=${MODEL_PREFIX} PRECISION=${PRECISION} FRAMEWORK=${FRAMEWORK}" >&2 @@ -213,6 +234,7 @@ inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 SRTCTL_PREFLIGHT_ARGS=() # These weights are staged on the Slurm compute nodes, not the login node. if [[ $MODEL_PREFIX == "kimik2.6" ]] || + [[ $MODEL_PREFIX == "kimik3" ]] || [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi