From f1bd4a3d5b2f075a16ce797711c88222de934376 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 25 Jul 2026 09:45:13 +0000 Subject: [PATCH 1/6] feat(agentx): add DSV4 GB300 MTP sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add three GB300 MTP3 AgentX topologies using the bedf79e image: aggregate TP8, 1P/1D DEP4/DEP8, and 1P/1D DEP8/DEP8. Follow PR #2157's aggregate/disaggregated organization, PR #2269's CI rack networking, and PR #2302's upstream srt-slurm per-node launch path with automatic hybrid load balancing. Align both disaggregated recipes on Dynamo 1.3.0.dev20260720, preserve prefill-only connector prefetch and breakable CUDA graph settings, and configure decode-side MooncakeStore as a synchronous KV consumer with lookup disabled. 中文：新增三个使用 bedf79e 镜像的 GB300 MTP3 AgentX 拓扑：聚合式 TP8、1P/1D DEP4/DEP8 和 1P/1D DEP8/DEP8。配置组织遵循 PR #2157 的聚合式/分离式结构，网络设置对齐 PR #2269 的 CI 机架环境，并依据 PR #2302 使用上游 srt-slurm 的单节点单进程启动模式及其自动启用的混合负载均衡。两个分离式配置统一使用 Dynamo 1.3.0.dev20260720，连接器预取和可中断 CUDA Graph 设置仅保留在预填充侧；解码侧 MooncakeStore 设为同步 KV 消费端并关闭查找功能。 --- benchmarks/benchmark_lib.sh | 6 +- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 151 +++++++++++++ ...gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml | 210 +++++++++++++++++ ...gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml | 211 ++++++++++++++++++ configs/nvidia-master.yaml | 73 ++++++ perf-changelog.yaml | 11 + runners/launch_gb300-nv.sh | 40 +--- 7 files changed, 670 insertions(+), 32 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index bda03e8b6..a41936ea1 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1797,7 +1797,11 @@ build_replay_cmd() { # X-Correlation-ID is useful tracing metadata but does not establish that # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. - if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then + # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this. + # aiperf's conv-aware routing emits nvext.session_control, a removed POC field + # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 + # (they moved to router/routing_constraints/agent_context). Default stays on. + if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml new file mode 100644 index 000000000..52f7797f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -0,0 +1,151 @@ +name: "svf-vllm-agg-gb300-tp8-mtp-agentic" + +# Validated GB300 AgentX aggregate topology: one TP8 worker spans two +# four-GPU nodes and serves both prefill and decode at concurrency 4. + +model: + path: "deepseek-v4-pro" + container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + frameworks: + dynamo: "1.2.1" + +dynamo: + wheel: "1.2.1" + install: true + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + enable-cumem-allocator: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml new file mode 100644 index 000000000..c71ca3e48 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml @@ -0,0 +1,210 @@ +name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic" + +# Validated GB300 AgentX MTP3 low-latency topology: one DEP4 prefill worker +# feeds one DEP8 decode worker at concurrency 128. + +model: + path: "deepseek-v4-pro" + container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":false,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml new file mode 100644 index 000000000..01320aa7d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml @@ -0,0 +1,211 @@ +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic" + +# Validated GB300 AgentX MTP3 efficiency topology: one DEP8 prefill worker and +# one DEP8 decode worker at concurrency 384. + +model: + path: "deepseek-v4-pro" + container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":false,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6e4fc8d34..7f5d78053 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7897,6 +7897,79 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: + image: inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: + image: inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [128] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: mtp + conc-list: [384] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c3602ff74..8705429c4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5085,3 +5085,14 @@ description: - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + description: + - "Add GB300 Dynamo-vLLM AgentX MTP3 configurations: aggregate TP8 at conc [1, 4], 1P/1D DEP4/DEP8 at conc 128, and 1P/1D DEP8/DEP8 at conc 384." + - "Use inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e for all three configurations." + - "Use NVIDIA/srt-slurm main per PR #2302; disaggregated configurations use per-node data-parallel launch (which enables hybrid load balancing) with NIXL plus MooncakeStore." + - "Pin all disaggregated recipes and router metadata to Dynamo 1.3.0.dev20260720 with stable cross-node prefix hashes, a 3600-second active-request expiry, 72 CPUs per task, and a 600-second cache warmup; keep breakable-CUDA-graph and connector-prefetch settings prefill-only." + - "Match the PR #2269 CI GB300 rack fabric by pinning Mooncake, UCX, and NCCL to mlx5_0,mlx5_1,mlx5_2,mlx5_3; disaggregated decode workers use MooncakeStore as a synchronous KV consumer with lookup disabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d450cdb86..e5a8d059b 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -150,38 +150,16 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic elif [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, - # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag - # (`srtctl apply --no-preflight`) — needed because: - # - # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local - # NVMe, fast) instead of the NFS path under /data/home/sa-shared. - # - /scratch only exists on GB300 compute nodes; it is NOT mounted - # on the GHA runner pod that invokes srtctl. - # - srtctl's pre-submit model check (_preflight_model in - # src/srtctl/core/validation.py) does a Path.is_dir() in-process - # on the invoking node — so it fails before sbatch is ever - # called with "Model alias 'X' resolved to '/scratch/...', - # but that path is unavailable". - # - --no-preflight skips just the optional Python-level FS check. - # vLLM still fails loudly at runtime if the path is genuinely - # missing on the compute node. - # - # All other upstream schema features we need are inherited from - # NVIDIA HEAD: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) - # - sbatch_directives / srun_options (top-level recipe fields) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - # 854b3fd = --no-preflight flag - # 6e34b8b = benchmark_stage propagates srun_options (needed for - # container-remap-root to reach the agentic_srt.sh srun) - git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 - mkdir -p recipes/vllm/deepseek-v4/agentic + # Agentic recipes use NVIDIA/srt-slurm v1.0.36. This is the upstream + # version validated in InferenceX PR #2302 and includes per-node DP, + # matching Dynamo health counts, multi-node TP port handling, and + # Mooncake compatibility. Keep it pinned so sweeps are reproducible. + git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + + mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + recipes/vllm/deepseek-v4/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From fc01c905992d2bd71a8194ec5707e16e3f5e7131 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 25 Jul 2026 18:53:35 +0000 Subject: [PATCH 2/6] fix: enable async Mooncake loads for decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set load_async=true for decode-side MooncakeStoreConnector while keeping asynchronous lookup and lookup disabled. This satisfies the current vLLM get_finished requirement in both disaggregated GB300 AgentX recipes. 中文：为解码侧 MooncakeStoreConnector 启用异步加载，同时保持异步查找和查找功能关闭。该修改使两个 GB300 AgentX 分离式推理配置满足当前 vLLM get_finished 的要求。 --- .../agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml | 2 +- .../agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml index c71ca3e48..bc2229817 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml @@ -106,7 +106,7 @@ backend: numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":false,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" kv-cache-dtype: "fp8" tensor-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml index 01320aa7d..22e2b5978 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml @@ -106,7 +106,7 @@ backend: numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":false,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" kv-cache-dtype: "fp8" tensor-parallel-size: 1 From 50e1f0215af3008e917cd99aa427a148f79356e6 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 26 Jul 2026 18:57:11 +0000 Subject: [PATCH 3/6] tune(agentx): expand TP4 MTP scheduler headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise the TP4 c4 scheduler and warmup limits to 16 sequences, extend MTP3 CUDA graph captures through 64 tokens, and lower GPU memory utilization to 0.93. 中文：扩大 TP4 c4 MTP 调度余量。将调度器和预热上限提高到 16 个序列，将 MTP3 CUDA Graph 捕获范围扩展到 64 个 token，并将 GPU 显存利用率下调至 0.93。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml new file mode 100644 index 000000000..d1e9aa7de --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -0,0 +1,147 @@ +name: "svf-vllm-agg-gb300-tp4-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node +# and serves both prefill and decode at concurrency 4. Scheduler, CUDA-graph, +# and memory settings match the B300 vLLM TP4 MTP agentic configuration. + +model: + path: "deepseek-v4-pro" + container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + frameworks: + dynamo: "1.2.1" + +dynamo: + wheel: "1.2.1" + install: true + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + enable-cumem-allocator: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + gpu-memory-utilization: 0.93 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" From 943bffa68151d9124f0a52d104a479da5b36d1f5 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 26 Jul 2026 18:50:33 +0000 Subject: [PATCH 4/6] feat(agentx): refine GB300 MTP sweep topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use random Dynamo session affinity and the upstream vLLM 426e59f image for aggregate and P/D recipes. Replace aggregate TP8 c4 with a B300-aligned TP4 c4 recipe and emit correct aggregate GPU metadata. Advance AIPerf for Dynamo session headers. 中文：优化 GB300 AgentX MTP 扫描拓扑。聚合式和 P/D 配置统一改用上游 vLLM 426e59f 镜像，P/D 配置启用 Dynamo 随机会话亲和路由；将聚合式 TP8 c4 替换为对齐 B300 参数的 TP4 c4，并修正聚合式结果的 GPU 拓扑标记；同时更新 AIPerf 以发送 Dynamo 会话请求头。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 4 ++-- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 13 +++++------ ...gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml | 16 +++++--------- ...gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml | 16 +++++--------- configs/nvidia-master.yaml | 22 ++++++++++++++++--- perf-changelog.yaml | 8 ++----- utils/aiperf | 2 +- 7 files changed, 40 insertions(+), 41 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index d1e9aa7de..6d1681793 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -6,14 +6,14 @@ name: "svf-vllm-agg-gb300-tp4-mtp-agentic" model: path: "deepseek-v4-pro" - container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.2.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 52f7797f9..06b916659 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -1,18 +1,18 @@ name: "svf-vllm-agg-gb300-tp8-mtp-agentic" # Validated GB300 AgentX aggregate topology: one TP8 worker spans two -# four-GPU nodes and serves both prefill and decode at concurrency 4. +# four-GPU nodes and serves both prefill and decode at concurrency 1. model: path: "deepseek-v4-pro" - container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.2.1" @@ -140,10 +140,9 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - EP_SIZE: "1" - DP_ATTENTION: "false" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml index bc2229817..cc706a4aa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml @@ -5,14 +5,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic" model: path: "deepseek-v4-pro" - container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.3.0.dev20260720" @@ -53,13 +53,8 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "kv" - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" + router-mode: "random" + router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" @@ -67,8 +62,6 @@ backend: type: vllm connector: null dp_launch_mode: per_node - kv_events_config: - prefill: true mooncake_kv_store: store_config: metadata_server: "P2PHANDSHAKE" @@ -204,6 +197,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml index 22e2b5978..fa856fe21 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml @@ -5,14 +5,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic" model: path: "deepseek-v4-pro" - container: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.3.0.dev20260720" @@ -53,13 +53,8 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "kv" - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" + router-mode: "random" + router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" @@ -67,8 +62,6 @@ backend: type: vllm connector: null dp_launch_mode: per_node - kv_events_config: - prefill: true mooncake_kv_store: store_config: metadata_server: "P2PHANDSHAKE" @@ -205,6 +198,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7f5d78053..c3ab2c64f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7898,7 +7898,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: dp-attn: true dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - image: inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv @@ -7911,7 +7911,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: agentic-coding: - search-space: - spec-decoding: mtp - conc-list: [1, 4] + conc-list: [1] prefill: num-worker: 1 tp: 8 @@ -7926,9 +7926,25 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: tp: 8 ep: 1 dp-attn: false + - spec-decoding: mtp + conc-list: [4] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts four GPUs only once. + decode: + num-worker: 0 + tp: 4 + ep: 1 + dp-attn: false dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8705429c4..967d1955c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5090,9 +5090,5 @@ - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg description: - - "Add GB300 Dynamo-vLLM AgentX MTP3 configurations: aggregate TP8 at conc [1, 4], 1P/1D DEP4/DEP8 at conc 128, and 1P/1D DEP8/DEP8 at conc 384." - - "Use inferactinc/public:sprint-agentx-fast-arm64-cu13.0.1-bedf79e for all three configurations." - - "Use NVIDIA/srt-slurm main per PR #2302; disaggregated configurations use per-node data-parallel launch (which enables hybrid load balancing) with NIXL plus MooncakeStore." - - "Pin all disaggregated recipes and router metadata to Dynamo 1.3.0.dev20260720 with stable cross-node prefix hashes, a 3600-second active-request expiry, 72 CPUs per task, and a 600-second cache warmup; keep breakable-CUDA-graph and connector-prefetch settings prefill-only." - - "Match the PR #2269 CI GB300 rack fabric by pinning Mooncake, UCX, and NCCL to mlx5_0,mlx5_1,mlx5_2,mlx5_3; disaggregated decode workers use MooncakeStore as a synchronous KV consumer with lookup disabled." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 + - "Add GB300 Dynamo-vLLM AgentX MTP3 points: aggregate TP8 c1, aggregate TP4 c4, P/D DEP4/DEP8 c128, and P/D DEP8/DEP8 c384; use upstream vLLM 426e59f, srt-slurm v1.0.36, and random session-affinity routing for P/D." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 diff --git a/utils/aiperf b/utils/aiperf index 0d2aa0572..ed2e040d0 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 0d2aa0572ac685943d38c580675c4a61023581d3 +Subproject commit ed2e040d0ed0976bbf632e5095d0b1ccebf8f6a2 From e1680ab8caa6d40a75703e8f4cde9563ba521771 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 26 Jul 2026 20:39:56 +0000 Subject: [PATCH 5/6] chore(agentx): link GB300 MTP changelog to PR 2341 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the dedicated GB300 AgentX MTP changelog entry to reference its new draft pull request. 中文：将独立的 GB300 AgentX MTP changelog 条目更新为新建的草稿 PR #2341。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 967d1955c..d8fd10265 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5091,4 +5091,4 @@ - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg description: - "Add GB300 Dynamo-vLLM AgentX MTP3 points: aggregate TP8 c1, aggregate TP4 c4, P/D DEP4/DEP8 c128, and P/D DEP8/DEP8 c384; use upstream vLLM 426e59f, srt-slurm v1.0.36, and random session-affinity routing for P/D." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2341 From afd24d5a3d6d3d1b49897bd772f465732fc0451e Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Mon, 27 Jul 2026 09:29:44 -0500 Subject: [PATCH 6/6] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a0e9adb90..075940940 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5108,4 +5108,4 @@ description: - "Add GB300 Dynamo-vLLM AgentX MTP3 points: aggregate TP8 c1, aggregate TP4 c4, P/D DEP4/DEP8 c128, and P/D DEP8/DEP8 c384; use upstream vLLM 426e59f, srt-slurm v1.0.36, and random session-affinity routing for P/D." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2341 - \ No newline at end of file +