From cf632d0df80b6f78ebe04d3dad4d8f1c965a21dc Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 16 Aug 2026 23:07:23 +0000 Subject: [PATCH 1/4] feat(agentx): add vLLM GB300 c736 TRT counterpart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增与 TensorRT-LLM c736 点相同的 2P1D DEP8/DEP32 拓扑。vLLM 使用合成接受长度 2.97，以对应 TensorRT-LLM 的 1.97 记法。 --- ...b300-2p1d-dep8-dep32-c736-mtp-agentic.yaml | 210 ++++++++++++++++++ configs/nvidia-master.yaml | 17 ++ perf-changelog.yaml | 9 + 3 files changed, 236 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic.yaml new file mode 100644 index 000000000..c65017188 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic.yaml @@ -0,0 +1,210 @@ +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP32 decode worker at concurrency 736. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 8 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 32 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 32 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 8 + max-num-batched-tokens: 32 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep32-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index cb636bf90..a9d367f3f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7649,6 +7649,23 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. + - spec-decoding: mtp + conc-list: [736] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.97" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: mtp conc-list: [1152] router: { name: dynamo-router, version: "1.3.0.dev20260720" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 48012c8c5..cac053a9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6103,3 +6103,12 @@ - "Use a 270 GB/rank target HiCache pool for the GLM-5.2 B200 TP8 c12 and c16 AgentX points to avoid the host-cache capacity cliff." - "Keep the existing ratio-0.75 pool for c1, c4, and c8 while retaining the SGLang EAGLE MTP and golden synthetic acceptance setup." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2652 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add GB300 vLLM AgentX c736 with TRT-matched topology and batching, decode graph capture 32, and synthetic AL 2.97." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 + append-only: true From dae25e98dd20ddd8ad110009de285cd144490552 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Mon, 17 Aug 2026 06:39:19 +0000 Subject: [PATCH 2/4] feat(agentx): append vLLM GB300 c388 point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GB300 vLLM AgentX 1P1D DEP8/DEP32 c388 点，并保留现有 c736 配置与结果历史。 --- ...b300-1p1d-dep8-dep32-c388-mtp-agentic.yaml | 210 ++++++++++++++++++ configs/nvidia-master.yaml | 17 ++ perf-changelog.yaml | 9 + 3 files changed, 236 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml new file mode 100644 index 000000000..c310f30ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml @@ -0,0 +1,210 @@ +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP32 decode worker at concurrency 388. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 8 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 32 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 32 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 4 + max-num-batched-tokens: 16 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a9d367f3f..19e7fc41a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7666,6 +7666,23 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 32 ep: 32 dp-attn: true + - spec-decoding: mtp + conc-list: [388] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.97" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: mtp conc-list: [1152] router: { name: dynamo-router, version: "1.3.0.dev20260720" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cac053a9e..60a7351c5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6112,3 +6112,12 @@ - "Add GB300 vLLM AgentX c736 with TRT-matched topology and batching, decode graph capture 32, and synthetic AL 2.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 append-only: true + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Append GB300 vLLM AgentX c388 with TRT-matched 1P1D DEP8/DEP32 topology and batching, decode graph capture 16, and synthetic AL 2.97." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 + append-only: true From 4acf09d4e8c69249687d6dacfbb81593fa0f589b Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Tue, 18 Aug 2026 09:55:55 +0000 Subject: [PATCH 3/4] feat(agentx): select vLLM GB300 low-latency points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 vLLM GB300 c4、c12 和 c24 低延迟配置，并从当前生成的扫描矩阵中移除 c736 和 c388。 --- ...g-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml | 208 ++++++++++++++++++ ...-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml | 208 ++++++++++++++++++ ...-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml | 208 ++++++++++++++++++ configs/nvidia-master.yaml | 51 +++-- perf-changelog.yaml | 10 +- 5 files changed, 667 insertions(+), 18 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml new file mode 100644 index 000000000..1bb6b2a1d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml @@ -0,0 +1,208 @@ +name: "svf-vllm-disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds four +# TEP8 decode workers at concurrency 4. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 8 + prefill_workers: 1 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 4096 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 1 + max-num-batched-tokens: 4 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 4 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p4d-dep4-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p4d-tep8-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml new file mode 100644 index 000000000..a78d83189 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml @@ -0,0 +1,208 @@ +name: "svf-vllm-disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds six +# TEP4 decode workers at concurrency 12. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 6 + prefill_workers: 1 + decode_workers: 6 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 4096 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 4 + max-num-batched-tokens: 16 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p6d-dep4-c12-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p6d-tep4-c12-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml new file mode 100644 index 000000000..3a17c4fa4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml @@ -0,0 +1,208 @@ +name: "svf-vllm-disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds six +# TEP4 decode workers at concurrency 24. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 6 + prefill_workers: 1 + decode_workers: 6 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 4096 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 4 + max-num-batched-tokens: 16 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p6d-dep4-c24-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p6d-tep4-c24-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 19e7fc41a..36942f2c6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7650,39 +7650,56 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp - conc-list: [736] + conc-list: [4] router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.97" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c736-mtp-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false - spec-decoding: mtp - conc-list: [388] + conc-list: [12] router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.97" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml" decode: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [24] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: num-worker: 1 - tp: 32 - ep: 32 + tp: 4 + ep: 4 dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml" + decode: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: false - spec-decoding: mtp conc-list: [1152] router: { name: dynamo-router, version: "1.3.0.dev20260720" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 60a7351c5..74d71d30b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6076,7 +6076,6 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 - - config-keys: - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp scenario-type: @@ -6121,3 +6120,12 @@ - "Append GB300 vLLM AgentX c388 with TRT-matched 1P1D DEP8/DEP32 topology and batching, decode graph capture 16, and synthetic AL 2.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 append-only: true + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Append GB300 vLLM AgentX low-latency c4/c12/c24 with TRT-matched 1P4D DEP4/TEP8 and 1P6D DEP4/TEP4 topologies and batching, decode graph capture 4/16, and synthetic AL 2.49." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 + append-only: true From c0fce8a2682364c68858e5ed8995bf94bbb70e30 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Tue, 18 Aug 2026 22:23:58 +0000 Subject: [PATCH 4/4] fix(agentx): correct GB300 low-concurrency settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正 c4、c12 和 c24 的解码批处理容量，并将 Dynamo 更新到 1.4.0。 --- .../disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml | 8 ++++---- .../disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml | 8 ++++---- .../disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml | 8 ++++---- configs/nvidia-master.yaml | 6 +++--- perf-changelog.yaml | 10 ++++++++++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml index 1bb6b2a1d..20c19c200 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tep8-c4-mtp-agentic.yaml @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.3.0.dev20260720" + dynamo: "1.4.0" dynamo: - wheel: "1.3.0.dev20260720" + wheel: "1.4.0" install: true environment: @@ -111,8 +111,8 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 1 - max-num-batched-tokens: 4 + max-num-seqs: 16 + max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml index a78d83189..96742ceaa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c12-mtp-agentic.yaml @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.3.0.dev20260720" + dynamo: "1.4.0" dynamo: - wheel: "1.3.0.dev20260720" + wheel: "1.4.0" install: true environment: @@ -111,8 +111,8 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 4 - max-num-batched-tokens: 16 + max-num-seqs: 48 + max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml index 3a17c4fa4..2603e9b7d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tep4-c24-mtp-agentic.yaml @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.3.0.dev20260720" + dynamo: "1.4.0" dynamo: - wheel: "1.3.0.dev20260720" + wheel: "1.4.0" install: true environment: @@ -111,8 +111,8 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 4 - max-num-batched-tokens: 16 + max-num-seqs: 96 + max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36942f2c6..99d564370 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7651,7 +7651,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp conc-list: [4] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 tp: 4 @@ -7668,7 +7668,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: false - spec-decoding: mtp conc-list: [12] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 tp: 4 @@ -7685,7 +7685,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: false - spec-decoding: mtp conc-list: [24] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 74d71d30b..acbef7049 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6076,6 +6076,7 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 + - config-keys: - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp scenario-type: @@ -6129,3 +6130,12 @@ - "Append GB300 vLLM AgentX low-latency c4/c12/c24 with TRT-matched 1P4D DEP4/TEP8 and 1P6D DEP4/TEP4 topologies and batching, decode graph capture 4/16, and synthetic AL 2.49." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 append-only: true + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Fix the GB300 vLLM AgentX c4/c12/c24 decode capacity at 4x concurrency with 8192 max batched tokens and update the low-concurrency Dynamo runtime and router to 1.4.0." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 + append-only: true