diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 2231e6fe60..c3223cbb19 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -202,7 +202,10 @@ Qwen3.5-397B-A17B-MXFP4: Qwen3.5-397B-A17B-FP8: base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" + # Qwen3.5 built-in MTP head uses EAGLE (in-checkpoint draft, no draft-model-path); + # build_server_config appends --speculative-num-steps=DECODE_MTP_SIZE and + # --speculative-num-draft-tokens=DECODE_MTP_SIZE+1. + mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" dp_flags: "--enable-dp-attention --enable-dp-lm-head" ep_flags: "--moe-a2a-backend mori" prefill: diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 6a1598d81d..2b4406affa 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -186,7 +186,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g16,mia1-p01-g17,mia1-p01-g19,mia1-p01-g31,mia1-p01-g37}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index cf5cdfeb6d..19c3a5dcef 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -312,6 +312,63 @@ qwen3.5-fp8-mi355x-sglang-disagg: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" +qwen3.5-fp8-mi355x-sglang-disagg-mtp: + image: lmsysorg/sglang:v0.5.16-rocm720-mi35x + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: mi355x-disagg + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1P+1D TP4P+TP8D/EP1, EAGLE MTP depth 3 — conc 8-64. + - spec-decoding: "mtp" + conc-list: [ 8, 16, 32, 64 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + # 1P+1D TP4P+TP4D/EP1, EAGLE MTP depth 3 — conc 128. + # At conc=128, TP8 decode bottlenecks MTP (draft verification + # saturates 8 GPUs, causing -38% tput/GPU and TTFT explosion). + # Switching to TP4D resolves the collapse: +83% tput/GPU, 7.5x + # better TTFT vs TP8D, while interactivity converges (39.5 vs 49.5). + - spec-decoding: "mtp" + conc-list: [ 128 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 model: amd/Qwen3.5-397B-A17B-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 52c74bf168..90a113e961 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6100,7 +6100,6 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 - - config-keys: - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp scenario-type: @@ -6195,3 +6194,11 @@ - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg-mtp + description: + - "Add new qwen3.5-fp8-mi355x-sglang-disagg-mtp config with native EAGLE MTP (depth 3, top-k 1) on image v0.5.16." + - "Two search-space arms: conc 8-64 with TP4P+TP8D, and conc 128 with TP4P+TP4D (avoids TP8D draft-verification bottleneck at high concurrency: +83% tput/GPU, 7.5x TTFT improvement)." + - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to Qwen3.5-397B-A17B-FP8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2607