From 74797f7c2d78248305b8d60168acf3e779cc79c2 Mon Sep 17 00:00:00 2001 From: ChangLiu0709 Date: Fri, 14 Aug 2026 13:40:11 +0000 Subject: [PATCH 1/2] [AMD] Qwen3.5-FP8 MI355X SGLang disagg MTP: EAGLE speculative decoding - Add new config qwen3.5-fp8-mi355x-sglang-disagg-mtp with native EAGLE MTP (depth 3, top-k 1) on image v0.5.16. - Two search-space arms: conc 8-64 with TP4P+TP8D, and conc 128 with TP4P+TP4D (avoids TP8D draft-verification bottleneck at high concurrency). - models.yaml: add mtp_flags for Qwen3.5-397B-A17B-FP8. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/models.yaml | 5 +- configs/amd-master.yaml | 57 +++++++++++++++++++++ perf-changelog.yaml | 8 +++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 2231e6fe60..c3223cbb19 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -202,7 +202,10 @@ Qwen3.5-397B-A17B-MXFP4: Qwen3.5-397B-A17B-FP8: base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" + # Qwen3.5 built-in MTP head uses EAGLE (in-checkpoint draft, no draft-model-path); + # build_server_config appends --speculative-num-steps=DECODE_MTP_SIZE and + # --speculative-num-draft-tokens=DECODE_MTP_SIZE+1. + mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" dp_flags: "--enable-dp-attention --enable-dp-lm-head" ep_flags: "--moe-a2a-backend mori" prefill: diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 552451fc7e..88416f6c9a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -326,6 +326,63 @@ qwen3.5-fp8-mi355x-sglang-disagg: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" +qwen3.5-fp8-mi355x-sglang-disagg-mtp: + image: lmsysorg/sglang:v0.5.16-rocm720-mi35x + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: mi355x-disagg + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1P+1D TP4P+TP8D/EP1, EAGLE MTP depth 3 — conc 8-64. + - spec-decoding: "mtp" + conc-list: [ 8, 16, 32, 64 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + # 1P+1D TP4P+TP4D/EP1, EAGLE MTP depth 3 — conc 128. + # At conc=128, TP8 decode bottlenecks MTP (draft verification + # saturates 8 GPUs, causing -38% tput/GPU and TTFT explosion). + # Switching to TP4D resolves the collapse: +83% tput/GPU, 7.5x + # better TTFT vs TP8D, while interactivity converges (39.5 vs 49.5). + - spec-decoding: "mtp" + conc-list: [ 128 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 model: amd/Qwen3.5-397B-A17B-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04ba9feb56..25239e47ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5953,3 +5953,11 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg-mtp + description: + - "Add new qwen3.5-fp8-mi355x-sglang-disagg-mtp config with native EAGLE MTP (depth 3, top-k 1) on image v0.5.16." + - "Two search-space arms: conc 8-64 with TP4P+TP8D, and conc 128 with TP4P+TP4D (avoids TP8D draft-verification bottleneck at high concurrency: +83% tput/GPU, 7.5x TTFT improvement)." + - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to Qwen3.5-397B-A17B-FP8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2557 From 4ffafba35d5fc8b4d1fc968a0197ba579ff5c424 Mon Sep 17 00:00:00 2001 From: ChangLiu0709 Date: Wed, 19 Aug 2026 11:05:51 +0000 Subject: [PATCH 2/2] submit.sh: expand node exclude list to match baseline PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add g14, g16, g17, g19, g31, g37 to SLURM_EXCLUDE_NODES — these nodes have broken docker supplementary group membership causing container launch timeouts. g17 caused the current CI failure. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/submit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 6a1598d81d..2b4406affa 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -186,7 +186,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g16,mia1-p01-g17,mia1-p01-g19,mia1-p01-g31,mia1-p01-g37}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi