diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3329f74b8..c63e3f985 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -270,7 +270,7 @@ qwen3.5-fp8-mi355x-atom-mtp: - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x + image: lmsysorg/sglang:v0.5.16-rocm720-mi35x model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x-disagg @@ -285,21 +285,20 @@ qwen3.5-fp8-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # 1P+1D TP8/EP1 low-concurrency sweep. - # dp-attn intentionally false (matches the 1k1k row): with - # --enable-dp-attention + --moe-a2a-backend mori, sglang auto-promotes - # moe_ep_size=tp_size=8, but is_deepep_class_backend() excludes MoRI, - # so num_shared_slots stays at the global value (1) and the + # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding). + # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode + # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs). + # dp-attn intentionally false: with --enable-dp-attention + + # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, + # but is_deepep_class_backend() excludes MoRI, so + # num_shared_slots stays at the global value (1) and the # (num_experts - num_shared_slots) % moe_ep_size assertion in # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - # Track upstream sglang for a fix; flip back to dp-attn=true once - # MoRI is added to is_deepep_class_backend() or shared-slot - # accounting is reconciled. - spec-decoding: "none" conc-list: [ 8, 16, 32, 64, 128 ] prefill: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4df66c266..621e346bf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6113,3 +6113,11 @@ description: - "Add Kimi-K3 GB300 AgentX configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2637 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + scenario-type: + - fixed-seq-len + description: + - "Bump qwen3.5-fp8-mi355x-sglang-disagg image from v0.5.14 to v0.5.16 (3-5% throughput improvement) and switch from TP8P+TP8D (16 GPU) to TP4P+TP8D (12 GPU) — TP4 prefill delivers identical decode interactivity with 24-31% better throughput/GPU." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2606