From 07d878898da2797f2c0708840361f70213f800cd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 17 Aug 2026 00:06:44 -0500 Subject: [PATCH 1/4] Port MI355X DSV4 fixed sequence to srt-slurm --- .../dsv4_fp4_mi355x_sglang-disagg.sh | 83 -------- .../fixed-seq/disagg-1p1d-dep8-mtp.yaml | 185 ++++++++++++++++++ .../fixed-seq/disagg-1p1d-dep8-stp.yaml | 178 +++++++++++++++++ .../mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml | 172 ++++++++++++++++ .../mi355x/fixed-seq/disagg-1p1d-tp8-stp.yaml | 170 ++++++++++++++++ configs/amd-master.yaml | 83 ++------ 6 files changed, 718 insertions(+), 153 deletions(-) delete mode 100755 benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-stp.yaml diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml new file mode 100644 index 0000000000..834ffe3581 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml @@ -0,0 +1,185 @@ +name: "mi355x-dsv4-disagg-1p1d-dep8-mtp-fixed-seq" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: {repo: "deepseek-ai/DeepSeek-V4-Pro"} + container: {image: "lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"} + frameworks: + sglang: "0.5.15.post1.dev20260719+ga03ca46a28" + sglang-router: "0.3.2" + amd-mori: "0.5.15.post1.dev20260719+ga03ca46a28" + +slurm: {time_limit: "04:00:00"} + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-prefill-delayer: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + moe-dense-tp-size: 1 + enable-dp-lm-head: true + watchdog-timeout: 3600 + load-balance-method: auto + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + speculative-algorithm: EAGLE + speculative-eagle-topk: 1 + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + log-level: info + log-level-http: error + context-length: 9217 + enable-metrics: true + enable-cache-report: true + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 131072 + max-total-tokens: 2097152 + disable-radix-cache: true + disable-cuda-graph: true + decode: + <<: *common_server + mem-fraction-static: 0.85 + max-total-tokens: null + disable-radix-cache: false + disable-cuda-graph: false + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128] + +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: {max_attempts: 720, interval_seconds: 5} + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}/fixed-seq" + mkdir -p "${result_root}" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 256 512; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --use-chat-template \ + --dsv4 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "$((concurrency * 10))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}" \ + --result-filename "dsv4-fp4-disagg-mtp1-dep8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml new file mode 100644 index 0000000000..ad5ca21005 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml @@ -0,0 +1,178 @@ +name: "mi355x-dsv4-disagg-1p1d-dep8-stp-fixed-seq" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: {repo: "deepseek-ai/DeepSeek-V4-Pro"} + container: {image: "lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"} + frameworks: + sglang: "0.5.15.post1.dev20260719+ga03ca46a28" + sglang-router: "0.3.2" + amd-mori: "0.5.15.post1.dev20260719+ga03ca46a28" + +slurm: {time_limit: "04:00:00"} + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-prefill-delayer: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + moe-dense-tp-size: 1 + enable-dp-lm-head: true + watchdog-timeout: 3600 + load-balance-method: auto + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + log-level: info + log-level-http: error + context-length: 9217 + enable-metrics: true + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 131072 + max-total-tokens: 2097152 + disable-radix-cache: true + disable-cuda-graph: true + decode: + <<: *common_server + mem-fraction-static: 0.85 + max-total-tokens: null + disable-radix-cache: false + disable-cuda-graph: false + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128] + +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: {max_attempts: 720, interval_seconds: 5} + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}/fixed-seq" + mkdir -p "${result_root}" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 256 512; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "$((concurrency * 10))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}" \ + --result-filename "dsv4-fp4-disagg-stp-dep8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml new file mode 100644 index 0000000000..a0e84b3d15 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml @@ -0,0 +1,172 @@ +name: "mi355x-dsv4-disagg-1p1d-tp8-mtp-fixed-seq" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: {repo: "deepseek-ai/DeepSeek-V4-Pro"} + container: {image: "lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"} + frameworks: + sglang: "0.5.15.post1.dev20260719+ga03ca46a28" + sglang-router: "0.3.2" + amd-mori: "0.5.15.post1.dev20260719+ga03ca46a28" + +slurm: {time_limit: "04:00:00"} + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + watchdog-timeout: 3600 + load-balance-method: auto + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + speculative-algorithm: EAGLE + speculative-eagle-topk: 1 + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + log-level: info + log-level-http: error + context-length: 9217 + enable-metrics: true + enable-cache-report: true + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 131072 + max-total-tokens: 2097152 + disable-radix-cache: true + disable-cuda-graph: true + decode: + <<: *common_server + mem-fraction-static: 0.85 + max-total-tokens: null + disable-radix-cache: false + disable-cuda-graph: false + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128] + +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: {max_attempts: 720, interval_seconds: 5} + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}/fixed-seq" + mkdir -p "${result_root}" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 2 4 8 16 32 64 128; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --use-chat-template \ + --dsv4 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}" \ + --result-filename "dsv4-fp4-disagg-mtp2-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-stp.yaml new file mode 100644 index 0000000000..f6bbc8a86c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-stp.yaml @@ -0,0 +1,170 @@ +name: "mi355x-dsv4-disagg-1p1d-tp8-stp-fixed-seq" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + frameworks: + sglang: "0.5.15.post1.dev20260719+ga03ca46a28" + sglang-router: "0.3.2" + amd-mori: "0.5.15.post1.dev20260719+ga03ca46a28" + +slurm: + time_limit: "04:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + watchdog-timeout: 3600 + load-balance-method: auto + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + log-level: info + log-level-http: error + context-length: 9217 + enable-metrics: true + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 131072 + max-total-tokens: 2097152 + disable-radix-cache: true + disable-cuda-graph: true + decode: + <<: *common_server + mem-fraction-static: 0.85 + max-total-tokens: null + disable-radix-cache: false + disable-cuda-graph: false + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128] + +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}/fixed-seq" + mkdir -p "${result_root}" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 2 4 8 16 32 64 128; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}" \ + --result-filename "dsv4-fp4-disagg-stp-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 1fee809497..6852d35c1c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -885,10 +885,10 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: dsv4-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260701 + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: mi355x-disagg + runner: cluster:mi355x-amds precision: fp4 framework: sglang-disagg router: { name: sglang-router, version: "0.3.2" } @@ -903,22 +903,19 @@ dsv4-fp4-mi355x-sglang-disagg: # non-MTP configurations # 1P1D pure TP8 (mori KV transfer) - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] + conc-list: [ 1 ] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-stp.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" conc-list: [ 256 ] @@ -928,50 +925,21 @@ dsv4-fp4-mi355x-sglang-disagg: ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - spec-decoding: "none" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MTP variant of dsv4-fp4-mi355x-sglang-disagg (base run 29821778206). Mirrors -# the base disagg search space and topologies exactly (same 20260701 image, -# mori KV transfer + MoE a2a, sglang-router --pd-disaggregation) and enables -# DeepSeek-V4 built-in MTP on the decode side via spec-decoding: mtp — the same -# enablement path as dsr1-fp8-mi355x-sglang-disagg-mtp. DECODE_MTP_SIZE=1 (draft -# depth 1) on every arm. Depth 2 was tried on the pure-TP8 (bottom-of-curve) arm, -# but this 20260701 build's 2-step EAGLE verification is lossy: greedy gsm8k fell -# to ~0.85 (below the 0.91 dsv4 floor), while depth 1 is lossless (~0.96, matching -# the spec-none base). So all arms use depth 1. The dsv4 ROCm MTP -# CUDA-graph fix (sgl-project/sglang#26383, merged 2026-05-27) is already -# present in the pinned 20260701 nightly, so no image bump is needed. +# MTP variant of dsv4-fp4-mi355x-sglang-disagg. The TP8 recipe preserves the +# existing two-step EAGLE grid, while the DEP8 recipe preserves one-step EAGLE. +# Each srt-slurm allocation iterates the original concurrency points directly. dsv4-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: mi355x-disagg + runner: cluster:mi355x-amds precision: fp4 framework: sglang-disagg router: { name: sglang-router, version: "0.3.2" } @@ -986,22 +954,19 @@ dsv4-fp4-mi355x-sglang-disagg-mtp: # MTP configurations # 1P1D pure TP8 (mori KV transfer) - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] + conc-list: [ 1 ] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "mtp" conc-list: [ 256 ] @@ -1011,33 +976,12 @@ dsv4-fp4-mi355x-sglang-disagg-mtp: ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - - spec-decoding: "mtp" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" dsv4-fp4-mi355x-sglang: @@ -1808,4 +1752,3 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } - From 3bf15cf4687f2aec7c80c50bd448d23aafa138c7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 17 Aug 2026 00:07:23 -0500 Subject: [PATCH 2/4] Document MI355X DSV4 srt-slurm migration --- perf-changelog.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 71b6332e6b..b6fdacdf93 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5816,3 +5816,14 @@ description: - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg + - dsv4-fp4-mi355x-sglang-disagg-mtp + description: + - "Port the production MI355X DeepSeek V4 fixed-sequence STP and MTP disaggregated search spaces from the model-specific AMD submit/helper stack to srt-slurm." + - "Preserve TP8 c1,c2,c4,c8,c16,c32,c64,c128 and DEP8/DPA c256,c512 for both STP and MTP; each of the four topology allocations now iterates its original concurrency points without reloading the model between points." + - "Preserve two-step EAGLE MTP on TP8 and one-step EAGLE MTP on DEP8, with the native SGLang Router frontend and AMD MoRI KV/MoE transport." + - "Use the runtime-verified lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 image and the exact srt-slurm commit c609754b5622f96d5c12a93149e245308d4f1e9b inherited from the validated DeepSeek V4 AgentX port." + - "Run fixed 8192-input/1024-output random workloads through the unchanged utils/bench_serving/benchmark_serving.py custom benchmark and remove the superseded dsv4_fp4_mi355x_sglang-disagg.sh wrapper." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2630 From 90e29a680200acc129bd0aa028d76213c49d8bab Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 17 Aug 2026 09:43:12 -0500 Subject: [PATCH 3/4] fix(mi355x): admit DSV4 MTP chat prompts --- .../deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml | 5 ++++- .../deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml | 5 ++++- perf-changelog.yaml | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml index 834ffe3581..737e3763ab 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml @@ -118,7 +118,10 @@ backend: speculative-num-draft-tokens: 2 log-level: info log-level-http: error - context-length: 9217 + # The DSV4 chat encoder adds three framing tokens to the nominal 8192 + # prompt. Match the workflow's 9472-token admission budget so the full + # 1024-token completion fits (8195 + 1024) instead of being rejected. + context-length: 9472 enable-metrics: true enable-cache-report: true mem-fraction-static: 0.80 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml index a0e84b3d15..161a20bd18 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml @@ -103,7 +103,10 @@ backend: speculative-num-draft-tokens: 3 log-level: info log-level-http: error - context-length: 9217 + # The DSV4 chat encoder adds three framing tokens to the nominal 8192 + # prompt. Match the workflow's 9472-token admission budget so the full + # 1024-token completion fits (8195 + 1024) instead of being rejected. + context-length: 9472 enable-metrics: true enable-cache-report: true mem-fraction-static: 0.80 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b6fdacdf93..6342951417 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5826,4 +5826,5 @@ - "Preserve two-step EAGLE MTP on TP8 and one-step EAGLE MTP on DEP8, with the native SGLang Router frontend and AMD MoRI KV/MoE transport." - "Use the runtime-verified lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 image and the exact srt-slurm commit c609754b5622f96d5c12a93149e245308d4f1e9b inherited from the validated DeepSeek V4 AgentX port." - "Run fixed 8192-input/1024-output random workloads through the unchanged utils/bench_serving/benchmark_serving.py custom benchmark and remove the superseded dsv4_fp4_mi355x_sglang-disagg.sh wrapper." + - "Set both MTP server roles to the workflow's 9472-token admission budget: DeepSeek V4 chat encoding adds three framing tokens, so the prior 9217 limit rejected every 8195-input/1024-output request before inference." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2630 From ea2d817be119d9f8d9b928125ce48437c1eff2ce Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 17 Aug 2026 09:47:35 -0500 Subject: [PATCH 4/4] fix(mi355x): query routed prefill DP rank --- .../deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml | 4 ++++ .../deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml | 4 ++++ perf-changelog.yaml | 1 + 3 files changed, 9 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml index 737e3763ab..a3dbc975fd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml @@ -85,6 +85,10 @@ backend: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + # The native router may select a different DPA rank from the rank encoded + # in bootstrap_room. Use SGLang's supported rank-query fallback so decode + # retrieves the actual prefill rank instead of rejecting the KV transfer. + SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK: "1" decode_environment: <<: *common_environment SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml index ad5ca21005..ca6b7a564c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml @@ -85,6 +85,10 @@ backend: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + # The native router may select a different DPA rank from the rank encoded + # in bootstrap_room. Use SGLang's supported rank-query fallback so decode + # retrieves the actual prefill rank instead of rejecting the KV transfer. + SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK: "1" decode_environment: <<: *common_environment SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6342951417..0e4c042e7d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5827,4 +5827,5 @@ - "Use the runtime-verified lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 image and the exact srt-slurm commit c609754b5622f96d5c12a93149e245308d4f1e9b inherited from the validated DeepSeek V4 AgentX port." - "Run fixed 8192-input/1024-output random workloads through the unchanged utils/bench_serving/benchmark_serving.py custom benchmark and remove the superseded dsv4_fp4_mi355x_sglang-disagg.sh wrapper." - "Set both MTP server roles to the workflow's 9472-token admission budget: DeepSeek V4 chat encoding adds three framing tokens, so the prior 9217 limit rejected every 8195-input/1024-output request before inference." + - "Enable SGLang's documented prefill-DP rank query fallback on both DEP8/DPA recipes; the first hardware run showed native-router rank selection diverging from bootstrap_room, which otherwise rejected every affected KV transfer before the later timeout." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2630