From 05e49305d54b92aa0770e6c8a7aa85c93b415ef8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 16 Aug 2026 23:51:32 -0500 Subject: [PATCH 1/2] port MI355X Qwen3.5 MXFP4 disagg sweep to srt-slurm --- .../qwen3.5_fp4_mi355x_sglang-disagg.sh | 84 ---------- .../disagg-1p1d-tp8-mxfp4-fixed-seq.yaml | 143 ++++++++++++++++++ configs/amd-master.yaml | 12 +- 3 files changed, 148 insertions(+), 91 deletions(-) delete mode 100755 benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml diff --git a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c7..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml new file mode 100644 index 0000000000..39bd143371 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml @@ -0,0 +1,143 @@ +# Production MI355X port of the legacy Qwen3.5 MXFP4 1P1D sweep. Each role +# owns one full 8-GPU node; SGLang Router carries requests and AMD MoRI moves KV. + +name: "mi355x-sglang-qwen3.5-mxfp4-disagg-1p1d-tp8-fixed-seq" + +model: + path: "hf:amd/Qwen3.5-397B-A17B-MXFP4" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp4" + +identity: + model: + repo: "amd/Qwen3.5-397B-A17B-MXFP4" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "08:00:00" + +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + decode_environment: *common_environment + sglang_config: + prefill: &common_config + served-model-name: "amd/Qwen3.5-397B-A17B-MXFP4" + tensor-parallel-size: 8 + disaggregation-transfer-backend: mori + disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + kv-cache-dtype: "fp8_e4m3" + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8] + enable-metrics: true + decode: + <<: *common_config + mem-fraction-static: 0.85 + prefill-round-robin-balance: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 8 16 32 64 128 256 512; do + num_prompts=$((concurrency * 10)) + if ((num_prompts < 16)); then + num_prompts=16 + fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model amd/Qwen3.5-397B-A17B-MXFP4 \ + --tokenizer amd/Qwen3.5-397B-A17B-MXFP4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3.5-mxfp4-disagg-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 1b2b2cbfac..e3a56092e3 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -347,10 +347,10 @@ qwen3.5-fp4-mi355x-sglang-mtp: - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 - runner: mi355x-disagg + runner: cluster:mi355x-amds precision: fp4 framework: sglang-disagg router: { name: sglang-router, version: "0.3.2" } @@ -363,22 +363,20 @@ qwen3.5-fp4-mi355x-sglang-disagg: osl: 1024 search-space: - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] + # The recipe owns the complete production search in one allocation. + conc-list: [1] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x From 13a64732e39f0f9f6854a26159a98808ef9a8ee4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 16 Aug 2026 23:52:27 -0500 Subject: [PATCH 2/2] document Qwen3.5 MXFP4 srt-slurm migration --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ae33f00aea..3efc0073ab 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5798,3 +5798,11 @@ - "Pin the launcher to SemiAnalysisAI/srt-slurm c609754b so native-router endpoint registration and AMD orchestration match the current integration branch." - "Reuse validated legacy MI355X Hugging Face cache trees through the standard $HF_HOME/hub layout under a per-model lock, avoiding duplicate production-checkpoint downloads or copies." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2628 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-disagg + description: + - "Replace the model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI for the existing 1P1D TP8 MI355X MXFP4 topology." + - "Run the complete 8k1k c8/c16/c32/c64/c128/c256/c512 search space in one eight-hour Slurm allocation through the repository's unchanged benchmark_serving.py custom benchmark." + - "Upgrade the disaggregated lane from SGLang ROCm v0.5.12.post1 to the current v0.5.17 MI35X image while retaining AITer attention, FP8 KV cache, and the production model identity." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2629