perf(amd): port MI355X Qwen3.5 FP8 disagg sweep to srt-slurm

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml

@@ -20,7 +20,7 @@ identity:
     amd-mori: "0.5.17.dev20260809+g7120f3ee13"
 
 slurm:
-  time_limit: "02:00:00"
+  time_limit: "08:00:00"
 
 # This topology owns every GPU on each MI355X node. Give SGLang the complete
 # 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which
@@ -118,27 +118,32 @@ benchmark:
     result_root="/results/${SLURM_JOB_ID}"
     mkdir -p "${result_root}/fixed-seq"
     trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
-    concurrency=8
-    python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
-      --backend openai-chat \
-      --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
-      --endpoint /v1/chat/completions \
-      --model Qwen/Qwen3.5-397B-A17B-FP8 \
-      --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \
-      --dataset-name random \
-      --random-input-len 8192 \
-      --random-output-len 1024 \
-      --random-range-ratio 1.0 \
-      --random-num-workers 1 \
-      --num-warmups "${concurrency}" \
-      --num-prompts "$((concurrency * 2))" \
-      --max-concurrency "${concurrency}" \
-      --request-rate inf \
-      --ignore-eos \
-      --disable-tqdm \
-      --save-result \
-      --result-dir "${result_root}/fixed-seq" \
-      --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json"
+    for concurrency in 8 16 32 64 128; do
+      num_prompts=$((concurrency * 10))
+      if ((num_prompts < 16)); then
+        num_prompts=16
+      fi
+      python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
+        --backend openai-chat \
+        --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
+        --endpoint /v1/chat/completions \
+        --model Qwen/Qwen3.5-397B-A17B-FP8 \
+        --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \
+        --dataset-name random \
+        --random-input-len 8192 \
+        --random-output-len 1024 \
+        --random-range-ratio 1.0 \
+        --random-num-workers 1 \
+        --num-warmups "$((concurrency * 2))" \
+        --num-prompts "${num_prompts}" \
+        --max-concurrency "${concurrency}" \
+        --request-rate inf \
+        --ignore-eos \
+        --disable-tqdm \
+        --save-result \
+        --result-dir "${result_root}/fixed-seq" \
+        --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json"
+    done
   env:
     HF_HOME: /hf_hub_cache
     HF_HUB_CACHE: /hf_hub_cache/hub

configs/amd-master.yaml

@@ -262,10 +262,10 @@ qwen3.5-fp8-mi355x-atom-mtp:
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp }
 
 qwen3.5-fp8-mi355x-sglang-disagg:
-  image: lmsysorg/sglang:v0.5.14-rocm720-mi35x
+  image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809
   model: Qwen/Qwen3.5-397B-A17B-FP8
   model-prefix: qwen3.5
-  runner: mi355x-disagg
+  runner: cluster:mi355x-amds
   precision: fp8
   framework: sglang-disagg
   router: { name: sglang-router, version: "0.3.2" }
@@ -277,33 +277,26 @@ qwen3.5-fp8-mi355x-sglang-disagg:
     - isl: 8192
       osl: 1024
       search-space:
-      # 1P+1D TP8/EP1 low-concurrency sweep.
-      # dp-attn intentionally false (matches the 1k1k row): with
-      # --enable-dp-attention + --moe-a2a-backend mori, sglang auto-promotes
+      # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm
+      # allocation so model initialization is paid once instead of per point.
+      # 1P+1D TP8/EP1 intentionally keeps DP attention disabled: with
+      # --enable-dp-attention + --moe-a2a-backend mori, SGLang auto-promotes
       # moe_ep_size=tp_size=8, but is_deepep_class_backend() excludes MoRI,
-      # so num_shared_slots stays at the global value (1) and the
-      # (num_experts - num_shared_slots) % moe_ep_size assertion in
-      # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared).
-      # Track upstream sglang for a fix; flip back to dp-attn=true once
-      # MoRI is added to is_deepep_class_backend() or shared-slot
-      # accounting is reconciled.
+      # so Qwen3.5 shared-slot accounting is invalid.
       - spec-decoding: "none"
-        conc-list: [ 8, 16, 32, 64, 128 ]
+        conc-list: [1]
         prefill:
           num-worker: 1
           tp: 8
           ep: 1
           dp-attn: false
           additional-settings:
-          - "PREFILL_NODES=1"
+          - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml"
         decode:
           num-worker: 1
           tp: 8
           ep: 1
           dp-attn: false
-          additional-settings:
-          - "DECODE_NODES=1"
-          - "DECODE_MTP_SIZE=0"
 
 qwen3.5-fp4-mi355x-sglang:
   image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730
@@ -1625,39 +1618,6 @@ qwen3-0.6b-fp16-mi355x-sglang-srt-disagg:
           ep: 1
           dp-attn: false
 
-# Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to
-# srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and
-# the request/KV planes remain native SGLang Router + AMD MoRI.
-qwen3.5-fp8-mi355x-sglang-srt-disagg:
-  image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809
-  model: Qwen/Qwen3.5-397B-A17B-FP8
-  model-prefix: qwen3.5
-  runner: cluster:mi355x-amds
-  precision: fp8
-  framework: sglang-disagg
-  multinode: true
-  disagg: true
-  router: { name: sglang-router, version: "0.3.2" }
-  kv-p2p-transfer: mori
-  scenarios:
-    fixed-seq-len:
-    - isl: 8192
-      osl: 1024
-      search-space:
-      - conc-list: [8]
-        prefill:
-          num-worker: 1
-          tp: 8
-          ep: 1
-          dp-attn: false
-          additional-settings:
-          - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml"
-        decode:
-          num-worker: 1
-          tp: 8
-          ep: 1
-          dp-attn: false
-
 minimaxm3-fp4-mi355x-vllm-agentic:
   image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf
   model: amd/MiniMax-M3-MXFP4

perf-changelog.yaml

@@ -5788,3 +5788,13 @@
     - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to DeepSeek-V4-Pro-AgentX; num-steps/num-draft-tokens are derived from DECODE_MTP_SIZE in server_sglang.sh. DeepSeek-V4's MTP head is a native NextN/EAGLE draft shipped with the model, so no --speculative-draft-model-path is needed."
     - "Two search-space arms: TP8/EP1 no-DP at conc-list [2,4,8,16,32], and TP8/EP8/DPA (ep=8, dp-attn=true) at conc-list [64,96,128], both on image lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2309
+
+- config-keys:
+    - qwen3.5-fp8-mi355x-sglang-disagg
+  description:
+    - "Replace the model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI for the existing 1P1D TP8 MI355X topology."
+    - "Run the complete 8k1k c8/c16/c32/c64/c128 search space in one eight-hour Slurm allocation through the repository's unchanged benchmark_serving.py custom benchmark."
+    - "Use lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809, retain FP8 KV cache and AITer attention, and remove the temporary duplicate qwen3.5-fp8-mi355x-sglang-srt-disagg config key."
+    - "Pin the launcher to SemiAnalysisAI/srt-slurm c609754b so native-router endpoint registration and AMD orchestration match the current integration branch."
+    - "Reuse validated legacy MI355X Hugging Face cache trees through the standard $HF_HOME/hub layout under a per-model lock, avoiding duplicate production-checkpoint downloads or copies."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2628

runners/launch_mi355x-amds-srt.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # in explicitly with CONFIG_FILE; all existing MI355X launch behavior remains
 # unchanged for every other row.
 SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git"
-SRT_SLURM_COMMIT="31e72da43ed21fe941c039be51b2cad1a3cf428a"
+SRT_SLURM_COMMIT="c609754b5622f96d5c12a93149e245308d4f1e9b"
 SLURM_PARTITION="compute"
 SGLANG_IMAGE="lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809"
 SHARED_BASE="/it-share/gharunners2/srt-slurm"
@@ -18,6 +18,8 @@ SHARED_RESULTS="${SHARED_BASE}/results"
 : "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}"
 : "${MODEL:?MODEL must identify the Hugging Face model}"
 
+MODEL_CACHE_KEY="models--${MODEL//\//--}"
+
 CONFIG_PATH="${CONFIG_FILE%%:*}"
 LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}"
 CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml"
@@ -29,9 +31,11 @@ WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}"
 SRT_REPO_DIR="${WORK_DIR}/srt-slurm"
 mkdir -p "$WORK_DIR" "$SHARED_RESULTS"
 
-# Materialize one immutable, shared squashfs and the small public validation
-# model. This job exits normally and never cancels or preempts another job.
-# The shared lock makes concurrent aggregate/disaggregated validations safe.
+# Materialize one immutable, shared squashfs and the requested model. Older
+# MI355X jobs populated Hugging Face's cache directly under SHARED_HF_CACHE;
+# expose a complete legacy cache through the current $HF_HOME/hub layout before
+# snapshot_download so large production checkpoints are reused without copies.
+# This job exits normally and never cancels or preempts another job.
 STAGE_SCRIPT="${WORK_DIR}/stage-mi355x-runtime.sbatch"
 cat > "$STAGE_SCRIPT" <<EOF
 #!/usr/bin/env bash
@@ -59,6 +63,15 @@ if ! unsquashfs -s "$SHARED_IMAGE" >/dev/null 2>&1; then
     mv "\$tmp" "$SHARED_IMAGE"
 fi
 flock -u 9
+mkdir -p "$SHARED_HF_CACHE/hub"
+exec 8>"$SHARED_HF_CACHE/.${MODEL_CACHE_KEY}.stage.lock"
+flock -w 2400 8
+legacy_model_dir="$SHARED_HF_CACHE/${MODEL_CACHE_KEY}"
+canonical_model_dir="$SHARED_HF_CACHE/hub/${MODEL_CACHE_KEY}"
+if [[ ! -e "\$canonical_model_dir" && -f "\$legacy_model_dir/refs/main" && -d "\$legacy_model_dir/snapshots" ]]; then
+    ln -s "../${MODEL_CACHE_KEY}" "\$canonical_model_dir"
+fi
+flock -u 8
 srun --nodes=1 --ntasks=1 \
     --container-image="$SHARED_IMAGE" \
     --container-mounts="$SHARED_HF_CACHE:/hf_hub_cache" \