diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c..000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml index 7af319853..717fcac66 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml @@ -20,7 +20,7 @@ identity: amd-mori: "0.5.17.dev20260809+g7120f3ee13" slurm: - time_limit: "02:00:00" + time_limit: "08:00:00" # This topology owns every GPU on each MI355X node. Give SGLang the complete # 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which @@ -118,27 +118,32 @@ benchmark: result_root="/results/${SLURM_JOB_ID}" mkdir -p "${result_root}/fixed-seq" trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT - concurrency=8 - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai-chat \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/chat/completions \ - --model Qwen/Qwen3.5-397B-A17B-FP8 \ - --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ - --dataset-name random \ - --random-input-len 8192 \ - --random-output-len 1024 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - --num-warmups "${concurrency}" \ - --num-prompts "$((concurrency * 2))" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_root}/fixed-seq" \ - --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json" + for concurrency in 8 16 32 64 128; do + num_prompts=$((concurrency * 10)) + if ((num_prompts < 16)); then + num_prompts=16 + fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3.5-397B-A17B-FP8 \ + --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json" + done env: HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 39b9e2d5d..1b2b2cbfa 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -262,10 +262,10 @@ qwen3.5-fp8-mi355x-atom-mtp: - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 - runner: mi355x-disagg + runner: cluster:mi355x-amds precision: fp8 framework: sglang-disagg router: { name: sglang-router, version: "0.3.2" } @@ -277,33 +277,26 @@ qwen3.5-fp8-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # 1P+1D TP8/EP1 low-concurrency sweep. - # dp-attn intentionally false (matches the 1k1k row): with - # --enable-dp-attention + --moe-a2a-backend mori, sglang auto-promotes + # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm + # allocation so model initialization is paid once instead of per point. + # 1P+1D TP8/EP1 intentionally keeps DP attention disabled: with + # --enable-dp-attention + --moe-a2a-backend mori, SGLang auto-promotes # moe_ep_size=tp_size=8, but is_deepep_class_backend() excludes MoRI, - # so num_shared_slots stays at the global value (1) and the - # (num_experts - num_shared_slots) % moe_ep_size assertion in - # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - # Track upstream sglang for a fix; flip back to dp-attn=true once - # MoRI is added to is_deepep_class_backend() or shared-slot - # accounting is reconciled. + # so Qwen3.5 shared-slot accounting is invalid. - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] + conc-list: [1] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 @@ -1625,39 +1618,6 @@ qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: ep: 1 dp-attn: false -# Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to -# srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and -# the request/KV planes remain native SGLang Router + AMD MoRI. -qwen3.5-fp8-mi355x-sglang-srt-disagg: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - multinode: true - disagg: true - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf model: amd/MiniMax-M3-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index efd8c51dd..ae33f00ae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5788,3 +5788,13 @@ - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to DeepSeek-V4-Pro-AgentX; num-steps/num-draft-tokens are derived from DECODE_MTP_SIZE in server_sglang.sh. DeepSeek-V4's MTP head is a native NextN/EAGLE draft shipped with the model, so no --speculative-draft-model-path is needed." - "Two search-space arms: TP8/EP1 no-DP at conc-list [2,4,8,16,32], and TP8/EP8/DPA (ep=8, dp-attn=true) at conc-list [64,96,128], both on image lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2309 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + description: + - "Replace the model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI for the existing 1P1D TP8 MI355X topology." + - "Run the complete 8k1k c8/c16/c32/c64/c128 search space in one eight-hour Slurm allocation through the repository's unchanged benchmark_serving.py custom benchmark." + - "Use lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809, retain FP8 KV cache and AITer attention, and remove the temporary duplicate qwen3.5-fp8-mi355x-sglang-srt-disagg config key." + - "Pin the launcher to SemiAnalysisAI/srt-slurm c609754b so native-router endpoint registration and AMD orchestration match the current integration branch." + - "Reuse validated legacy MI355X Hugging Face cache trees through the standard $HF_HOME/hub layout under a per-model lock, avoiding duplicate production-checkpoint downloads or copies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2628 diff --git a/runners/launch_mi355x-amds-srt.sh b/runners/launch_mi355x-amds-srt.sh index a5e94c6a0..5fef8f7c6 100755 --- a/runners/launch_mi355x-amds-srt.sh +++ b/runners/launch_mi355x-amds-srt.sh @@ -5,7 +5,7 @@ set -euo pipefail # in explicitly with CONFIG_FILE; all existing MI355X launch behavior remains # unchanged for every other row. SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" -SRT_SLURM_COMMIT="31e72da43ed21fe941c039be51b2cad1a3cf428a" +SRT_SLURM_COMMIT="c609754b5622f96d5c12a93149e245308d4f1e9b" SLURM_PARTITION="compute" SGLANG_IMAGE="lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" SHARED_BASE="/it-share/gharunners2/srt-slurm" @@ -18,6 +18,8 @@ SHARED_RESULTS="${SHARED_BASE}/results" : "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}" : "${MODEL:?MODEL must identify the Hugging Face model}" +MODEL_CACHE_KEY="models--${MODEL//\//--}" + CONFIG_PATH="${CONFIG_FILE%%:*}" LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml" @@ -29,9 +31,11 @@ WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" SRT_REPO_DIR="${WORK_DIR}/srt-slurm" mkdir -p "$WORK_DIR" "$SHARED_RESULTS" -# Materialize one immutable, shared squashfs and the small public validation -# model. This job exits normally and never cancels or preempts another job. -# The shared lock makes concurrent aggregate/disaggregated validations safe. +# Materialize one immutable, shared squashfs and the requested model. Older +# MI355X jobs populated Hugging Face's cache directly under SHARED_HF_CACHE; +# expose a complete legacy cache through the current $HF_HOME/hub layout before +# snapshot_download so large production checkpoints are reused without copies. +# This job exits normally and never cancels or preempts another job. STAGE_SCRIPT="${WORK_DIR}/stage-mi355x-runtime.sbatch" cat > "$STAGE_SCRIPT" </dev/null 2>&1; then mv "\$tmp" "$SHARED_IMAGE" fi flock -u 9 +mkdir -p "$SHARED_HF_CACHE/hub" +exec 8>"$SHARED_HF_CACHE/.${MODEL_CACHE_KEY}.stage.lock" +flock -w 2400 8 +legacy_model_dir="$SHARED_HF_CACHE/${MODEL_CACHE_KEY}" +canonical_model_dir="$SHARED_HF_CACHE/hub/${MODEL_CACHE_KEY}" +if [[ ! -e "\$canonical_model_dir" && -f "\$legacy_model_dir/refs/main" && -d "\$legacy_model_dir/snapshots" ]]; then + ln -s "../${MODEL_CACHE_KEY}" "\$canonical_model_dir" +fi +flock -u 8 srun --nodes=1 --ntasks=1 \ --container-image="$SHARED_IMAGE" \ --container-mounts="$SHARED_HF_CACHE:/hf_hub_cache" \