diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 95653207b6..0000000000 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash - -# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (DeepSeek-V4-Pro FP4, 1P1D TP8). -# -# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by -# environment variables and submits a SLURM job via submit.sh. The agentic / -# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro -# specific flags (dsv4 attention backend, page-size 256, SWA settings). - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - DURATION \ - KV_OFFLOADING \ - IS_AGENTIC \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# ── Identity / result naming ── -export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" -export PRECISION="${PRECISION:-fp4}" -export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" - -# ── Agentic benchmark params ── -export DURATION="${DURATION:-1800}" -# DSV4-Pro max model len for agentic traces (matches single-node recipe). -export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" - -# ── In-tree sglang patches ── -# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a -# pure MoE/DSA architecture without hybrid state; skip to avoid interference. -export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" - -# ── Aiter fault mitigation ── -# --disable-custom-all-reduce avoids a known aiter fault on MI355X. -export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" - -# ── KV cache offloading (HiCache) ── -# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe -# only implements HiCache, so "hicache" is the only supported value. -# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. -export KV_OFFLOADING="${KV_OFFLOADING:-none}" -if [[ "$KV_OFFLOADING" != "none" ]]; then - export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" -fi -# HiCache/Mooncake tunables only matter when KV offloading is enabled. -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then - export HICACHE_TIER="${HICACHE_TIER:-L2}" - export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" - # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. - export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" - # HiCache ratio (host pool = ratio * GPU KV pool). - export HICACHE_RATIO="${HICACHE_RATIO:-3}" - # DSv4 wants the ratio-based pool, but server_sglang.sh prefers - # --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB is set. - # Opt out via FORCE_HICACHE_RATIO instead of unsetting TOTAL_CPU_DRAM_GB - # (also required client-side by benchmark_lib.sh when KV_OFFLOADING=dram). - export FORCE_HICACHE_RATIO=1 - - # ── HiCache layout/backend by tier ── - # L3 (Mooncake): page_first + direct + write_through + storage=mooncake - # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none - # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding - # the mori RDMA race that causes GPU memory access faults with write_through. - if [[ "${HICACHE_TIER^^}" == "L3" ]]; then - export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" - else - export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - fi - export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" - # Shared nodes: use non-default Mooncake ports to avoid collisions. - export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" - export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" - export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" - export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" - export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" - export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" - export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" - export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" - export MC_DEVICE="${MC_DEVICE:-}" - export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" - export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" -fi - -# ── MoRIIO RDMA Send Queue tuning ── -export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" -export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" - -# ── SGLang PD router policy + server metrics ── -export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" -export ENABLE_METRICS="${ENABLE_METRICS:-1}" - -# ── MTP ── -export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# Derive EP/DP enable flags from the topology inputs. -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/bench.sh b/benchmarks/multi_node/amd_utils/bench.sh deleted file mode 100755 index 3dde0a68a5..0000000000 --- a/benchmarks/multi_node/amd_utils/bench.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# Dual-Engine Disaggregated Benchmark Runner -# -# ENGINE=sglang (default): SGLang benchmark -# ENGINE=vllm: vLLM benchmark -# -# Produces JSON result files via benchmark_serving.py so that the CI pipeline -# can collect and process results. -# -# Usage: bash bench.sh \ -# \ -# - -ENGINE="${ENGINE:-sglang-disagg}" - -n_prefill=$1 -n_decode=$2 -prefill_gpus=$3 -decode_gpus=$4 -model_path=$5 -model_name=$6 -MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - BENCH_MODEL="${MODEL_NAME:-${MODEL_PATH}}" -else - BENCH_MODEL="${MODEL_PATH}" -fi -log_path=$7 - -chosen_isl=${8:-1024} -chosen_osl=${9:-1024} -concurrency_list=${10:-"512x1"} -if [[ "$ENGINE" == "vllm-disagg" ]]; then - chosen_req_rate=${11:-inf} -else - chosen_req_rate=${11:-1} -fi -random_range_ratio=${12:-0.8} -num_prompts_multiplier=${13:-10} - -IFS='x' read -r -a chosen_concurrencies <<< "$concurrency_list" - -ROUTER_PORT="${ROUTER_PORT:-30000}" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -profile_folder="${log_path}/${ENGINE}_isl_${chosen_isl}_osl_${chosen_osl}" -mkdir -p "$profile_folder" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -for max_concurrency in "${chosen_concurrencies[@]}"; do - - export_file="${profile_folder}/concurrency_${max_concurrency}_req_rate_${chosen_req_rate}_gpus_$((prefill_gpus+decode_gpus))_ctx_${prefill_gpus}_gen_${decode_gpus}" - - num_prompts=$(( max_concurrency * num_prompts_multiplier )) - if [[ "$num_prompts" -lt 16 ]]; then - num_prompts=16 - fi - - echo "profile_folder: $profile_folder" - echo "max_concurrency: $max_concurrency" - echo "chosen_req_rate: $chosen_req_rate" - echo "MODEL_PATH: $MODEL_PATH" - echo "ROUTER_PORT: $ROUTER_PORT" - echo "chosen_isl: $chosen_isl" - echo "chosen_osl: $chosen_osl" - echo "num_prompts: $num_prompts" - echo "export_file: $export_file" - - # Engine-specific extra flags - extra_flags="" - # vllm - if [[ "$ENGINE" == "vllm-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - # atom - elif [[ "$ENGINE" == "atom-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - if [ "$IS_MTP" = "true" ]; then - # just override extra_flags as dsv3 use different tokenizer path - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - extra_flags="--dsv4" - else - extra_flags="--use-chat-template" - fi - fi - # sglang - else - if [ "$IS_MTP" = "true" ]; then - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - extra_flags="--dsv4" - else - extra_flags="--use-chat-template" - fi - fi - fi - - run_benchmark_serving \ - --bench-serving-dir "$REPO_ROOT" \ - --model "$BENCH_MODEL" \ - --port "$ROUTER_PORT" \ - --backend openai \ - --input-len "$chosen_isl" \ - --output-len "$chosen_osl" \ - --random-range-ratio "$random_range_ratio" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "$export_file" \ - --result-dir /workspace/ \ - $extra_flags - - echo "-----------------------------------------" - - # vLLM: cooldown between rounds for idle KV block reaper - if [[ "$ENGINE" == "vllm-disagg" ]]; then - echo "[BENCH] Cooldown: waiting 10s for idle KV block reaper..." - sleep 10 - fi -done diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh deleted file mode 100755 index 0cf3978bbf..0000000000 --- a/benchmarks/multi_node/amd_utils/env.sh +++ /dev/null @@ -1,377 +0,0 @@ -#!/bin/bash -# Dual-engine environment setup for multi-node disaggregated serving. -# -# ENGINE=sglang (default): SGLang/MoRI environment -# ENGINE=vllm: vLLM/Nixl environment -# -# REQUIRED ENVIRONMENT VARIABLES: -# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,... or mlx5_0,mlx5_1,...) -# Set by runner or auto-detected from hostname. -set -x - -ENGINE="${ENGINE:-sglang-disagg}" -export PYTHONDONTWRITEBYTECODE=1 - -# ============================================================================= -# HiCache / Mooncake settings from job.slurm -# ============================================================================= -# job.slurm writes the recipe-provided HiCache/Mooncake tunables to -# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source -# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container -# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this -# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, -# overriding the recipe's --page-size. Empty values in the file are harmless: -# the "${VAR:-default}" fallbacks still treat "" as unset. -if [[ -f /config/hicache_mc.env ]]; then - set -a - source /config/hicache_mc.env - set +a - echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" -fi - -# ============================================================================= -# Shared: IBDEVICES detection -# ============================================================================= - -# Prefer IBDEVICES set by runner (runners/launch_mi355x-amds.sh) -# Fall back to hostname detection if not set (for direct script execution) -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - echo "ERROR: Unable to detect RDMA devices. Set IBDEVICES explicitly." >&2 - exit 1 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - -# Shared: Auto-detect default network interface (portable across clusters) -# Only auto-detect if not already set by the runner/environment -if [[ -z "$GLOO_SOCKET_IFNAME" ]]; then - export GLOO_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi -if [[ -z "$NCCL_SOCKET_IFNAME" ]]; then - export NCCL_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi - -set +x - -export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} - -# ============================================================================= -# MoRI-specific environment -# ============================================================================= -# Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. - -export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" -export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" -export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" -export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" -export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" - -# QoS/DSCP configuration -# Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname -if [[ -n "$MORI_RDMA_TC" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC (set by runner or environment)" -elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null| awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - TC=$(( 4 * ND_DSCP )) - export MORI_RDMA_SL=$ND_PRIO - export MORI_IO_SL=$ND_PRIO - export MORI_RDMA_TC=$TC - export MORI_IO_TC=$TC - echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL, MORI_IO_TC=$MORI_IO_TC, MORI_IO_SL=$MORI_IO_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - # Fall back to hostname-based detection - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration." - fi - fi -else - # nicctl not available, try hostname-based detection - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration." - echo " This is normal for clusters without QoS or outside Docker containers." - fi -fi - -# ============================================================================= -# Engine-specific environment -# ============================================================================= - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # ========================================================================= - # vLLM/Nixl-specific environment - # ========================================================================= - export VLLM_USE_V1=1 - export VLLM_SERVER_DEV_MODE=0 - export VLLM_DISABLE_REQUEST_ID_RANDOMIZATION=1 - - set -x - - # UCX_NET_DEVICES: Use the first tw-eth interface for UCX TCP transport - if [[ -z "$UCX_NET_DEVICES" ]]; then - UCX_NET_DEV=$(ip -o link show 2>/dev/null | awk -F': ' '/tw-eth/{print $2}' | head -1) - if [[ -n "$UCX_NET_DEV" ]]; then - export UCX_NET_DEVICES="$UCX_NET_DEV" - else - FIRST_IB=$(echo "$IBDEVICES" | cut -d',' -f1) - if [[ -n "$FIRST_IB" ]]; then - export UCX_NET_DEVICES="${FIRST_IB}:1" - fi - fi - echo "[INFO] Auto-set UCX_NET_DEVICES=$UCX_NET_DEVICES" - else - echo "[INFO] Using UCX_NET_DEVICES=$UCX_NET_DEVICES (set by environment)" - fi - - # RoCEv2: use IPv4-mapped GID (index 1) for inter-node RDMA routing - export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-1} - - # QoS/DSCP configuration for lossless RoCEv2 fabric. - if [[ -n "$UCX_IB_TRAFFIC_CLASS" ]]; then - echo "[INFO] Using UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS (set by environment)" - elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null | awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) - export UCX_IB_SL=$ND_PRIO - echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - fi - fi - else - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - else - echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration." - fi - fi - - set +x - echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" - -else - # ========================================================================= - # SGLang-specific environment - # ========================================================================= - - export SGLANG_USE_AITER=1 - export AITER_LOG_LEVEL=ERROR - - export SGLANG_MORI_DISPATCH_DTYPE=auto - # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast - # export MORI_COMBINE_DTYPE_DECODE=fp8 - export MORI_COMBINE_DTYPE_PREFILL="" - export MORI_COMBINE_DTYPE_DECODE="" - export SGLANG_MORI_QP_PER_TRANSFER=4 - export SGLANG_MORI_NUM_WORKERS=4 - # Keep these as overridable defaults (not hard assignments), otherwise - # later tuning blocks cannot raise them for high-concurrency runs. - # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" - - # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" - # export MORI_IO_QP_MAX_CQE=32768 - # export MORI_IO_QP_MAX_SGE=1 - - # export MORI_IO_TC_DISABLE=0 - - export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 - export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 - - export SGLANG_HEALTH_CHECK_TIMEOUT=600 - - # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading - if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then - export SGLANG_ROCM_FUSED_DECODE_MLA=0 - export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - export SAFETENSORS_FAST_GPU=1 - fi - - # Disable allocating memory in one pass - export MORI_SHMEM_MODE=ISOLATION - - # Enable spec v2 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - export SGLANG_LOG_MS=true - export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32 - - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=512 - - export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768 - export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703 - - # set MTP size=1 when EP16 - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - - export MORI_EP_LAUNCH_CONFIG_MODE=AUTO - - # Default to WARNING to cut per-op MoRI log spam on long multinode/eval - # runs; override with MORI_APP_LOG_LEVEL=INFO when debugging. - export MORI_APP_LOG_LEVEL="${MORI_APP_LOG_LEVEL:-WARNING}" - - # Router logging control: - # 0 (default) keeps noisy per-request access logs out of stdout while still logging to file. - # 1 mirrors router logs to stdout via tee (useful for live debugging). - export SGLANG_ROUTER_STDOUT_LOGS="${SGLANG_ROUTER_STDOUT_LOGS:-0}" - - # FIXME: WA for latest upstream 0305 image - export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} - - # Decode CUDA-graph capture crash on ROCm 7.2.0 (TP8+EP8, mori a2a). - # Symptom: during decode cuda-graph capture, the torch ProcessGroupNCCL - # *watchdog* thread calls hipEventQuery() to poll in-flight NCCL work. - # ROCm <= 7.2.0's HIP runtime does NOT honor cudaStreamCaptureModeThreadLocal, - # so the watchdog's cross-thread query touches the main thread's active - # capture and invalidates it -> "HIP error: operation not permitted on an - # event last recorded in a capturing stream (hipErrorCapturedEvent)" -> - # watchdog aborts -> "Rank 0 scheduler died during initialization" (-6). - # This is a HIP runtime bug, not OOM and not a mori/deepep-mode bug (it - # fires for --deepep-mode normal and auto alike; EP8+mori just adds NCCL - # PGs that make the watchdog race fire). Refs: sgl-project/sglang#29235, - # #24011; ROCm/hip#3876; pytorch/pytorch#176251. - # Real fix = ROCm 7.2.2+ (honors THREAD_LOCAL). Until the base image is - # bumped, TORCH_NCCL_BLOCKING_WAIT=true makes NCCL work completion use a - # blocking wait instead of the async watchdog hipEventQuery poll, so no - # event is queried during capture. CUDA graph stays fully enabled. - export TORCH_NCCL_BLOCKING_WAIT="${TORCH_NCCL_BLOCKING_WAIT:-1}" - export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}" - # export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" - - # ========================================================================= - # DeepSeek-V4-Pro PD recipe overrides - # Placed at the end of the SGLang env block so it wins over the global - # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD - # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / - # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK - # fused-MoE path, which raises "Unsupported kernel config for moe heuristic - # dispatch" at decode time on this fp4 model (job 19034 crash). Only the - # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and - # the cluster NIC/socket vars stay runner-derived. - # ========================================================================= - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - export SGLANG_AITER_MLA_PERSIST=0 - ## resolve the OOR issue - export HSA_NO_SCRATCH_RECLAIM=0 - # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). - export MORI_IO_QP_MAX_SEND_WR=32767 - # Unified radix tree: cache impl with per-component (full-attn / SWA) - # management for hybrid-attention models. Set unconditionally (not gated on - # hicache) so all SGLang runs use it. - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 - # Proactively free out-of-window SWA KV slots during chunked prefill. - # Without it, in-flight requests pin SWA KV for their whole context, keeping - # the SWA pool under constant eviction pressure; under LRU the trailing - # window of cached sessions gets flushed, making prefix-cache hits bimodal - # and collapsing the effective hit rate on multi-turn agentic workloads. - export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - - # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) - export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=auto - export MORI_COMBINE_DTYPE_DECODE=auto - - # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=64 - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE - - # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 - # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* - # coupling when set. - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 - - # Fixed inter-kernel switch threshold (not derived). - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 - - # Overlap plan stream on for DSv4 (global default is 0) - # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) - export SGLANG_DEFAULT_THINKING=1 - export SGLANG_DSV4_REASONING_EFFORT=high - export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false - export SGLANG_USE_AITER=1 - export SGLANG_USE_ROCM700A=0 - export SGLANG_OPT_USE_FUSED_COMPRESS=true - export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton - export SGLANG_OPT_FP8_WO_A_GEMM=false - export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false - export SGLANG_OPT_USE_TOPK_V2=false - export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} - export SGLANG_OPT_USE_TILELANG_INDEXER=false - export SGLANG_OPT_USE_TILELANG_MHC_PRE=false - export SGLANG_OPT_USE_TILELANG_MHC_POST=false - export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 - export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true - export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false - export SGLANG_ROCM_USE_MULTI_STREAM=false - export AITER_BF16_FP8_MOE_BOUND=0 - export SGLANG_EAGER_INPUT_NO_COPY=true - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 - fi - -fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/env_atom.sh b/benchmarks/multi_node/amd_utils/env_atom.sh deleted file mode 100644 index 089594cbe4..0000000000 --- a/benchmarks/multi_node/amd_utils/env_atom.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# ATOM/mooncake-specific environment setup for multi-node disaggregated serving. -# -# Sourced by server_atom.sh in place of env.sh (which is SGLang/MoRI-specific). -# -# REQUIRED ENVIRONMENT VARIABLES: -# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,...) -# Set by runner or auto-detected from hostname. - -set -x - -export PYTHONUNBUFFERED=1 -export PYTHONDONTWRITEBYTECODE=1 - -# ============================================================================= -# IBDEVICES detection (same as env.sh) -# ============================================================================= - -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - # ATOM uses mooncake proxy_ip/handshake_port for KV transfer — IBDEVICES is - # not passed as a server argument (unlike SGLang --disaggregation-ib-device). - # Log a warning but do not fail; mooncake will use its own RDMA device selection. - echo "[WARN] Unable to detect RDMA devices via ibv_devinfo; IBDEVICES unset (non-fatal for ATOM/mooncake)" >&2 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - -# ============================================================================= -# ATOM/mooncake-specific environment -# ============================================================================= - -# mooncake RDMA KV transfer library path -export LD_LIBRARY_PATH=/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-} - -# faster model loading (safetensors only) -export SAFETENSORS_FAST_GPU=1 - -# aiter logging (WARNING to reduce noise; use DEBUG for troubleshooting) -export VLLM_LOG_LEVEL=WARNING -export ATOM_LOG_LEVEL=WARNING -export AITER_LOG_LEVEL=WARNING -export LOG_LEVEL=WARNING -export LOGLEVEL=WARNING - -set +x - -# ATOM_HOST_IP is set per-node in server_atom.sh (= host_ip, used as handshake IP) -echo "[INFO] ATOM env: IBDEVICES=$IBDEVICES LD_LIBRARY_PATH includes mooncake" \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh deleted file mode 100755 index 86acabbab0..0000000000 --- a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -# Pre-flight RDMA QoS/DCQCN validation for a single node, run by job.slurm via -# `srun` across every allocated node BEFORE any container/GPU time is spent. -# -# A misconfigured NIC (PFC not covering the RoCE priority, DCQCN disabled, ...) -# doesn't make MoRI's cross-node EP/RDMA transfers error out -- it just quietly -# degrades, and the only symptom is unexplained tail latency or throughput -# variance in the benchmark numbers hours later. Failing fast here, before the -# job burns node-hours, is much cheaper than debugging that after the fact. -# -# check_qos()/check_dcqcn() below are trimmed/adapted from ROCm/mori's -# tools/env_check.sh: -# https://github.com/ROCm/mori/blob/main/tools/env_check.sh -# The upstream script's expensive ib_write_bw/ib_write_lat full-mesh bandwidth -# and latency tests are intentionally NOT ported here -- this runs before -# every single job, so it must be fast (seconds), not a multi-minute -# fabric-wide benchmark of its own. -# -# Scoped to AMD Pollara (ionic) NICs via nicctl, matching this repo's current -# fleet (see env.sh's nicctl-based MORI_RDMA_TC/SL detection). If bnxt_re or -# mlx5 NICs are added to the fleet, port the bnxt_*/mlx5_* check_* functions -# from the upstream script following the same pattern. -# -# Exit code: 0 = OK (or gracefully skipped, e.g. no ionic NICs on this host), -# 1 = hard QoS/DCQCN misconfiguration -- do not proceed with the run. -set -uo pipefail - -AINIC_MIN_VER="1.117.5-a-45" # minimum recommended AINIC firmware for IBGDA - -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[0;33m' -NC='\033[0m' - -log_ok() { echo -e "[$(hostname -s)] ${GREEN}[OK]${NC} $*"; } -log_fail() { echo -e "[$(hostname -s)] ${RED}[FAIL]${NC} $*"; } -log_warn() { echo -e "[$(hostname -s)] ${YELLOW}[WARN]${NC} $*"; } -die() { log_fail "$@"; exit 1; } - -# version_ge -> true if >= (dotted/hyphenated, via sort -V) -version_ge() { - local cand="$1" min="$2" - [[ "$cand" == "$min" ]] && return 0 - [[ "$(printf '%s\n%s\n' "$cand" "$min" | sort -V | head -1)" == "$min" ]] -} - -# check_ainic_version_recommendation -# warns if firmware is on the IBGDA-incapable 1.117.1 branch, or below the -# recommended minimum for cross-node MORI (EP over RDMA / IBGDA). -check_ainic_version_recommendation() { - local ver="$1" - [[ -n "$ver" ]] || { log_warn "cannot verify AINIC firmware version against recommendation (empty)"; return; } - if [[ "$ver" =~ ^1\.117\.1([.-]|$) ]]; then - log_warn "AINIC firmware $ver is on the 1.117.1 branch, which does NOT support IBGDA -- upgrade to >= $AINIC_MIN_VER" - elif version_ge "$ver" "$AINIC_MIN_VER"; then - log_ok "AINIC firmware $ver meets the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" - else - log_warn "AINIC firmware $ver is below the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" - fi -} - -# check_versions() -- informational only (never hard-fails the job). -check_versions() { - local fw_output sw_output - fw_output=$(sudo nicctl show version firmware 2>/dev/null) - sw_output=$(sudo nicctl show version host-software 2>/dev/null) - - local fw_versions fw_count - fw_versions=$(echo "$fw_output" | grep -i "firmware" | awk '{print $NF}' | sort -u) - fw_count=$(echo "$fw_versions" | grep -c . || true) - if [[ "$fw_count" -ne 1 ]]; then - log_warn "firmware versions not consistent across NICs:" - echo "$fw_versions" - local v - while read -r v; do [[ -n "$v" ]] && check_ainic_version_recommendation "$v"; done <<< "$fw_versions" - else - log_ok "firmware : $fw_versions" - check_ainic_version_recommendation "$fw_versions" - fi - - local nicctl_ver - nicctl_ver=$(echo "$sw_output" | grep "nicctl" | awk '{print $NF}') - [[ -n "$nicctl_ver" ]] && log_ok "nicctl : $nicctl_ver" || log_warn "cannot determine nicctl version" -} - -# check_qos() -- HARD gate: classification type must be DSCP, and PFC no-drop -# must be enabled and cover every no-drop priority. Dies (exit 1) otherwise. -check_qos() { - local qos_output - qos_output=$(sudo nicctl show qos 2>/dev/null) - [[ -n "$qos_output" ]] || die "sudo nicctl show qos returned nothing" - - local class_type - class_type=$(echo "$qos_output" | grep "Classification type" | head -1 | awk '{print $NF}') - [[ "$class_type" == "DSCP" ]] || die "classification type is '$class_type', expected 'DSCP'" - log_ok "classification type : DSCP" - - local nd_prio_raw - nd_prio_raw=$(echo "$qos_output" | grep "PFC no-drop priorities" | head -1 | awk '{print $NF}') - [[ -n "$nd_prio_raw" ]] || die "cannot find PFC no-drop priority" - local nd_prios=() - IFS=',' read -ra nd_prios <<< "$nd_prio_raw" - log_ok "no-drop priorities : ${nd_prios[*]}" - - local pfc_bitmap - pfc_bitmap=$(echo "$qos_output" | grep "PFC priority bitmap" | head -1 | awk '{print $NF}') - [[ -n "$pfc_bitmap" && "$pfc_bitmap" != "0x0" ]] || die "PFC is not enabled (bitmap=$pfc_bitmap)" - local p - for p in "${nd_prios[@]}"; do - (( pfc_bitmap & (1 << p) )) || die "PFC bitmap $pfc_bitmap does not cover priority $p" - done - log_ok "PFC enabled for priorities ${nd_prios[*]} (bitmap=$pfc_bitmap)" -} - -# check_dcqcn() -- HARD gate: DCQCN must be enabled on every ROCE device, and -# the CNP DSCP must be consistent across NICs. Exits 1 otherwise. -check_dcqcn() { - local dcqcn_output - dcqcn_output=$(sudo nicctl show dcqcn 2>/dev/null) - [[ -n "$dcqcn_output" ]] || die "sudo nicctl show dcqcn returned nothing" - - local total - total=$(echo "$dcqcn_output" | grep -c "ROCE device") - [[ "$total" -gt 0 ]] || die "no ROCE devices found in dcqcn output" - - local disabled - disabled=$(echo "$dcqcn_output" | grep "Status" | grep -v "Enabled" || true) - if [[ -n "$disabled" ]]; then - log_fail "some ROCE devices have DCQCN disabled:" - echo "$disabled" - exit 1 - fi - log_ok "DCQCN enabled on all $total ROCE devices" - - local cnp_values cnp_count - cnp_values=$(echo "$dcqcn_output" | grep "DSCP value used for CNP" | awk '{print $NF}' | sort -u) - cnp_count=$(echo "$cnp_values" | grep -c . || true) - [[ "$cnp_count" -eq 1 ]] || die "CNP DSCP not consistent across NICs: $cnp_values" - log_ok "CNP DSCP = $cnp_values (consistent across all NICs)" -} - -# ============================= main ============================= - -if ! command -v nicctl &>/dev/null; then - log_warn "nicctl not found on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check (not an ionic NIC host, or nicctl not on PATH)" - exit 0 -fi - -# nicctl exits 0 even with no NIC present, so check its output rather than its exit code. -_nicctl_probe=$(sudo nicctl show version firmware 2>&1 || true) -if echo "$_nicctl_probe" | grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC'; then - log_warn "nicctl present but no ionic NIC detected/accessible on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check" - exit 0 -fi - -check_versions -check_qos -check_dcqcn - -log_ok "RDMA QoS/DCQCN pre-flight check passed on $(hostname -s)" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm deleted file mode 100755 index edf83015a9..0000000000 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ /dev/null @@ -1,789 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=disagg-bench -#SBATCH -N 3 # Overridden by submit.sh -N flag -#SBATCH -n 3 # Overridden by submit.sh -n flag -#SBATCH --ntasks-per-node=1 -#SBATCH --spread-job -#SBATCH --gres=gpu:8 -#SBATCH --time=24:00:00 -# --output and --error are set by submit.sh via BENCHMARK_LOGS_DIR - -ENGINE="${ENGINE:-sglang-disagg}" - -echo "=== Job Start Time ===" -echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" -echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" -echo "ENGINE: $ENGINE" -echo "=======================" -echo "" - -# ============================================================================= -# Model Validation -# ============================================================================= - -# Use $(pwd) not BASH_SOURCE — sbatch copies the script to /var/spool/slurmd/ -# at runtime, but the CWD remains the submit-time directory (amd_utils/). -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODELS_YAML="$(pwd)/models_vllm.yaml" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - MODELS_YAML="$(pwd)/models_atom.yaml" -else - MODELS_YAML="$(pwd)/models.yaml" -fi - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "Error: models YAML not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then - echo "Error: DOCKER_IMAGE_NAME is not set." - exit 1 -fi - -MODEL_NAME="${MODEL_NAME:-None}" -# Resolve the models.yaml entry the same way server_sglang.sh does: agentic runs -# (IS_AGENTIC) use the '-AgentX' recipe, non-agentic disaggregated runs use -# '-DI'. Fall back to the bare model name if the variant key is absent. -# MODEL_NAME itself is left unchanged so env.sh/server_sglang.sh still see the base name. -if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - MODEL_YAML_KEY="${MODEL_NAME}-AgentX" -else - MODEL_YAML_KEY="${MODEL_NAME}-DI" -fi -if ! grep -q "^${MODEL_YAML_KEY}:" "$MODELS_YAML"; then - if grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - MODEL_YAML_KEY="$MODEL_NAME" - else - echo "Error: Model '$MODEL_YAML_KEY' (nor bare '$MODEL_NAME') not found in $MODELS_YAML" - echo "Available models:" - grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' - exit 1 - fi -fi -echo "Model found: $MODEL_YAML_KEY (MODEL_NAME=$MODEL_NAME, IS_AGENTIC=${IS_AGENTIC:-0})" - -RUN_FILE="server.sh" -echo "Runfile set: $RUN_FILE" - -# DI_REPO_DIR points to the repo root. -# $(pwd) is amd_utils/ (the sbatch submit dir); go up 3 levels to reach the repo root. -export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) - -xP="${xP:-1}" -yD="${yD:-1}" - -# Benchmark configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" - -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# Engine-specific defaults -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-false}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-false}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-false}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-false}" -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_MTP_SIZE=${DECODE_MTP_SIZE:-0} - -# Router selection: "vllm-router" (external container) or "moriio" (in-container proxy) -ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" -ROUTER_PORT="${ROUTER_PORT:-30000}" -PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" - -# ============================================================================= -# Model Path Resolution -# ============================================================================= - -# MODEL_DIR detection: prefer env var, fall back to hostname detection -if [[ -z "$MODEL_DIR" ]]; then - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - MODEL_DIR="/nfsdata" - elif [[ $NODENAME == mia1* ]]; then - MODEL_DIR="/it-share/data" - else - MODEL_DIR="/nfsdata" - fi - echo "[INFO] Auto-detected MODEL_DIR=$MODEL_DIR from hostname $(hostname -s)" -fi -export MODEL_DIR - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots - DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}" - echo "Looking for model: $MODEL_NAME (disk dir: $DISK_DIR_NAME)" - - resolve_hf_cache_path() { - local base_path=$1 - if [[ -d "${base_path}/snapshots" ]]; then - local snapshot=$(ls -1 "${base_path}/snapshots" 2>/dev/null | head -1) - if [[ -n "$snapshot" ]]; then - echo "${base_path}/snapshots/${snapshot}" - return 0 - fi - fi - echo "$base_path" - return 1 - } - - MODEL_PATH="" - SEARCH_PATHS=( - "${MODEL_DIR}/${DISK_DIR_NAME}" - "${MODEL_DIR}/${MODEL_NAME}" - "/nfsdata/hf_hub_cache-0/${DISK_DIR_NAME}" - "/nfsdata/hf_hub_cache-0/${MODEL_NAME}" - ) - - for search_path in "${SEARCH_PATHS[@]}"; do - if [[ -d "$search_path" ]]; then - RESOLVED=$(resolve_hf_cache_path "$search_path") - MODEL_PATH="$RESOLVED" - echo "Found MODEL_PATH: $MODEL_PATH" - break - fi - done - - if [[ -z "$MODEL_PATH" ]]; then - echo "FATAL: Model '$MODEL_NAME' not found. Searched:" - for p in "${SEARCH_PATHS[@]}"; do echo " - $p"; done - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -else - # SGLang: Validate model path across all allocated nodes - echo "Looking for model: $MODEL_NAME" - echo "Checking model availability across all allocated nodes..." - - ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") - TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) - echo "Total allocated nodes: $TOTAL_NODES" - echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" - - check_model_path() { - local path=$1 - local check_name=$2 - echo "Checking $check_name: $path" - srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " - if [ -d '$path' ]; then - echo \"\$(hostname): Found $path\" - exit 0 - else - echo \"\$(hostname): Missing $path\" - exit 1 - fi - " - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "$check_name available on ALL nodes" - return 0 - else - echo "$check_name NOT available on all nodes" - return 1 - fi - } - - # Extract hf_dir from models.yaml (same as vllm-disagg path above) - SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" - - # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); - # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. - if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then - # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) - # Use it directly if it exists on all nodes, otherwise try subdirectory combos. - if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then - echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" - elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then - MODEL_PATH="$MODEL_PATH/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_PATH" - echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" - echo " - $MODEL_PATH/$MODEL_NAME" - exit 1 - fi - elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then - MODEL_PATH="$MODEL_DIR/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" - echo " - $MODEL_DIR/$MODEL_NAME" - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -fi - -# ============================================================================= -# Node Selection -# ============================================================================= - -NUM_NODES=$((xP + yD)) -echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" - -FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) -SELECTED_NODELIST_STR=$(echo "$SELECTED_NODES" | tr '\n' ',' | sed 's/,$//') - -# Docker privilege detection — evaluated per-node since group membership varies. -# Exported as a snippet so every srun participant resolves it locally. -export DOCKER_CMD_DETECT='if docker ps &>/dev/null 2>&1; then DOCKER_CMD=docker; else DOCKER_CMD="sudo docker"; fi' - -# Update SLURM environment variables -export SLURM_NNODES=$NUM_NODES -export SLURM_NTASKS=$NUM_NODES -export SLURM_JOB_NUM_NODES=$NUM_NODES -export SLURM_NPROCS=$NUM_NODES -export SLURM_JOB_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" -export SLURM_NTASKS_PER_NODE=1 - -echo "" -echo "Selected nodes: $SELECTED_NODELIST_STR" - -# ============================================================================= -# IP Resolution -# ============================================================================= - -USER_NAME=$(whoami) -MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -NODE0_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'ip route get 1.1.1.1') -NODE0_ADDR=$(echo "$NODE0_ADDR" | awk '/src/ {print $7}') - -IPS=() -for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'ip route get 1.1.1.1') - IP=$(echo "$IP" | awk '/src/ {print $7}') - IPS+=("$IP") -done - -echo "Node IPs: ${IPS[*]}" - -DOCKER_MOUNT_PATH="/workspace" -WS_PATH="${DOCKER_MOUNT_PATH}/benchmarks/multi_node/amd_utils" - -NNODES=$NUM_NODES - -echo "MASTER_NODE: ${MASTER_NODE}" -echo "NODE0_ADDR: ${NODE0_ADDR}" -echo "NNODES: ${NNODES}" -echo "REPO DIR: ${DI_REPO_DIR}" -echo "USER: ${USER_NAME}" - -# Reduce log spam -export TQDM_MININTERVAL=20 - -# Translate the host-resolved MODEL_PATH to the Docker mount namespace -DOCKER_MODEL_PATH="${MODEL_PATH/#$MODEL_DIR//models}" - -export DI_REPO_DIR=$DI_REPO_DIR -export WS_PATH=$WS_PATH -export NNODES=$NNODES -export NODE0_ADDR=$NODE0_ADDR -export MODEL_PATH=$MODEL_PATH -export MODEL_DIR=$MODEL_DIR -export xP=$xP -export yD=$yD -export MODEL_NAME=$MODEL_NAME -export USER_NAME=$USER_NAME -export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" -export GPUS_PER_NODE=$GPUS_PER_NODE -export BENCH_INPUT_LEN=$BENCH_INPUT_LEN -export BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN -export BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO -export BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER -export BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY -export BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE -export DRY_RUN="${DRY_RUN:-0}" -export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" -export KEEP_CONTAINERS="${KEEP_CONTAINERS:-0}" -export ENGINE=$ENGINE - -# Eval-related env vars (threaded from submit.sh) -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE="${IS_MULTINODE:-false}" -export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" -export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" -export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" -export HF_TOKEN="${HF_TOKEN:-}" -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export EVAL_LIMIT="${EVAL_LIMIT:-}" - -SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') -export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" - -# vLLM external router container. -# NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}" -ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" -# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). -CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" -export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" - -SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) - -# ============================================================================= -# RDMA QoS / DCQCN Pre-flight Check -# ============================================================================= -# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before -# any container/GPU time is spent. Runs on the bare host (nicctl is a host -# tool). See rdma_check.sh for details. -RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh" -if [[ "${SKIP_RDMA_CHECK:-0}" == "1" ]]; then - echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check" -elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then - echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..." - srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT" - RDMA_CHECK_RC=$? - if [[ $RDMA_CHECK_RC -ne 0 ]]; then - echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)." - echo " Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)." - exit 1 - fi - echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)" -else - echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" -fi - -cleanup() { - echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." - # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` - # client is killed before --rm can fire, so the container (and its GPU/ports/ - # :30000) leaks and the step never returns. Force-remove THIS job's container - # on every allocated node. Scoped to $DOCKER_CONT_NAME so it never touches - # other users' containers. (Ported from InferenceY 51ebfa88.) - srun --nodelist="$SELECTED_NODELIST_SRUN" \ - bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' 2>/dev/null || true - rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true - echo "[${SLURM_JOB_ID}] cleanup done." -} - -trap cleanup INT TERM HUP - -# Force NFS cache refresh on all nodes -echo "Refreshing NFS caches on all nodes..." -srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' - sync - ls -la '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils > /dev/null 2>&1 - stat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server.sh > /dev/null 2>&1 - cat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server.sh > /dev/null 2>&1 - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true - echo "NFS cache refreshed on $(hostname)" -' - -# ============================================================================= -# Build engine-specific Docker environment variables -# ============================================================================= - -# Common env vars (always passed) -DOCKER_ENV_COMMON=( - -e SLURM_JOB_ID=\$SLURM_JOB_ID - -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST - -e NNODES=\$NNODES - -e NODE_RANK=\$SLURM_PROCID - -e NODE0_ADDR=\$NODE0_ADDR - -e MODEL_DIR=/models - -e MODEL_NAME=\$MODEL_NAME - -e GPUS_PER_NODE=\$GPUS_PER_NODE - -e xP=\$xP - -e yD=\$yD - -e IPADDRS=\$IPADDRS - -e BENCH_INPUT_LEN=\$BENCH_INPUT_LEN - -e BENCH_OUTPUT_LEN=\$BENCH_OUTPUT_LEN - -e BENCH_RANDOM_RANGE_RATIO=\$BENCH_RANDOM_RANGE_RATIO - -e BENCH_NUM_PROMPTS_MULTIPLIER=\$BENCH_NUM_PROMPTS_MULTIPLIER - -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY - -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE - -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e BENCHMARK_LOGS_DIR=/benchmark_logs - -e ENGINE=\$ENGINE - -e WS_PATH=${WS_PATH} - -e RUN_EVAL=\$RUN_EVAL - -e EVAL_ONLY=\$EVAL_ONLY - -e \"EVAL_CONC=\$EVAL_CONC\" - -e FRAMEWORK=\$FRAMEWORK - -e PRECISION=\$PRECISION - -e MODEL_PREFIX=\$MODEL_PREFIX - -e RUNNER_TYPE=\$RUNNER_TYPE - -e RESULT_FILENAME=\$RESULT_FILENAME - -e SPEC_DECODING=\$SPEC_DECODING - # DISAGG was never forwarded into the container at all (not even under a - # different name), so process_agentic_result.py's env_bool("DISAGG") always - # defaulted to false in the result JSON regardless of the actual topology. - -e DISAGG=\${DISAGG:-false} - -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE - # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the - # *_SIZE vars server_sglang.sh uses for launch args) are what - # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without - # these the container never sees them (only *_SIZE was passed), so agentic - # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 - # for every multinode run. - -e PREFILL_TP=\$PREFILL_TP - -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS - -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP - -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP - # process_agentic_result.py reads these (distinct from the ENABLE_* launch - # booleans above) for agg_bmk.json's parallelism metadata; without them it - # defaulted to ep=1/dp_attention=false even for DEP8 arms. - -e PREFILL_EP=\${PREFILL_EP:-1} - -e PREFILL_DP_ATTN=\${PREFILL_DP_ATTN:-false} - -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} - -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} - -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-} - -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} - -e DURATION=\${DURATION:-1800} - -e IS_AGENTIC=\${IS_AGENTIC:-0} - -e KV_OFFLOADING=\${KV_OFFLOADING:-none} - -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} - -e KV_OFFLOAD_BACKEND_METADATA=\"\${KV_OFFLOAD_BACKEND_METADATA:-}\" - -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} - -e ENABLE_METRICS=\${ENABLE_METRICS:-0} - -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} - -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} - -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} - -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} - -e DECODE_TP_SIZE=\$DECODE_TP_SIZE - -e DECODE_TP=\$DECODE_TP - -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS - -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP - -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP - # See PREFILL_EP/PREFILL_DP_ATTN comment above -- decode counterpart. - -e DECODE_EP=\${DECODE_EP:-1} - -e DECODE_DP_ATTN=\${DECODE_DP_ATTN:-false} - -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE - -e IS_MULTINODE=\$IS_MULTINODE - -e DRY_RUN=\${DRY_RUN:-0} - # SWE-bench agentic eval runs inside this container and needs Modal/HF - # credentials to launch sandboxes and download datasets. - -e SWEBENCH_USE_MODAL=\${SWEBENCH_USE_MODAL:-false} - -e MODAL_TOKEN_ID=\${MODAL_TOKEN_ID:-} - -e MODAL_TOKEN_SECRET=\${MODAL_TOKEN_SECRET:-} - -e HF_TOKEN=\${HF_TOKEN:-} - -e SCENARIO_TYPE=\${SCENARIO_TYPE:-} - -e \"EVAL_LIMIT=\${EVAL_LIMIT:-}\" -) - -# Engine-specific env vars -if [[ "$ENGINE" == "vllm-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e VLLM_WS_PATH=${WS_PATH} - -e MODEL_PATH=$DOCKER_MODEL_PATH - -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma - -e UCX_SOCKADDR_TLS_PRIORITY=tcp - -e UCX_MEMTYPE_CACHE=y - -e UCX_RNDV_SCHEME=get_zcopy - -e UCX_RNDV_THRESH=4k - -e UCX_ROCM_IPC_MIN_ZCOPY=0 - -e UCX_LOG_LEVEL=warn - -e HSA_ENABLE_SDMA=1 - -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT:-300} - -e PYTHONPYCACHEPREFIX=/tmp/pycache - ) -elif [[ "$ENGINE" == "atom-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e ATOM_WS_PATH=${WS_PATH} - -e PREFILL_PORT=${PREFILL_PORT:-8010} - -e DECODE_PORT=${DECODE_PORT:-8020} - -e ROUTER_PORT=${ROUTER_PORT:-30000} - -e HANDSHAKE_PORT=${HANDSHAKE_PORT:-6301} - -e MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.85} - -e KV_CACHE_DTYPE=${KV_CACHE_DTYPE:-fp8} - -e BLOCK_SIZE=${BLOCK_SIZE:-16} - -e MAX_NUM_SEQS=${MAX_NUM_SEQS:-256} - -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} - -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} - -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} - -e IBDEVICES=${IBDEVICES:-} - ) -else - DOCKER_ENV_ENGINE=( - -e SGLANG_WS_PATH=${WS_PATH} - ) -fi - -# HiCache / Mooncake settings are delivered via a bind-mounted config file rather -# than a long list of docker -e flags. Write it once to the shared benchmark-logs -# dir (already a host path, visible on every node) and mount it read-only at -# /config/hicache_mc.env, where env.sh sources it before applying its defaults. -# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. -HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" -cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" - -# Engine-specific container filter for pre-clean -CONT_FILTER="name=^container_${ENGINE}_" - -# ============================================================================= -# Optional: separate benchmark-client image (agentic runs) — node-0 sibling -# ============================================================================= -# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own -# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + -# deps), instead of rebuilding the aiperf venv inside the server container every -# run. Give the server container access to the host docker socket + CLI and the -# host paths the sibling needs for its bind mounts. These fragments are expanded -# at submit time and injected into the server `docker run` below; empty (no-op) -# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. -CLIENT_DOCKER_MOUNTS="" -CLIENT_DOCKER_ENV="" -if [[ -n "${CLIENT_IMAGE:-}" ]]; then - HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" - CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" - CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" - echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" - # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true -fi - -srun \ - --nodelist="$SELECTED_NODELIST_SRUN" \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - bash -lc " -set -euo pipefail - -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -# Per-node docker privilege detection -eval \"\$DOCKER_CMD_DETECT\" -echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" - -# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) -RDMA_MOUNTS=() -if [[ "$ENGINE" == "atom-disagg" ]]; then - -# When the container base OS differs from the host (e.g. Ubuntu 24.04 image -# on a 22.04 host), the container's bundled libibverbs/libionic may be -# ABI-incompatible with the host kernel drivers. Detect the NIC type and -# bind-mount the host's out-of-tree RDMA userspace libraries into the -# container so the RDMA stack always matches the running kernel. -_detect_nic_type() { - if [[ -n \"\${MORI_NIC_TYPE:-}\" ]]; then echo \"\$MORI_NIC_TYPE\"; return; fi - local bnxt=0 mlx5=0 ionic=0 - if [[ -d /sys/class/infiniband ]]; then - for dev in /sys/class/infiniband/*; do - local name; name=\$(basename \"\$dev\") - case \"\$name\" in - bnxt_re*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; - *) - local drv; drv=\$(basename \"\$(readlink -f \"\$dev/device/driver\" 2>/dev/null)\" 2>/dev/null || true) - case \"\$drv\" in bnxt*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; esac ;; - esac - done - fi - if (( bnxt >= mlx5 && bnxt >= ionic && bnxt > 0 )); then echo bnxt - elif (( ionic >= mlx5 && ionic > 0 )); then echo ionic - else echo mlx5; fi -} - -_find_host_ibverbs() { - for c in /usr/lib64/libibverbs.so.1 /lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1.14.39.0 /usr/lib/x86_64-linux-gnu/libibverbs.so.1; do - local r; r=\$(readlink -f \"\$c\" 2>/dev/null || true) - [[ \"\$r\" == *libibverbs.so.1.14.57.0 ]] && continue - if [[ -f \"\$r\" ]]; then echo \"\$r\"; return; fi - done -} - -_NIC_TYPE=\$(_detect_nic_type) -echo \"[rdma] NIC type: \${_NIC_TYPE} on \$(hostname)\" - -if [[ \"\$_NIC_TYPE\" == \"ionic\" || \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - _host_ibv=\$(_find_host_ibverbs) - if [[ -n \"\$_host_ibv\" ]]; then - RDMA_MOUNTS+=(-v \"\$_host_ibv:/lib/x86_64-linux-gnu/libibverbs.so.1\") - fi -fi - -if [[ \"\$_NIC_TYPE\" == \"ionic\" ]]; then - for _dir in /usr/local/lib /usr/lib/x86_64-linux-gnu; do - for _lib in \"\$_dir\"/libionic*.so; do - [[ -f \"\$_lib\" ]] || continue - _real=\$(readlink -f \"\$_lib\") - [[ -f \"\$_real\" ]] && RDMA_MOUNTS+=(-v \"\$_real:\$_real\") - RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - done - if [[ -d /usr/lib/x86_64-linux-gnu/libibverbs ]]; then - for _lib in /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:\$_lib\") - done - fi - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -elif [[ \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - for _lib in /usr/local/lib/libbnxt_re-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/libibverbs/\$(basename \"\$_lib\")\") - done - for _lib in /usr/local/lib/libbnxt_re.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -fi - -if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then - echo \"[rdma] bind-mounts: \${RDMA_MOUNTS[*]}\" -else - echo \"[rdma] no out-of-tree RDMA mounts needed\" -fi -fi # end: if ENGINE == atom-disagg - -# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released -# before the drain gate. stop-only left containers in Created/Exited state -# on some nodes. -\$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true -sleep 2 - -# GPU drain gate: fail fast on leftover VRAM use instead of OOMing in model -# load ~15 min later. Reuses wait_for_amd_gpu_clean from benchmark_lib.sh. -if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then - echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU pre-flight drain check\" -else - # Unset so benchmark_lib.sh's unrelated agentic KV_OFFLOADING check doesn't exit 1 here. - bash -c \"unset IS_AGENTIC SCENARIO_TYPE; source $DI_REPO_DIR/benchmarks/benchmark_lib.sh && wait_for_amd_gpu_clean\" -fi - -# Start vLLM external router container on node 0 -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true - \$DOCKER_CMD run -d \ - --name \"$ROUTER_CONT_NAME\" \ - --network host \ - --ulimit nofile=1048576:1048576 \ - -v /tmp:/run_logs \ - \"$VLLM_ROUTER_IMAGE\" \ - bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ - --vllm-pd-disaggregation \ - --kv-connector moriio \ - --vllm-discovery-address 0.0.0.0:${PROXY_PING_PORT} \ - --port ${ROUTER_PORT} \ - --host 0.0.0.0 \ - --policy consistent_hash \ - --prefill-policy consistent_hash \ - --decode-policy consistent_hash \ - --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" -fi - -# Skip exec on vllm-disagg rank 0 so we can stop the router after the main -# container exits. Without this, decode nodes block forever waiting for the -# router port to close (the router is a separate container). -MAYBE_EXEC=exec -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - MAYBE_EXEC= - set +e -fi - -\$MAYBE_EXEC \$DOCKER_CMD run \ - --init \ - --stop-timeout 10 \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --device=/dev/infiniband/rdma_cm \ - --device=/dev/infiniband/uverbs0 \ - --device=/dev/infiniband/uverbs1 \ - --device=/dev/infiniband/uverbs2 \ - --device=/dev/infiniband/uverbs3 \ - --device=/dev/infiniband/uverbs4 \ - --device=/dev/infiniband/uverbs5 \ - --device=/dev/infiniband/uverbs6 \ - --device=/dev/infiniband/uverbs7 \ - --ulimit memlock=-1 \ - --ulimit stack=67108864 \ - --ulimit nofile=1048576:1048576 \ - --network host \ - --ipc host \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v /sys:/sys \ - $(command -v nicctl >/dev/null 2>&1 && echo "-v $(which nicctl):/usr/sbin/nicctl") \ - -v ${MODEL_DIR}:/models \ - -v \$HOME/.ssh:/root/.ssh \ - --shm-size 128G \ - -v /tmp:/run_logs \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ - ${EXTRA_DOCKER_MOUNTS:-} \ - ${CLIENT_DOCKER_MOUNTS} \ - \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ - ${DOCKER_ENV_COMMON[*]} \ - ${DOCKER_ENV_ENGINE[*]} \ - ${CLIENT_DOCKER_ENV} \ - --name \"$DOCKER_CONT_NAME\" \ - --entrypoint \"\" \ - \"$DOCKER_IMAGE_NAME\" bash -lc ' - set -o pipefail - mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"' - '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log - ' - -# Only reached when exec was skipped (vllm-disagg rank 0) -DOCKER_EXIT_CODE=\$? -echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\" -\$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true -exit \$DOCKER_EXIT_CODE -" - -if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' - - # Clean up vLLM external router container on node 0 - if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then - srun --nodes=1 --ntasks=1 --nodelist="$MASTER_NODE" bash -c ' - eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$ROUTER_CONT_NAME"' 2>/dev/null || true - ' - fi -fi diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml deleted file mode 100644 index 88a61ccc10..0000000000 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ /dev/null @@ -1,436 +0,0 @@ -# Model-specific SGLang server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# base_flags: str # Common flags for both prefill and decode -# mtp_flags: str # Appended to decode when DECODE_MTP_SIZE > 0 -# dp_flags: str # Appended when DP attention is enabled (prefill or decode) -# ep_flags: str # Appended when EP is enabled. EP-specific MoE knobs only -# # (a2a backend, deepep mode, ep-dispatch algorithm). With -# # ep=1 these are dropped so the MoE runs tensor-parallel (TP). -# prefill: -# mem_fraction_static: float -# disable_radix_cache: bool -# dp: # Config when data-parallel attention is enabled -# max_running_requests: int -# chunked_prefill_size: str # Can be integer or bash arithmetic expression -# cuda_graph_bs: str # Space-separated values -# no_dp: # Config when data-parallel attention is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str # "start-end" expanded via seq -# decode: -# mem_fraction_static: float -# prefill_round_robin_balance: bool -# dp: -# max_running_requests: int -# chunked_prefill_size: str -# cuda_graph_bs_range: str -# ep_only: # Config when EP is enabled but DP is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str -# no_dp: -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str - -DeepSeek-V3: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V3-0324: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -GLM-5-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" - mtp_flags: "" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-Preview: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: 16384 - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-v2: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode " - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 " - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - context_length: 9217 - max_total_tokens: 131072 - enable_two_batch_overlap: true - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-512" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V4-Pro-AgentX: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" - dp_flags: "--enable-dp-attention --enable-prefill-delayer" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" - prefill: - mem_fraction_static: 0.85 - disable_radix_cache: false - disable_cuda_graph: true - dp: - max_running_requests: 1024 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" # dsv4 compressor kernel uint16 token cap (255*256) - context_length: 1048576 - # max_total_tokens: 1048576 - no_dp: - max_running_requests: 64 - # Small prefill chunks interleave long-context agentic prefills across - # requests instead of letting one ~100K-token prefill monopolize the - # engine (the conc>=16 queue-saturation / decode-stall failure mode). - # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). - # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 - # (32*256) stays a page-size multiple well under that cap. - chunked_prefill_size: 8192 - context_length: 1048576 - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - disagg_decode_enable_radix_cache: false - dp: - max_running_requests: 1024 - cuda_graph_bs_range: "1-128" - no_dp: - max_running_requests: 128 - cuda_graph_bs_range: "1-128" - -DeepSeek-V4-Pro-DI: - base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" - # DeepSeek-V4 built-in MTP: load via the EAGLE spec path with eagle-topk 1 (a single - # MTP chain). NOT NEXTN -- the V3/R1 NEXTN loader crashes the dsv4 decode server at - # init on the V4 architecture. Mirrors the sglang dsv4 recipes - # (srt-slurm-recipes/sglang/deepseek-v4/*: EAGLE + eagle-topk 1, in-checkpoint draft, - # no draft-model-path). Consumed by build_server_config only when DECODE_MTP_SIZE > 0 - # (spec-decoding: mtp); a no-op for the spec-none base scenario (DECODE_MTP_SIZE=0). - # build_server_config appends --speculative-num-steps/--speculative-num-draft-tokens - # from DECODE_MTP_SIZE. - mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - disable_cuda_graph: true - dp: - max_running_requests: 1024 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 2097152 - no_dp: - max_running_requests: 128 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 2097152 - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 1024 - cuda_graph_bs_range: "1-128" - no_dp: - max_running_requests: 128 - cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/models_atom.yaml b/benchmarks/multi_node/amd_utils/models_atom.yaml deleted file mode 100644 index 485320dffa..0000000000 --- a/benchmarks/multi_node/amd_utils/models_atom.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Model-specific ATOM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# env: str # Space-separated KEY=VALUE pairs exported unconditionally -# tp_dp_flags: str # Shared TP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_tp_dp_flags: str # TP+DPA flags for prefill only (overrides tp_dp_flags) -# decode_tp_dp_flags: str # TP+DPA flags for decode only (overrides tp_dp_flags) -# tp_dp_env: str # Space-separated KEY=VALUE pairs exported only in TP+DPA mode -# ep_dp_flags: str # Shared EP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_ep_dp_flags: str # EP+DPA flags for prefill only (overrides ep_dp_flags) -# decode_ep_dp_flags: str # EP+DPA flags for decode only (overrides ep_dp_flags) -# ep_dp_env: str # Space-separated KEY=VALUE pairs exported only in EP+DPA mode -# mtp_flags: str # Flags passed to SPEC_ARGS before $DECODE_MTP_SIZE (e.g. "--method mtp --num-speculative-tokens") -# kv_cache_flags: str # Full --kv_cache_dtype flag string (e.g. "--kv_cache_dtype fp8", or "" for none) -# online_quant_config: str # JSON string passed to --online_quant_config (used when DPA is disabled) -# online_quant_dpa_config: str # JSON string passed to --online_quant_config when DPA is enabled (falls back to online_quant_config) -# block_size: str # --block-size value (overrides server_atom.sh default of 16) -# mem_frac_static: str # --gpu-memory-utilization value (overrides default of 0.85) -# max_model_len: str # --max-model-len value (overrides default of unset) -# max_num_seqs: str # --max-num-seqs value (overrides default of 256) -# max_num_batched_tokens: str # --max-num-batched-tokens value (overrides default of unset) -# scheduler_delay_factor: str # --scheduler-delay-factor value (overrides default of unset) - -DeepSeek-V4-Pro: - env: "ATOM_MOE_GU_ITLV=1 AITER_BF16_FP8_MOE_BOUND=0" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention --enable-tbo" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo" - decode_tp_dp_flags: "--enable-dp-attention --enable-tbo" - tp_dp_env: "GPU_MAX_HW_QUEUES=5 ATOM_CPU_AFFINITY=1" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method mtp --num-speculative-tokens" - -MiniMax-M3-MXFP4: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill" - decode_tp_dp_flags: "--enable-dp-attention" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens" - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - block_size: "128" - mem_frac_static: "0.8" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" - -Kimi-K2.5-MXFP4: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 AITER_MXFP4_INTERMEDIATE=1" - kv_cache_flags: "--kv_cache_dtype fp8" - mem_frac_static: "0.9" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" - -MiniMax-M3-MXFP8: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill" - decode_tp_dp_flags: "--enable-dp-attention" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens" - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*.gate.*","*.block_sparse_moe.experts*"]}' - block_size: "128" - mem_frac_static: "0.8" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml deleted file mode 100644 index 79d4f4d8f8..0000000000 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Model-specific vLLM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the model identifier -# used in amd-master.yaml and the directory/HF-cache name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# prefill_flags: str # vLLM CLI flags for prefill workers -# decode_flags: str # vLLM CLI flags for decode workers -# env: str # Space-separated KEY=VALUE pairs exported before vllm serve -# hf_dir: str # (optional) On-disk directory name if it differs from the key -# # e.g. HF cache layout: models--amd--Kimi-K2.5-MXFP4 - -Llama-3.1-405B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" - -amd-Llama-3.3-70B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" - -Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" - hf_dir: "models--amd--Kimi-K2.5-MXFP4" - -MiniMax-M2.5: - # AITER fused-MoE kernel fmoe_bf16_blockscaleFp8_g1u1_vs_silu_32x384 for gfx950 writes OOB when run with MiniMax's shapes at M=8K(=num batched tokens), crashing vllm during AITER warmup. - # Set token budget to 4k to avoid using that shape, instead of disabling AITER_MOE. - prefill_flags: "--max-num-batched-tokens 4K --tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --no-enable-prefix-caching --gpu-memory-utilization 0.95 --block-size 32" - decode_flags: "--max-num-batched-tokens 4K --tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --no-enable-prefix-caching --gpu-memory-utilization 0.95 --block-size 32" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ENGINE_READY_TIMEOUT_S=3600 VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1" - hf_dir: "models--MiniMaxAI--MiniMax-M2.5" - -MiniMax-M3-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" - prefill_env: "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048" - hf_dir: "models--amd--MiniMax-M3-MXFP4" - -gpt-oss-120b: - prefill_flags: "--tensor-parallel-size 8" - decode_flags: "--tensor-parallel-size 8" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_TRITON_BF16_GEMM=0 VLLM_USE_AITER_UNIFIED_ATTENTION=1 VLLM_ROCM_USE_AITER_MHA=0 ROCM_TRITON_MOE_PRESHUFFLE_SCALES=0" - -MiniMax-M3-MXFP8: - # MiniMax-M3 MXFP8 disagg, no EP. The --tensor-parallel-size 8 below is just a - # placeholder: server_vllm.sh sed-rewrites it to PREFILL_TP_SIZE/DECODE_TP_SIZE - # from the master-config prefill/decode tp (the sweep uses TP4 workers only, - # varying the prefill:decode worker ratio rather than TP). - # --block-size 128 is mandatory (MSA sparse/index cache); text-only benchmark - # so --language-model-only frees the vision encoder. gfx950 uses FP8 KV cache. - # Serve flags + env kept in sync with the single-node recipe - # benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x.sh: - # --moe-backend aiter (AITER fused MoE), --linear-backend emulation, and the - # larger --max-num-batched-tokens 32768 prefill budget; plus the AITER - # router-append shared-experts fusion (self-disables under EP; the sweep is - # EP1 so it stays active) and INT6 quick all-reduce quantization. - prefill_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" - hf_dir: "models--MiniMaxAI--MiniMax-M3-MXFP8" diff --git a/benchmarks/multi_node/amd_utils/node_excludes.yaml b/benchmarks/multi_node/amd_utils/node_excludes.yaml deleted file mode 100644 index 6ba840328e..0000000000 --- a/benchmarks/multi_node/amd_utils/node_excludes.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Per (framework, model) SLURM node exclusions, consumed by submit.sh. -# -# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is -# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any -# (framework, model) combination with no matching rule gets an empty exclude -# list (i.e. no nodes excluded). -# -# SLURM_EXCLUDE_NODES env var, if set, always overrides this file. - -rules: - - framework: vllm-disagg - models: - - Kimi-K2.5-MXFP4 - exclude_nodes: mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12 diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh deleted file mode 100755 index b62ca58160..0000000000 --- a/benchmarks/multi_node/amd_utils/server.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Multi-Engine Disaggregated Server Dispatcher -# ============================================================================= -# Dispatches to the engine-specific server launcher based on ENGINE env var. -# ENGINE=sglang-disagg (default) -> server_sglang.sh (SGLang + MoRI) -# ENGINE=vllm-disagg -> server_vllm.sh (vLLM + Nixl/MoRI-IO) -# ENGINE=atom-disagg -> server_atom.sh (ATOM + mooncake) -# ============================================================================= - -ENGINE="${ENGINE:-sglang-disagg}" -WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname "${BASH_SOURCE[0]}")}}}}" -export WS_PATH ENGINE - -echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - source "$WS_PATH/server_vllm.sh" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - export ATOM_WS_PATH="$WS_PATH" - source "$WS_PATH/server_atom.sh" -else - source "$WS_PATH/server_sglang.sh" -fi diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh deleted file mode 100755 index 2fee5679ca..0000000000 --- a/benchmarks/multi_node/amd_utils/server_atom.sh +++ /dev/null @@ -1,622 +0,0 @@ -#!/bin/bash -# ATOM Disaggregated Server Launcher -# ============================================================================= -# Uses atom.entrypoints.openai_server with mooncake RDMA KV transfer. -# Mirrors server_sglang.sh topology (dynamic xP/yD) but adapts to ATOM's -# explicit kv-transfer-config and atomesh router. -# -# Key differences from server_sglang.sh: -# - Engine: atom.entrypoints.openai_server (not sglang.launch_server) -# - KV transfer: mooncake (--kv-transfer-config JSON) -# - Router: atomesh (not sglang_router) -# - Prefill port: $PREFILL_PORT (default 8010) / Decode port: $DECODE_PORT (default 8020) -# - Router port: $ROUTER_PORT (default 8000) -# ============================================================================= - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" -yD="${yD:-1}" - -IPADDRS="${IPADDRS:-localhost}" - -# Parallelism -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP}" - -# MTP -DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# ATOM server ports (different from SGLang which uses 8000 for all) -PREFILL_PORT="${PREFILL_PORT:-8010}" -DECODE_PORT="${DECODE_PORT:-8020}" -ROUTER_PORT="${ROUTER_PORT:-8000}" -HANDSHAKE_PORT="${HANDSHAKE_PORT:-6301}" - -# ATOM server tuning — defaults applied after YAML load (env var > YAML > shell default) -EXTRA_SERVER_ARGS="${EXTRA_SERVER_ARGS:-}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -DRY_RUN="${DRY_RUN:-0}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= - -source $ATOM_WS_PATH/setup_deps.sh -source $ATOM_WS_PATH/env_atom.sh - -# Raise FD limit — lm-eval with high num_concurrent can exhaust the default 1024 -ulimit -n 65536 2>/dev/null || ulimit -n 8192 2>/dev/null || true -echo "ulimit -n (open files): $(ulimit -n)" - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -if [[ -z "$host_ip" ]]; then - host_ip=$(hostname -I 2>/dev/null | awk '{print $1}') -fi -host_name=$(hostname) - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -# Load model-specific config from YAML (single parse for all fields) -set -x -_yaml_tmp=$(mktemp) -python3 << PYEOF > "$_yaml_tmp" -import yaml -with open('${ATOM_WS_PATH}/models_atom.yaml') as f: - m = yaml.safe_load(f).get('${MODEL_NAME}', {}) -def sh(v): return v.replace("'", "'\\''") -print(f"MODEL_ENVS='{sh(m.get('env', ''))}'") -_tp_dp = m.get('tp_dp_flags', '') -print(f"PREFILL_MODEL_TP_DP_FLAGS='{sh(m.get('prefill_tp_dp_flags', _tp_dp))}'") -print(f"DECODE_MODEL_TP_DP_FLAGS='{sh(m.get('decode_tp_dp_flags', _tp_dp))}'") -_ep_dp = m.get('ep_dp_flags', '') -print(f"PREFILL_MODEL_EP_DP_FLAGS='{sh(m.get('prefill_ep_dp_flags', _ep_dp))}'") -print(f"DECODE_MODEL_EP_DP_FLAGS='{sh(m.get('decode_ep_dp_flags', _ep_dp))}'") -print(f"MODEL_TP_DP_ENV='{sh(m.get('tp_dp_env', ''))}'") -print(f"MODEL_EP_DP_ENV='{sh(m.get('ep_dp_env', ''))}'") -print(f"MODEL_MTP_FLAGS='{sh(m.get('mtp_flags', ''))}'") -print(f"MODEL_KV_ARG='{sh(m.get('kv_cache_flags', ''))}'") -print(f"_ONLINE_QUANT_CONFIG='{sh(m.get('online_quant_config', ''))}'") -print(f"_ONLINE_QUANT_DPA_CONFIG='{sh(m.get('online_quant_dpa_config', m.get('online_quant_config', '')))}'") -print(f"_YAML_BLOCK_SIZE='{sh(m.get('block_size', ''))}'") -print(f"_YAML_MEM_FRAC_STATIC='{sh(m.get('mem_frac_static', ''))}'") -print(f"_YAML_MAX_MODEL_LEN='{sh(m.get('max_model_len', ''))}'") -print(f"_YAML_MAX_NUM_SEQS='{sh(m.get('max_num_seqs', ''))}'") -print(f"_YAML_MAX_NUM_BATCHED_TOKENS='{sh(m.get('max_num_batched_tokens', ''))}'") -print(f"_YAML_SCHEDULER_DELAY_FACTOR='{sh(m.get('scheduler_delay_factor', ''))}'") -PYEOF -# shellcheck source=/dev/null -source "$_yaml_tmp" -rm -f "$_yaml_tmp" -unset _yaml_tmp - -# Apply server-tuning: YAML > env var > shell default -# (job.slurm injects BLOCK_SIZE/MEM_FRAC_STATIC/MAX_NUM_SEQS with hardcoded -# defaults into the Docker env, so env-first would always shadow the YAML.) -BLOCK_SIZE="${_YAML_BLOCK_SIZE:-${BLOCK_SIZE:-16}}" -MEM_FRAC_STATIC="${_YAML_MEM_FRAC_STATIC:-${MEM_FRAC_STATIC:-0.85}}" -MAX_MODEL_LEN="${_YAML_MAX_MODEL_LEN:-${MAX_MODEL_LEN:-}}" -MAX_NUM_SEQS="${_YAML_MAX_NUM_SEQS:-${MAX_NUM_SEQS:-256}}" -MAX_NUM_BATCHED_TOKENS="${_YAML_MAX_NUM_BATCHED_TOKENS:-${MAX_NUM_BATCHED_TOKENS:-}}" -SCHEDULER_DELAY_FACTOR="${_YAML_SCHEDULER_DELAY_FACTOR:-${SCHEDULER_DELAY_FACTOR:-}}" -unset _YAML_BLOCK_SIZE _YAML_MEM_FRAC_STATIC _YAML_MAX_MODEL_LEN _YAML_MAX_NUM_SEQS _YAML_MAX_NUM_BATCHED_TOKENS _YAML_SCHEDULER_DELAY_FACTOR - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= - -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -# Build prefill IP list and atomesh --prefill args -PREFILL_ARGS="" -PREFILL_IPS=() -for i in $(seq 0 $((xP - 1))); do - idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_IPS[$i]="${IP_ARRAY[$idx]}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$idx]}:${PREFILL_PORT}" -done - -# Build decode IP list and atomesh --decode args -DECODE_ARGS="" -DECODE_IPS=() -for i in $(seq 0 $((yD - 1))); do - idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_IPS[$i]="${IP_ARRAY[$idx]}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$idx]}:${DECODE_PORT}" -done - -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP}" - - - - -# Parallel args -PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE") #TP -ONLINE_QUANT_ARG="" -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - if [ "$PREFILL_ENABLE_EP" = "true" ]; then #EP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi - if [[ -n "$_ONLINE_QUANT_DPA_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_DPA_CONFIG}'" - fi -else - if [[ -n "$_ONLINE_QUANT_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_CONFIG}'" - fi -fi - -DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE") #TP -if [ "$DECODE_ENABLE_DP" = "true" ]; then - if [ "$DECODE_ENABLE_EP" = "true" ]; then #EP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi -fi -unset _dp_env_pair -unset _ONLINE_QUANT_CONFIG _ONLINE_QUANT_DPA_CONFIG - -for _env_pair in ${MODEL_ENVS}; do - export "$_env_pair" -done -unset _env_pair - -# MTP args -SPEC_ARGS=() -if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE:-0}" -gt 0 ]]; then - SPEC_ARGS=(${MODEL_MTP_FLAGS} "$DECODE_MTP_SIZE") -fi - -# KV cache arg - full flag string from YAML -KV_CACHE_ARG="${MODEL_KV_ARG}" - -# Optional model length / batched-token cap -MODEL_LEN_ARGS="" -if [[ -n "$MAX_MODEL_LEN" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-model-len ${MAX_MODEL_LEN}" -fi -if [[ -n "$MAX_NUM_BATCHED_TOKENS" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}" -fi -if [[ -n "$SCHEDULER_DELAY_FACTOR" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --scheduler-delay-factor ${SCHEDULER_DELAY_FACTOR}" -fi - - -cat < prefill node 0 + router -# rank 1 .. (NODE_OFFSET-1) -> remaining prefill nodes -# rank NODE_OFFSET .. -> decode nodes -# ============================================================================= -if [ "$NODE_RANK" -eq 0 ]; then - # ────────────────────────────────────────────────────────────────────────── - # Node 0: prefill server (producer) + atomesh router - # ────────────────────────────────────────────────────────────────────────── - echo "NODE INFO =======================================" - echo "${host_name}:${host_ip} is Prefill Node 0 + Router" - echo "Prefill TP=${PREFILL_TP_SIZE}, Decode TP=${DECODE_TP_SIZE}" - echo "Prefill servers: ${PREFILL_ARGS}" - echo "Decode servers: ${DECODE_ARGS}" - echo "================================================" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill0_${host_name}.log & - set +x - prefill0_pid=$! - fi - - # Wait for all prefill and decode servers to be ready - WAIT_SERVER_TIMEOUT="${WAIT_SERVER_TIMEOUT:-2500}" - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for all servers to be up (timeout=${WAIT_SERVER_TIMEOUT}s)..." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for prefill/decode /health endpoints" - else - _deadline=$(( $(date +%s) + WAIT_SERVER_TIMEOUT )) - for _ip in "${PREFILL_IPS[@]}"; do - echo "[wait] prefill http://${_ip}:${PREFILL_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${PREFILL_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] prefill ${_ip}:${PREFILL_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] prefill ${_ip}:${PREFILL_PORT} ready" - done - for _ip in "${DECODE_IPS[@]}"; do - echo "[wait] decode http://${_ip}:${DECODE_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${DECODE_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] decode ${_ip}:${DECODE_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] decode ${_ip}:${DECODE_PORT} ready" - done - fi - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "All servers up. Starting atomesh router..." - - ROUTER_CMD="/usr/local/bin/atomesh launch \ - --host 0.0.0.0 --port ${ROUTER_PORT} \ - --pd-disaggregation \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS} \ - --policy random \ - --backend atom \ - --log-level info \ - --disable-health-check \ - --disable-circuit-breaker \ - --prometheus-port 29100" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/tmp/slurm_job-${SLURM_JOB_ID}_router_${host_name}.log" - set -x - eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & - set +x - proxy_pid=$! - - # Wait for router to accept connections - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-300}" - echo "[wait] router http://0.0.0.0:${ROUTER_PORT}/v1/models (timeout=${WAIT_ROUTER_TIMEOUT}s)" - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/v1/models" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${ROUTER_PORT}/v1/models not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router /v1/models ready" - - echo "Router is ready for benchmarking" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - cd $ATOM_WS_PATH - - export IS_MTP="false" - if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE:-0}" -gt 0 ]]; then - export IS_MTP="true" - fi - - BENCH_CMD="bash $ATOM_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - # Health check: verify the router is still serving before running eval. - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port ${ROUTER_PORT} (conc=${EVAL_CONCURRENT_REQUESTS})" - else - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" run_eval --framework lm-eval --port ${ROUTER_PORT} - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - export DECODE_NUM_WORKERS="${yD}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" append_lm_eval_summary - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy results - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Waiting 60s before killing router and prefill server..." - sleep 60 - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing router and prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $proxy_pid - kill $prefill0_pid - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - # ────────────────────────────────────────────────────────────────────────── - # Prefill nodes 1..N (kv_producer) - # ────────────────────────────────────────────────────────────────────────── - echo "${host_name}:${host_ip} is Prefill Node (rank ${NODE_RANK})" - - # Determine which prefill worker this node belongs to, and its headnode IP - prefill_worker_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_IP="${PREFILL_IPS[$prefill_worker_idx]}" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & - set +x - prefill_pid=$! - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-2800}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing prefill server (rank ${NODE_RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $prefill_pid 2>/dev/null; fi - -else - # ────────────────────────────────────────────────────────────────────────── - # Decode nodes (kv_consumer) - # ────────────────────────────────────────────────────────────────────────── - RANK=$((NODE_RANK - NODE_OFFSET)) - echo "${host_name}:${host_ip} is Decode Node (rank ${RANK})" - - _MAX_CONC=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - CUDAGRAPH_SIZES='[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' - - DECODE_MAX_NUM_SEQS="${_MAX_CONC}" - - DECODE_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${DECODE_PORT} \ - --trust-remote-code \ - ${DECODE_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${DECODE_MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_consumer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - eval "$DECODE_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log & - set +x - decode_pid=$! - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-2800}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing decode server (rank ${RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $decode_pid 2>/dev/null; fi -fi - -echo "Script completed successfully" -exit 0 \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh deleted file mode 100755 index aaaca61ef5..0000000000 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ /dev/null @@ -1,1374 +0,0 @@ -#!/bin/bash -# SGLang Disaggregated Server Launcher with Model-Specific Configurations -# ============================================================================= - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" #-> Number of Prefill Workers -yD="${yD:-1}" #-> Number of Decode Workers - -IPADDRS="${IPADDRS:-localhost}" -HEADNODE_PORT="${HEADNODE_PORT:-20000}" -# Parallelism Configuration -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-true}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-true}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-true}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-true}" -DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -# Extract the maximum concurrency from the x-delimited list -BENCH_MAX_CONC_VALUE=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - -# Dry Run for debugging purpose -DRY_RUN="${DRY_RUN:-0}" - -# GPU count (expandable for different hardware) -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= -source $SGLANG_WS_PATH/setup_deps.sh -source $SGLANG_WS_PATH/env.sh - -host_ip=$(ip route get 1.1.1.1 | awk '/src/ {print $7}') -host_name=$(hostname) - -# MORI_RDMA_TC configuration (optional) -# If set by runner, use it for RDMA traffic class configuration -# If not set, RDMA operations will proceed without QoS/traffic class settings -if [[ -n "${MORI_RDMA_TC}" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC for RDMA traffic class configuration" - echo "[INFO] Host '$host_name' configured with MORI_RDMA_TC=$MORI_RDMA_TC" -else - echo "[INFO] MORI_RDMA_TC not set. Skipping RDMA traffic class configuration." - echo "[INFO] This is normal for clusters without QoS requirements." -fi - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -MODELS_YAML="${SGLANG_WS_PATH}/models.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -# Load model config via inline Python (PyYAML is available in SGLang containers) -# Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP") -# is done here in Python to avoid bash glob-expanding the * characters. -eval "$(python3 -c " -import yaml, sys, os - -config_path = '${MODELS_YAML}' -model_name = '${MODEL_NAME}' - -# Select the models.yaml recipe variant by run type: agentic runs (IS_AGENTIC) -# use the '-AgentX' entry, non-agentic disaggregated runs use '-DI'. -# Fall back to the bare model name if the variant-specific key is absent. -is_agentic = '${IS_AGENTIC:-0}'.strip().lower() in ('1', 'true') -model_key = f'{model_name}-AgentX' if is_agentic else f'{model_name}-DI' - -with open(config_path) as f: - models = yaml.safe_load(f) - -if model_key not in models: - if model_name in models: - model_key = model_name - else: - print(f'echo \"ERROR: Model {model_key} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_key] -print(f'echo \"Selected models.yaml entry: {model_key} (IS_AGENTIC={is_agentic})\"') - -def eval_formula(val): - \"\"\"Evaluate chunked_prefill_size: if string, resolve variable names from env and compute.\"\"\" - if isinstance(val, (int, float)): - return int(val) - s = str(val) - # Build a namespace from env vars (convert numeric values to int) - ns = {} - for k, v in os.environ.items(): - try: - ns[k] = int(v) - except (ValueError, TypeError): - pass - try: - return int(eval(s, {'__builtins__': {}}, ns)) - except Exception as e: - print(f'echo \"WARNING: Cannot evaluate formula: {s} ({e})\"', file=sys.stderr) - return val - -def parse_range(cuda_range, default_start, default_end): - if '-' in str(cuda_range): - s, e = str(cuda_range).split('-') - return s, e - return str(default_start), str(default_end) - -# Output shell variables -print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') -print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') -print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') -print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') - -prefill = m.get('prefill', {}) -decode = m.get('decode', {}) - -print(f'PREFILL_MEM_FRACTION_STATIC=\"{prefill.get(\"mem_fraction_static\", 0.8)}\"') -print(f'PREFILL_DISABLE_RADIX_CACHE=\"{prefill.get(\"disable_radix_cache\", True)}\"') -print(f'PREFILL_DISABLE_CUDA_GRAPH=\"{prefill.get(\"disable_cuda_graph\", False)}\"') - -dp = prefill.get('dp', {}) -no_dp = prefill.get('no_dp', {}) -print(f'PREFILL_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 24)}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CUDA_GRAPH_BS_DP=\"{dp.get(\"cuda_graph_bs\", \"1 2 3\")}\"') -print(f'PREFILL_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_DP=\"{dp.get(\"max_total_tokens\", \"\")}\"') -print(f'PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP=\"{dp.get(\"enable_two_batch_overlap\", False)}\"') -print(f'PREFILL_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_NO_DP=\"{no_dp.get(\"max_total_tokens\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') - -print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') -print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') -print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') - -dp = decode.get('dp', {}) -ep_only = decode.get('ep_only', {}) -no_dp = decode.get('no_dp', {}) - -# Decode DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) -print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') - -# Decode EP-only config (EP enabled but DP disabled) -print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_EP_ONLY=\"{ep_only.get(\"context_length\", \"\")}\"') -s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') - -# Decode no-DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -# Compute DP-dependent prefill parameters -if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then - prefill_cuda_graph_bs=($PREFILL_CUDA_GRAPH_BS_DP) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_DP - prefill_enable_two_batch_overlap=$PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP -else - prefill_cuda_graph_bs=($(seq $PREFILL_CUDA_GRAPH_BS_NO_DP_START $PREFILL_CUDA_GRAPH_BS_NO_DP_END)) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_NO_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_NO_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_NO_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_NO_DP - prefill_enable_two_batch_overlap="false" -fi - -# When both DP and EP are enabled, override max-running-requests with max bench concurrency -if [[ "$PREFILL_ENABLE_DP" == "true" ]] && [[ "$PREFILL_ENABLE_EP" == "true" ]]; then - prefill_max_running_requests=$BENCH_MAX_CONC_VALUE - prefill_dp_ranks=$PREFILL_TP_SIZE - # MORI_MAX_DISPATCH_TOKENS_PREFILL stays at 8192 (no change) - # MORI_MOE_MAX_INPUT_TOKENS_PREFILL=$((MORI_MAX_DISPATCH_TOKENS_PREFILL * prefill_dp_ranks / 2)) - echo "[DP+EP override] Prefill: max-running-requests=$prefill_max_running_requests, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" -fi - -# Compute DP-dependent decode parameters (3-way: DP > EP-only > no_dp) -if [[ "$DECODE_ENABLE_DP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) - decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) - decode_context_length=$DECODE_CONTEXT_LENGTH_DP -elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY - decode_context_length=$DECODE_CONTEXT_LENGTH_EP_ONLY -else - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP - decode_context_length=$DECODE_CONTEXT_LENGTH_NO_DP -fi -# In PD-disaggregation the decode must admit requests against the SAME context -# length as prefill; otherwise decode accepts over-length requests that prefill -# rejects, and those requests hang forever waiting for a KV transfer that never -# comes (the 8k1k conc-500 straggler). Fall back to the prefill value if the -# decode context_length is not set in the model config, so the two always agree. -if [[ -z "$decode_context_length" ]]; then - decode_context_length=$prefill_context_length -fi - -# When both DP and EP are enabled, override max-running-requests and dispatch tokens -if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_max_running_requests=$BENCH_MAX_CONC_VALUE - decode_dp_ranks=$DECODE_TP_SIZE - MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks)) - # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10)) - # Update derived variable - SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD - echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE, INTER_KERNEL_SWITCH=$SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD" -fi - -# Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS) -# disable_cuda_graph (model-level) routes prefill to --disable-cuda-graph instead of --cuda-graph-bs. -if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "true" ]]; then - PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --disable-cuda-graph " -else - PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " -fi - -if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" -fi -# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. -if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" -fi -if [[ -n "$prefill_context_length" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" -fi -if [[ -n "$prefill_max_total_tokens" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --max-total-tokens ${prefill_max_total_tokens}" -fi -if [[ "$prefill_enable_two_batch_overlap" == "True" ]] || [[ "$prefill_enable_two_batch_overlap" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --enable-two-batch-overlap" - PREFILL_SDMA_ENV="MORI_ENABLE_SDMA=true" -fi - -DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-running-requests ${decode_max_running_requests} --cuda-graph-bs ${decode_cuda_graph_bs[*]} " - -if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" -fi -if [[ -n "$decode_context_length" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --context-length ${decode_context_length}" -fi - -if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" -fi - -if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then - MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) - # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MOE_MAX_INPUT_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) -fi - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -# Ceiling division by GPUS_PER_NODE for nodes-per-worker -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + 7) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + 7) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -# Build prefill arguments dynamically based on xP -PREFILL_HEADNODE_URLS=() -PREFILL_ARGS="" -# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's -# --server-metrics scrape. The router on :30000 does not serve Prometheus, so -# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). -SERVER_METRICS_URLS=() -# Per-worker base URLs (port 8000) for direct cache flushing between -# concurrency points. The router (:30000) does not fan /flush_cache out, so -# trace_replay.sh must POST to each prefill/decode worker directly. -SERVER_FLUSH_URLS=() -for i in $(seq 0 $((xP - 1))); do - prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") -done - -# Build decode arguments dynamically based on yD -DECODE_HEADNODE_URLS=() -DECODE_ARGS="" -for i in $(seq 0 $((yD - 1))); do - decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") -done - -echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" -echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" -echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" -echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" - -# ============================================================================= -# Configuration Builder Functions -# ============================================================================= - -# KV_P2P_TRANSFER (from amd-master.yaml kv-p2p-transfer) overrides the -# --disaggregation-transfer-backend baked into models.yaml base_flags. -apply_kv_p2p_transfer_override() { - local flags="$1" - if [[ -z "${KV_P2P_TRANSFER:-}" ]]; then - printf '%s' "$flags" - return 0 - fi - local stripped - stripped="$(echo "$flags" | sed -E 's/--disaggregation-transfer-backend[[:space:]]+[^[:space:]]+//g')" - stripped="${stripped#"${stripped%%[![:space:]]*}"}" - stripped="${stripped%"${stripped##*[![:space:]]}"}" - echo "[KV_P2P] Using disaggregation-transfer-backend=${KV_P2P_TRANSFER} (KV_P2P_TRANSFER env)" >&2 - printf '%s --disaggregation-transfer-backend %s' "$stripped" "$KV_P2P_TRANSFER" -} - -build_server_config() { - local mode="$1" - local model_name="$2" - local tp_size="$3" - local enable_ep="$4" - local enable_dp="$5" - local decode_mtp_size="$6" - - # Calculate EP and DP sizes based on enable flags - local ep_size=1 - local dp_size=1 - - if [[ "$enable_ep" == "true" ]]; then - ep_size=$tp_size - fi - - if [[ "$enable_dp" == "true" ]]; then - dp_size=$tp_size - fi - - # Build parallelism arguments - local parallel_args="--tp-size ${tp_size}" - - if [[ "$enable_ep" == "true" ]]; then - parallel_args="$parallel_args --ep-size ${ep_size}" - fi - - if [[ "$enable_dp" == "true" ]]; then - parallel_args="$parallel_args --dp-size ${dp_size}" - fi - - # Get model-specific configuration from YAML-loaded variables - local base_config - base_config="$(apply_kv_p2p_transfer_override "$MODEL_BASE_FLAGS")" - local mtp_config="" - local dp_config="" - local ep_config="" - local specific_config="" - - # MTP config (only if MTP is enabled and mode is decode) - if [ "$decode_mtp_size" -gt 0 ]; then - mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))" - fi - - # DP config (only if DP is enabled) - if [[ "$enable_dp" == "true" ]]; then - dp_config="$MODEL_DP_FLAGS" - fi - - # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo. - # With ep=1 (EP disabled) these are dropped, so the MoE runs tensor-parallel (TP) - # instead of expert-parallel — even when dp-attention is on. - if [[ "$enable_ep" == "true" ]]; then - ep_config="$MODEL_EP_FLAGS" - fi - - # Mode-specific config - if [[ "$mode" == "prefill" ]]; then - specific_config="$PREFILL_MODE_FLAGS" - elif [[ "$mode" == "decode" ]]; then - specific_config="$DECODE_MODE_FLAGS" - fi - - # Combine: parallel args + base config + ep config + mtp config + dp config + specific config - local full_config="$parallel_args" - if [[ -n "$base_config" ]]; then - full_config="$full_config $base_config" - fi - if [[ -n "$ep_config" ]]; then - full_config="$full_config $ep_config" - fi - # MTP/speculative flags go to BOTH prefill and decode. In PD-disaggregation the - # draft (nextn) layers participate in prefill KV computation as well as decode - # verification, so the speculative config must match on both roles. Gating this to - # decode only left prefill without the nextn KV layer: prefill registered one fewer - # PD state component than decode, which newer sglang (v0.5.15+) rejects outright - # ("state component count mismatch") and older builds tolerated silently while - # feeding the decode's nextn verification uninitialized state (lossy greedy MTP). - if [[ -n "$mtp_config" ]]; then - full_config="$full_config $mtp_config" - fi - if [[ -n "$dp_config" ]]; then - full_config="$full_config $dp_config" - fi - if [[ -n "$specific_config" ]]; then - full_config="$full_config $specific_config" - fi - - echo "$full_config" -} - -# Build complete server configurations -PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") -DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") - -# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). -if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then - [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" - [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" -fi - -if [[ -n "$MODEL_NAME" ]]; then - echo "Using model-specific configuration for: $MODEL_NAME" -fi - -# sync.py barrier timeout for server-up (port 8000). DSV4 needs more headroom. -# Override via SYNC_BARRIER_TIMEOUT if needed. -if [[ -z "${SYNC_BARRIER_TIMEOUT:-}" ]]; then - case "${MODEL_NAME}" in - *DeepSeek-V4*) SYNC_BARRIER_TIMEOUT=3000 ;; - *) SYNC_BARRIER_TIMEOUT=1800 ;; - esac -fi -echo "SYNC_BARRIER_TIMEOUT=${SYNC_BARRIER_TIMEOUT}s (model=${MODEL_NAME:-unset})" - -# ============================================================================= -# Optional KV cache offloading (HiCache) — enabled when -# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. -# HiCache extends RadixAttention, so radix cache MUST stay on (drop -# --disable-radix-cache). The --hicache-* flags are appended to BOTH the -# prefill and decode server configs. -# ============================================================================= -KV_OFFLOADING="${KV_OFFLOADING:-none}" -KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" -if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" - HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" - HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" - - # Optional L3 storage tier behind the CPU-DRAM (L2) cache. - # "" -> CPU DRAM only (default) - # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) - HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - - # Layout / IO backend / write policy are backend-specific: - # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake - # store maps a page-contiguous segment for RDMA/zero-copy). This layout - # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. - # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first - # has no host>device constraint (the "direct" IO backend REQUIRES a - # page_first layout, so it cannot be paired with layer_first). - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - else - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - fi - - # Mooncake master/connection settings (used only when storage=mooncake). - # The master runs once on node 0; every prefill/decode server connects to - # it via NODE0_ADDR so it is reachable across nodes. - MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" - MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" - MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" - MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" - MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" - MC_PROTOCOL="${MC_PROTOCOL:-tcp}" - MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" - MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" - MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" - MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" - - # Emit the --hicache-storage-backend flags (empty unless mooncake). The - # extra-config JSON is single-quoted so it survives the later `eval` of the - # launch command as a single argument. - build_storage_flags() { - [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 - local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" - echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" - } - - # HiCache capacity. Prefer an absolute per-rank pool derived from the - # per-node DRAM budget computed by the sweep generator (enforcement); fall - # back to --hicache-ratio (relative to the GPU KV pool) when no budget is - # provided, keeping configs that predate the budget unchanged. - # FORCE_HICACHE_RATIO lets a recipe opt into ratio-based sizing without - # unsetting TOTAL_CPU_DRAM_GB — that var is also the shared client-side - # gate (benchmark_lib.sh requires it whenever KV_OFFLOADING=dram) and is - # forwarded verbatim into client.env below, so unsetting it here would - # make the aiperf client container fail its own env validation before - # ever sending a request. - HICACHE_RATIO="${HICACHE_RATIO:-5}" - HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" - # DeepSeek V4's hybrid HiCache pool rejects --hicache-size (requires - # --hicache-ratio), so the absolute per-node budget cannot be applied to it. - # See sglang _deepseek_v4_num_host_pages() (raises ValueError when - # server_args.hicache_size > 0): - # https://github.com/sgl-project/sglang/blob/9dd57ef8c48e2cd82292d849f01e2130c5203e67/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py#L262-L266 - # FORCE_HICACHE_RATIO additionally lets a recipe opt into ratio-based sizing - # for any other model without unsetting TOTAL_CPU_DRAM_GB (see comment above). - if [[ "${FORCE_HICACHE_RATIO:-0}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then - # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget (only prefill - # offloads KV to CPU DRAM today); --hicache-size is per rank per host - # pool. A prefill server may span nodes (PREFILL_TP_SIZE is its total - # ranks), so divide by the ranks that land on one node. - prefill_ranks_per_node=$(( PREFILL_TP_SIZE < GPUS_PER_NODE ? PREFILL_TP_SIZE : GPUS_PER_NODE )) - prefill_hicache_size_gb=$(( TOTAL_CPU_DRAM_GB / prefill_ranks_per_node / HICACHE_HOST_POOL_COUNT )) - if (( prefill_hicache_size_gb < 1 )); then - echo "Error: TOTAL_CPU_DRAM_GB=${TOTAL_CPU_DRAM_GB} / ranks_per_node=${prefill_ranks_per_node} / host_pools=${HICACHE_HOST_POOL_COUNT} rounds below 1 GB" >&2 - exit 1 - fi - HICACHE_SIZING_FLAGS="--hicache-size ${prefill_hicache_size_gb}" - echo "[HiCache] prefill CPU pool capped at ${prefill_hicache_size_gb} GB/rank (budget ${TOTAL_CPU_DRAM_GB} GB / ranks_per_node ${prefill_ranks_per_node} / host_pools ${HICACHE_HOST_POOL_COUNT})" - fi - - build_hicache_flags() { - echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache ${HICACHE_SIZING_FLAGS} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" - } - - # HiCache requires RadixAttention; strip any --disable-radix-cache. - PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" - DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" - - # Prefill always gets HiCache. - PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" - - - DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" - echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" - fi -else - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" -fi - -if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE -fi - -# ============================================================================= -# Container Synchronization -# ============================================================================= - -# sync.py barrier/health-barrier exits 1 on timeout (and prints which -# node/port never became ready), but without an explicit check here the -# script would silently continue past a timed-out barrier -- printing a -# misleading "success" message and launching the next stage against -# servers/routers that never actually came up, instead of failing fast. -run_barrier_or_die() { - local desc="$1" cmd="$2" - if ! eval "$cmd"; then - echo "FATAL: ${desc} failed — see the sync.py timeout output above for which node/port never became ready." >&2 - exit 1 - fi -} - -echo "Waiting at the container creation barrier on $host_name" -run_barrier_or_die "container creation barrier" "python3 $SGLANG_WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 300" - - -# ============================================================================= -# Node Role Assignment and Server Launch -# ============================================================================= - -# Run a blocking command while watching the local server PID. If the server dies -# (crash / OOM / killed) the blocking command is aborted and we return non-zero, -# so the srun task exits non-zero and SLURM's --kill-on-bad-exit tears the whole -# job down in seconds instead of waiting out the ~1800s barrier timeout. -wait_or_die() { # $1 = server pid to watch; rest = blocking command - local watch=$1; shift - "$@" & local cmd=$! - while kill -0 "$cmd" 2>/dev/null; do - kill -0 "$watch" 2>/dev/null || { - echo "FATAL: $(hostname) local sglang server (pid $watch) died; tearing down job" >&2 - kill "$cmd" 2>/dev/null || true - return 1 - } - sleep 5 - done - wait "$cmd" -} - -if [ "$NODE_RANK" -eq 0 ]; then - echo "NODE INFO =======================================" - echo "================================================" - echo "Node List : ${SLURM_JOB_NODELIST}" - echo "Node IPs : ${IPADDRS}" - echo "Model Name : ${MODEL_NAME:-'Not specified'}" - echo "================================================" - - echo "CLUSTER INFO ====================================" - echo "================================================" - echo "${host_name}:${host_ip} is Proxy Node and Prefill Node" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Prefill servers ($((PREFILL_TP_SIZE/GPUS_PER_NODE)) nodes): ${PREFILL_ARGS}" - echo "Decode servers ($((DECODE_TP_SIZE/GPUS_PER_NODE)) nodes): ${DECODE_ARGS}" - echo "Prefill env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL}" - echo "Decode env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_DECODE} " - echo "Decode env: SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE} " - - echo "================================================" - - # Dump all resolved commands to a text file for debugging / reproducibility. - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" - - # Start the Mooncake store master (L3 HiCache backend) on node 0 only. - # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. - if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then - echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" - MC_MASTER_CMD="mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port=${MC_METADATA_PORT} \ - --rpc_port=${MC_MASTER_PORT} \ - --rpc_thread_num=${MC_MASTER_THREADS} \ - --metrics_port=${MC_METRICS_PORT} \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" - dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $MC_MASTER_CMD" - else - MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" - mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port="${MC_METADATA_PORT}" \ - --rpc_port="${MC_MASTER_PORT}" \ - --rpc_thread_num="${MC_MASTER_THREADS}" \ - --metrics_port="${MC_METRICS_PORT}" \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ - > "${MC_MASTER_LOG}" 2>&1 & - mc_master_pid=$! - sleep 3 - # Fail loudly on a port collision. On shared nodes the Mooncake RPC - # port may already be taken by another user's master; in that case the - # metrics-port health check below can still pass against the foreign - # master while our RPC port is dead, and the prefill then hangs. - if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then - echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." - echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." - grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 - exit 1 - fi - for ((i=3; i<=60; i+=3)); do - if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then - echo " mooncake master OK at ${i}s" - break - fi - sleep 3 - done - fi - fi - - # start the head prefill server - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/$MODEL_NAME \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[0]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank 0" - fi - - - dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - # Launch under `setsid` so the server (python + its TP-scheduler - # children) sits in a dedicated process group; teardown can then - # `kill -- -$pgid` the WHOLE tree. Killing $prefill0_pid alone leaves - # children holding the process-sub tee's pipe, so the container's outer - # `| tee` never gets EOF and the container never exits (srun/CI hangs). - # Process substitution (not `| tee`) keeps $! as the setsid group leader, - # not tee's. Mirrors the router launch below. - setsid bash -c "$PREFILL_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log >/dev/null) 2>&1 & - set +x - prefill0_pid=$! - prefill0_pgid=$(ps -o pgid= -p "$prefill0_pid" 2>/dev/null | tr -d ' ') - : "${prefill0_pgid:=$prefill0_pid}" - fi - - - echo "Waiting for all prefill and decode servers to be up . . ." - - - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports 8000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$prefill0_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - echo "Congratulations!!! All prefill and decode servers are up . . ." - - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - # Agentic router config (main): long-context prefills can look unhealthy to - # the default circuit breaker during a concurrent burst. Disable the breaker - # and relax health-check sensitivity so a busy-but-alive worker is not - # ejected. cache_aware prefill routing exploits HiCache/radix prefix reuse - # across the agentic trace; round_robin decode keeps the single decode worker - # fed evenly. Override via ROUTER_RESILIENCE_FLAGS / ROUTER_POLICY_FLAGS. - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" - # server_sglang.sh previously read ROUTER_PREFILL_POLICY, but the recipe - # scripts export PREFILL_ROUTER_POLICY, so the recipe's policy override was - # silently ignored and the router always fell back to this hardcoded - # default. Also comment out ROUTER_DECODE_POLICY for now (superseded by - # --dp-aware below). - ROUTER_PREFILL_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" - # ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" - ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" - ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" - ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --dp-aware --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" - else - # DI router config (8k1k branch, run 28696443568): with defaults the per-worker - # circuit stays OPEN for cb-timeout-duration-secs=60 before a half-open retrial. - # In that run every request 503'd from request #1 ("all circuits open or - # unhealthy") for ~31s and lm_eval (max_retries=5) then gave up -- i.e. the - # circuit was still open when the client budget ran out, so 0 result files were - # produced. Shortening the open->half-open window (and letting the router itself - # retry a failed worker selection) lets a transient trip re-close INSIDE the - # client retry budget instead of nuking the whole eval. The breaker stays fully - # ENABLED (thresholds unchanged); this only speeds recovery. Override via - # ROUTER_CB_ARGS / ROUTER_POLICY_FLAGS. - ROUTER_CB_ARGS="${ROUTER_CB_ARGS:---cb-timeout-duration-secs 15 --retry-max-retries 3}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy random --prefill-policy random --decode-policy random}" - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:-${ROUTER_CB_ARGS}}" - fi - - echo "Router config: IS_AGENTIC=${IS_AGENTIC:-0} policy/resilience=${ROUTER_POLICY_FLAGS} ${ROUTER_RESILIENCE_FLAGS}" - - ROUTER_CMD="python -m sglang_router.launch_router \ - --pd-disaggregation \ - --port 30000 \ - ${ROUTER_POLICY_FLAGS} \ - ${ROUTER_RESILIENCE_FLAGS} \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS}" - - - dump_cmd "ROUTER" "$ROUTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" - # sgl-router (Rust/tracing) emits ANSI color codes. NO_COLOR asks it to - # skip them at the source; the sed strip guarantees a clean file even if - # it doesn't honor NO_COLOR. Both branches use process substitution so - # $! stays the router pid, not sed's/tee's pid. - # - # Newer sglang-router (>=0.5.14) spawns the actual Rust worker - # (`sglang::router`, which binds :30000) as a child and lets the python - # launcher exit, so the worker reparents to init. It KEEPS its process - # group, though. We therefore launch under `setsid` to isolate the - # launcher+worker in a dedicated process group and record that pgid, so - # teardown can `kill -- -$proxy_pgid` the whole group even after the - # launcher is gone. `kill $proxy_pid` alone would miss the worker. - set -x - if [[ "${SGLANG_ROUTER_STDOUT_LOGS:-0}" == "1" ]]; then - NO_COLOR=1 setsid bash -c "exec $ROUTER_CMD" > >(sed -u -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' | tee "$ROUTER_LOG_FILE") 2>&1 & - else - NO_COLOR=1 setsid bash -c "exec $ROUTER_CMD" > >(sed -u -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' >"$ROUTER_LOG_FILE") 2>&1 & - fi - set +x - proxy_pid=$! - proxy_pgid=$(ps -o pgid= -p "$proxy_pid" 2>/dev/null | tr -d ' ') - : "${proxy_pgid:=$proxy_pid}" - - # Wait for router to be ready via health endpoint - HEALTH_BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-health \ - --health-endpoint /readiness \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1 - fi - - # ---- End-to-end router readiness canary (run 28696443568) ---- - # The /readiness barrier above only proves the router PROCESS is up; it does - # NOT prove the router can reach a prefill worker and complete a generation. - # In that run the eval fired the instant /readiness passed and EVERY request - # 503'd ("No available prefill workers (all circuits open or unhealthy)") from - # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed. - # Gate the benchmark on ONE successful generation THROUGH the router so the eval - # never starts against a router whose prefill path is not yet actually serving. - # - # Run the poll loop under wait_or_die (like the barriers above) rather than - # inline: the canary is the FIRST real generation through prefill, so a - # prefill crash right after /readiness passes is plausible. Without - # wait_or_die watching prefill0_pid, that just looks like repeated 503s and - # the loop burns the full ROUTER_CANARY_TIMEOUT (default 600s) instead of - # detecting the dead pid and aborting in ~5s. - run_router_canary() { - local canary_url="http://${NODE0_ADDR}:30000/v1/chat/completions" - local canary_model="${MODEL_DIR}/${MODEL_NAME}" - local canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} )) - local canary_code - while [ "$(date +%s)" -lt "$canary_deadline" ]; do - canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \ - -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \ - -X POST "$canary_url" -H 'Content-Type: application/json' \ - -d "{\"model\":\"${canary_model}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) - if [ "$canary_code" = "200" ] && \ - ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then - echo "Router readiness canary passed (end-to-end generation OK)" - return 0 - fi - echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ." - sleep 5 - done - echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." - head -c 800 /tmp/router_canary.out 2>/dev/null - return 1 - } - if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then - wait_or_die "$prefill0_pid" run_router_canary || exit 1 - fi - - echo "Router is ready for benchmarking" - fi - - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - echo "Benchmarking on ${host_name}:${host_ip}" - cd $SGLANG_WS_PATH - - # Export IS_MTP based on whether MTP is enabled - if [ "$DECODE_MTP_SIZE" -gt 0 ]; then - export IS_MTP=true - else - export IS_MTP=false - fi - - # Select the benchmark runner. - # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) - # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - # Point aiperf's server-metrics scrape at the per-worker Prometheus - # /metrics endpoints. The router (:30000) that aiperf auto-detects from - # --url does not expose Prometheus, so without this the scrape finds no - # reachable endpoint and all server-side cache/KV fields come out null. - # Only set it when the workers were actually started with --enable-metrics. - if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then - AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") - export AIPERF_SERVER_METRICS_URLS - echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" - fi - # Per-worker base URLs for cache flushing between concurrency points. - # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. - if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then - SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") - export SERVER_FLUSH_URLS_CSV - echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" - fi - # trace_replay.sh signature: model_path model_name concurrency_list log_path - BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ - $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" - echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" - else - # bench.sh signature: - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path - # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - echo "Benchmark runner: bench.sh (fixed-seq-len)" - fi - - IS_AGENTIC_RUN=0 - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - IS_AGENTIC_RUN=1 - fi - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - elif [[ -n "${CLIENT_IMAGE:-}" && "$IS_AGENTIC_RUN" == "1" ]]; then - # Separate client image (node-0 sibling container): run the aiperf trace - # replay in its own sibling container built from CLIENT_IMAGE (which ships - # a pre-baked aiperf + deps) instead of rebuilding the aiperf venv inside - # this server container. The server/router stay up in this container while - # the client container drives the benchmark against the router on - # localhost (--network host). job.slurm mounts the host docker socket + CLI - # into this container and forwards HOST_REPO_DIR / HOST_MODEL_DIR / - # HOST_BENCH_LOGS / CLIENT_CONT_NAME so the sibling can be launched here. - CLIENT_ENV_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/client.env" - mkdir -p "/run_logs/slurm_job-${SLURM_JOB_ID}" - # Forward the benchmark-relevant env (incl. runtime-computed metrics/flush - # URLs) to the client container; override the few paths/flags that differ - # inside the pre-baked image. Unset vars are skipped, so the client keeps - # its own defaults for anything not exported here. - { - for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ - DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME RUNNER_TYPE IMAGE \ - AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ - ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ - DISAGG IS_MULTINODE \ - TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ - PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_ENABLE_DP PREFILL_HARDWARE \ - DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_ENABLE_DP DECODE_HARDWARE \ - KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ - WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ - AIPERF_WARMUP_REQUESTS_PER_LANE AIPERF_TRACE_IDLE_GAP_CAP_SECONDS \ - AIPERF_EXPERIMENTAL_FAST AIPERF_UNSAFE_OVERRIDE \ - AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do - if [[ -n "${!_v+x}" ]]; then - _val="${!_v}" - # docker --env-file requires one KEY=VALUE per line with no - # embedded newlines; KV_OFFLOAD_BACKEND_METADATA carries - # pretty-printed multi-line JSON, which otherwise splits - # into unparseable lines (e.g. '"name": "hicache",') and - # aborts the client container launch. Re-serialize it to - # compact single-line JSON (round-tripping through - # json.loads/json.dumps) instead of naively stripping - # newlines, so this stays correct even if a value ever - # contained a literal newline inside a string. Empty/ - # "none"/"null" is the normal case when KV offloading is - # disabled (job.slurm always sets this var, even to ""), - # and must pass through untouched -- matching how - # optional_kv_offload_backend_metadata() in - # process_agentic_result.py treats those as "no metadata" - # rather than invalid JSON. - if [[ "$_v" == "KV_OFFLOAD_BACKEND_METADATA" && -n "$_val" && "$_val" != "null" ]]; then - _val="$(python3 -c 'import json, sys -print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { - echo "KV_OFFLOAD_BACKEND_METADATA must contain valid JSON" >&2 - exit 1 - } - fi - printf '%s=%s\n' "$_v" "$_val" - fi - done - echo "INFMAX_CONTAINER_WORKSPACE=/workspace" - # Do NOT pin AGENTIC_OUTPUT_DIR: it must default to /workspace (the - # host repo mount == GITHUB_WORKSPACE) so the aggregated - # ${RESULT_FILENAME}_conc.json lands where the workflow guard globs - # it. /workspace is bind-mounted writable, same as the co-located path. - echo "HF_HOME=/run_logs/hf_cache" - echo "MODEL_DIR=/models" - # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when - # unset (e.g. reusing the server image, which carries no pre-baked - # venv), trace_replay builds aiperf on the fly from - # /workspace/utils/aiperf — same as the co-located path. - if [[ -n "${CLIENT_AIPERF_VENV:-}" ]]; then - echo "AIPERF_USE_PREBUILT=1" - echo "AIPERF_VENV=${CLIENT_AIPERF_VENV}" - fi - } > "$CLIENT_ENV_FILE" - - echo "Launching agentic benchmark in separate client container: ${CLIENT_IMAGE}" - docker rm -f "${CLIENT_CONT_NAME}" 2>/dev/null || true - set -x - docker run --rm --network host \ - --name "${CLIENT_CONT_NAME}" \ - --shm-size 32G \ - -v "${HOST_REPO_DIR}:/workspace" \ - -v "${HOST_MODEL_DIR}:/models" \ - -v /tmp:/run_logs \ - -v "${HOST_BENCH_LOGS}:/benchmark_logs" \ - --env-file "${CLIENT_ENV_FILE}" \ - --entrypoint "" \ - "${CLIENT_IMAGE}" \ - bash -lc "cd /workspace/benchmarks/multi_node/amd_utils && bash trace_replay.sh /models ${MODEL_NAME} \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" - set +x - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval (GSM8K) evaluation on Node 0..." - - # Health check: verify the router is still serving before running eval. - # The throughput benchmark may have crashed/exhausted decode workers. - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:30000/readiness" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - # Must run from repo root so utils/evals/gsm8k.yaml resolves - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list. - # Export CONC before run_eval so meta_env.json matches validate_scores.py. - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - export CONC="${EVAL_CONCURRENT_REQUESTS}" - - # Override eval context length with model's configured context_length - if [[ -n "$prefill_context_length" ]]; then - export EVAL_MAX_MODEL_LEN="$prefill_context_length" - fi - - export ISL="${BENCH_INPUT_LEN:-0}" - export OSL="${BENCH_OUTPUT_LEN:-0}" - bridge_disagg_eval_metadata - # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, - # RESULT_FILENAME are already set via Docker -e flags from job.slurm - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --framework lm-eval --port 30000 - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - # Always rewrite meta_env.json so EP/DPA match the workflow - # topology even when run_eval() staged artifacts internally. - rewrite_lm_eval_meta_env - - # Fixed-seq-len post-bench eval still needs append to move - # results out of the temp EVAL_RESULT_DIR. - if [[ "${EVAL_ONLY:-false}" != "true" || "$IS_AGENTIC_RUN" != "1" ]]; then - append_lm_eval_summary - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy benchmark results to BENCHMARK_LOGS_DIR (mounted from host) - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the proxy server and prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - # Kill the router's entire process group (isolated via setsid at launch). - # The python launcher (proxy_pid) has usually already exited after - # spawning the detached Rust worker; the worker reparents to init but - # stays in this process group, so a group-kill reliably closes :30000. - # `kill $proxy_pid` alone misses the worker and hangs decode/prefill. - kill -TERM -"${proxy_pgid:-$proxy_pid}" 2>/dev/null || true - # Group-kill the prefill server tree (setsid at launch) so its - # TP-scheduler children die too and release the process-sub tee -> - # the container's outer `| tee` gets EOF and the container can exit. - kill -TERM -"${prefill0_pgid:-$prefill0_pid}" 2>/dev/null || true - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - echo "${host_name}:${host_ip} is Prefill Node (Model: ${MODEL_NAME:-'default'})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/${MODEL_NAME} \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - rank=$((NODE_RANK % PREFILL_NODES_PER_WORKER)) - prefill_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - # setsid isolates the server tree in its own process group so teardown - # can group-kill it (python + TP-scheduler children); otherwise the - # children hold the process-sub tee's pipe and the container never exits. - setsid bash -c "$PREFILL_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log >/dev/null) 2>&1 & - set +x - prefill_pid=$! - prefill_pgid=$(ps -o pgid= -p "$prefill_pid" 2>/dev/null | tr -d ' ') - : "${prefill_pgid:=$prefill_pid}" - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$prefill_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - wait_or_die "$prefill_pid" bash -c "$WAIT_CMD" || exit 1 - fi - - echo "Killing the rank $NODE_RANK prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - # Group-kill the whole server tree (setsid at launch) so TP-scheduler - # children die and the process-sub tee gets EOF -> container can exit. - kill -TERM -"${prefill_pgid:-$prefill_pid}" 2>/dev/null || true - fi - -else - RANK=$((NODE_RANK - xP * PREFILL_NODES_PER_WORKER)) - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME:-'default'})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - echo "Decode node rank: $RANK" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - DECODE_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then - DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" - fi - set +x - - # Agentic trace replay doesn't reproduce real token-by-token traffic, so - # measured MTP/EAGLE acceptance there isn't representative (PR #2309 - # review: https://github.com/SemiAnalysisAI/InferenceX/pull/2309#pullrequestreview-4778348624). - # Per the AgentX fairness guidelines (golden_al_distribution/README.md), - # agentic throughput benchmarks simulate acceptance at the model's - # committed golden AL instead of measuring real (non-representative) - # acceptance. Eval runs (RUN_EVAL / EVAL_ONLY) need real acceptance so - # GSM8K scores reflect actual MTP behavior. Golden curve source: - # golden_al_distribution/dsv4_mtp.yaml (thinking_on). - DECODE_SIM_ACC_ENV="" - if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && { [[ "${IS_AGENTIC:-0}" == "1" ]] || [[ "${IS_AGENTIC:-}" == "true" ]]; }; then - if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "[INFO] Eval mode: synthetic MTP disabled (using real acceptance)" - else - DSV4_GOLDEN_AL="" - case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in - *DeepSeek-V4*:1) DSV4_GOLDEN_AL=1.79 ;; - *DeepSeek-V4*:2) DSV4_GOLDEN_AL=2.27 ;; - *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; - esac - if [[ -n "$DSV4_GOLDEN_AL" ]]; then - DECODE_SIM_ACC_ENV="SGLANG_SIMULATE_ACC_LEN=${DSV4_GOLDEN_AL} SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token" - else - echo "WARNING: agentic MTP run (model=${MODEL_NAME}, DECODE_MTP_SIZE=${DECODE_MTP_SIZE}) has no golden AL wired in server_sglang.sh -- falling back to real (unsimulated, non-representative) acceptance. Add a case in server_sglang.sh and golden_al_distribution/ before shipping this arm. See golden_al_distribution/README.md." >&2 - fi - fi - fi - - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${DECODE_SIM_ACC_ENV} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path ${MODEL_DIR}/${MODEL_NAME} \ - --disaggregation-mode decode \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${DECODE_SERVER_CONFIG} " - - if [ "$DECODE_NODES_PER_WORKER" -gt 1 ]; then - rank=$((RANK % DECODE_NODES_PER_WORKER)) - decode_idx=$((RANK / DECODE_NODES_PER_WORKER)) - DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - # setsid isolates the server tree in its own process group so teardown - # can group-kill it (python + TP-scheduler children); otherwise the - # children hold the process-sub tee's pipe and the container never exits. - setsid bash -c "$DECODE_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log >/dev/null) 2>&1 & - - set +x - decode_pid=$! - decode_pgid=$(ps -o pgid= -p "$decode_pid" 2>/dev/null | tr -d ' ') - : "${decode_pgid:=$decode_pid}" - fi - - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$decode_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - wait_or_die "$decode_pid" bash -c "$WAIT_CMD" || exit 1 - fi - - echo "Killing the rank $RANK decode server" - if [[ "$DRY_RUN" -eq 0 ]]; then - # Group-kill the whole server tree (setsid at launch) so TP-scheduler - # children die and the process-sub tee gets EOF -> container can exit. - kill -TERM -"${decode_pgid:-$decode_pid}" 2>/dev/null || true - fi - -fi - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh deleted file mode 100755 index 55154cd015..0000000000 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ /dev/null @@ -1,542 +0,0 @@ -#!/bin/bash -# vLLM Disaggregated Server Launcher with Model-Specific Configurations -# ============================================================================= -# -# Node role assignment (by NODE_RANK): -# 0 -> Proxy/Router + first Prefill node (kv_producer) -# 1..xP-1 -> Additional Prefill nodes (kv_producer) -# xP..xP+yD-1 -> Decode nodes (kv_consumer) -# -# Total nodes = xP + yD (router co-located with first prefill, like SGLang). - -# ============================================================================= -# Dependency Setup (idempotent; required when using base vLLM image) -# ============================================================================= -source "$(dirname "${BASH_SOURCE[0]}")/setup_deps.sh" - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" -yD="${yD:-1}" - -IPADDRS="${IPADDRS:-localhost}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -DRY_RUN="${DRY_RUN:-0}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-$GPUS_PER_NODE}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-$GPUS_PER_NODE}" - -ROUTER_PORT="${ROUTER_PORT:-30000}" -SERVER_PORT="${SERVER_PORT:-2584}" -ENGINE_ID="${ENGINE_ID:-${MODEL_NAME}-pd-run}" - -# Prefer MODEL_PATH from job.slurm (handles HF cache snapshot resolution) -MODEL_PATH="${MODEL_PATH:-${MODEL_DIR}/${MODEL_NAME}}" - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= -source $WS_PATH/env.sh - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -# RDMA IP for Nixl KV transfer (prefer 192.168.x.x subnet if available) -rdma_ip=$(hostname -I | tr ' ' '\n' | grep '^192\.168\.' | head -1) -rdma_ip="${rdma_ip:-$host_ip}" -host_name=$(hostname) - -echo "[INFO] Management IP (barriers/proxy): $host_ip" -echo "[INFO] RDMA IP (Nixl KV transfer): $rdma_ip" - -# ============================================================================= -# RDMA / Nixl Workarounds -# ============================================================================= - -setup_rdma_env() { - # Pensando ionic (RoCEv2) point-to-point /31 route fix. - # Each benic interface has a /31 to the TOR switch. Without explicit routes, - # traffic to other nodes' RDMA IPs falls through to the management network. - if [[ "$rdma_ip" =~ ^192\.168\.([0-9]+)\.([0-9]+)$ ]]; then - local rdma_subnet="${BASH_REMATCH[1]}" - local rdma_host="${BASH_REMATCH[2]}" - local rdma_gw="192.168.${rdma_subnet}.$(( rdma_host | 1 ))" - local rdma_iface - rdma_iface=$(ip -o addr show | awk -v ip="$rdma_ip" '$4 ~ ip {print $2}' | head -1) - if [[ -n "$rdma_iface" ]]; then - ip route replace "192.168.${rdma_subnet}.0/24" via "$rdma_gw" dev "$rdma_iface" 2>/dev/null && \ - echo "[RDMA-ROUTE] Added 192.168.${rdma_subnet}.0/24 via $rdma_gw dev $rdma_iface" || \ - echo "[RDMA-ROUTE] Route add failed for 192.168.${rdma_subnet}.0/24" - fi - fi - - # Patch Nixl UCX backend: set ucx_error_handling_mode=none. - # Required for ALL NIC types under high concurrency (C512+). Without this, - # UCX's default UCP_ERR_HANDLING_MODE_PEER triggers transport-level error - # recovery on ibv_post_send failures, preventing RIXL RDMA READ retries from - # recovering gracefully. This causes the prefill KV cache to fill to 100% - # and deadlock the pipeline. On ionic NICs this was already applied (rdmacm - # incompatibility); on mlx5 NICs it was incorrectly skipped. - local nixl_api - nixl_api=$(python3 -c "import rixl._api; print(rixl._api.__file__)" 2>/dev/null) - if [[ -n "$nixl_api" ]]; then - if ! grep -q 'ucx_error_handling_mode' "$nixl_api"; then - sed -i '/self\.create_backend(bknd, init)/i\ init["ucx_error_handling_mode"] = "none"' "$nixl_api" - echo "[PATCH] Added ucx_error_handling_mode=none to $nixl_api (IBDEVICES=${IBDEVICES:-unset})" - else - echo "[PATCH] ucx_error_handling_mode already set in $nixl_api" - fi - fi -} - -setup_rdma_env - -if [[ -z "$UCX_NET_DEVICES" ]]; then - echo "Error: UCX_NET_DEVICES is empty after env.sh detection" >&2 - exit 1 -fi - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -MODELS_YAML="${WS_PATH}/models_vllm.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "$MODEL_NAME" ]]; then - echo "ERROR: MODEL_NAME is not set"; exit 1 -fi - -eval "$(python3 -c " -import yaml, sys - -with open('${MODELS_YAML}') as f: - models = yaml.safe_load(f) - -model_name = '${MODEL_NAME}' -if model_name not in models: - print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_name] - -def bash_escape(s): - \"\"\"Escape a value for safe embedding in a bash double-quoted assignment.\"\"\" - return s.replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"').replace('\$', '\\\\\$').replace('\`', '\\\\\`') - -pf = bash_escape(m.get('prefill_flags', '--tensor-parallel-size 8')) -df = bash_escape(m.get('decode_flags', '--tensor-parallel-size 8')) -ev = bash_escape(m.get('env', '')) -dev = bash_escape(m.get('decode_env', '')) -pev = bash_escape(m.get('prefill_env', '')) -print(f'PREFILL_SERVER_CONFIG=\"{pf}\"') -print(f'DECODE_SERVER_CONFIG=\"{df}\"') -print(f'MODEL_ENVS=\"{ev}\"') -print(f'DECODE_MODEL_ENVS=\"{dev}\"') -print(f'PREFILL_MODEL_ENVS=\"{pev}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -# Apply tensor-parallel size and EP/DP flags from submit pipeline. -if [[ -n "${PREFILL_TP_SIZE:-}" ]]; then - if echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${PREFILL_TP_SIZE}/g") - else - PREFILL_SERVER_CONFIG+=" --tensor-parallel-size ${PREFILL_TP_SIZE}" - fi -fi -if [[ -n "${DECODE_TP_SIZE:-}" ]]; then - if echo "$DECODE_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${DECODE_TP_SIZE}/g") - else - DECODE_SERVER_CONFIG+=" --tensor-parallel-size ${DECODE_TP_SIZE}" - fi -fi -if [[ "${PREFILL_ENABLE_EP:-false}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - PREFILL_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${PREFILL_ENABLE_DP:-false}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - PREFILL_SERVER_CONFIG+=" --enable-dp-attention" -fi -if [[ "${DECODE_ENABLE_EP:-false}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - DECODE_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${DECODE_ENABLE_DP:-false}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - DECODE_SERVER_CONFIG+=" --enable-dp-attention" -fi - -echo "PREFILL_SERVER_CONFIG (after TP/EP/DP): $PREFILL_SERVER_CONFIG" -echo "DECODE_SERVER_CONFIG (after TP/EP/DP): $DECODE_SERVER_CONFIG" - -# ============================================================================= -# Container Synchronization -# ============================================================================= - -echo "Waiting at the container creation barrier on $host_name" -python3 $WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 600 - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_ARGS="" -DECODE_ARGS="" - -for ((i=0; i "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for all prefill and decode servers to be up . . ." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: skipping barrier (wait-for-all-ports)" - else - python3 $WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports $SERVER_PORT \ - --wait-for-all-ports \ - --timeout 1800 - fi - - echo "Congratulations!!! All prefill and decode servers are up . . ." - - # Wait for proxy /health to confirm it is accepting requests - HEALTH_BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-health \ - --health-endpoint /health \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - eval "$HEALTH_BARRIER_CMD" - echo "MoRI-IO proxy is ready for benchmarking" - fi - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - echo "Benchmarking on ${host_name}:${host_ip}" - cd $WS_PATH - - export ROUTER_PORT=$ROUTER_PORT - BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port $ROUTER_PORT (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --framework lm-eval --port "$ROUTER_PORT" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export EP_SIZE=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${yD}" - export DP_ATTENTION="${PREFILL_ENABLE_DP}" - export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" - export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - append_lm_eval_summary - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy benchmark/eval results to BENCHMARK_LOGS_DIR (mounted from host) - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - [[ -n "${prefill_pid:-}" ]] && kill $prefill_pid 2>/dev/null || true - sleep 2 - pkill -f "vllm serve" 2>/dev/null || true - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then - echo "${host_name}:${host_ip} is Additional Prefill Node (Model: ${MODEL_NAME})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${PREFILL_MODEL_ENVS}; do - export "$env_pair" - echo "[PREFILL_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - PREFILL_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${PREFILL_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - PREFILL_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log" - set -x - eval "$PREFILL_CMD" > "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the prefill server" - [[ "$DRY_RUN" -eq 0 ]] && kill $prefill_pid 2>/dev/null || true - -else - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${DECODE_MODEL_ENVS}; do - export "$env_pair" - echo "[DECODE_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - DECODE_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${DECODE_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - DECODE_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log" - set -x - eval "$DECODE_CMD" > "$DECODE_LOG_FILE" 2>&1 & - set +x - decode_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the decode server" - [[ "$DRY_RUN" -eq 0 ]] && kill $decode_pid 2>/dev/null || true -fi - -# echo "Killing the etcd server" -# kill $etcd_pid 2>/dev/null || true -# pkill -f etcd 2>/dev/null || true - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh deleted file mode 100644 index 8fa49dfb27..0000000000 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# ============================================================================= -# setup_deps.sh — Install missing disagg dependencies at container start. -# -# Dispatched by $ENGINE (set by server.sh dispatcher): -# vllm-disagg -> recipe deps + amd-quark + UCX/RIXL path exports -# (base image: vllm/vllm-openai-rocm:nightly) -# sglang-disagg -> SGLang aiter gluon patch + per-model installs -# (base image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-*) -# -# Sourced by server_vllm.sh and server_sglang.sh so PATH / LD_LIBRARY_PATH -# exports persist. Each patch is idempotent: skipped if already applied. -# -# Build steps run in subshells to avoid CWD pollution between installers. -# ============================================================================= - -ROCM_PATH="${ROCM_PATH:-/opt/rocm}" -UCX_HOME="${UCX_HOME:-/usr/local/ucx}" -RIXL_HOME="${RIXL_HOME:-/usr/local/rixl}" - -_SETUP_START=$(date +%s) -_SETUP_INSTALLED=() - -git_clone_retry() { - local url="$1" dest="$2" max_tries=3 try=1 - while (( try <= max_tries )); do - if git clone --quiet "$url" "$dest" 2>/dev/null; then return 0; fi - echo "[SETUP] git clone attempt $try/$max_tries failed for $url, retrying in 10s..." - rm -rf "$dest" - sleep 10 - (( try++ )) - done - echo "[SETUP] git clone failed after $max_tries attempts: $url" - return 1 -} - -# --------------------------------------------------------------------------- -# 5. Container RDMA/net tools -# - ibv_devinfo comes from ibverbs-utils -# - iproute2 provides the `ip` command -# Used for in-container NIC/RDMA validation and routing checks. -# --------------------------------------------------------------------------- -install_recipe_deps() { - if command -v ibv_devinfo >/dev/null 2>&1 && command -v ip >/dev/null 2>&1; then - echo "[SETUP] Container RDMA/net tools already present" - return 0 - fi - - echo "[SETUP] Installing ibv_devinfo + iproute2 in container..." - apt-get update -q -y && apt-get install -q -y \ - ibverbs-utils iproute2 \ - && rm -rf /var/lib/apt/lists/* - - if ! command -v ibv_devinfo >/dev/null 2>&1 || ! command -v ip >/dev/null 2>&1; then - echo "[SETUP] ERROR: Failed to install ibv_devinfo/iproute2"; exit 1 - fi - _SETUP_INSTALLED+=("ibverbs-utils+iproute2") -} - -# --------------------------------------------------------------------------- -# 6b. amd-quark (MXFP4 quantization support for Kimi-K2.5-MXFP4 and similar) -# Required due to ROCm vLLM missing the quark dependency: -# https://github.com/vllm-project/vllm/issues/35633 -# --------------------------------------------------------------------------- -install_amd_quark() { - if python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] amd-quark already present" - return 0 - fi - - echo "[SETUP] Installing amd-quark for MXFP4 quantization support..." - pip install --quiet amd-quark - - if ! python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] WARN: amd-quark install failed (non-fatal for non-MXFP4 models)" - return 0 - fi - _SETUP_INSTALLED+=("amd-quark") -} - -# --------------------------------------------------------------------------- -# SGLang: Install latest transformers for GLM model type support. -# -# GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes -# the glm_moe_dsa model type. The mori images do not ship it. Gated on any -# GLM model name (not just GLM-5-FP8) so other GLM variants pick up the same -# fix; only installs when a GLM model is active (avoid overhead otherwise). -# --------------------------------------------------------------------------- -install_transformers_glm5() { - if [[ "$MODEL_NAME" != *GLM* ]]; then - return 0 - fi - - if python3 -c "from transformers import AutoConfig; AutoConfig.from_pretrained('zai-org/GLM-5-FP8', trust_remote_code=True)" 2>/dev/null; then - echo "[SETUP] transformers already supports GLM-5 model type" - return 0 - fi - - echo "[SETUP] Installing transformers with GLM-5 (glm_moe_dsa) support..." - pip install --quiet -U --no-cache-dir \ - "git+https://github.com/huggingface/transformers.git@6ed9ee36f608fd145168377345bfc4a5de12e1e2" - _SETUP_INSTALLED+=("transformers-glm5") -} - -# ============================================================================= -# Run installers (engine-gated) -# ============================================================================= - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - install_recipe_deps - install_amd_quark - - # ========================================================================= - # vLLM: Export UCX/RIXL paths (persists since this file is sourced) - # ========================================================================= - export ROCM_PATH="${ROCM_PATH}" - export UCX_HOME="${UCX_HOME}" - export RIXL_HOME="${RIXL_HOME}" - export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" - export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" -else - install_transformers_glm5 -fi - -_SETUP_END=$(date +%s) -if [[ ${#_SETUP_INSTALLED[@]} -eq 0 ]]; then - echo "[SETUP] All dependencies already present ($(( _SETUP_END - _SETUP_START ))s wallclock)" -else - echo "[SETUP] Installed: ${_SETUP_INSTALLED[*]} in $(( _SETUP_END - _SETUP_START ))s" -fi diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh deleted file mode 100755 index e7ecc8d913..0000000000 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/bin/bash -# -# Cluster Configuration Template for Multi-Node Disaggregated Serving -# -# This script submits a multi-node disaggregated benchmark job to SLURM. -# It must be configured for your specific cluster before use. -# -# ENGINE=sglang (default): SGLang disaggregated serving -# ENGINE=vllm: vLLM disaggregated serving -# -# Router is co-located with the first prefill node (same for both engines), -# so NUM_NODES = PREFILL_NODES + DECODE_NODES. - -usage() { - cat << 'USAGE' -Usage: - bash submit.sh \ - \ - \ - \ - \ - [NODE_LIST] - -Arguments: - PREFILL_NODES Number of prefill nodes - PREFILL_WORKERS Number of prefill workers (usually 1) - DECODE_NODES Number of decode nodes - DECODE_WORKERS Number of decode workers (usually 1) - ISL Input sequence length - OSL Output sequence length - CONCURRENCIES Concurrency levels, delimited by 'x' (e.g., "8x16x32") - REQUEST_RATE Request rate ("inf" for max throughput) - PREFILL_ENABLE_EP true/false or 1/0 (expert parallelism on prefill) - PREFILL_ENABLE_DP true/false or 1/0 (data-parallel attention on prefill) - DECODE_ENABLE_EP true/false or 1/0 (expert parallelism on decode) - DECODE_ENABLE_DP true/false or 1/0 (data-parallel attention on decode) - PREFILL_TP Tensor parallel size per prefill node - DECODE_TP Tensor parallel size per decode node - RANDOM_RANGE_RATIO Random range ratio for benchmark client - NODE_LIST Optional: comma-separated hostnames (must match NUM_NODES) - -Required environment variables: - SLURM_ACCOUNT SLURM account name - SLURM_PARTITION SLURM partition - TIME_LIMIT Job time limit (e.g., "08:00:00") - MODEL_PATH Path to model directory (e.g., /nfsdata) - MODEL_NAME Model name directory - CONTAINER_IMAGE Docker image name (e.g., vllm_disagg_pd:latest) - RUNNER_NAME Runner identifier (for job name) - -Optional environment variables: - DRY_RUN 1 = echo composed server/router launch commands instead of - running them (preview a recipe against a real allocation). -USAGE -} - -check_env() { - local name="$1" - if [[ -z "${!name:-}" ]]; then - echo "Error: ${name} not specified" >&2 - usage >&2 - exit 1 - fi -} - -check_env SLURM_ACCOUNT -check_env SLURM_PARTITION -check_env TIME_LIMIT - -check_env MODEL_PATH -check_env MODEL_NAME -check_env CONTAINER_IMAGE -check_env RUNNER_NAME -check_env FRAMEWORK - -# GPUS_PER_NODE defaults to 8 (MI355X). Set to 4 for MI325X if needed. -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# COMMAND_LINE ARGS -PREFILL_NODES=$1 -PREFILL_WORKERS=${2:-1} -DECODE_NODES=$3 -DECODE_WORKERS=${4:-1} -ISL=$5 -OSL=$6 -CONCURRENCIES=$7 -REQUEST_RATE=$8 -PREFILL_ENABLE_EP=${9:-true} -PREFILL_ENABLE_DP=${10:-true} -DECODE_ENABLE_EP=${11:-true} -DECODE_ENABLE_DP=${12:-true} -PREFILL_TP=${13:-8} -DECODE_TP=${14:-8} -RANDOM_RANGE_RATIO=${15:-0.8} -NODE_LIST=${16} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) -profiler_args="${ISL} ${OSL} ${CONCURRENCIES} ${REQUEST_RATE}" - -# Export variables for the SLURM job -export ENGINE="${FRAMEWORK:-sglang}" -export MODEL_DIR=$MODEL_PATH -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export PROFILER_ARGS=$profiler_args - -# Engine-specific xP/yD semantics and TP exports -if [[ "$ENGINE" == "vllm-disagg" ]]; then - export PROXY_STREAM_IDLE_TIMEOUT=${PROXY_STREAM_IDLE_TIMEOUT:-300} -fi -# xP = prefill workers, yD = decode workers (may span multiple nodes) -export xP=$PREFILL_WORKERS -export yD=$DECODE_WORKERS -export PREFILL_TP_SIZE=$(( $PREFILL_NODES * $PREFILL_TP / $PREFILL_WORKERS )) -export PREFILL_ENABLE_EP=${PREFILL_ENABLE_EP} -export PREFILL_ENABLE_DP=${PREFILL_ENABLE_DP} -export PREFILL_TP -export PREFILL_EP=${PREFILL_EP:-1} -export PREFILL_DP_ATTN=${PREFILL_DP_ATTN:-false} -export PREFILL_NUM_WORKERS=${PREFILL_NUM_WORKERS:-$PREFILL_WORKERS} -export PREFILL_PP_SIZE=${PREFILL_PP_SIZE:-1} -export PREFILL_DCP_SIZE=${PREFILL_DCP_SIZE:-1} -export PREFILL_PCP_SIZE=${PREFILL_PCP_SIZE:-1} -export DECODE_TP_SIZE=$(( $DECODE_NODES * $DECODE_TP / $DECODE_WORKERS )) -export DECODE_ENABLE_EP=${DECODE_ENABLE_EP} -export DECODE_ENABLE_DP=${DECODE_ENABLE_DP} -export DECODE_TP -export DECODE_EP=${DECODE_EP:-1} -export DECODE_DP_ATTN=${DECODE_DP_ATTN:-false} -export DECODE_NUM_WORKERS=${DECODE_NUM_WORKERS:-$DECODE_WORKERS} -export DECODE_PP_SIZE=${DECODE_PP_SIZE:-1} -export DECODE_DCP_SIZE=${DECODE_DCP_SIZE:-1} -export DECODE_PCP_SIZE=${DECODE_PCP_SIZE:-1} -export DECODE_MTP_SIZE=${DECODE_MTP_SIZE} - -export NUM_NODES=$NUM_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -export MODEL_NAME=$MODEL_NAME -export BENCH_INPUT_LEN=${ISL} -export BENCH_OUTPUT_LEN=${OSL} -export BENCH_NUM_PROMPTS_MULTIPLIER=${BENCH_NUM_PROMPTS_MULTIPLIER:-10} -export BENCH_MAX_CONCURRENCY=${CONCURRENCIES} -export BENCH_REQUEST_RATE=${REQUEST_RATE} -export BENCH_RANDOM_RANGE_RATIO=${RANDOM_RANGE_RATIO:-0.8} - -# DRY_RUN=1 makes server_sglang.sh echo the composed prefill/decode/router launch -# commands instead of executing them (useful for previewing a recipe against a real -# allocation). Threaded here → job.slurm → Docker (-e DRY_RUN) → server_sglang.sh. -# sbatch defaults to --export=ALL, so exporting it is what carries it into the job. -export DRY_RUN="${DRY_RUN:-0}" - -# Eval-related env vars (threaded from workflow → runner → here → job.slurm → Docker) -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE="${IS_MULTINODE:-false}" -export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" -export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" -export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" -export HF_TOKEN="${HF_TOKEN:-}" -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export EVAL_LIMIT="${EVAL_LIMIT:-}" - -# Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. -export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" -mkdir -p "$BENCHMARK_LOGS_DIR" - -# Optional: pass an explicit node list to sbatch. -NODELIST_OPT=() -if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then - IFS=',' read -r -a NODE_ARR <<< "$NODE_LIST" - if [[ "${#NODE_ARR[@]}" -ne "$NUM_NODES" ]]; then - echo "Error: NODE_LIST has ${#NODE_ARR[@]} nodes but NUM_NODES=${NUM_NODES}" >&2 - echo "Error: NODE_LIST='${NODE_LIST}'" >&2 - exit 1 - fi - NODELIST_CSV="$(IFS=,; echo "${NODE_ARR[*]}")" - NODELIST_OPT=(--nodelist "$NODELIST_CSV") -fi - -# Optional: exclude specific nodes for known-bad (FRAMEWORK, MODEL_NAME) -# combos (e.g. nodes with broken Docker sockets), looked up from -# node_excludes.yaml. Set SLURM_EXCLUDE_NODES to override with an explicit -# comma-separated hostname list (takes precedence over the file). -# -# Resolution must fail loudly (not silently yield an empty exclude list) if -# it can't be trusted: a submit host missing python3/PyYAML, or a genuine -# parse error, must not silently reintroduce the known-bad-node issue this -# exclusion mechanism exists to prevent. -EXCLUDE_OPT=() -NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml" -if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then - RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES" -elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then - if command -v python3 >/dev/null 2>&1 && python3 -c "import yaml" >/dev/null 2>&1; then - RESOLVED_EXCLUDE_NODES=$(python3 -c " -import yaml - -with open('${NODE_EXCLUDES_YAML}') as f: - cfg = yaml.safe_load(f) or {} - -framework = '${FRAMEWORK}' -model = '${MODEL_NAME}' -for rule in cfg.get('rules', []): - if rule.get('framework') == framework and model in (rule.get('models') or []): - print(rule.get('exclude_nodes', '')) - break -") - PYTHON_EXCLUDE_RC=$? - if [[ $PYTHON_EXCLUDE_RC -ne 0 ]]; then - echo "Error: python3 failed (exit ${PYTHON_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 - echo "Error: fix the YAML, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 - exit 1 - fi - else - # Fall back to an awk parser (mirrors job.slurm's awk-based models.yaml - # parsing) matched to node_excludes.yaml's fixed rule/models/exclude_nodes - # shape. Only exercised when python3 or its yaml module is unavailable. - echo "Warning: python3/PyYAML unavailable on submit host; falling back to awk parsing of ${NODE_EXCLUDES_YAML}" >&2 - RESOLVED_EXCLUDE_NODES=$(awk -v fw="$FRAMEWORK" -v model="$MODEL_NAME" ' - /^ - framework:/ { - line = $0 - sub(/^ - framework: */, "", line) - fw_match = (line == fw) - model_match = 0 - next - } - fw_match && /^ - / { - m = $0 - sub(/^ - */, "", m) - gsub(/^"|"$/, "", m) - if (m == model) model_match = 1 - next - } - fw_match && model_match && /^ exclude_nodes:/ { - val = $0 - sub(/^ *exclude_nodes: */, "", val) - gsub(/^"|"$/, "", val) - print val - exit - } - ' "$NODE_EXCLUDES_YAML") - AWK_EXCLUDE_RC=$? - if [[ $AWK_EXCLUDE_RC -ne 0 ]]; then - echo "Error: awk fallback failed (exit ${AWK_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 - echo "Error: fix the YAML/parser, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 - exit 1 - fi - fi -else - RESOLVED_EXCLUDE_NODES="" -fi -if [[ -n "$RESOLVED_EXCLUDE_NODES" ]]; then - EXCLUDE_OPT=(--exclude "$RESOLVED_EXCLUDE_NODES") -fi - -# ============================================================================= -# Reuse existing allocation (skip sbatch) -# ============================================================================= -# When SLURM_REUSE_JOBID is set, run job.slurm directly in the current shell, -# attaching to the existing allocation. Inner `srun` calls pick up the -# allocation via SLURM_JOB_ID; SLURM_OVERLAP=1 lets them share task slots with -# the interactive shell already holding the allocation. -if [[ -n "${SLURM_REUSE_JOBID:-}" ]]; then - REUSE_JID="$SLURM_REUSE_JOBID" - echo "Reusing existing Slurm allocation ${REUSE_JID} (skipping sbatch)" >&2 - - # Resolve allocation's nodelist if not already provided. - ALLOC_NODELIST="${SLURM_JOB_NODELIST:-$(squeue -h -j "$REUSE_JID" -o '%N' 2>/dev/null)}" - if [[ -z "$ALLOC_NODELIST" ]]; then - echo "Error: could not resolve nodelist for job ${REUSE_JID}" >&2 - exit 1 - fi - ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" | wc -l) - if [[ "$ALLOC_NNODES" -lt "$NUM_NODES" ]]; then - echo "Error: allocation ${REUSE_JID} has ${ALLOC_NNODES} nodes, need ${NUM_NODES}" >&2 - exit 1 - fi - - export SLURM_JOB_ID="$REUSE_JID" - export SLURM_JOBID="$REUSE_JID" - export SLURM_JOB_NODELIST="$ALLOC_NODELIST" - export SLURM_NODELIST="$ALLOC_NODELIST" - export SLURM_NNODES="$ALLOC_NNODES" - export SLURM_JOB_NUM_NODES="$ALLOC_NNODES" - export SLURM_NTASKS="$ALLOC_NNODES" - export SLURM_NPROCS="$ALLOC_NNODES" - export SLURM_NTASKS_PER_NODE=1 - export SLURM_TASKS_PER_NODE="1(x${ALLOC_NNODES})" - export SLURM_OVERLAP=1 - export SLURM_SUBMIT_DIR="$(pwd)" - - STDOUT_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.out" - STDERR_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.err" - rm -f "$STDOUT_LOG" "$STDERR_LOG" - - nohup bash "$(dirname "$0")/job.slurm" >"$STDOUT_LOG" 2>"$STDERR_LOG" & - INLINE_PID=$! - echo "$INLINE_PID" > "${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.pid" - echo "Started job.slurm (pid=${INLINE_PID}); logs: ${STDOUT_LOG}" >&2 - - echo "$REUSE_JID" - exit 0 -fi - -# Construct the sbatch command -sbatch_cmd=( - sbatch - --parsable - --exclusive - -N "$NUM_NODES" - -n "$NUM_NODES" - "${NODELIST_OPT[@]}" - "${EXCLUDE_OPT[@]}" - --time "$TIME_LIMIT" - --partition "$SLURM_PARTITION" - --account "$SLURM_ACCOUNT" - --job-name "$RUNNER_NAME" - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" - "$(dirname "$0")/job.slurm" -) - -JOB_ID=$("${sbatch_cmd[@]}") -if [[ $? -ne 0 ]]; then - echo "Error: Failed to submit job with sbatch" >&2 - exit 1 -fi -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/sync.py b/benchmarks/multi_node/amd_utils/sync.py deleted file mode 100755 index 96e94c1b08..0000000000 --- a/benchmarks/multi_node/amd_utils/sync.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -""" -Multi-node synchronization utilities for disaggregated inference. - -Subcommands: - barrier - Wait until all specified nodes have opened their ports (TCP barrier) - Optionally wait for HTTP health endpoints to return 200 - wait - Block until a remote port closes (shutdown coordination) -""" - -import socket -import time -import threading -import argparse -import sys -import urllib.request -import urllib.error - - -def is_port_open(ip, port, timeout=2): - """Check if a given IP and port are accessible.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(timeout) - return s.connect_ex((ip, port)) == 0 - - -def check_health(ip, port, path="/health", timeout=2): - """Return True if http://ip:port/path returns HTTP 200.""" - try: - url = f"http://{ip}:{port}{path}" - req = urllib.request.Request(url) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return getattr(resp, "status", 200) == 200 - except (urllib.error.URLError, urllib.error.HTTPError, OSError): - return False - - -# ============================================================================= -# barrier subcommand -# ============================================================================= - -def cmd_barrier(args): - """Wait until all nodes have opened the specified ports.""" - NODE_IPS = [ip.strip() for ip in args.node_ips.split(",") if ip.strip()] - NODE_PORTS = [int(p.strip()) for p in args.node_ports.split(",") if p.strip()] - - if not NODE_IPS: - print("Error: NODE_IPS argument is empty or not set.") - sys.exit(1) - - if len(NODE_PORTS) == 1: - NODE_PORTS *= len(NODE_IPS) - elif len(NODE_PORTS) != len(NODE_IPS): - print("Error: Number of ports must match number of node IPs or only one port should be given for all.") - sys.exit(1) - - server_socket = None - - def open_port(): - nonlocal server_socket - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind((args.local_ip, args.local_port)) - server_socket.listen(5) - print(f"Port {args.local_port} is now open on {args.local_ip}.") - while True: - conn, addr = server_socket.accept() - conn.close() - - def close_port(): - nonlocal server_socket - if server_socket: - server_socket.close() - print(f"Port {args.local_port} has been closed on {args.local_ip}.") - - if args.enable_port: - threading.Thread(target=open_port, daemon=True).start() - - # Wait for all ports (TCP check) - if args.wait_for_all_ports: - start_time = time.time() - timeout = args.timeout - seen_open = set() - - while True: - status = {(ip, port): is_port_open(ip, port) - for ip, port in zip(NODE_IPS, NODE_PORTS)} - - # Fail fast: a port that was open and is now closed means that - # server died/was killed; don't wait out the full timeout. - regressed = [t for t in seen_open if not status[t]] - if regressed: - print("ERROR: the following ports were open then went down (server died):", flush=True) - for ip, port in regressed: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - seen_open.update(t for t, ok in status.items() if ok) - - if all(status.values()): - break - - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_open = [t for t, ok in status.items() if not ok] - print(f"ERROR: Timeout after {timeout} seconds waiting for ports to open.", flush=True) - print("The following nodes/ports are still not responding:", flush=True) - for ip, port in not_open: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - remaining = timeout - (time.time() - start_time) - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . . ({remaining:.0f}s remaining)", flush=True) - else: - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . .", flush=True) - time.sleep(5) - - # Wait for all health endpoints (HTTP check) - if args.wait_for_all_health: - health_path = args.health_endpoint - start_time = time.time() - timeout = args.timeout - seen_ready = set() - - while True: - status = { - (ip, port): check_health(ip, port, health_path) - for ip, port in zip(NODE_IPS, NODE_PORTS) - } - - # Fail fast: an endpoint that was healthy and is now down means the - # server died/was killed; don't wait out the full timeout. - regressed = [t for t in seen_ready if not status[t]] - if regressed: - print(f"ERROR: the following ({health_path}) were healthy then went down (server died):", flush=True) - for ip, port in regressed: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - seen_ready.update(t for t, ok in status.items() if ok) - - if all(status.values()): - break - - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_ready = [t for t, ok in status.items() if not ok] - print(f"ERROR: Timeout after {timeout} seconds waiting for health endpoints.", flush=True) - print(f"The following (http://ip:port{health_path}) are still not responding:", flush=True) - for ip, port in not_ready: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - remaining = timeout - (time.time() - start_time) - print( - f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) .. ({remaining:.0f}s remaining)", - flush=True, - ) - else: - print(f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) ..", flush=True) - time.sleep(30) - - if args.enable_port: - # Keep the port open long enough for slow nodes to pass their barrier. - # The previous 30s was too short when setup times vary by minutes. - grace = max(60, args.timeout // 2) if args.timeout > 0 else 300 - time.sleep(grace) - close_port() - - -# ============================================================================= -# wait subcommand -# ============================================================================= - -def cmd_wait(args): - """Wait while a remote port remains open, exit when it closes.""" - print( - f"Waiting while port {args.remote_port} on {args.remote_ip} is open...", - flush=True, - ) - while is_port_open(args.remote_ip, args.remote_port): - time.sleep(5) - print(f"Port {args.remote_port} on {args.remote_ip} is now closed.", flush=True) - - -# ============================================================================= -# CLI -# ============================================================================= - -def main(): - parser = argparse.ArgumentParser(description="Multi-node synchronization utilities.") - subparsers = parser.add_subparsers(dest="command", required=True) - - # barrier subcommand - bp = subparsers.add_parser("barrier", help="Wait for all nodes to open specified ports.") - bp.add_argument("--local-ip", required=False, help="Local IP address to bind the server.") - bp.add_argument("--local-port", type=int, required=False, help="Port number to bind the server.") - bp.add_argument("--enable-port", action="store_true", help="Enable opening and closing of local port.") - bp.add_argument("--node-ips", required=True, help="Comma-separated list of node IPs.") - bp.add_argument("--node-ports", required=True, help="Comma-separated list of ports to check.") - bp.add_argument("--timeout", type=int, default=600, - help="Timeout in seconds (default: 600). Set to 0 for no timeout.") - bp.add_argument("--wait-for-all-ports", action="store_true", - help="Wait until all node ports are open (TCP).") - bp.add_argument("--wait-for-all-health", action="store_true", - help="Wait until http://ip:port/health returns 200 for all nodes.") - bp.add_argument("--health-endpoint", default="/health", - help="Path for health check (default: /health).") - bp.set_defaults(func=cmd_barrier) - - # wait subcommand - wp = subparsers.add_parser("wait", help="Wait while a remote port remains open.") - wp.add_argument("--remote-ip", required=True, help="Remote server IP address.") - wp.add_argument("--remote-port", type=int, required=True, help="Remote port number.") - wp.set_defaults(func=cmd_wait) - - args = parser.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh deleted file mode 100644 index f4e9f84c23..0000000000 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash -# Dual-Engine Disaggregated Benchmark Runner -# -# ENGINE=sglang (default): SGLang benchmark -# ENGINE=vllm: vLLM benchmark -# -# Produces JSON result files via benchmark_serving.py so that the CI pipeline -# can collect and process results. -# -# Usage: bash bench.sh \ -# \ -# - -ENGINE="${ENGINE:-sglang-disagg}" - -model_path=$1 -model_name=$2 -concurrency_list=${3:-"1"} -MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODEL="${MODEL_NAME:-${MODEL_PATH}}" -else - MODEL="${MODEL_PATH}" -fi -log_path=${4:-/run_logs} - -# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. -# Falls back to 1 if unset so the loop always runs at least once. -IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" - - -ROUTER_PORT="${ROUTER_PORT:-30000}" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" -mkdir -p "$RESULT_DIR" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# clear_kv_caches — wipe all KV cache tiers on every backend worker before a -# concurrency point, so each conc is measured cold (no prefix reuse bleeding in -# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ -# clear_caches.sh, but the worker base URLs are already resolved by -# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. -# -# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not -# fan /flush_cache out): -# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any -# request is in flight, so we drain-retry until "Cache flushed" or -# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. -# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear -# — HTTP != 200 when L3 is off, tolerated. -# Best-effort: logs WARN, never hard-fails the sweep. -clear_kv_caches() { - local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" - local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" - if [[ -z "$urls_csv" ]]; then - echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 - return 0 - fi - local -a urls - IFS=',' read -r -a urls <<< "$urls_csv" - local url start ok resp code - for url in "${urls[@]}"; do - [[ -n "$url" ]] || continue - # L1 + L2: drain-retry until flushed (no-op while requests in flight). - start=$(date +%s); ok=0; resp="" - while :; do - resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) - echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } - (( $(date +%s) - start >= drain_tmo )) && break - sleep 3 - done - if [[ "$ok" == 1 ]]; then - echo "[clear_caches] ${url}: L1+L2 flushed" - else - echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 - fi - # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. - code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) - if [[ "$code" == 200 ]]; then - echo "[clear_caches] ${url}: L3 store cleared" - else - echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" - fi - done -} - -# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -PORT="${ROUTER_PORT}" -MODEL="${MODEL:-${BENCH_MODEL}}" -DURATION="${DURATION:-1800}" -export MODEL DURATION MAX_MODEL_LEN -RESULT_DIR="${RESULT_DIR:-${profile_folder}}" -# Base name for the per-conc aggregate written by the existing -# utils.agentic.aggregation.process_agentic_result module. -# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" -# file per concurrency, so each concurrency below is always suffixed with -# _conc (matching agentic_srt.sh on the gb200 path). -RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" - -mkdir -p "$RESULT_DIR" - -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - set -x - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - set +x -fi - -resolve_trace_source -install_agentic_deps - -ANY_FAILED=0 -for max_concurrency in "${chosen_concurrencies[@]}"; do - - echo "==========================================" - echo "Agentic trace replay: conc=$max_concurrency" - echo "==========================================" - - # Clear all KV cache tiers on every backend before this conc point so it is - # measured cold (no prefix reuse from the previous conc). Default on; set - # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. - if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then - echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" - clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 - fi - - # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes - # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes - # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml - # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the - # _conc result-file suffix, so we always nest to keep the layout identical - # across runners and avoid overwriting earlier runs in local multi-conc sweeps. - CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" - mkdir -p "$CONC_RESULT_DIR" - - CONC="$max_concurrency" - USERS="$max_concurrency" - export CONC USERS - build_replay_cmd "$CONC_RESULT_DIR" - - # Per-conc result name consumed by write_agentic_result_json. Always suffix - # with _conc so the file matches - # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / - # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. - export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" - if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then - echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 - ANY_FAILED=1 - fi - - echo "-----------------------------------------" - -done - -export RESULT_FILENAME="$RESULT_FILENAME_BASE" - -if [ "$ANY_FAILED" -ne 0 ]; then - echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 -fi diff --git a/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index d92dad7503..0000000000 --- a/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index 1505b905de..0000000000 --- a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index d7995fb250..0000000000 --- a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Same EP/DP booleans as dsr1_fp8_mi355x_sglang-disagg.sh → amd_utils/submit.sh -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -# Parameter order matches SGLang disagg submit.sh; arg 16 is optional NODELIST. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index a9a28d889b..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index e86e044a45..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index ac0f0a27d7..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODE_LIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh deleted file mode 100644 index e86e044a45..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index 91628d632f..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -# MiniMax-M3 MXFP8 (~414 GB) is pre-staged in this cluster's shared HF cache -# (/it-share/hf-hub-cache/models--MiniMaxAI--MiniMax-M3-MXFP8), not the default -# /it-share/data the launcher sets. Point the disagg model dir there for M3 only; -# submit.sh exports MODEL_DIR=$MODEL_PATH and job.slurm resolves the snapshot under -# it and bind-mounts MODEL_DIR into the prefill/decode serving containers. -export MODEL_PATH=/it-share/hf-hub-cache -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh deleted file mode 100644 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh deleted file mode 100644 index a8c0d2743b..0000000000 --- a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" \ No newline at end of file diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md index 81dbd51995..d6acbc30d2 100644 --- a/benchmarks/multi_node/llm-d/README.md +++ b/benchmarks/multi_node/llm-d/README.md @@ -2,7 +2,7 @@ This directory holds the SLURM-side orchestration for the `llmd-vllm` benchmark framework. It mirrors the AMD `sglang-disagg` pattern under -`benchmarks/multi_node/amd_utils/` (NOT the Dynamo / srt-slurm pattern): +the standalone llm-d Slurm path (not the srt-slurm pattern): InferenceX itself owns the SLURM job, no vendor multi-node tool involved. | File | Role | diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 7e189767bc..227bcf0478 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -556,7 +556,7 @@ PY IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" # GPU counts embedded in the result filename as _gpus_/_ctx_/_gen_ tokens so the # CI "Process result" step (benchmark-multinode-tmpl.yml) can parse them and run - # process_result.py for llm-d -- same filename convention as amd_utils/bench.sh. + # process_result.py for llm-d using the standard InferenceX filename convention. # ctx = prefill GPUs, gen = decode GPUs; nodes*GPUS_PER_NODE is correct for any # PREFILL_WORKERS/DECODE_WORKERS split (e.g. high-tpt 2P -> 16 prefill GPUs). _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 11c34736f8..49a873fe4a 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash # # Submit a multi-node llmd-vllm wide-EP P/D disagg benchmark job to SLURM. -# Modeled after benchmarks/multi_node/amd_utils/submit.sh; prints JOB_ID on -# stdout so the runner can poll for completion. +# Prints JOB_ID on stdout so the runner can poll for completion. # # Topology (matches the llm-d wide-EP guide reference): # 1 prefill instance with DP=PREFILL_NODES * GPUS_PER_NODE diff --git a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c7..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c7..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml new file mode 100644 index 0000000000..e1b878bc2b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml @@ -0,0 +1,35 @@ +# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and +# compute nodes share /it-share, so source, output, image, and result paths do +# not need node-local transport. + +cluster: mi355x-amds +default_partition: compute +default_time_limit: "01:00:00" +output_dir: /it-share/gharunners2/srt-slurm/outputs + +gpus_per_node: 8 +accelerator_vendor: amd +network_interface: eno0 + +gpu_sbatch_directive: gres +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false +runtime_config_transport: shared-filesystem + +default_sbatch_directives: + cpus-per-task: "32" + mem: "128G" + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /it-share/hf-hub-cache: /hf_hub_cache + # Legacy production checkpoints live here. The launcher may expose one as a + # zero-copy seed under the canonical /hf_hub_cache/hub layout. + /it-share/hf_home: /it-share/hf_home + # The launcher appends the exact shared workspace and result roots. + +containers: + sglang-rocm-v0.5.17-mi35x: /it-share/gharunners2/srt-slurm/containers/sglang-rocm-v0.5.17-mi35x-20260809.sqsh + +nginx_raise_ulimit: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml new file mode 100644 index 0000000000..ba0e13364f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml @@ -0,0 +1,485 @@ +base: + name: "mi355x-dsr1-fp4-disagg-fixed-seq" + + model: + path: "/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp4" + + identity: + model: {repo: "amd/DeepSeek-R1-0528-MXFP4-v2"} + container: {image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809"} + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + + slurm: {time_limit: "08:00:00"} + + resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + + frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + + backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + MORI_ENABLE_SDMA: "false" + MORI_EP_LAUNCH_CONFIG_MODE: "AUTO" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "8192" + SGLANG_MORI_MOE_MAX_INPUT_TOKENS: "32768" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "1024" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MORI_MOE_MAX_INPUT_TOKENS: "2703" + sglang_config: + prefill: &tp8_stp + served-model-name: "amd/DeepSeek-R1-0528-MXFP4-v2" + trust-remote-code: true + tp-size: 8 + disaggregation-transfer-backend: mori + kv-cache-dtype: fp8_e4m3 + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 16384 + disable-radix-cache: true + cuda-graph-bs: &cg128 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + enable-metrics: true + enable-cache-report: true + decode: + <<: *tp8_stp + mem-fraction-static: 0.85 + chunked-prefill-size: 262144 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg128 + + sbatch_directives: + cpus-per-task: "128" + mem: "0" + + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + + health_check: {max_attempts: 720, interval_seconds: 5} + + benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}/fixed-seq" + mkdir -p "${result_root}" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + chat_args=() + if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then + chat_args+=(--use-chat-template) + fi + for concurrency in ${CONCURRENCY_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model amd/DeepSeek-R1-0528-MXFP4-v2 \ + --tokenizer /model \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + "${chat_args[@]}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}" \ + --result-filename "dsr1-fp4-${BENCHMARK_VARIANT}-isl8192-osl1024-c${concurrency}.json" + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + CONCURRENCY_LIST: "1 2 4 8" + BENCHMARK_VARIANT: "stp-1p1d-tp8" + USE_CHAT_TEMPLATE: "0" + +override_stp_1p1d_tp8: + name: "mi355x-dsr1-fp4-stp-1p1d-tp8-fixed-seq" + benchmark: + env: + CONCURRENCY_LIST: "1 2 4 8" + BENCHMARK_VARIANT: "stp-1p1d-tp8" + +override_stp_1p2d_tp8: + name: "mi355x-dsr1-fp4-stp-1p2d-tp8-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + benchmark: + env: + CONCURRENCY_LIST: "2 4 8 16 32 64 128 256" + BENCHMARK_VARIANT: "stp-1p2d-tp8" + +override_stp_1p2d_tp4_tp8: + name: "mi355x-dsr1-fp4-stp-1p2d-tp4-tp8-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 8 + backend: + sglang_config: + prefill: + tp-size: 4 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256" + BENCHMARK_VARIANT: "stp-1p2d-tp4-tp8" + +override_stp_1p1d_dep8: + name: "mi355x-dsr1-fp4-stp-1p1d-dep8-fixed-seq" + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "64" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "128" + sglang_config: + prefill: &dep8_stp_c512 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_stp_c512 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: &cg512 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + benchmark: + env: + CONCURRENCY_LIST: "128 256 512" + BENCHMARK_VARIANT: "stp-1p1d-dep8" + +override_stp_2p1d_dep8: + name: "mi355x-dsr1-fp4-stp-2p1d-dep8-fixed-seq" + resources: + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "1024" + sglang_config: + prefill: &dep8_stp_c4096 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_stp_c4096 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "1024 2048 4096" + BENCHMARK_VARIANT: "stp-2p1d-dep8" + +override_mtp3_1p1d_tp8: + name: "mi355x-dsr1-fp4-mtp3-1p1d-tp8-fixed-seq" + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "2048" + sglang_config: + prefill: &tp8_mtp3 + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + <<: *tp8_mtp3 + benchmark: + env: + CONCURRENCY_LIST: "1 2 4 8" + BENCHMARK_VARIANT: "mtp3-1p1d-tp8" + USE_CHAT_TEMPLATE: "1" + +override_mtp3_1p2d_tp8_wide: + name: "mi355x-dsr1-fp4-mtp3-1p2d-tp8-wide-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "2048" + sglang_config: + prefill: &mtp3_tp8 + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + <<: *mtp3_tp8 + benchmark: + env: + CONCURRENCY_LIST: "2 4 8 16 32 64" + BENCHMARK_VARIANT: "mtp3-1p2d-tp8-wide" + USE_CHAT_TEMPLATE: "1" + +override_mtp3_1p2d_tp8_narrow: + name: "mi355x-dsr1-fp4-mtp3-1p2d-tp8-narrow-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "2048" + sglang_config: + prefill: *mtp3_tp8 + decode: *mtp3_tp8 + benchmark: + env: + CONCURRENCY_LIST: "2 4 8 16 32" + BENCHMARK_VARIANT: "mtp3-1p2d-tp8-narrow" + USE_CHAT_TEMPLATE: "1" + +override_mtp2_1p2d_tp8: + name: "mi355x-dsr1-fp4-mtp2-1p2d-tp8-fixed-seq" + resources: + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + backend: + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1536" + sglang_config: + prefill: &mtp2_tp8 + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + decode: + <<: *mtp2_tp8 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256" + BENCHMARK_VARIANT: "mtp2-1p2d-tp8" + USE_CHAT_TEMPLATE: "1" + +override_mtp3_1p1d_dep8: + name: "mi355x-dsr1-fp4-mtp3-1p1d-dep8-fixed-seq" + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "320" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "160" + sglang_config: + prefill: &dep8_mtp3_c640 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + max-running-requests: 640 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_mtp3_c640 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256 512 640" + BENCHMARK_VARIANT: "mtp3-1p1d-dep8" + USE_CHAT_TEMPLATE: "1" + +override_mtp1_1p1d_dep8: + name: "mi355x-dsr1-fp4-mtp1-1p1d-dep8-fixed-seq" + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "128" + sglang_config: + prefill: &dep8_mtp1_c512 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_mtp1_c512 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "64 128 256 512" + BENCHMARK_VARIANT: "mtp1-1p1d-dep8" + USE_CHAT_TEMPLATE: "1" + +override_mtp1_2p1d_dep8: + name: "mi355x-dsr1-fp4-mtp1-2p1d-dep8-fixed-seq" + resources: + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + backend: + prefill_environment: + MORI_ENABLE_SDMA: "true" + decode_environment: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "1024" + sglang_config: + prefill: &dep8_mtp1_c4096 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: "SGLang/DeepSeek-R1-NextN" + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + <<: *dep8_mtp1_c4096 + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + cuda-graph-bs: *cg512 + benchmark: + env: + CONCURRENCY_LIST: "1024 2048 4096" + BENCHMARK_VARIANT: "mtp1-2p1d-dep8" + USE_CHAT_TEMPLATE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml new file mode 100644 index 0000000000..beb4777979 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml @@ -0,0 +1,94 @@ +# Minimal MI355X aggregate validation. It exercises native SGLang Router and +# one aggregate SGLang worker; this is orchestration validation, not tuning. + +name: "mi355x-sglang-qwen3-0.6b-agg-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 1 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + +backend: + type: sglang + aggregated_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + sglang_config: + aggregated: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..b630a0ef65 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,107 @@ +# Minimal two-node MI355X P/D validation. SGLang Router provides the request +# plane and the bundled AMD MoRI transport moves KV between prefill and decode. + +name: "mi355x-sglang-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 1 + gpus_per_decode: 1 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: *worker_environment + sglang_config: + prefill: &worker_config + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + disaggregation-transfer-backend: mori + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + decode: *worker_config + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index acf6efef1a..eab63d0975 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -268,50 +268,6 @@ qwen3.5-fp8-mi355x-atom-mtp: search-space: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.16-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding). - # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode - # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs). - # dp-attn intentionally false: with --enable-dp-attention + - # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, - # but is_deepep_class_backend() excludes MoRI, so - # num_shared_slots stays at the global value (1) and the - # (num_experts - num_shared_slots) % moe_ep_size assertion in - # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 model: amd/Qwen3.5-397B-A17B-MXFP4 @@ -376,41 +332,6 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 48, 56, 64] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32] } - -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -473,157 +394,6 @@ dsr1-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - kimik3-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-cb8104839c141609d99f1254459ef3a4f1bd4263 model: moonshotai/Kimi-K3 @@ -667,502 +437,44 @@ kimik3-fp4-mi355x-atom-agentic-mtp: search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp } - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [8, 10], spec-decoding: mtp } - -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds +dsv4-fp4-mi355x-sglang: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: mi355x precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true + framework: sglang + multinode: false scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # non-MTP configurations - # 1P1D pure TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" + - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048 } + - { tp: 4, dp-attn: true, conc-start: 16, conc-end: 128 } + - { tp: 4, dp-attn: false, conc-start: 1, conc-end: 32 } - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 32, 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 640, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - -dsv4-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260701 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D pure TP8 (mori KV transfer) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "none" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MTP variant of dsv4-fp4-mi355x-sglang-disagg (base run 29821778206). Mirrors -# the base disagg search space and topologies exactly (same 20260701 image, -# mori KV transfer + MoE a2a, sglang-router --pd-disaggregation) and enables -# DeepSeek-V4 built-in MTP on the decode side via spec-decoding: mtp — the same -# enablement path as dsr1-fp8-mi355x-sglang-disagg-mtp. DECODE_MTP_SIZE=1 (draft -# depth 1) on every arm. Depth 2 was tried on the pure-TP8 (bottom-of-curve) arm, -# but this 20260701 build's 2-step EAGLE verification is lossy: greedy gsm8k fell -# to ~0.85 (below the 0.91 dsv4 floor), while depth 1 is lossless (~0.96, matching -# the spec-none base). So all arms use depth 1. The dsv4 ROCm MTP -# CUDA-graph fix (sgl-project/sglang#26383, merged 2026-05-27) is already -# present in the pinned 20260701 nightly, so no image bump is needed. -dsv4-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 (mori KV transfer) - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - - spec-decoding: "mtp" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - -dsv4-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048 } - - { tp: 4, dp-attn: true, conc-start: 16, conc-end: 128 } - - { tp: 4, dp-attn: false, conc-start: 1, conc-end: 32 } - -# MTP variant of dsv4-fp4-mi355x-sglang. Mirrors the base search space and adds -# spec-decoding: mtp, which routes to dsv4_fp4_mi355x_sglang_mtp.sh (EAGLE -# speculative decoding), per sgl-project/sglang#26383 ([AMD][DSV4] DSV4 MTP -# graph + sparse triton attn optimizations, merged to main 2026-05-27). That PR -# fixes the ROCm HIP-radix MTP CUDA-graph bug (the false-EOS symptom in sgl -# #20404) and validates GSM8K 0.950 with MTP on. -dsv4-fp4-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048, spec-decoding: mtp } - - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32, spec-decoding: mtp } +# MTP variant of dsv4-fp4-mi355x-sglang. Mirrors the base search space and adds +# spec-decoding: mtp, which routes to dsv4_fp4_mi355x_sglang_mtp.sh (EAGLE +# speculative decoding), per sgl-project/sglang#26383 ([AMD][DSV4] DSV4 MTP +# graph + sparse triton attn optimizations, merged to main 2026-05-27). That PR +# fixes the ROCm HIP-radix MTP CUDA-graph bug (the false-EOS symptom in sgl +# #20404) and validates GSM8K 0.950 with MTP on. +dsv4-fp4-mi355x-sglang-mtp: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048, spec-decoding: mtp } + - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32, spec-decoding: mtp } # DSv4 on MI355X via vLLM, using the official vllm/vllm-openai-rocm # nightly image. DSv4 base ROCm support (vllm-project/vllm#40871) merged @@ -1319,201 +631,17 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # uses the thinking_on golden AL 2.49 for three draft tokens; eval uses real # MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. dsv4-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202608201032 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 48] } - -dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 + image: rocm/atom-dev:nightly_202608201032 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: mi355x + runner: cluster:mi355x-amds precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true + framework: atom + multinode: false scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 2P1D DPA+TP8 - - conc-list: [ 256, 512, 768, 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 - - conc-list: [ 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 + agentic-coding: + - search-space: + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 48] } minimaxm3-fp8-mi300x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.27.1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1559,70 +687,6 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: - dram-utilization: 0.20 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } - -dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: "mtp" - conc-list: [ 1,2,4,8,16,32,48,64 ] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - spec-decoding: "mtp" - conc-list: [ 64, 96, 128, 192 ] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - -# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. -# EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no -# gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the -# checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), -# the H200 dsv4 vLLM path. Config mirrors the same-model dsv4-fp4-mi355x-vllm -# (TP8, conc 4-512); 8x256GB (2TB) has ample headroom for the ~1.05TB FP8 -# footprint. Launch script dsv4_fp8_mi325x.sh carries the deepseek_v4 + gfx942 -# AITER flags. dsv4-fp8-mi325x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: deepseek-ai/DeepSeek-V4-Pro @@ -1733,3 +797,276 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } + +qwen3-0.6b-fp16-mi355x-sglang-srt-agg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang + multinode: true + disagg: false + router: { name: sglang-router, version: "0.3.2" } + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + num-nodes: 1 + worker: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml" + +qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang-disagg + multinode: true + disagg: true + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + +# Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to +# srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and +# the request/KV planes remain native SGLang Router + AMD MoRI. + +dsr1-fp4-mi355x-sglang-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: amd/DeepSeek-R1-0528-MXFP4-v2 + model-prefix: dsr1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p1d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p2d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p2d_tp4_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_2p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + +dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: amd/DeepSeek-R1-0528-MXFP4-v2 + model-prefix: dsr1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p2d_tp8_wide" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_2p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + +dsr1-fp4-mi355x-sglang-disagg-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: amd/DeepSeek-R1-0528-MXFP4-v2 + model-prefix: dsr1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p2d_tp8_narrow" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp2_1p2d_tp8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_1p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_2p1d_dep8" + - "SRT_MODEL_LOCAL_PATH=/it-share/data/DeepSeek-R1-0528-MXFP4-v2" + - "SRT_DRAFT_MODEL=SGLang/DeepSeek-R1-NextN" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} diff --git a/configs/deprecated/amd-1k1k-master.yaml b/configs/deprecated/amd-1k1k-master.yaml index 6a64e965ae..fb6ef21300 100644 --- a/configs/deprecated/amd-1k1k-master.yaml +++ b/configs/deprecated/amd-1k1k-master.yaml @@ -274,41 +274,6 @@ qwen3.5-fp8-mi355x-atom-mtp: search-space: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Matches qwen3.5-fp8-mi355x-sglang TP8/EP1 low-concurrency sweep - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 @@ -354,41 +319,6 @@ qwen3.5-fp4-mi355x-sglang-mtp: search-space: - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP8/EP1, dp-attn false; MoRI conn.py overlay via job.slurm. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -432,41 +362,6 @@ glm5-fp8-mi355x-sglang-mtp: search-space: - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep (aligned with glm5-fp8-mi355x-sglang conc range) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - glm5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: zai-org/GLM-5-FP8 @@ -612,452 +507,6 @@ dsr1-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Middle of curve" (1 prefill workers each at TP8 and 2 decode workers at DEP8) - - spec-decoding: "none" - conc-list: [ 1536, 1024, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Middle of curve" (1 prefill worker at TP8 and 2 decode workers each at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1536, 1024, 512, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - -kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:v0.24.0 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - dsv4-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro @@ -1219,153 +668,6 @@ glm5-fp8-mi325x-sglang-mtp: osl: 1024 search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } -dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [ 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. minimaxm3-fp8-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1487,82 +789,6 @@ minimaxm3-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } -minimaxm3-fp8-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -minimaxm3-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MiniMax-M3 MXFP8 MI300X day-zero recipe. Reuse the dedicated ROCm image and -# MI355X serving shape, but retain the default BF16 KV cache because this -# checkpoint lacks calibrated ROCm FP8 attention scales. TP8-only, plain -# (non-expert-parallel) search space across the full conc range: EP8 -# (--enable-expert-parallel) produces garbage/incoherent output on this -# MXFP8+gfx942 combination (confirmed locally: TP8/EP8 returns garbled tokens -# even on trivial prompts, TP8/EP1 answers correctly), and was already the -# lower-throughput topology where measured. minimaxm3-fp8-mi300x-vllm: image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1628,62 +854,6 @@ minimaxm3-fp8-mi325x-vllm-mtp: - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } -minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Decode-heavy 1P TP4 + 2D TP4 (3 nodes): double the decode engines to - # absorb the decode-bound 1k1k tail at high concurrency. - - spec-decoding: "none" - conc-list: [ 256, 512, 1024, 2048 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - # 8k1k is prefill-bound (8192-token prompts vs 1024 decode steps): pair the - # balanced layout with a prefill-heavy 2P/1D layout. Concurrency is capped at - # 512 so the multi-node eval policy (8k1k + conc >= 16, highest eligible conc) - # marks lm-eval at conc 512 — matching the range NVIDIA's aggregated 8k1k - # sweep tops out at and keeping the lm-eval async client stable. dsv4-fp8-mi325x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: deepseek-ai/DeepSeek-V4-Pro diff --git a/configs/deprecated/amd-glm5-glm5.1-master.yaml b/configs/deprecated/amd-glm5-glm5.1-master.yaml index 8906a0a89b..7a7cf845eb 100644 --- a/configs/deprecated/amd-glm5-glm5.1-master.yaml +++ b/configs/deprecated/amd-glm5-glm5.1-master.yaml @@ -32,44 +32,6 @@ glm5-fp8-mi355x-sglang-mtp: search-space: - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } - - -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep; dp-attn false (NSA / MoRI path) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - glm5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: zai-org/GLM-5-FP8 diff --git a/configs/deprecated/amd-kimik2.5-8k1k-master.yaml b/configs/deprecated/amd-kimik2.5-8k1k-master.yaml index 252848fc69..73aed4fa27 100644 --- a/configs/deprecated/amd-kimik2.5-8k1k-master.yaml +++ b/configs/deprecated/amd-kimik2.5-8k1k-master.yaml @@ -81,110 +81,3 @@ kimik2.5-fp4-mi355x-atom: osl: 1024 search-space: - { tp: 4, conc-start: 4, conc-end: 128 } - -kimik2.5-fp4-mi355x-atom-disagg: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atomesh_202607121715 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1P(tp4)1D(tp4) - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P(tp4)2D(tp8) - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # All workers TP4 (real-weight sweep: TP8 decode is no better than TP4). - # Split across P/D topologies: 1D is stable at low conc; the high-conc tail - # runs on 2D so decode load is spread across two engines. - - isl: 8192 - osl: 1024 - search-space: - # 1P(TP4) 1D(TP4) = 2 nodes. Low concurrency only. - - spec-decoding: "none" - conc-list: [ 1, 2, 4 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P(TP4) 2D(TP4) = 3 nodes. High concurrency (decode KV/load spread over 2D). - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" diff --git a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml index ed6b003afc..5fbe55925b 100644 --- a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml @@ -143,63 +143,3 @@ minimaxm2.5-fp8-mi325x-vllm: search-space: - { tp: 2, conc-start: 4, conc-end: 64 } - { tp: 8, ep: 8, conc-start: 4, conc-end: 256 } - -# Diverged from minimaxm2.5-fp8-mi325x-vllm (agentic-coding sibling). Reasons below; -# the original minimaxm2.5-fp8-mi325x-vllm entry is left identical to origin/main so -# its fixed-seq-len sweep is unaffected. -# - image: 'vllm/vllm-openai-rocm:v0.18.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf' -minimaxm2.5-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-a6682d1d259cca69a9ae737ea5608fbbe7520031 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - # Prefill also needs EP=8: MiniMax M2.5 expert intermediate_size=1536, - # TP8 shards to 192 which is not divisible by FP8 block_n=128. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" diff --git a/configs/deprecated/amd-minimaxm3-8k1k-master.yaml b/configs/deprecated/amd-minimaxm3-8k1k-master.yaml index ad509a518b..c8f5386738 100644 --- a/configs/deprecated/amd-minimaxm3-8k1k-master.yaml +++ b/configs/deprecated/amd-minimaxm3-8k1k-master.yaml @@ -42,45 +42,6 @@ minimaxm3-fp8-mi355x-vllm-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 512, spec-decoding: mtp } - -# MiniMax-M3 MXFP4 MI355X vLLM disaggregated (prefill/decode) config. -minimaxm3-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P TP4 + 1D TP4 (2 nodes total), conc sweep 1..256 (single job, looped) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" -# MiniMax-M3 MXFP4 MI355X vLLM recipe. The pinned nightly includes upstream -# MiniMax-M3 Quark MXFP4 support (vllm-project/vllm#45794). Use the text-only -# language-model path and mirror the MXFP8 MI355X search space for a direct -# precision comparison. minimaxm3-fp4-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: amd/MiniMax-M3-MXFP4 @@ -181,109 +142,6 @@ minimaxm3-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - -minimaxm3-fp8-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 2P1D, DPA TP4 - - conc-list: [ 256, 512, 768, 1024 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D TP4 -minimaxm3-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 2P1D, DPA TP4 - - conc-list: [ 256, 512, 768, 1024 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D TP4 minimaxm3-fp8-mi300x-vllm: image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -370,68 +228,3 @@ minimaxm3-fp8-mi325x-vllm-mtp: - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - -# MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) sweep on the -# day-zero ROCm image, over the MoRI-IO KV-transfer pipeline (MoRI-patch-removal -# infra #1585). All workers are TP4, no EP: the single-node M3 MXFP8 recipe -# (minimaxm3-fp8-mi355x-vllm, PR #2003) found plain TP4 beats both TP8 and -# TP4/EP4 on tok/s/GPU for this model on gfx950, so prefill and decode both use -# TP4 and we tune the prefill:decode worker ratio (xP:yD) instead of TP. The -# mi355x-disagg pool has 3 nodes and the launcher places one worker per node -# (NUM_NODES = xP + yD), so every layout keeps xP + yD <= 3: -# - 1P-TP4 / 1D-TP4 (2 nodes): balanced, full concurrency curve. -# - 1P-TP4 / 2D-TP4 (3 nodes): decode-heavy, for the decode-bound 1k1k tail. -# - 2P-TP4 / 1D-TP4 (3 nodes): prefill-heavy, for the prefill-bound 8k1k tail. -# Per-worker serve flags live in -# benchmarks/multi_node/amd_utils/models_vllm.yaml (MiniMax-M3-MXFP8). -minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Prefill-heavy 2P TP4 + 1D TP4 (3 nodes): two half-node TP4 prefill workers - # keep the single TP4 decode engine fed for the prefill-bound 8k1k tail. - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9fcda3df36..d83fd7297d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6547,3 +6547,29 @@ - "Filter AgentX traces at the same 202,752-token context limit used by both TileRT roles so oversized Weka trajectories are excluded before replay." - "Pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, stacked on the AMD/native-router PR #7 and base runtime PR #1, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, backend-declared conversion GPU resources, pre-container NVIDIA driver-hook activation, and lossless Slurm container-environment exports." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2750 + +- config-keys: + - qwen3-0.6b-fp16-mi355x-sglang-srt-agg + - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg + scenario-type: + - fixed-seq-len + description: + - "Add small-model aggregate and 1P1D disaggregated correctness lanes for the shared MI355X srt-slurm launcher." + - "Use the native SGLang router for request routing and AMD MoRI for disaggregated KV transfer, without Dynamo, NIXL, or the deleted amd_utils control plane." + - "Remove the active and deprecated MI355X matrix entries that depended on amd_utils; configuration-specific child PRs reintroduce the active production lanes as native srt-slurm recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp + - dsr1-fp4-mi355x-sglang-disagg-mtp + scenario-type: + - fixed-seq-len + description: + - "Replace the DeepSeek-R1 MXFP4 model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI on MI355X." + - "Preserve the complete production STP and MTP 8k1k search space as 12 explicit runtime selectors: TP8 and DEP8/DPA 1P1D, 1P2D, and 2P1D topologies spanning c1 through c4096 with one-, two-, and three-step NEXTN variants." + - "Use lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809, the existing shared 78-shard MXFP4 checkpoint, FP8 KV cache, AITer attention, role-specific MoRI tuning, and the SGLang/DeepSeek-R1-NextN draft model." + - "Run the repository's unchanged benchmark_serving.py through srt-slurm's custom benchmark contract, with one allocation per topology and tuning family to amortize model initialization across each concurrency sweep." + - "Validate local production checkpoints before submission and prefetch optional external draft repositories once into the canonical shared Hugging Face cache instead of downloading them independently on every prefill and decode worker." + - "Pin SemiAnalysisAI/srt-slurm PR #7 at 10a58d2ebb4d756c423424049c736b909b72e14f so DP-attention SGLang workers inherit the validated P/D rank-routing contract, multi-node rank-offset preservation, and ATOM recipe flag normalization." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2633 diff --git a/runners/launch_mi355x-amds-srt.sh b/runners/launch_mi355x-amds-srt.sh new file mode 100755 index 0000000000..1b7c76c851 --- /dev/null +++ b/runners/launch_mi355x-amds-srt.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Shared MI355X entry point for AMD-capable srt-slurm recipes. Matrix rows opt +# in explicitly with CONFIG_FILE; recipe files own model- and topology-specific +# behavior while this launcher owns staging, submission, logs, and results. +SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" +SRT_SLURM_COMMIT="c87d7b34b009be920896126013ad6dc74c5a99d5" +SLURM_PARTITION="compute" +SHARED_BASE="/it-share/gharunners2/srt-slurm" +SHARED_HF_CACHE="/it-share/hf-hub-cache" +LEGACY_HF_CACHE="/it-share/hf_home" +SHARED_AIPERF_CACHE="/it-share/aiperf-cache" +SHARED_RESULTS="${SHARED_BASE}/results" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set by Actions}" +: "${RESULT_FILENAME:?RESULT_FILENAME must be set by the benchmark workflow}" +: "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}" +: "${IMAGE:?IMAGE must identify the SGLang container image}" +: "${MODEL:?MODEL must identify the Hugging Face model}" + +SGLANG_IMAGE="$IMAGE" +IMAGE_KEY="${SGLANG_IMAGE//\//_}" +IMAGE_KEY="${IMAGE_KEY//:/_}" +SHARED_IMAGE="${SHARED_BASE}/containers/${IMAGE_KEY}.sqsh" +LOCAL_IMAGE="/var/lib/squash/${IMAGE_KEY}.sqsh" +SRT_MODEL_LOCAL_PATH="${SRT_MODEL_LOCAL_PATH:-}" +SRT_DRAFT_MODEL="${SRT_DRAFT_MODEL:-}" + +CONFIG_PATH="${CONFIG_FILE%%:*}" +LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" +CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml" +[[ -f "$LOCAL_RECIPE" ]] || { echo "Missing recipe: $LOCAL_RECIPE" >&2; exit 1; } +[[ -f "$CLUSTER_PROFILE" ]] || { echo "Missing cluster profile: $CLUSTER_PROFILE" >&2; exit 1; } + +RUN_KEY="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUNNER_NAME:-runner}" +WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" +SRT_REPO_DIR="${WORK_DIR}/srt-slurm" +mkdir -p "$WORK_DIR" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" + +# Materialize one immutable shared squashfs and the requested model inputs. +# Production checkpoints may be Hugging Face repositories or an existing +# shared local directory. Legacy cache layouts are linked into the canonical +# HF_HOME/hub layout so large checkpoints are reused without copies. +STAGE_SCRIPT="${WORK_DIR}/stage-mi355x-runtime.sbatch" +cat > "$STAGE_SCRIPT" <"${SHARED_IMAGE}.lock" +flock -w 2400 9 +if ! unsquashfs -s "$SHARED_IMAGE" >/dev/null 2>&1; then + tmp="${SHARED_IMAGE}.tmp.\${SLURM_JOB_ID}" + rm -f "\$tmp" + local_image="${LOCAL_IMAGE}" + if unsquashfs -s "\$local_image" >/dev/null 2>&1; then + cp --sparse=always "\$local_image" "\$tmp" + else + enroot import -o "\$tmp" "docker://${SGLANG_IMAGE}" + fi + unsquashfs -s "\$tmp" >/dev/null + mv "\$tmp" "$SHARED_IMAGE" +fi +flock -u 9 +mkdir -p "$SHARED_HF_CACHE/hub" +seed_legacy_cache() { + local repo="\$1" + local cache_key="models--\${repo//\//--}" + local legacy_model_dir="$SHARED_HF_CACHE/\${cache_key}" + local canonical_model_dir="$SHARED_HF_CACHE/hub/\${cache_key}" + exec 8>"$SHARED_HF_CACHE/.\${cache_key}.stage.lock" + flock -w 2400 8 + if [[ ! -e "\$canonical_model_dir" ]]; then + if [[ -f "\$legacy_model_dir/refs/main" && -d "\$legacy_model_dir/snapshots" ]]; then + ln -s "../\${cache_key}" "\$canonical_model_dir" + elif [[ -f "$LEGACY_HF_CACHE/\${cache_key}/refs/main" && -d "$LEGACY_HF_CACHE/\${cache_key}/snapshots" ]]; then + ln -s "$LEGACY_HF_CACHE/\${cache_key}" "\$canonical_model_dir" + fi + fi + flock -u 8 +} + +model_repos=() +if [[ -n "$SRT_MODEL_LOCAL_PATH" ]]; then + python3 - "$SRT_MODEL_LOCAL_PATH" <<'PYMODEL' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +if not root.is_dir(): + raise SystemExit(f"local model directory does not exist: {root}") +for required in ("config.json", "tokenizer_config.json", "model.safetensors.index.json"): + if not (root / required).is_file(): + raise SystemExit(f"local model is missing {required}: {root}") +index = json.loads((root / "model.safetensors.index.json").read_text()) +shards = sorted(set(index.get("weight_map", {}).values())) +if not shards: + raise SystemExit(f"local model index has no shards: {root}") +missing = [shard for shard in shards if not (root / shard).is_file()] +if missing: + raise SystemExit(f"local model is missing {len(missing)} indexed shards: {missing[:5]}") +print(f"validated local model {root}: {len(shards)} indexed shards") +PYMODEL +else + model_repos+=("$MODEL") +fi +if [[ -n "$SRT_DRAFT_MODEL" ]]; then + model_repos+=("$SRT_DRAFT_MODEL") +fi +for repo in "\${model_repos[@]}"; do + seed_legacy_cache "\$repo" +done + +if (( \${#model_repos[@]} == 0 )); then + exit 0 +fi +model_repo_list="\$(IFS=,; echo "\${model_repos[*]}")" +srun --nodes=1 --ntasks=1 \ + --container-image="$SHARED_IMAGE" \ + --container-mounts="$SHARED_HF_CACHE:/hf_hub_cache,$LEGACY_HF_CACHE:$LEGACY_HF_CACHE" \ + --container-writable --container-remap-root --no-container-entrypoint \ + --export=ALL,HF_HOME=/hf_hub_cache,HF_HUB_CACHE=/hf_hub_cache/hub,HUGGINGFACE_HUB_CACHE=/hf_hub_cache/hub,MODEL_REPOS="\$model_repo_list" \ + python3 -c 'import os; from huggingface_hub import snapshot_download; [snapshot_download(repo) for repo in os.environ["MODEL_REPOS"].split(",") if repo]' +EOF +STAGE_JOB_ID=$(sbatch --wait --parsable "$STAGE_SCRIPT") +STAGE_JOB_ID="${STAGE_JOB_ID%%;*}" +echo "MI355X runtime prerequisites verified with Slurm job ${STAGE_JOB_ID}" + +git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" +git -C "$SRT_REPO_DIR" checkout "$SRT_SLURM_COMMIT" +ACTUAL_SRT_COMMIT=$(git -C "$SRT_REPO_DIR" rev-parse HEAD) +[[ "$ACTUAL_SRT_COMMIT" == "$SRT_SLURM_COMMIT" ]] || { + echo "srt-slurm checkout mismatch: $ACTUAL_SRT_COMMIT" >&2 + exit 1 +} + +mkdir -p "${SRT_REPO_DIR}/$(dirname "$CONFIG_PATH")" +cp "$LOCAL_RECIPE" "${SRT_REPO_DIR}/${CONFIG_PATH}" +cp "$CLUSTER_PROFILE" "${WORK_DIR}/srtslurm.yaml" +python3 - "${WORK_DIR}/srtslurm.yaml" "${SRT_REPO_DIR}/${CONFIG_PATH}" \ + "$GITHUB_WORKSPACE" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" "$SHARED_IMAGE" <<'PY' +import os +import sys +from pathlib import Path + +import yaml + +profile_path = Path(sys.argv[1]) +recipe_path = Path(sys.argv[2]) +workspace, results, aiperf_cache, image_path = sys.argv[3:] +needle = " /it-share/hf-hub-cache: /hf_hub_cache\n" +text = profile_path.read_text() +if text.count(needle) != 1: + raise SystemExit("expected exactly one Hugging Face cache mount") +profile_path.write_text( + text.replace( + needle, + needle + + f" {aiperf_cache}: /aiperf_mmap_cache\n" + + f" {workspace}: /infmax-workspace\n" + + f" {results}: /results\n", + ) +) + +recipe = yaml.safe_load(recipe_path.read_text()) +container_alias = recipe["model"]["container"] +profile = yaml.safe_load(profile_path.read_text()) +profile.setdefault("containers", {})[container_alias] = image_path +profile_path.write_text(yaml.safe_dump(profile, sort_keys=False)) + +benchmark_env = recipe.setdefault("benchmark", {}).setdefault("env", {}) +forwarded = ( + "AIPERF_EXPERIMENTAL_FAST", + "CONC", + "CONC_LIST", + "DURATION", + "EVAL_CONC", + "EVAL_LIMIT", + "EVAL_ONLY", + "FRAMEWORK", + "IS_AGENTIC", + "KV_OFFLOADING", + "MAX_MODEL_LEN", + "MODEL", + "MODEL_PREFIX", + "PRECISION", + "RESULT_FILENAME", + "RUN_EVAL", + "RUNNER_TYPE", + "SPEC_DECODING", + "TOTAL_CPU_DRAM_GB", +) +for key in forwarded: + value = os.environ.get(key) + if value: + benchmark_env[key] = value + +# The legacy MI355X launcher sized DP+EP admission and MoRI decode dispatch +# from each matrix point, rather than from a fixed recipe-wide ceiling. Keep +# that behavior when a single recipe is reused across multiple concurrencies. +if ( + os.environ.get("PREFILL_DP_ATTN", "false").lower() == "true" + and int(os.environ.get("PREFILL_EP", "1")) > 1 +): + concurrency = int(os.environ["CONC"]) + prefill = recipe["backend"]["sglang_config"]["prefill"] + decode = recipe["backend"]["sglang_config"]["decode"] + prefill["max-running-requests"] = concurrency + decode["max-running-requests"] = concurrency + + decode_tp = int(os.environ["DECODE_TP"]) + mtp_size = int(os.environ.get("DECODE_MTP_SIZE", "0")) + decode_environment = recipe["backend"]["decode_environment"] + decode_environment["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = str( + concurrency // decode_tp * (mtp_size + 1) + ) + +if os.environ.get("EVAL_ONLY", "false").lower() == "true" or os.environ.get("RUN_EVAL", "false").lower() == "true": + decode_env = recipe.get("backend", {}).get("decode_environment", {}) + for key in ( + "SGLANG_SIMULATE_ACC_LEN", + "SGLANG_SIMULATE_ACC_METHOD", + "SGLANG_SIMULATE_ACC_TOKEN_MODE", + ): + decode_env.pop(key, None) + server_config = recipe.get("backend", {}).get("sglang_config", {}) + for mode in ("prefill", "decode"): + server_config.get(mode, {}).pop("ep-dispatch-algorithm", None) +recipe_path.write_text(yaml.safe_dump(recipe, sort_keys=False)) +PY + +export PATH="$HOME/.local/bin:$PATH" +cd "$SRT_REPO_DIR" +uv venv --python 3.12 +uv pip install -e . +make setup-compute ARCH=x86_64 +source .venv/bin/activate +export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" +export SRTCTL_RUNTIME_SOURCE_DIR="$SRT_REPO_DIR" + +echo "Submitting ${CONFIG_PATH} with srt-slurm ${SRT_SLURM_COMMIT}" +set +e +SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \ + --tags "mi355x,inferencex,github-actions,${RUN_KEY}" 2>&1) +SRTCTL_RC=$? +set -e +echo "$SRTCTL_OUTPUT" +if [[ $SRTCTL_RC -ne 0 ]]; then + echo "srtctl apply failed with exit code ${SRTCTL_RC}" >&2 + exit "$SRTCTL_RC" +fi +JOB_ID=$(grep -oE 'Job [0-9]+' <<< "$SRTCTL_OUTPUT" | awk '{print $2}' | tail -1) +[[ -n "$JOB_ID" ]] || { echo "Unable to parse srt-slurm job ID" >&2; exit 1; } +echo "SRT_SLURM_JOB_ID=$JOB_ID" + +while squeue --noheader --jobs "$JOB_ID" | grep -q .; do + squeue --noheader --jobs "$JOB_ID" --format='srt-slurm %i %T %M %R' + sleep 15 +done + +read -r JOB_STATE JOB_EXIT JOB_NODELIST < <( + sacct -X --noheader --parsable2 --jobs "$JOB_ID" \ + --format=State,ExitCode,NodeList | head -1 | tr '|' ' ' +) +echo "srt-slurm job ${JOB_ID}: state=${JOB_STATE} exit=${JOB_EXIT} nodes=${JOB_NODELIST}" + +RESULT_DIR="${SHARED_RESULTS}/${JOB_ID}" +OUTPUT_LOG_DIR="${SHARED_BASE}/outputs/${JOB_ID}/logs" +mkdir -p "$GITHUB_WORKSPACE/LOGS" +if [[ -d "$OUTPUT_LOG_DIR" ]]; then + tar -C "$OUTPUT_LOG_DIR" -czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" . +fi +if [[ -d "$RESULT_DIR" ]]; then + cp -R "$RESULT_DIR/." "$GITHUB_WORKSPACE/LOGS/" +fi + +if [[ "${DISAGG:-false}" == "true" ]]; then + PREFILL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + DECODE_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP)) + TOTAL_GPUS=$((PREFILL_GPUS + DECODE_GPUS)) +else + TOTAL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP * ${PREFILL_PP_SIZE:-1} * ${PREFILL_PCP_SIZE:-1})) +fi + +if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + shopt -s nullglob + RESULTS=("$GITHUB_WORKSPACE/${RESULT_FILENAME}"_conc*.json) + shopt -u nullglob + [[ ${#RESULTS[@]} -gt 0 ]] || { + echo "No AgentX aggregate results found for ${RESULT_FILENAME}" >&2 + exit 1 + } + printf 'Collected %s\n' "${RESULTS[@]}" +else + shopt -s nullglob + RESULTS=("$RESULT_DIR"/fixed-seq/*.json) + shopt -u nullglob + [[ ${#RESULTS[@]} -gt 0 ]] || { echo "No fixed-sequence results found in $RESULT_DIR" >&2; exit 1; } + for result in "${RESULTS[@]}"; do + concurrency=$(basename "$result" | sed -n 's/.*-c\([0-9][0-9]*\)\.json/\1/p') + [[ -n "$concurrency" ]] || { echo "Cannot parse concurrency from $result" >&2; exit 1; } + if [[ "${DISAGG:-false}" == "true" ]]; then + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS}_gen_${DECODE_GPUS}.json" + else + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}.json" + fi + cp "$result" "$output" + echo "Collected $output" + done +fi + +if [[ "$JOB_STATE" != COMPLETED || "$JOB_EXIT" != 0:0 ]]; then + echo "srt-slurm validation failed: ${JOB_STATE} (${JOB_EXIT})" >&2 + exit 1 +fi + +printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" +echo "MI355X srt-slurm validation completed successfully" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 49c3af0cf5..a111f19637 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +if [[ -n "${CONFIG_FILE:-}" ]]; then + exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_mi355x-amds-srt.sh" +fi + scancel_sync() { local jobid=$1 local timeout=${2:-600} diff --git a/utils/evals/test_batched_eval.py b/utils/evals/test_batched_eval.py index a5d6df0085..11b7542197 100644 --- a/utils/evals/test_batched_eval.py +++ b/utils/evals/test_batched_eval.py @@ -228,17 +228,15 @@ def test_validate_scores_checks_threshold_for_every_concurrency( def test_amd_multinode_container_forwards_eval_concurrency_list() -> None: - job_slurm = ( + launcher = ( Path(__file__).resolve().parents[2] - / "benchmarks" - / "multi_node" - / "amd_utils" - / "job.slurm" + / "runners" + / "launch_mi355x-amds-srt.sh" ) - contents = job_slurm.read_text() + contents = launcher.read_text() - assert r'-e \"EVAL_CONC=\$EVAL_CONC\"' in contents - assert "-e EVAL_CONC\n" not in contents + assert ' "EVAL_CONC",' in contents + assert 'benchmark_env[key] = value' in contents workflow = ( Path(__file__).resolve().parents[2]