From 7877188969de32ba8c84d4dfc6f25ef1d0f31702 Mon Sep 17 00:00:00 2001 From: billishyahao Date: Thu, 13 Aug 2026 11:21:05 +0000 Subject: [PATCH 1/2] [AMD] add dsv4 agentx sglang mtp --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 273 ++++++++++++++++++ configs/amd-master.yaml | 25 ++ perf-changelog.yaml | 8 + 3 files changed, 306 insertions(+) create mode 100644 benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh new file mode 100644 index 000000000..61add3eaf --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using SGLang +# with EAGLE/MTP speculative decoding. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +# ROCR/HIP visibility under slurm cgroups. +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# A server killed on this node minutes earlier (previous job, crashed run) +# can still be draining its HBM: KFD reclaim takes minutes, and booting into a +# half-drained node fails RCCL init with HIP 'unhandled cuda error' / +# 'invalid argument'. DeepSeek-V4-Pro is an 805 GiB checkpoint, so the drain +# window here is at the long end. Wait for the GPUs to come back before +# launching. Per-GPU threshold: idle nodes hold a small driver/firmware VRAM +# baseline (observed up to ~4%/GPU), while a draining or occupied GPU sits at +# 50-90%. Require every GPU <= 10%. +GPU_CLEAN=false +for i in $(seq 1 90); do + VRAM_MAX=$(rocm-smi --showmemuse 2>/dev/null | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" | awk '{if ($NF > m) m = $NF} END {print m+0}') + if [ "${VRAM_MAX:-0}" -le 10 ]; then echo "GPUs clean (vram%max=$VRAM_MAX after $((i*10))s)"; GPU_CLEAN=true; break; fi + echo "waiting for prior-job GPU memory reclaim: vram%max=$VRAM_MAX"; sleep 10 +done +[ "$GPU_CLEAN" = "true" ] || { echo "Error: GPUs still draining prior job's memory after 15min" >&2; exit 1; } + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +mkdir -p "$RESULT_DIR" + +# ---- Client config ---------------------------------------------------------- +export PYTHONNOUSERSITE=1 +# Agentic warmup dispatches hundreds of large prompts at once; allow up to +# 15 minutes of TCP progress before AIPerf declares a connection dead. +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +# AIPerf pins one pooled keep-alive connection per session (client-side +# keep-alive 300s) while uvicorn's default SGLANG_TIMEOUT_KEEP_ALIVE is 5s; +# inter-turn idle gaps can reuse a socket exactly as the server closes it. +# Outlast the client pool so the race cannot occur. +export SGLANG_TIMEOUT_KEEP_ALIVE=900 + +# ---- DSv4 kernel routing / thinking mode ------------------------------------ +# Mirrors the deleted spec-none sibling plus the DSv4 block in +# benchmarks/multi_node/amd_utils/env.sh. AgentX measures the thinking-on +# regime, which is also the golden-AL curve committed for this model. +export SGLANG_DEFAULT_THINKING=1 +export SGLANG_DSV4_REASONING_EFFORT=high +export SGLANG_USE_ROCM700A=0 +export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton +export AITER_BF16_FP8_MOE_BOUND=0 + +# Unified radix tree: per-component (full-attn / SWA) cache management for +# hybrid-attention models, plus proactive release of out-of-window SWA KV +# slots during chunked prefill. Without the latter, in-flight requests pin SWA +# KV for their whole context and the trailing window of cached sessions gets +# flushed under LRU, collapsing the effective prefix-cache hit rate on +# multi-turn agentic workloads. +export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 +export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + +# ---- HiCache (host DRAM KV tier) -------------------------------------------- +# Per-arm L2 sizing: host pinned memory is roughly +# HICACHE_RATIO * (per-rank device KV pool) * TP, which must stay under the +# node's ~2.7 TB of DRAM. The deleted spec-none sibling used ratio 4 with a +# smaller device pool; at TP8 with mem-fraction-static 0.85 that would +# oversubscribe host DRAM, so this recipe starts from 1.5 (the value validated +# on this cluster by glm5.2_fp4_mi355x_sglang_mtp.sh) and leaves every knob +# overridable for tuning. +CACHE_ARGS=() +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + hicache) + HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected: hicache)" >&2 + exit 1 + ;; + esac +fi + +# ---- Parallelism ------------------------------------------------------------ +# NOTE: the DP-attention path below is currently DORMANT (no dp-attn arms in +# amd-master.yaml for this key). It is kept so a future arm can enable it +# without rebuilding the router plumbing: sglang-router fronts the DP ranks +# with consistent hashing on the AIPerf correlation id, keeping multi-turn +# sessions on the DP rank that holds their radix/hicache prefix. +USE_SGLANG_ROUTER=false +SGLANG_BACKEND_PORT="$PORT" +# Small prefill chunks interleave long-context agentic prefills across +# requests instead of letting one ~100K-token prefill monopolize the engine +# (the conc>=16 queue-saturation / decode-stall failure mode). 8192 = 32*256, +# a page-size multiple well under the dsv4 compressor kernel's uint16 token +# cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. +CHUNKED_PREFILL_SIZE=8192 +# MTP adds a draft KV pool and extra graph captures on top of the spec-none +# footprint, which ran at 0.90. +MEM_FRACTION_STATIC=0.85 +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "$DP_ATTENTION" = "true" ]; then + USE_SGLANG_ROUTER=true + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + SGLANG_BACKEND_PORT=$((PORT + 1)) + SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) + SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) + + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + + # Chunked prefill is a whole-engine budget, so widen it by the DP degree. + CHUNKED_PREFILL_SIZE=$((8192 * TP)) + PARALLEL_ARGS+=( + --dp "$TP" + --enable-dp-attention + --enable-prefill-delayer + ) +fi + +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--ep-size "$EP_SIZE") +fi + +# AgentX concurrency counts live session trees, not individual requests. +# Subagent fan-out can push instantaneous request concurrency above CONC, so +# leave 2x headroom rather than clipping those bursts at the scheduler. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +[ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 +CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS +[ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 + +# Saturation arms carry a larger in-flight working set than the 30-minute +# default warmup drain allows. +if [ "$CONC" -ge 32 ]; then + export AGENTIC_WARMUP_GRACE_PERIOD=3600 +fi + +# ---- Speculative decoding --------------------------------------------------- +# DeepSeek-V4 ships a built-in MTP head, loaded through the EAGLE spec path +# with eagle-topk 1 (a single MTP chain); NOT NEXTN, whose V3/R1 loader +# crashes on the V4 architecture. Depth 3 matches the vLLM agentic sibling +# (dsv4-fp4-mi355x-vllm-agentic-mtp) and the fixed-seq-len SGLang MTP recipe. +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +# Throughput runs pin acceptance to the committed golden AL for this model, +# thinking mode, and draft length (golden_al_distribution/dsv4_mtp.yaml: +# thinking_on, 3 -> 2.49). Eval-only runs keep real target verification so +# accuracy stays meaningful. +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +# ---- Launch ----------------------------------------------------------------- +# No --chat-template: the AgentX traces are tool-heavy, and +# chat_templates/deepseek_v4_thinking.jinja renders only system/user/assistant +# (tool definitions and role: tool messages are silently dropped, which would +# truncate prompts and distort ISL). The multi-node DeepSeek-V4-Pro-AgentX +# profile and the vLLM agentic sibling both serve DSv4 without an override. +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$SGLANG_BACKEND_PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --attention-backend dsv4 + --page-size 256 + --swa-full-tokens-ratio 0.10 + --kv-cache-dtype fp8_e4m3 + --disable-shared-experts-fusion + --tool-call-parser deepseekv4 + --reasoning-parser deepseek-v4 + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${SPEC_ARGS[@]}" + "${CACHE_ARGS[@]}" + # MTP draft-token forward passes under long-context agentic load block the + # scheduler long enough to trip the 1800s watchdog mid-warmup. + --watchdog-timeout 3600 + --enable-metrics +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" + +{ + echo "=== SGLANG_* env vars at launch ===" + env | grep -E '^SGLANG_' | sort + echo "===================================" +} | tee "$SERVER_LOG" + +echo "Starting SGLang server for MI355X..." +"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_SGLANG_ROUTER" = "true" ]; then + echo "Starting SGLang router on port $PORT for $TP DP ranks..." + "${SGLANG_ROUTER_CMD[@]}" \ + --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ + --policy consistent_hashing \ + --request-id-headers x-correlation-id \ + --dp-aware \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ + --connect-timeout-secs 900 \ + --request-timeout-secs 14400 \ + --disable-health-check \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c94029484..d6f65d9b9 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1718,3 +1718,28 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + +# DeepSeek-V4-Pro FP4 agentic-coding benchmark on MI355X via SGLang with the +# in-checkpoint MTP head (EAGLE path, depth 3 -> golden AL 2.49 for +# thinking_on, golden_al_distribution/dsv4_mtp.yaml). Restores the single-node +# SGLang AgentX coverage removed in de493d859 (PR #2531, which deleted the +# spec-none dsv4-fp4-mi355x-sglang-agentic-hicache key and its script) so the +# aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp +# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image matches the +# already-green disagg AgentX entry. Pure TP8 only: DSA + dp-attention hangs a +# collective under long-context prefill, so no DEP arm ships until that path is +# validated. conc 16 appears on both arms to isolate the host KV tier's gain. +dsv4-fp4-mi355x-sglang-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 16], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6082f2b86..76937dac9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,3 +5918,11 @@ - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + description: + - "Add DeepSeek-V4-Pro FP4 single-node SGLang AgentX recipe on MI355X with EAGLE/MTP (num-steps 3, num-draft-tokens 4) and SGLANG_SIMULATE_ACC_LEN=2.49 from the committed thinking_on golden AL curve" + - "Restores the single-node SGLang agentic coverage deleted in PR #2531; TP8 arms with and without HiCache DRAM offload" + - "Image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 021153c3387d3e7ed5cf742654eb4a371fef4d20 Mon Sep 17 00:00:00 2001 From: billishyahao Date: Thu, 13 Aug 2026 15:38:57 +0000 Subject: [PATCH 2/2] bump image --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d6f65d9b9..84c2a8fbb 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1730,7 +1730,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: # collective under long-context prefill, so no DEP arm ships until that path is # validated. conc 16 appears on both arms to isolate the host KV tier's gain. dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds